diff --git a/cmd/cli-river/main.go b/cmd/cli-river/main.go index 8923c719..55e11ff1 100644 --- a/cmd/cli-river/main.go +++ b/cmd/cli-river/main.go @@ -3,7 +3,7 @@ package main import ( "encoding/json" "fmt" - "io/ioutil" + "io" "os" "path/filepath" "strings" @@ -54,7 +54,7 @@ func main() { connInfoPath := filepath.Join(_DbPath, fmt.Sprintf("connInfo.%s", _DbID)) file, err := os.Open(connInfoPath) if err == nil { - b, _ := ioutil.ReadAll(file) + b, _ := io.ReadAll(file) err := json.Unmarshal(b, connInfo) if err != nil { _Shell.Print(err.Error()) diff --git a/go.mod b/go.mod index 10a06082..ad53528a 100644 --- a/go.mod +++ b/go.mod @@ -3,86 +3,93 @@ module github.com/ronaksoft/river-sdk go 1.18 require ( - github.com/Workiva/go-datastructures v1.0.53 + github.com/Workiva/go-datastructures v1.1.1 github.com/beeker1121/goque v2.1.0+incompatible - github.com/blevesearch/bleve v1.0.14 + github.com/blevesearch/bleve/v2 v2.3.10 github.com/boltdb/bolt v1.3.1 github.com/dgraph-io/badger/v2 v2.2007.4 github.com/dustin/go-humanize v1.0.1 github.com/fatih/color v1.16.0 github.com/getsentry/sentry-go v0.27.0 - github.com/gobwas/ws v1.1.0 - github.com/juju/ratelimit v1.0.1 + github.com/gobwas/ws v1.3.2 + github.com/juju/ratelimit v1.0.2 github.com/monnand/dhkx v0.0.0-20180522003156-9e5b033f1ac4 - github.com/nyaruka/phonenumbers v1.0.70 - github.com/olekukonko/tablewriter v0.0.4 + github.com/nyaruka/phonenumbers v1.3.1 + github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 github.com/ronaksoft/river-msg v0.0.0-20240204095508-6868e75c8967 github.com/ronaksoft/rony v0.12.29 - github.com/siongui/instago v0.0.0-20200627103523-54424cb565ea + github.com/siongui/instago v0.0.0-20230301150703-9da679ec808b github.com/smartystreets/goconvey v1.6.4 - github.com/spf13/cobra v1.2.1 - github.com/spf13/viper v1.8.1 + github.com/spf13/cobra v1.8.0 + github.com/spf13/viper v1.18.2 github.com/tidwall/buntdb v1.1.7 github.com/valyala/fasthttp v1.40.0 github.com/valyala/tcplisten v1.0.0 - go.uber.org/zap v1.18.1 + go.uber.org/zap v1.21.0 golang.org/x/mobile v0.0.0-20210527171505-7e972142eb43 - golang.org/x/sync v0.1.0 - google.golang.org/protobuf v1.29.1 + golang.org/x/sync v0.5.0 + google.golang.org/protobuf v1.31.0 gopkg.in/abiosoft/ishell.v2 v2.0.0 ) require ( - github.com/RoaringBitmap/roaring v0.5.1 // indirect + github.com/RoaringBitmap/roaring v1.2.3 // indirect github.com/abiosoft/ishell v2.0.0+incompatible // indirect github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db // indirect github.com/allegro/bigcache/v2 v2.2.5 // indirect github.com/andybalholm/brotli v1.0.5 // indirect - github.com/armon/go-metrics v0.3.9 // indirect + github.com/armon/go-metrics v0.4.1 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/bits-and-blooms/bitset v1.2.0 // indirect + github.com/blevesearch/bleve_index_api v1.0.6 // indirect + github.com/blevesearch/geo v0.1.18 // indirect github.com/blevesearch/go-porterstemmer v1.0.3 // indirect - github.com/blevesearch/mmap-go v1.0.2 // indirect - github.com/blevesearch/segment v0.9.0 // indirect + github.com/blevesearch/gtreap v0.1.1 // indirect + github.com/blevesearch/mmap-go v1.0.4 // indirect + github.com/blevesearch/scorch_segment_api/v2 v2.1.6 // indirect + github.com/blevesearch/segment v0.9.1 // indirect github.com/blevesearch/snowballstem v0.9.0 // indirect - github.com/blevesearch/zap/v11 v11.0.14 // indirect - github.com/blevesearch/zap/v12 v12.0.14 // indirect - github.com/blevesearch/zap/v13 v13.0.6 // indirect - github.com/blevesearch/zap/v14 v14.0.5 // indirect - github.com/blevesearch/zap/v15 v15.0.3 // indirect + github.com/blevesearch/upsidedown_store_api v1.0.2 // indirect + github.com/blevesearch/vellum v1.0.10 // indirect + github.com/blevesearch/zapx/v11 v11.3.10 // indirect + github.com/blevesearch/zapx/v12 v12.3.10 // indirect + github.com/blevesearch/zapx/v13 v13.3.10 // indirect + github.com/blevesearch/zapx/v14 v14.3.10 // indirect + github.com/blevesearch/zapx/v15 v15.3.13 // indirect github.com/c-bata/go-prompt v0.2.5 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.1.1 // indirect - github.com/couchbase/vellum v1.0.2 // indirect github.com/dgraph-io/badger/v3 v3.2103.1 // indirect github.com/dgraph-io/ristretto v0.1.0 // indirect github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect - github.com/fsnotify/fsnotify v1.4.9 // indirect - github.com/glycerine/go-unsnap-stream v0.0.0-20190901134440-81cf024a9e0a // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/gobwas/httphead v0.1.0 // indirect github.com/gobwas/pool v0.2.1 // indirect github.com/gocql/gocql v0.0.0-20200131111108-92af2e088537 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect - github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/btree v1.0.1 // indirect github.com/google/flatbuffers v1.12.0 // indirect github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99 // indirect github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/hashicorp/errwrap v1.0.0 // indirect - github.com/hashicorp/go-immutable-radix v1.1.0 // indirect + github.com/hashicorp/go-immutable-radix v1.3.1 // indirect github.com/hashicorp/go-msgpack v1.1.5 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-sockaddr v1.0.2 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/jtolds/gls v4.20.0+incompatible // indirect - github.com/klauspost/compress v1.16.0 // indirect - github.com/magiconair/properties v1.8.5 // indirect + github.com/klauspost/compress v1.17.0 // indirect + github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easygo v0.0.0-20190618140210-3c14a0dc985f // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect @@ -90,12 +97,13 @@ require ( github.com/mattn/go-tty v0.0.3 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/miekg/dns v1.1.43 // indirect - github.com/mitchellh/mapstructure v1.4.1 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mschoch/smat v0.2.0 // indirect github.com/panjf2000/ants/v2 v2.4.6 // indirect github.com/panjf2000/gnet v1.5.0 // indirect - github.com/pelletier/go-toml v1.9.3 // indirect - github.com/philhofer/fwd v1.1.1 // indirect + github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/pkg/term v1.1.0 // indirect github.com/prometheus/client_golang v1.11.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect @@ -103,16 +111,17 @@ require ( github.com/prometheus/procfs v0.6.0 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/ronaksoft/memberlist v0.2.7 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/scylladb/go-reflectx v1.0.1 // indirect github.com/scylladb/gocqlx/v2 v2.4.0 // indirect github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d // indirect - github.com/spf13/afero v1.6.0 // indirect - github.com/spf13/cast v1.3.1 // indirect - github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/steveyen/gtreap v0.1.0 // indirect - github.com/subosito/gotenv v1.2.0 // indirect + github.com/subosito/gotenv v1.6.0 // indirect github.com/syndtr/goleveldb v1.0.0 // indirect github.com/tidwall/btree v0.3.0 // indirect github.com/tidwall/gjson v1.6.7 // indirect @@ -121,20 +130,19 @@ require ( github.com/tidwall/pretty v1.0.2 // indirect github.com/tidwall/rtree v0.0.0-20201103190202-0d877048965d // indirect github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563 // indirect - github.com/tinylib/msgp v1.1.5 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect - github.com/willf/bitset v1.1.11 // indirect - go.etcd.io/bbolt v1.3.5 // indirect - go.opencensus.io v0.23.0 // indirect - go.uber.org/atomic v1.8.0 // indirect - go.uber.org/multierr v1.7.0 // indirect - golang.org/x/mod v0.8.0 // indirect - golang.org/x/net v0.8.0 // indirect - golang.org/x/sys v0.14.0 // indirect - golang.org/x/text v0.8.0 // indirect - golang.org/x/tools v0.6.0 // indirect + go.etcd.io/bbolt v1.3.7 // indirect + go.opencensus.io v0.24.0 // indirect + go.uber.org/atomic v1.9.0 // indirect + go.uber.org/multierr v1.9.0 // indirect + golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 // indirect + golang.org/x/mod v0.14.0 // indirect + golang.org/x/net v0.19.0 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect + golang.org/x/tools v0.16.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 4f7ab572..8cbf4ce1 100644 --- a/go.sum +++ b/go.sum @@ -49,12 +49,11 @@ github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKz github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.5 h1:zl/OfRA6nftbBK9qTohYBJ5xvw6C/oNKizR7cZGl3cI= github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= -github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= -github.com/RoaringBitmap/roaring v0.5.1 h1:ugdwntNygzk1FZnmtxUr+jM9AYrpU3I3zpt49npDWVo= -github.com/RoaringBitmap/roaring v0.5.1/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= +github.com/RoaringBitmap/roaring v1.2.3 h1:yqreLINqIrX22ErkKI0vY47/ivtJr6n+kMhVOVmhWBY= +github.com/RoaringBitmap/roaring v1.2.3/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= -github.com/Workiva/go-datastructures v1.0.53 h1:J6Y/52yX10Xc5JjXmGtWoSSxs3mZnGSaq37xZZh7Yig= -github.com/Workiva/go-datastructures v1.0.53/go.mod h1:1yZL+zfsztete+ePzZz/Zb1/t5BnDuE2Ya2MMGhzP6A= +github.com/Workiva/go-datastructures v1.1.1 h1:9G5u1UqKt6ABseAffHGNfbNQd7omRlWE5QaxNruzhE0= +github.com/Workiva/go-datastructures v1.1.1/go.mod h1:1yZL+zfsztete+ePzZz/Zb1/t5BnDuE2Ya2MMGhzP6A= github.com/abiosoft/ishell v2.0.0+incompatible h1:zpwIuEHc37EzrsIYah3cpevrIc8Oma7oZPxr03tlmmw= github.com/abiosoft/ishell v2.0.0+incompatible/go.mod h1:HQR9AqF2R3P4XXpMpI0NAzgHf/aS6+zVXRj14cVk9qg= github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db h1:CjPUSXOiYptLbTdr1RceuZgSFDQ7U15ITERUGrUORx8= @@ -75,8 +74,9 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-metrics v0.3.9 h1:O2sNqxBdvq8Eq5xmzljcYzAORli6RWCvEym4cJf9m18= github.com/armon/go-metrics v0.3.9/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= +github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= +github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/beeker1121/goque v2.1.0+incompatible h1:m5pZ5b8nqzojS2DF2ioZphFYQUqGYsDORq6uefUItPM= @@ -90,30 +90,41 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYEDvkta6I8/rnYM5gSdSV2tJ6XbZuEtY= github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= +github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= +github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= -github.com/blevesearch/bleve v1.0.14 h1:Q8r+fHTt35jtGXJUM0ULwM3Tzg+MRfyai4ZkWDy2xO4= -github.com/blevesearch/bleve v1.0.14/go.mod h1:e/LJTr+E7EaoVdkQZTfoz7dt4KoDNvDbLb8MSKuNTLQ= -github.com/blevesearch/blevex v1.0.0 h1:pnilj2Qi3YSEGdWgLj1Pn9Io7ukfXPoQcpAI1Bv8n/o= -github.com/blevesearch/blevex v1.0.0/go.mod h1:2rNVqoG2BZI8t1/P1awgTKnGlx5MP9ZbtEciQaNhswc= -github.com/blevesearch/cld2 v0.0.0-20200327141045-8b5f551d37f5/go.mod h1:PN0QNTLs9+j1bKy3d/GB/59wsNBFC4sWLWG3k69lWbc= +github.com/blevesearch/bleve/v2 v2.3.10 h1:z8V0wwGoL4rp7nG/O3qVVLYxUqCbEwskMt4iRJsPLgg= +github.com/blevesearch/bleve/v2 v2.3.10/go.mod h1:RJzeoeHC+vNHsoLR54+crS1HmOWpnH87fL70HAUCzIA= +github.com/blevesearch/bleve_index_api v1.0.6 h1:gyUUxdsrvmW3jVhhYdCVL6h9dCjNT/geNU7PxGn37p8= +github.com/blevesearch/bleve_index_api v1.0.6/go.mod h1:YXMDwaXFFXwncRS8UobWs7nvo0DmusriM1nztTlj1ms= +github.com/blevesearch/geo v0.1.18 h1:Np8jycHTZ5scFe7VEPLrDoHnnb9C4j636ue/CGrhtDw= +github.com/blevesearch/geo v0.1.18/go.mod h1:uRMGWG0HJYfWfFJpK3zTdnnr1K+ksZTuWKhXeSokfnM= github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo= github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M= -github.com/blevesearch/mmap-go v1.0.2 h1:JtMHb+FgQCTTYIhtMvimw15dJwu1Y5lrZDMOFXVWPk0= -github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA= -github.com/blevesearch/segment v0.9.0 h1:5lG7yBCx98or7gK2cHMKPukPZ/31Kag7nONpoBt22Ac= -github.com/blevesearch/segment v0.9.0/go.mod h1:9PfHYUdQCgHktBgvtUOF4x+pc4/l8rdH0u5spnW85UQ= +github.com/blevesearch/gtreap v0.1.1 h1:2JWigFrzDMR+42WGIN/V2p0cUvn4UP3C4Q5nmaZGW8Y= +github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgYICSZ3w0tYk= +github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc= +github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs= +github.com/blevesearch/scorch_segment_api/v2 v2.1.6 h1:CdekX/Ob6YCYmeHzD72cKpwzBjvkOGegHOqhAkXp6yA= +github.com/blevesearch/scorch_segment_api/v2 v2.1.6/go.mod h1:nQQYlp51XvoSVxcciBjtvuHPIVjlWrN1hX4qwK2cqdc= +github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU= +github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw= github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s= github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs= -github.com/blevesearch/zap/v11 v11.0.14 h1:IrDAvtlzDylh6H2QCmS0OGcN9Hpf6mISJlfKjcwJs7k= -github.com/blevesearch/zap/v11 v11.0.14/go.mod h1:MUEZh6VHGXv1PKx3WnCbdP404LGG2IZVa/L66pyFwnY= -github.com/blevesearch/zap/v12 v12.0.14 h1:2o9iRtl1xaRjsJ1xcqTyLX414qPAwykHNV7wNVmbp3w= -github.com/blevesearch/zap/v12 v12.0.14/go.mod h1:rOnuZOiMKPQj18AEKEHJxuI14236tTQ1ZJz4PAnWlUg= -github.com/blevesearch/zap/v13 v13.0.6 h1:r+VNSVImi9cBhTNNR+Kfl5uiGy8kIbb0JMz/h8r6+O4= -github.com/blevesearch/zap/v13 v13.0.6/go.mod h1:L89gsjdRKGyGrRN6nCpIScCvvkyxvmeDCwZRcjjPCrw= -github.com/blevesearch/zap/v14 v14.0.5 h1:NdcT+81Nvmp2zL+NhwSvGSLh7xNgGL8QRVZ67njR0NU= -github.com/blevesearch/zap/v14 v14.0.5/go.mod h1:bWe8S7tRrSBTIaZ6cLRbgNH4TUDaC9LZSpRGs85AsGY= -github.com/blevesearch/zap/v15 v15.0.3 h1:Ylj8Oe+mo0P25tr9iLPp33lN6d4qcztGjaIsP51UxaY= -github.com/blevesearch/zap/v15 v15.0.3/go.mod h1:iuwQrImsh1WjWJ0Ue2kBqY83a0rFtJTqfa9fp1rbVVU= +github.com/blevesearch/upsidedown_store_api v1.0.2 h1:U53Q6YoWEARVLd1OYNc9kvhBMGZzVrdmaozG2MfoB+A= +github.com/blevesearch/upsidedown_store_api v1.0.2/go.mod h1:M01mh3Gpfy56Ps/UXHjEO/knbqyQ1Oamg8If49gRwrQ= +github.com/blevesearch/vellum v1.0.10 h1:HGPJDT2bTva12hrHepVT3rOyIKFFF4t7Gf6yMxyMIPI= +github.com/blevesearch/vellum v1.0.10/go.mod h1:ul1oT0FhSMDIExNjIxHqJoGpVrBpKCdgDQNxfqgJt7k= +github.com/blevesearch/zapx/v11 v11.3.10 h1:hvjgj9tZ9DeIqBCxKhi70TtSZYMdcFn7gDb71Xo/fvk= +github.com/blevesearch/zapx/v11 v11.3.10/go.mod h1:0+gW+FaE48fNxoVtMY5ugtNHHof/PxCqh7CnhYdnMzQ= +github.com/blevesearch/zapx/v12 v12.3.10 h1:yHfj3vXLSYmmsBleJFROXuO08mS3L1qDCdDK81jDl8s= +github.com/blevesearch/zapx/v12 v12.3.10/go.mod h1:0yeZg6JhaGxITlsS5co73aqPtM04+ycnI6D1v0mhbCs= +github.com/blevesearch/zapx/v13 v13.3.10 h1:0KY9tuxg06rXxOZHg3DwPJBjniSlqEgVpxIqMGahDE8= +github.com/blevesearch/zapx/v13 v13.3.10/go.mod h1:w2wjSDQ/WBVeEIvP0fvMJZAzDwqwIEzVPnCPrz93yAk= +github.com/blevesearch/zapx/v14 v14.3.10 h1:SG6xlsL+W6YjhX5N3aEiL/2tcWh3DO75Bnz77pSwwKU= +github.com/blevesearch/zapx/v14 v14.3.10/go.mod h1:qqyuR0u230jN1yMmE4FIAuCxmahRQEOehF78m6oTgns= +github.com/blevesearch/zapx/v15 v15.3.13 h1:6EkfaZiPlAxqXz0neniq35my6S48QI94W/wyhnpDHHQ= +github.com/blevesearch/zapx/v15 v15.3.13/go.mod h1:Turk/TNRKj9es7ZpKK95PS7f6D44Y7fAFy8F4LXQtGg= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= @@ -142,19 +153,12 @@ github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8Nz github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/couchbase/ghistogram v0.1.0/go.mod h1:s1Jhy76zqfEecpNWJfWUiKZookAFaiGOEoyzgHt9i7k= -github.com/couchbase/moss v0.1.0/go.mod h1:9MaHIaRuy9pvLPUJxB8sh8OrLfyDczECVL37grCIubs= -github.com/couchbase/vellum v1.0.2 h1:BrbP0NKiyDdndMPec8Jjhy0U47CZ0Lgx3xUC2r9rZqw= -github.com/couchbase/vellum v1.0.2/go.mod h1:FcwrEivFpNi24R3jLOs3n+fs5RnuQnQqCLBJ1uAg1W4= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d h1:SwD98825d6bdB+pEuTxWOXiSjBrHdOl/UVp75eI7JT8= -github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d/go.mod h1:URriBxXwVq5ijiJ12C7iIZqlA69nTlI+LgI6/pwftG8= -github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= -github.com/cznic/strutil v0.0.0-20181122101858-275e90344537/go.mod h1:AHHPPPXTw0h6pVabbcbyGRK1DckRn7r/STdZEeIDzZc= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgraph-io/badger/v2 v2.2007.4 h1:TRWBQg8UrlUhaFdco01nO2uXwzKS7zd+HVdwV/GHc4o= github.com/dgraph-io/badger/v2 v2.2007.4/go.mod h1:vSw/ax2qojzbN6eXHIx6KPKtCSHJN/Uz0X0VPruTIhk= @@ -179,9 +183,6 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= -github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= -github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= -github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= @@ -189,9 +190,11 @@ github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4Nij github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BMXYYRWTLOJKlh+lOBt6nUQgXAfB7oVIQt5cNreqSLI= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:rZfgFAXFS/z/lEd6LJmf9HVZ1LkgYiHx5pHhV5DR16M= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= github.com/getsentry/sentry-go v0.11.0/go.mod h1:KBQIxiZAetw62Cj8Ri964vAEWVdgfaUCn30Q3bCvANo= github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= @@ -201,11 +204,6 @@ github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NB github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= github.com/gin-gonic/gin v1.5.0/go.mod h1:Nd6IXA8m5kNZdNEHMBd93KT+mdY3+bewLgRvmCsR2Do= -github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= -github.com/glycerine/go-unsnap-stream v0.0.0-20190901134440-81cf024a9e0a h1:FQqoVvjbiUioBBFUL5up+h+GdCa/AnJsL/1bIs/veSI= -github.com/glycerine/go-unsnap-stream v0.0.0-20190901134440-81cf024a9e0a/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= -github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 h1:gclg6gY70GLy3PbkQ1AERPfmLMMagS60DKF78eWwLn8= -github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= @@ -242,8 +240,8 @@ github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/gobwas/ws v1.0.3/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= -github.com/gobwas/ws v1.1.0 h1:7RFti/xnNkMJnrK7D1yQ/iCIB5OrrY/54/H930kIbHA= -github.com/gobwas/ws v1.1.0/go.mod h1:nzvNcVha5eUziGrbxFCo6qFIojQHjJV5cLYIbezhfL0= +github.com/gobwas/ws v1.3.2 h1:zlnbNHxumkRvfPWgfXu8RBwyNR1x8wh9cf5PTOCqs9Q= +github.com/gobwas/ws v1.3.2/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY= github.com/gocql/gocql v0.0.0-20200131111108-92af2e088537 h1:NaMut1fdw76YYX/TPinSAbai4DShF5tPort3bHpET6g= github.com/gocql/gocql v0.0.0-20200131111108-92af2e088537/go.mod h1:DL0ekTmBSTdlNF25Orwt/JMzqIq3EJ4MVa/J/uK64OY= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -251,12 +249,15 @@ github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRx github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 h1:gtexQ/VGyN+VVFRXSFiguSNcXmS6rkKT+X7FdIrTtfo= +github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -281,8 +282,9 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -342,8 +344,9 @@ github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brv github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-immutable-radix v1.1.0 h1:vN9wG1D6KG6YHRTWr8512cxGOVgTMEfgEdSj/hr8MPc= github.com/hashicorp/go-immutable-radix v1.1.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= +github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-msgpack v1.1.5 h1:9byZdVjKTe5mce63pRVNP1L7UAmdHOTEMGehn6KvJWs= github.com/hashicorp/go-msgpack v1.1.5/go.mod h1:gWVc3sv/wbDmR3rQsj1CAktEZzoz1YNK9NfGLXJ69/4= @@ -375,30 +378,30 @@ github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ikawaha/kagome.ipadic v1.1.2/go.mod h1:DPSBbU0czaJhAb/5uKQZHMc9MTVRpDugJfX+HddPHHg= github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI= github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= github.com/iris-contrib/jade v1.1.3/go.mod h1:H/geBymxJhShH5kecoiOCSssPX7QWYH7UaeZTSWddIk= github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0GqwkjqxNd0u65g= github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= -github.com/jmhodges/levigo v1.0.0 h1:q5EC36kV79HWeTBWsod3mG11EgStG3qArTKcvlksN1U= -github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/juju/ratelimit v1.0.1 h1:+7AIFJVQ0EQgq/K9+0Krm7m530Du7tIz0METWzN0RgY= -github.com/juju/ratelimit v1.0.1/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= +github.com/juju/ratelimit v1.0.2 h1:sRxmtRiajbvrcLQT7S+JbqU0ntsb9W2yhSdNN8tWfaI= +github.com/juju/ratelimit v1.0.2/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= @@ -414,26 +417,26 @@ github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0 github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4= -github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM= +github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/kljensen/snowball v0.6.0/go.mod h1:27N7E8fVU5H68RlUmnWwZCfxgt4POBJfENGMvNRhldw= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easygo v0.0.0-20190618140210-3c14a0dc985f h1:4+gHs0jJFJ06bfN8PshnM6cHcxGjRUVRLo5jndDiKRQ= github.com/mailru/easygo v0.0.0-20190618140210-3c14a0dc985f/go.mod h1:tHCZHV8b2A90ObojrEAzY0Lb03gxUxjDHr5IJyAh4ew= github.com/markbates/oncer v1.0.0/go.mod h1:Z59JA581E9GP6w96jai+TGqafHPW+cPfRxz2aSZ0mcI= @@ -456,7 +459,6 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.10 h1:CoZ3S2P7pvtP45xOtBw+/mDL2z0RKI576gSkzRRpdGg= github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= @@ -479,16 +481,19 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/monnand/dhkx v0.0.0-20180522003156-9e5b033f1ac4 h1:UsjqpfLSsCM5SVN5OGhiWJnxDokyT74E6Ahj6kVZxh8= github.com/monnand/dhkx v0.0.0-20180522003156-9e5b033f1ac4/go.mod h1:/cxRiYq8L/bpGLJJJ7mN66Qv2nj915TfJdujDVyYVGA= github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= -github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -497,10 +502,11 @@ github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5Vgl github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/nyaruka/phonenumbers v1.0.70 h1:AecpfeQ8/qWKzkZM2NZZEoq5B+qAuH9MUmHK397U7CQ= github.com/nyaruka/phonenumbers v1.0.70/go.mod h1:sDaTZ/KPX5f8qyV9qN+hIm+4ZBARJrupC6LuhshJq1U= -github.com/olekukonko/tablewriter v0.0.4 h1:vHD/YYe1Wolo78koG299f7V/VAS08c6IpCLn+Ejf/w8= -github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= +github.com/nyaruka/phonenumbers v1.3.1 h1:QcPRnL8hhZeSS6mB0p/Rlv7ma5ASmcgF33YLY0g0A8k= +github.com/nyaruka/phonenumbers v1.3.1/go.mod h1:4jyKp/BFUokLbCHyoZag+T3S1KezFVoEKtgnbpzItC4= +github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.3 h1:OoxbjfXVZyod1fmWYhI7SEyaD8B00ynP3T+D5GiyHOY= @@ -516,10 +522,9 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= -github.com/philhofer/fwd v1.1.1 h1:GdGcTjf5RNAxwS4QLsiMzJYj5KEvPJD3Abr261yRQXQ= +github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= +github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= @@ -530,8 +535,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pkg/term v1.1.0 h1:xIAAdCMh3QIAy+5FrE8Ad8XoDhEU4ufwbaSozViP9kk= github.com/pkg/term v1.1.0/go.mod h1:E25nymQcrSllhX42Ok8MRm1+hyBdHY0dCeiKZ9jpNGw= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= @@ -556,13 +561,12 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/psanford/memfs v0.0.0-20210214183328-a001468d78ef/go.mod h1:tcaRap0jS3eifrEEllL6ZMd9dg8IlDpi2S1oARrQ+NI= -github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/ronaksoft/memberlist v0.2.7 h1:Y4bZjKPc1sJ4qCdTnnLW/HdWvOiZugm9qlg25ZnAr/A= github.com/ronaksoft/memberlist v0.2.7/go.mod h1:VBpJGpiaUma9qsGzwF7eTHrsC/MZriNQSA+5S3A+kAQ= github.com/ronaksoft/river-msg v0.0.0-20240204095508-6868e75c8967 h1:AsuiBXi/RcXv7AkaOfxl7Kad5q0uTV03Kt98B35EUDM= @@ -573,9 +577,14 @@ github.com/ronaksoft/rony v0.12.29/go.mod h1:zFTRu/B1z4njwi28FfpyF9unZLfR3tr2gAy github.com/rs/cors v1.8.0/go.mod h1:EBwu+T5AvHOcXwvZIkQFjUN6s8Czyqw12GL/Y0tUyRM= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/columnize v2.1.2+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g= github.com/scylladb/go-reflectx v1.0.1 h1:b917wZM7189pZdlND9PbIJ6NQxfDPfBvUaQ7cjj1iZQ= github.com/scylladb/go-reflectx v1.0.1/go.mod h1:rWnOfDIRWBGN0miMLIcoPt/Dhi2doCMZqwMCJ3KupFc= @@ -586,8 +595,8 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/siongui/instago v0.0.0-20200627103523-54424cb565ea h1:chI/USCexc1dvONjp4yo4P9V7gEB4dzVt8LM9lEAK8k= -github.com/siongui/instago v0.0.0-20200627103523-54424cb565ea/go.mod h1:GBLEvaysQs2760s073rF44j8GW1a3aetjj+/t/k44rs= +github.com/siongui/instago v0.0.0-20230301150703-9da679ec808b h1:tXHTsJBDeJqNA3NKKJ4RfQIx9kXS9WlQQ+hms7TP7Qw= +github.com/siongui/instago v0.0.0-20230301150703-9da679ec808b/go.mod h1:vHEuPOuXnHEG4r4HDBYGpSY4rgQt2bhpXMg4tKdjj2g= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= @@ -597,46 +606,53 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1 github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw= github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/spf13/viper v1.8.1 h1:Kq1fyeebqsBfbjZj4EL7gj2IO0mMaiyjYUWcUsl2O44= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= -github.com/steveyen/gtreap v0.1.0 h1:CjhzTa274PyJLJuMZwIzCO1PfC00oRa8d1Kc78bFXJM= -github.com/steveyen/gtreap v0.1.0/go.mod h1:kl/5J7XbrOmlIbYIXdRHDDE5QxHqpk0cmkT7Z4dM9/Y= +github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ= +github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= -github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= -github.com/tebeka/snowball v0.4.2/go.mod h1:4IfL14h1lvwZcp1sfXuuc7/7yCsvVffTWxWxCLfFpYg= -github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c h1:g+WoO5jjkqGAzHWCjJB1zZfXPIAaDpzXIEJ0eS6B5Ok= -github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c/go.mod h1:ahpPrc7HpcfEWDQRZEmnXMzHY03mLDYMCxeDzy46i+8= github.com/tidwall/btree v0.3.0 h1:LcwmLI5kv+AaH/xnBgOuKfbu5eLBWVdWTpD2o+qSRdU= github.com/tidwall/btree v0.3.0/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8= github.com/tidwall/buntdb v1.1.7 h1:Dk7KO7RcGgWJTNHt9nYgLxxztgHXWm2C5SfOCXBNrz4= @@ -653,8 +669,6 @@ github.com/tidwall/rtree v0.0.0-20201103190202-0d877048965d h1:RfyBHwIzDt48a28uB github.com/tidwall/rtree v0.0.0-20201103190202-0d877048965d/go.mod h1:/h+UnNGt0IhNNJLkGikcdcJqm66zGD/uJGMRxK/9+Ao= github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563 h1:Otn9S136ELckZ3KKDyCkxapfufrqDqwmGjcHfAyXRrE= github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563/go.mod h1:mLqSmt7Dv/CNneF2wfcChfN1rvapyQr01LGKnKex0DQ= -github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= -github.com/tinylib/msgp v1.1.5 h1:2gXmtWueD2HefZHQe1QOy9HVzmFrLOVvsXwXBQ0ayy0= github.com/tinylib/msgp v1.1.5/go.mod h1:eQsjooMTnV42mHu917E26IogZ2930nFyBQdofk10Udg= github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31/go.mod h1:onvgF043R+lC5RZ8IT9rBXDaEDnpnw/Cl+HFiw+v/7Q= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= @@ -673,9 +687,6 @@ github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPU github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= -github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= -github.com/willf/bitset v1.1.11 h1:N7Z7E9UvjW+sGsEl7k/SJrvY2reP1A07MrGuCjIOjRE= -github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= @@ -689,8 +700,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= -go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= +go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= @@ -700,19 +711,24 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.8.0 h1:CUhrE4N1rqSE6FM9ecihEjRkLQu8cDfgDyoOs83mEY4= go.uber.org/atomic v1.8.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= -go.uber.org/zap v1.18.1 h1:CSUJ2mjFszzEWt4CdKISEuChVIXGBn3lAPwkRGyVrc4= go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= +go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -738,6 +754,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 h1:qCEDpW1G+vcj3Y7Fy52pEM1AWm3abj8WimGYejI3SC4= +golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -751,7 +769,6 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -766,8 +783,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= +golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -813,8 +830,8 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -838,8 +855,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -847,7 +864,6 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -889,7 +905,6 @@ golang.org/x/sys v0.0.0-20200918174421-af09f7315aff/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -908,10 +923,11 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= -golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -923,8 +939,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -986,8 +1002,9 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM= +golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1095,8 +1112,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.29.1 h1:7QBf+IK2gx70Ap/hDsOmam3GE0v9HicjfEdAxE62UoM= -google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/abiosoft/ishell.v2 v2.0.0 h1:/J5yh3nWYSSGFjALcitTI9CLE0Tu27vBYHX0srotqOc= gopkg.in/abiosoft/ishell.v2 v2.0.0/go.mod h1:sFp+cGtH6o4s1FtpVPTMcHq2yue+c4DGOVohJCPUzwY= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -1135,6 +1152,7 @@ gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/internal/repo/groups.go b/internal/repo/groups.go index 092b8734..c53376c4 100644 --- a/internal/repo/groups.go +++ b/internal/repo/groups.go @@ -5,8 +5,8 @@ import ( "strings" "time" - "github.com/blevesearch/bleve" - "github.com/blevesearch/bleve/search/query" + "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/search/query" "github.com/dgraph-io/badger/v2" "github.com/ronaksoft/river-msg/go/msg" "github.com/ronaksoft/river-sdk/internal/domain" @@ -459,6 +459,7 @@ func (r *repoGroups) Search(teamID int64, searchPhrase string) []*msg.Group { if r.peerSearch == nil { return groups } + t1 := bleve.NewTermQuery("group") t1.SetField("type") terms := strings.Fields(searchPhrase) diff --git a/internal/repo/messages.go b/internal/repo/messages.go index 54b398d9..ce6af7e0 100644 --- a/internal/repo/messages.go +++ b/internal/repo/messages.go @@ -9,8 +9,8 @@ import ( "sync" "time" - "github.com/blevesearch/bleve" - "github.com/blevesearch/bleve/search/query" + "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/search/query" "github.com/dgraph-io/badger/v2" "github.com/dgraph-io/badger/v2/pb" "github.com/ronaksoft/river-msg/go/msg" diff --git a/internal/repo/repo.go b/internal/repo/repo.go index 31638c98..12a0a602 100644 --- a/internal/repo/repo.go +++ b/internal/repo/repo.go @@ -8,18 +8,17 @@ import ( "sync" "time" - "github.com/blevesearch/bleve" - "github.com/blevesearch/bleve/analysis/analyzer/keyword" - "github.com/blevesearch/bleve/analysis/lang/en" - "github.com/blevesearch/bleve/mapping" + "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" + "github.com/blevesearch/bleve/v2/analysis/lang/en" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/dgraph-io/badger/v2" "github.com/dgraph-io/badger/v2/options" "github.com/pkg/errors" "github.com/ronaksoft/river-sdk/internal/domain" + "github.com/ronaksoft/river-sdk/internal/logs" "github.com/ronaksoft/rony/tools" "github.com/tidwall/buntdb" - - "github.com/dgraph-io/badger/v2" - "github.com/ronaksoft/river-sdk/internal/logs" "go.uber.org/zap" ) @@ -62,6 +61,7 @@ type repository struct { } func MustInit(dbPath string, lowMemory bool) { + bleve.NewIndexMapping() err := Init(dbPath, lowMemory) if err != nil { panic(err) diff --git a/internal/repo/users.go b/internal/repo/users.go index 371b6072..b36d5424 100644 --- a/internal/repo/users.go +++ b/internal/repo/users.go @@ -6,8 +6,8 @@ import ( "strings" "time" - "github.com/blevesearch/bleve" - "github.com/blevesearch/bleve/search/query" + "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/search/query" "github.com/dgraph-io/badger/v2" "github.com/ronaksoft/river-msg/go/msg" "github.com/ronaksoft/river-sdk/internal/domain" diff --git a/vendor/github.com/RoaringBitmap/roaring/.gitignore b/vendor/github.com/RoaringBitmap/roaring/.gitignore index b7943ab2..851f323d 100644 --- a/vendor/github.com/RoaringBitmap/roaring/.gitignore +++ b/vendor/github.com/RoaringBitmap/roaring/.gitignore @@ -3,4 +3,3 @@ roaring-fuzz.zip workdir coverage.out testdata/all3.classic -testdata/all3.msgp.snappy diff --git a/vendor/github.com/RoaringBitmap/roaring/.travis.yml b/vendor/github.com/RoaringBitmap/roaring/.travis.yml deleted file mode 100644 index 9ce8cadc..00000000 --- a/vendor/github.com/RoaringBitmap/roaring/.travis.yml +++ /dev/null @@ -1,33 +0,0 @@ -language: go -sudo: false -install: -- go get -t github.com/RoaringBitmap/roaring -- go get -t golang.org/x/tools/cmd/cover -- go get -t github.com/mattn/goveralls -- go get -t github.com/mschoch/smat -notifications: - email: false -go: -- "1.12.x" -- "1.13.x" -- "1.14.x" -- tip - -# whitelist -branches: - only: - - master -script: -- goveralls -v -service travis-ci -ignore arraycontainer_gen.go,bitmapcontainer_gen.go,rle16_gen.go,rle_gen.go,roaringarray_gen.go,rle.go || go test -- go test -race -run TestConcurrent* -- go build -tags appengine -- go test -tags appengine -- GOARCH=arm64 go build -- GOARCH=386 go build -- GOARCH=386 go test -- GOARCH=arm go build -- GOARCH=arm64 go build - -matrix: - allow_failures: - - go: tip diff --git a/vendor/github.com/RoaringBitmap/roaring/CONTRIBUTORS b/vendor/github.com/RoaringBitmap/roaring/CONTRIBUTORS index b1e3a379..1a8da9cc 100644 --- a/vendor/github.com/RoaringBitmap/roaring/CONTRIBUTORS +++ b/vendor/github.com/RoaringBitmap/roaring/CONTRIBUTORS @@ -13,4 +13,6 @@ Forud Ghafouri (@fzerorubigd), Joe Nall (@joenall), (@fredim), Edd Robinson (@e-dard), -Alexander Petrov (@alldroll) +Alexander Petrov (@alldroll), +Guy Molinari (@guymolinari), +Ling Jin (@JinLingChristopher) diff --git a/vendor/github.com/RoaringBitmap/roaring/Makefile b/vendor/github.com/RoaringBitmap/roaring/Makefile index 906bd725..0a4f9f0a 100644 --- a/vendor/github.com/RoaringBitmap/roaring/Makefile +++ b/vendor/github.com/RoaringBitmap/roaring/Makefile @@ -64,7 +64,7 @@ qa: fmtcheck test vet lint # Get the dependencies deps: GOPATH=$(GOPATH) go get github.com/stretchr/testify - GOPATH=$(GOPATH) go get github.com/willf/bitset + GOPATH=$(GOPATH) go get github.com/bits-and-blooms/bitset GOPATH=$(GOPATH) go get github.com/golang/lint/golint GOPATH=$(GOPATH) go get github.com/mschoch/smat GOPATH=$(GOPATH) go get github.com/dvyukov/go-fuzz/go-fuzz @@ -97,10 +97,6 @@ nuke: rm -rf ./target GOPATH=$(GOPATH) go clean -i ./... - -ser: - go generate - cover: go test -coverprofile=coverage.out go tool cover -html=coverage.out diff --git a/vendor/github.com/RoaringBitmap/roaring/README.md b/vendor/github.com/RoaringBitmap/roaring/README.md index 39b48420..753b8068 100644 --- a/vendor/github.com/RoaringBitmap/roaring/README.md +++ b/vendor/github.com/RoaringBitmap/roaring/README.md @@ -1,17 +1,14 @@ -roaring [![Build Status](https://travis-ci.org/RoaringBitmap/roaring.png)](https://travis-ci.org/RoaringBitmap/roaring) [![GoDoc](https://godoc.org/github.com/RoaringBitmap/roaring?status.svg)](https://godoc.org/github.com/RoaringBitmap/roaring) [![GoDoc](https://godoc.org/github.com/RoaringBitmap/roaring/roaring64?status.svg)](https://godoc.org/github.com/RoaringBitmap/roaring/roaring64) [![Go Report Card](https://goreportcard.com/badge/RoaringBitmap/roaring)](https://goreportcard.com/report/github.com/RoaringBitmap/roaring) +roaring [![GoDoc](https://godoc.org/github.com/RoaringBitmap/roaring/roaring64?status.svg)](https://godoc.org/github.com/RoaringBitmap/roaring/roaring64) [![Go Report Card](https://goreportcard.com/badge/RoaringBitmap/roaring)](https://goreportcard.com/report/github.com/RoaringBitmap/roaring) [![Build Status](https://cloud.drone.io/api/badges/RoaringBitmap/roaring/status.svg)](https://cloud.drone.io/RoaringBitmap/roaring) ![Go-CI](https://github.com/RoaringBitmap/roaring/workflows/Go-CI/badge.svg) ![Go-ARM-CI](https://github.com/RoaringBitmap/roaring/workflows/Go-ARM-CI/badge.svg) ![Go-Windows-CI](https://github.com/RoaringBitmap/roaring/workflows/Go-Windows-CI/badge.svg) -![Go-macos-CI](https://github.com/RoaringBitmap/roaring/workflows/Go-macos-CI/badge.svg) ============= This is a go version of the Roaring bitmap data structure. - - Roaring bitmaps are used by several major systems such as [Apache Lucene][lucene] and derivative systems such as [Solr][solr] and -[Elasticsearch][elasticsearch], [Apache Druid (Incubating)][druid], [LinkedIn Pinot][pinot], [Netflix Atlas][atlas], [Apache Spark][spark], [OpenSearchServer][opensearchserver], [Cloud Torrent][cloudtorrent], [Whoosh][whoosh], [Pilosa][pilosa], [Microsoft Visual Studio Team Services (VSTS)][vsts], and eBay's [Apache Kylin][kylin]. The YouTube SQL Engine, [Google Procella](https://research.google/pubs/pub48388/), uses Roaring bitmaps for indexing. +[Elasticsearch][elasticsearch], [Apache Druid (Incubating)][druid], [LinkedIn Pinot][pinot], [Netflix Atlas][atlas], [Apache Spark][spark], [OpenSearchServer][opensearchserver], [anacrolix/torrent][anacrolix/torrent], [Whoosh][whoosh], [Pilosa][pilosa], [Microsoft Visual Studio Team Services (VSTS)][vsts], and eBay's [Apache Kylin][kylin]. The YouTube SQL Engine, [Google Procella](https://research.google/pubs/pub48388/), uses Roaring bitmaps for indexing. [lucene]: https://lucene.apache.org/ [solr]: https://lucene.apache.org/solr/ @@ -19,7 +16,7 @@ Roaring bitmaps are used by several major systems such as [Apache Lucene][lucene [druid]: https://druid.apache.org/ [spark]: https://spark.apache.org/ [opensearchserver]: http://www.opensearchserver.com -[cloudtorrent]: https://github.com/jpillora/cloud-torrent +[anacrolix/torrent]: https://github.com/anacrolix/torrent [whoosh]: https://bitbucket.org/mchaput/whoosh/wiki/Home [pilosa]: https://www.pilosa.com/ [kylin]: http://kylin.apache.org/ @@ -33,7 +30,7 @@ Roaring bitmaps are found to work well in many important applications: The ``roaring`` Go library is used by -* [Cloud Torrent](https://github.com/jpillora/cloud-torrent) +* [anacrolix/torrent] * [runv](https://github.com/hyperhq/runv) * [InfluxDB](https://www.influxdata.com) * [Pilosa](https://www.pilosa.com/) @@ -43,6 +40,7 @@ The ``roaring`` Go library is used by * [SourceGraph](https://github.com/sourcegraph/sourcegraph) * [M3](https://github.com/m3db/m3) * [trident](https://github.com/NetApp/trident) +* [Husky](https://www.datadoghq.com/blog/engineering/introducing-husky/) This library is used in production in several systems, it is part of the [Awesome Go collection](https://awesome-go.com). @@ -56,23 +54,108 @@ This code is licensed under Apache License, Version 2.0 (ASL2.0). Copyright 2016-... by the authors. +When should you use a bitmap? +=================================== + + +Sets are a fundamental abstraction in +software. They can be implemented in various +ways, as hash sets, as trees, and so forth. +In databases and search engines, sets are often an integral +part of indexes. For example, we may need to maintain a set +of all documents or rows (represented by numerical identifier) +that satisfy some property. Besides adding or removing +elements from the set, we need fast functions +to compute the intersection, the union, the difference between sets, and so on. + + +To implement a set +of integers, a particularly appealing strategy is the +bitmap (also called bitset or bit vector). Using n bits, +we can represent any set made of the integers from the range +[0,n): the ith bit is set to one if integer i is present in the set. +Commodity processors use words of W=32 or W=64 bits. By combining many such words, we can +support large values of n. Intersections, unions and differences can then be implemented + as bitwise AND, OR and ANDNOT operations. +More complicated set functions can also be implemented as bitwise operations. + +When the bitset approach is applicable, it can be orders of +magnitude faster than other possible implementation of a set (e.g., as a hash set) +while using several times less memory. + +However, a bitset, even a compressed one is not always applicable. For example, if +you have 1000 random-looking integers, then a simple array might be the best representation. +We refer to this case as the "sparse" scenario. + +When should you use compressed bitmaps? +=================================== + +An uncompressed BitSet can use a lot of memory. For example, if you take a BitSet +and set the bit at position 1,000,000 to true and you have just over 100kB. That is over 100kB +to store the position of one bit. This is wasteful even if you do not care about memory: +suppose that you need to compute the intersection between this BitSet and another one +that has a bit at position 1,000,001 to true, then you need to go through all these zeroes, +whether you like it or not. That can become very wasteful. + +This being said, there are definitively cases where attempting to use compressed bitmaps is wasteful. +For example, if you have a small universe size. E.g., your bitmaps represent sets of integers +from [0,n) where n is small (e.g., n=64 or n=128). If you are able to uncompressed BitSet and +it does not blow up your memory usage, then compressed bitmaps are probably not useful +to you. In fact, if you do not need compression, then a BitSet offers remarkable speed. + +The sparse scenario is another use case where compressed bitmaps should not be used. +Keep in mind that random-looking data is usually not compressible. E.g., if you have a small set of +32-bit random integers, it is not mathematically possible to use far less than 32 bits per integer, +and attempts at compression can be counterproductive. + +How does Roaring compares with the alternatives? +================================================== + + +Most alternatives to Roaring are part of a larger family of compressed bitmaps that are run-length-encoded +bitmaps. They identify long runs of 1s or 0s and they represent them with a marker word. +If you have a local mix of 1s and 0, you use an uncompressed word. + +There are many formats in this family: + +* Oracle's BBC is an obsolete format at this point: though it may provide good compression, +it is likely much slower than more recent alternatives due to excessive branching. +* WAH is a patented variation on BBC that provides better performance. +* Concise is a variation on the patented WAH. It some specific instances, it can compress +much better than WAH (up to 2x better), but it is generally slower. +* EWAH is both free of patent, and it is faster than all the above. On the downside, it +does not compress quite as well. It is faster because it allows some form of "skipping" +over uncompressed words. So though none of these formats are great at random access, EWAH +is better than the alternatives. + + + +There is a big problem with these formats however that can hurt you badly in some cases: there is no random access. If you want to check whether a given value is present in the set, you have to start from the beginning and "uncompress" the whole thing. This means that if you want to intersect a big set with a large set, you still have to uncompress the whole big set in the worst case... + +Roaring solves this problem. It works in the following manner. It divides the data into chunks of 216 integers +(e.g., [0, 216), [216, 2 x 216), ...). Within a chunk, it can use an uncompressed bitmap, a simple list of integers, +or a list of runs. Whatever format it uses, they all allow you to check for the present of any one value quickly +(e.g., with a binary search). The net result is that Roaring can compute many operations much faster than run-length-encoded +formats like WAH, EWAH, Concise... Maybe surprisingly, Roaring also generally offers better compression ratios. + + + + ### References - Daniel Lemire, Owen Kaser, Nathan Kurz, Luca Deri, Chris O'Hara, François Saint-Jacques, Gregory Ssi-Yan-Kai, Roaring Bitmaps: Implementation of an Optimized Software Library, Software: Practice and Experience 48 (4), 2018 [arXiv:1709.07821](https://arxiv.org/abs/1709.07821) - Samy Chambi, Daniel Lemire, Owen Kaser, Robert Godin, Better bitmap performance with Roaring bitmaps, -Software: Practice and Experience 46 (5), 2016. -http://arxiv.org/abs/1402.6407 This paper used data from http://lemire.me/data/realroaring2014.html -- Daniel Lemire, Gregory Ssi-Yan-Kai, Owen Kaser, Consistently faster and smaller compressed bitmaps with Roaring, Software: Practice and Experience 46 (11), 2016. http://arxiv.org/abs/1603.06549 - +Software: Practice and Experience 46 (5), 2016.[arXiv:1402.6407](http://arxiv.org/abs/1402.6407) This paper used data from http://lemire.me/data/realroaring2014.html +- Daniel Lemire, Gregory Ssi-Yan-Kai, Owen Kaser, Consistently faster and smaller compressed bitmaps with Roaring, Software: Practice and Experience 46 (11), 2016. [arXiv:1603.06549](http://arxiv.org/abs/1603.06549) ### Dependencies Dependencies are fetched automatically by giving the `-t` flag to `go get`. they include - - github.com/willf/bitset + - github.com/bits-and-blooms/bitset - github.com/mschoch/smat - github.com/glycerine/go-unsnap-stream - github.com/philhofer/fwd @@ -84,6 +167,15 @@ Note that the smat library requires Go 1.6 or better. - go get -t github.com/RoaringBitmap/roaring +### Instructions for contributors + +Using bash or other common shells: +``` +$ git clone git@github.com:RoaringBitmap/roaring.git +$ export GO111MODULE=on +$ go mod tidy +$ go test -v +``` ### Example @@ -239,7 +331,7 @@ Only the 32-bit roaring format is standard and cross-operable between Java, C++, ### Documentation -Current documentation is available at http://godoc.org/github.com/RoaringBitmap/roaring and http://godoc.org/github.com/RoaringBitmap/roaring64 +Current documentation is available at https://pkg.go.dev/github.com/RoaringBitmap/roaring and https://pkg.go.dev/github.com/RoaringBitmap/roaring/roaring64 ### Goroutine safety @@ -298,12 +390,14 @@ You can help us test further the library with fuzzy testing: go get github.com/dvyukov/go-fuzz/go-fuzz-build go test -tags=gofuzz -run=TestGenerateSmatCorpus go-fuzz-build github.com/RoaringBitmap/roaring - go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200 + go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200 -func FuzzSmat Let it run, and if the # of crashers is > 0, check out the reports in the workdir where you should be able to find the panic goroutine stack traces. +You may also replace `-func FuzzSmat` by `-func FuzzSerializationBuffer` or `-func FuzzSerializationStream`. + ### Alternative in Go There is a Go version wrapping the C/C++ implementation https://github.com/RoaringBitmap/gocroaring diff --git a/vendor/github.com/RoaringBitmap/roaring/arraycontainer.go b/vendor/github.com/RoaringBitmap/roaring/arraycontainer.go index 12fe6cf5..9541fd53 100644 --- a/vendor/github.com/RoaringBitmap/roaring/arraycontainer.go +++ b/vendor/github.com/RoaringBitmap/roaring/arraycontainer.go @@ -4,8 +4,6 @@ import ( "fmt" ) -//go:generate msgp -unexported - type arrayContainer struct { content []uint16 } @@ -18,10 +16,11 @@ func (ac *arrayContainer) String() string { return s + "}" } -func (ac *arrayContainer) fillLeastSignificant16bits(x []uint32, i int, mask uint32) { +func (ac *arrayContainer) fillLeastSignificant16bits(x []uint32, i int, mask uint32) int { for k := 0; k < len(ac.content); k++ { x[k+i] = uint32(ac.content[k]) | mask } + return i + len(ac.content) } func (ac *arrayContainer) iterate(cb func(x uint16) bool) bool { @@ -359,28 +358,17 @@ func (ac *arrayContainer) iorArray(value2 *arrayContainer) container { len1 := value1.getCardinality() len2 := value2.getCardinality() maxPossibleCardinality := len1 + len2 - if maxPossibleCardinality > arrayDefaultMaxSize { // it could be a bitmap! - bc := newBitmapContainer() - for k := 0; k < len(value2.content); k++ { - v := value2.content[k] - i := uint(v) >> 6 - mask := uint64(1) << (v % 64) - bc.bitmap[i] |= mask - } - for k := 0; k < len(ac.content); k++ { - v := ac.content[k] - i := uint(v) >> 6 - mask := uint64(1) << (v % 64) - bc.bitmap[i] |= mask - } - bc.cardinality = int(popcntSlice(bc.bitmap)) - if bc.cardinality <= arrayDefaultMaxSize { - return bc.toArrayContainer() - } - return bc - } if maxPossibleCardinality > cap(value1.content) { - newcontent := make([]uint16, 0, maxPossibleCardinality) + // doubling the capacity reduces new slice allocations in the case of + // repeated calls to iorArray(). + newSize := 2 * maxPossibleCardinality + // the second check is to handle overly large array containers + // and should not occur in normal usage, + // as all array containers should be at most arrayDefaultMaxSize + if newSize > 2*arrayDefaultMaxSize && maxPossibleCardinality <= 2*arrayDefaultMaxSize { + newSize = 2 * arrayDefaultMaxSize + } + newcontent := make([]uint16, 0, newSize) copy(newcontent[len2:maxPossibleCardinality], ac.content[0:len1]) ac.content = newcontent } else { @@ -388,6 +376,13 @@ func (ac *arrayContainer) iorArray(value2 *arrayContainer) container { } nl := union2by2(value1.content[len2:maxPossibleCardinality], value2.content, ac.content) ac.content = ac.content[:nl] // reslice to match actual used capacity + + if nl > arrayDefaultMaxSize { + // Only converting to a bitmap when arrayDefaultMaxSize + // is actually exceeded minimizes conversions in the case of repeated + // calls to iorArray(). + return ac.toBitmapContainer() + } return ac } @@ -400,11 +395,19 @@ func (ac *arrayContainer) iorBitmap(bc2 *bitmapContainer) container { } func (ac *arrayContainer) iorRun16(rc *runContainer16) container { - bc1 := ac.toBitmapContainer() - bc2 := rc.toBitmapContainer() - bc1.iorBitmap(bc2) - *ac = *newArrayContainerFromBitmap(bc1) - return ac + runCardinality := rc.getCardinality() + // heuristic for if the container should maybe be an + // array container. + if runCardinality < ac.getCardinality() && + runCardinality+ac.getCardinality() < arrayDefaultMaxSize { + var result container + result = ac + for _, run := range rc.iv { + result = result.iaddRange(int(run.start), int(run.start)+int(run.length)+1) + } + return result + } + return rc.orArray(ac) } func (ac *arrayContainer) lazyIOR(a container) container { @@ -489,7 +492,7 @@ func (ac *arrayContainer) orArrayCardinality(value2 *arrayContainer) int { func (ac *arrayContainer) lazyorArray(value2 *arrayContainer) container { value1 := ac maxPossibleCardinality := value1.getCardinality() + value2.getCardinality() - if maxPossibleCardinality > arrayLazyLowerBound { // it could be a bitmap!^M + if maxPossibleCardinality > arrayLazyLowerBound { // it could be a bitmap! bc := newBitmapContainer() for k := 0; k < len(value2.content); k++ { v := value2.content[k] @@ -849,6 +852,10 @@ func (ac *arrayContainer) getCardinality() int { return len(ac.content) } +func (ac *arrayContainer) isEmpty() bool { + return len(ac.content) == 0 +} + func (ac *arrayContainer) rank(x uint16) int { answer := binarySearch(ac.content, x) if answer >= 0 { @@ -888,7 +895,7 @@ func (ac *arrayContainer) resetTo(a container) { x.fillArray(ac.content) case *runContainer16: - card := int(x.cardinality()) + card := int(x.getCardinality()) ac.realloc(card) cur := 0 for _, r := range x.iv { @@ -962,10 +969,10 @@ func (ac *arrayContainer) numberOfRuns() (nr int) { runlen++ } else { if cur < prev { - panic("then fundamental arrayContainer assumption of sorted ac.content was broken") + panic("the fundamental arrayContainer assumption of sorted ac.content was broken") } if cur == prev { - panic("then fundamental arrayContainer assumption of deduplicated content was broken") + panic("the fundamental arrayContainer assumption of deduplicated content was broken") } else { nr++ runlen = 0 @@ -1000,16 +1007,42 @@ func (ac *arrayContainer) containerType() contype { return arrayContype } -func (ac *arrayContainer) addOffset(x uint16) []container { - low := &arrayContainer{} - high := &arrayContainer{} +func (ac *arrayContainer) addOffset(x uint16) (container, container) { + var low, high *arrayContainer + + if len(ac.content) == 0 { + return nil, nil + } + + if y := uint32(ac.content[0]) + uint32(x); highbits(y) == 0 { + // Some elements will fall into low part, allocate a container. + // Checking the first one is enough because they are ordered. + low = &arrayContainer{} + } + if y := uint32(ac.content[len(ac.content)-1]) + uint32(x); highbits(y) > 0 { + // Some elements will fall into high part, allocate a container. + // Checking the last one is enough because they are ordered. + high = &arrayContainer{} + } + for _, val := range ac.content { y := uint32(val) + uint32(x) if highbits(y) > 0 { + // OK, if high == nil then highbits(y) == 0 for all y. high.content = append(high.content, lowbits(y)) } else { + // OK, if low == nil then highbits(y) > 0 for all y. low.content = append(low.content, lowbits(y)) } } - return []container{low, high} + + // Ensure proper nil interface. + if low == nil { + return nil, high + } + if high == nil { + return low, nil + } + + return low, high } diff --git a/vendor/github.com/RoaringBitmap/roaring/arraycontainer_gen.go b/vendor/github.com/RoaringBitmap/roaring/arraycontainer_gen.go deleted file mode 100644 index 6ee670ee..00000000 --- a/vendor/github.com/RoaringBitmap/roaring/arraycontainer_gen.go +++ /dev/null @@ -1,134 +0,0 @@ -package roaring - -// NOTE: THIS FILE WAS PRODUCED BY THE -// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp) -// DO NOT EDIT - -import "github.com/tinylib/msgp/msgp" - -// Deprecated: DecodeMsg implements msgp.Decodable -func (z *arrayContainer) DecodeMsg(dc *msgp.Reader) (err error) { - var field []byte - _ = field - var zbzg uint32 - zbzg, err = dc.ReadMapHeader() - if err != nil { - return - } - for zbzg > 0 { - zbzg-- - field, err = dc.ReadMapKeyPtr() - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "content": - var zbai uint32 - zbai, err = dc.ReadArrayHeader() - if err != nil { - return - } - if cap(z.content) >= int(zbai) { - z.content = (z.content)[:zbai] - } else { - z.content = make([]uint16, zbai) - } - for zxvk := range z.content { - z.content[zxvk], err = dc.ReadUint16() - if err != nil { - return - } - } - default: - err = dc.Skip() - if err != nil { - return - } - } - } - return -} - -// Deprecated: EncodeMsg implements msgp.Encodable -func (z *arrayContainer) EncodeMsg(en *msgp.Writer) (err error) { - // map header, size 1 - // write "content" - err = en.Append(0x81, 0xa7, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74) - if err != nil { - return err - } - err = en.WriteArrayHeader(uint32(len(z.content))) - if err != nil { - return - } - for zxvk := range z.content { - err = en.WriteUint16(z.content[zxvk]) - if err != nil { - return - } - } - return -} - -// Deprecated: MarshalMsg implements msgp.Marshaler -func (z *arrayContainer) MarshalMsg(b []byte) (o []byte, err error) { - o = msgp.Require(b, z.Msgsize()) - // map header, size 1 - // string "content" - o = append(o, 0x81, 0xa7, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74) - o = msgp.AppendArrayHeader(o, uint32(len(z.content))) - for zxvk := range z.content { - o = msgp.AppendUint16(o, z.content[zxvk]) - } - return -} - -// Deprecated: UnmarshalMsg implements msgp.Unmarshaler -func (z *arrayContainer) UnmarshalMsg(bts []byte) (o []byte, err error) { - var field []byte - _ = field - var zcmr uint32 - zcmr, bts, err = msgp.ReadMapHeaderBytes(bts) - if err != nil { - return - } - for zcmr > 0 { - zcmr-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "content": - var zajw uint32 - zajw, bts, err = msgp.ReadArrayHeaderBytes(bts) - if err != nil { - return - } - if cap(z.content) >= int(zajw) { - z.content = (z.content)[:zajw] - } else { - z.content = make([]uint16, zajw) - } - for zxvk := range z.content { - z.content[zxvk], bts, err = msgp.ReadUint16Bytes(bts) - if err != nil { - return - } - } - default: - bts, err = msgp.Skip(bts) - if err != nil { - return - } - } - } - o = bts - return -} - -// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z *arrayContainer) Msgsize() (s int) { - s = 1 + 8 + msgp.ArrayHeaderSize + (len(z.content) * (msgp.Uint16Size)) - return -} diff --git a/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer.go b/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer.go index e09b091f..71029f4f 100644 --- a/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer.go +++ b/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer.go @@ -5,8 +5,6 @@ import ( "unsafe" ) -//go:generate msgp -unexported - type bitmapContainer struct { cardinality int bitmap []uint64 @@ -115,7 +113,7 @@ type bitmapContainerShortIterator struct { func (bcsi *bitmapContainerShortIterator) next() uint16 { j := bcsi.i - bcsi.i = bcsi.ptr.NextSetBit(bcsi.i + 1) + bcsi.i = bcsi.ptr.NextSetBit(uint(bcsi.i) + 1) return uint16(j) } func (bcsi *bitmapContainerShortIterator) hasNext() bool { @@ -128,7 +126,7 @@ func (bcsi *bitmapContainerShortIterator) peekNext() uint16 { func (bcsi *bitmapContainerShortIterator) advanceIfNeeded(minval uint16) { if bcsi.hasNext() && bcsi.peekNext() < minval { - bcsi.i = bcsi.ptr.NextSetBit(int(minval)) + bcsi.i = bcsi.ptr.NextSetBit(uint(minval)) } } @@ -266,7 +264,7 @@ func bitmapEquals(a, b []uint64) bool { return true } -func (bc *bitmapContainer) fillLeastSignificant16bits(x []uint32, i int, mask uint32) { +func (bc *bitmapContainer) fillLeastSignificant16bits(x []uint32, i int, mask uint32) int { // TODO: should be written as optimized assembly pos := i base := mask @@ -280,6 +278,7 @@ func (bc *bitmapContainer) fillLeastSignificant16bits(x []uint32, i int, mask ui } base += 64 } + return pos } func (bc *bitmapContainer) equals(o container) bool { @@ -351,6 +350,10 @@ func (bc *bitmapContainer) getCardinality() int { return bc.cardinality } +func (bc *bitmapContainer) isEmpty() bool { + return bc.cardinality == 0 +} + func (bc *bitmapContainer) clone() container { ptr := bitmapContainer{bc.cardinality, make([]uint64, len(bc.bitmap))} copy(ptr.bitmap, bc.bitmap[:]) @@ -1009,20 +1012,23 @@ func (bc *bitmapContainer) fillArray(container []uint16) { } } -func (bc *bitmapContainer) NextSetBit(i int) int { - x := i / 64 - if x >= len(bc.bitmap) { +func (bc *bitmapContainer) NextSetBit(i uint) int { + var ( + x = i / 64 + length = uint(len(bc.bitmap)) + ) + if x >= length { return -1 } w := bc.bitmap[x] w = w >> uint(i%64) if w != 0 { - return i + countTrailingZeros(w) + return int(i) + countTrailingZeros(w) } x++ - for ; x < len(bc.bitmap); x++ { + for ; x < length; x++ { if bc.bitmap[x] != 0 { - return (x * 64) + countTrailingZeros(bc.bitmap[x]) + return int(x*64) + countTrailingZeros(bc.bitmap[x]) } } return -1 @@ -1118,34 +1124,60 @@ func (bc *bitmapContainer) containerType() contype { return bitmapContype } -func (bc *bitmapContainer) addOffset(x uint16) []container { - low := newBitmapContainer() - high := newBitmapContainer() +func (bc *bitmapContainer) addOffset(x uint16) (container, container) { + var low, high *bitmapContainer + + if bc.cardinality == 0 { + return nil, nil + } + b := uint32(x) >> 6 i := uint32(x) % 64 end := uint32(1024) - b + + low = newBitmapContainer() if i == 0 { copy(low.bitmap[b:], bc.bitmap[:end]) - copy(high.bitmap[:b], bc.bitmap[end:]) } else { low.bitmap[b] = bc.bitmap[0] << i for k := uint32(1); k < end; k++ { newval := bc.bitmap[k] << i - if newval == 0 { - newval = bc.bitmap[k-1] >> (64 - i) - } + newval |= bc.bitmap[k-1] >> (64 - i) low.bitmap[b+k] = newval } + } + low.computeCardinality() + + if low.cardinality == bc.cardinality { + // All elements from bc ended up in low, meaning high will be empty. + return low, nil + } + + if low.cardinality == 0 { + // low is empty, let's reuse the container for high. + high = low + low = nil + } else { + // None of the containers will be empty, so allocate both. + high = newBitmapContainer() + } + + if i == 0 { + copy(high.bitmap[:b], bc.bitmap[end:]) + } else { for k := end; k < 1024; k++ { newval := bc.bitmap[k] << i - if newval == 0 { - newval = bc.bitmap[k-1] >> (64 - i) - } + newval |= bc.bitmap[k-1] >> (64 - i) high.bitmap[k-end] = newval } high.bitmap[b] = bc.bitmap[1023] >> (64 - i) } - low.computeCardinality() high.computeCardinality() - return []container{low, high} + + // Ensure proper nil interface. + if low == nil { + return nil, high + } + + return low, high } diff --git a/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer_gen.go b/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer_gen.go deleted file mode 100644 index 9b5a465f..00000000 --- a/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer_gen.go +++ /dev/null @@ -1,415 +0,0 @@ -package roaring - -// NOTE: THIS FILE WAS PRODUCED BY THE -// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp) -// DO NOT EDIT - -import "github.com/tinylib/msgp/msgp" - -// Deprecated: DecodeMsg implements msgp.Decodable -func (z *bitmapContainer) DecodeMsg(dc *msgp.Reader) (err error) { - var field []byte - _ = field - var zbzg uint32 - zbzg, err = dc.ReadMapHeader() - if err != nil { - return - } - for zbzg > 0 { - zbzg-- - field, err = dc.ReadMapKeyPtr() - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "cardinality": - z.cardinality, err = dc.ReadInt() - if err != nil { - return - } - case "bitmap": - var zbai uint32 - zbai, err = dc.ReadArrayHeader() - if err != nil { - return - } - if cap(z.bitmap) >= int(zbai) { - z.bitmap = (z.bitmap)[:zbai] - } else { - z.bitmap = make([]uint64, zbai) - } - for zxvk := range z.bitmap { - z.bitmap[zxvk], err = dc.ReadUint64() - if err != nil { - return - } - } - default: - err = dc.Skip() - if err != nil { - return - } - } - } - return -} - -// Deprecated: EncodeMsg implements msgp.Encodable -func (z *bitmapContainer) EncodeMsg(en *msgp.Writer) (err error) { - // map header, size 2 - // write "cardinality" - err = en.Append(0x82, 0xab, 0x63, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x74, 0x79) - if err != nil { - return err - } - err = en.WriteInt(z.cardinality) - if err != nil { - return - } - // write "bitmap" - err = en.Append(0xa6, 0x62, 0x69, 0x74, 0x6d, 0x61, 0x70) - if err != nil { - return err - } - err = en.WriteArrayHeader(uint32(len(z.bitmap))) - if err != nil { - return - } - for zxvk := range z.bitmap { - err = en.WriteUint64(z.bitmap[zxvk]) - if err != nil { - return - } - } - return -} - -// Deprecated: MarshalMsg implements msgp.Marshaler -func (z *bitmapContainer) MarshalMsg(b []byte) (o []byte, err error) { - o = msgp.Require(b, z.Msgsize()) - // map header, size 2 - // string "cardinality" - o = append(o, 0x82, 0xab, 0x63, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x74, 0x79) - o = msgp.AppendInt(o, z.cardinality) - // string "bitmap" - o = append(o, 0xa6, 0x62, 0x69, 0x74, 0x6d, 0x61, 0x70) - o = msgp.AppendArrayHeader(o, uint32(len(z.bitmap))) - for zxvk := range z.bitmap { - o = msgp.AppendUint64(o, z.bitmap[zxvk]) - } - return -} - -// Deprecated: UnmarshalMsg implements msgp.Unmarshaler -func (z *bitmapContainer) UnmarshalMsg(bts []byte) (o []byte, err error) { - var field []byte - _ = field - var zcmr uint32 - zcmr, bts, err = msgp.ReadMapHeaderBytes(bts) - if err != nil { - return - } - for zcmr > 0 { - zcmr-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "cardinality": - z.cardinality, bts, err = msgp.ReadIntBytes(bts) - if err != nil { - return - } - case "bitmap": - var zajw uint32 - zajw, bts, err = msgp.ReadArrayHeaderBytes(bts) - if err != nil { - return - } - if cap(z.bitmap) >= int(zajw) { - z.bitmap = (z.bitmap)[:zajw] - } else { - z.bitmap = make([]uint64, zajw) - } - for zxvk := range z.bitmap { - z.bitmap[zxvk], bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - return - } - } - default: - bts, err = msgp.Skip(bts) - if err != nil { - return - } - } - } - o = bts - return -} - -// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z *bitmapContainer) Msgsize() (s int) { - s = 1 + 12 + msgp.IntSize + 7 + msgp.ArrayHeaderSize + (len(z.bitmap) * (msgp.Uint64Size)) - return -} - -// Deprecated: DecodeMsg implements msgp.Decodable -func (z *bitmapContainerShortIterator) DecodeMsg(dc *msgp.Reader) (err error) { - var field []byte - _ = field - var zhct uint32 - zhct, err = dc.ReadMapHeader() - if err != nil { - return - } - for zhct > 0 { - zhct-- - field, err = dc.ReadMapKeyPtr() - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "ptr": - if dc.IsNil() { - err = dc.ReadNil() - if err != nil { - return - } - z.ptr = nil - } else { - if z.ptr == nil { - z.ptr = new(bitmapContainer) - } - var zcua uint32 - zcua, err = dc.ReadMapHeader() - if err != nil { - return - } - for zcua > 0 { - zcua-- - field, err = dc.ReadMapKeyPtr() - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "cardinality": - z.ptr.cardinality, err = dc.ReadInt() - if err != nil { - return - } - case "bitmap": - var zxhx uint32 - zxhx, err = dc.ReadArrayHeader() - if err != nil { - return - } - if cap(z.ptr.bitmap) >= int(zxhx) { - z.ptr.bitmap = (z.ptr.bitmap)[:zxhx] - } else { - z.ptr.bitmap = make([]uint64, zxhx) - } - for zwht := range z.ptr.bitmap { - z.ptr.bitmap[zwht], err = dc.ReadUint64() - if err != nil { - return - } - } - default: - err = dc.Skip() - if err != nil { - return - } - } - } - } - case "i": - z.i, err = dc.ReadInt() - if err != nil { - return - } - default: - err = dc.Skip() - if err != nil { - return - } - } - } - return -} - -// Deprecated: EncodeMsg implements msgp.Encodable -func (z *bitmapContainerShortIterator) EncodeMsg(en *msgp.Writer) (err error) { - // map header, size 2 - // write "ptr" - err = en.Append(0x82, 0xa3, 0x70, 0x74, 0x72) - if err != nil { - return err - } - if z.ptr == nil { - err = en.WriteNil() - if err != nil { - return - } - } else { - // map header, size 2 - // write "cardinality" - err = en.Append(0x82, 0xab, 0x63, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x74, 0x79) - if err != nil { - return err - } - err = en.WriteInt(z.ptr.cardinality) - if err != nil { - return - } - // write "bitmap" - err = en.Append(0xa6, 0x62, 0x69, 0x74, 0x6d, 0x61, 0x70) - if err != nil { - return err - } - err = en.WriteArrayHeader(uint32(len(z.ptr.bitmap))) - if err != nil { - return - } - for zwht := range z.ptr.bitmap { - err = en.WriteUint64(z.ptr.bitmap[zwht]) - if err != nil { - return - } - } - } - // write "i" - err = en.Append(0xa1, 0x69) - if err != nil { - return err - } - err = en.WriteInt(z.i) - if err != nil { - return - } - return -} - -// Deprecated: MarshalMsg implements msgp.Marshaler -func (z *bitmapContainerShortIterator) MarshalMsg(b []byte) (o []byte, err error) { - o = msgp.Require(b, z.Msgsize()) - // map header, size 2 - // string "ptr" - o = append(o, 0x82, 0xa3, 0x70, 0x74, 0x72) - if z.ptr == nil { - o = msgp.AppendNil(o) - } else { - // map header, size 2 - // string "cardinality" - o = append(o, 0x82, 0xab, 0x63, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x74, 0x79) - o = msgp.AppendInt(o, z.ptr.cardinality) - // string "bitmap" - o = append(o, 0xa6, 0x62, 0x69, 0x74, 0x6d, 0x61, 0x70) - o = msgp.AppendArrayHeader(o, uint32(len(z.ptr.bitmap))) - for zwht := range z.ptr.bitmap { - o = msgp.AppendUint64(o, z.ptr.bitmap[zwht]) - } - } - // string "i" - o = append(o, 0xa1, 0x69) - o = msgp.AppendInt(o, z.i) - return -} - -// Deprecated: UnmarshalMsg implements msgp.Unmarshaler -func (z *bitmapContainerShortIterator) UnmarshalMsg(bts []byte) (o []byte, err error) { - var field []byte - _ = field - var zlqf uint32 - zlqf, bts, err = msgp.ReadMapHeaderBytes(bts) - if err != nil { - return - } - for zlqf > 0 { - zlqf-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "ptr": - if msgp.IsNil(bts) { - bts, err = msgp.ReadNilBytes(bts) - if err != nil { - return - } - z.ptr = nil - } else { - if z.ptr == nil { - z.ptr = new(bitmapContainer) - } - var zdaf uint32 - zdaf, bts, err = msgp.ReadMapHeaderBytes(bts) - if err != nil { - return - } - for zdaf > 0 { - zdaf-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "cardinality": - z.ptr.cardinality, bts, err = msgp.ReadIntBytes(bts) - if err != nil { - return - } - case "bitmap": - var zpks uint32 - zpks, bts, err = msgp.ReadArrayHeaderBytes(bts) - if err != nil { - return - } - if cap(z.ptr.bitmap) >= int(zpks) { - z.ptr.bitmap = (z.ptr.bitmap)[:zpks] - } else { - z.ptr.bitmap = make([]uint64, zpks) - } - for zwht := range z.ptr.bitmap { - z.ptr.bitmap[zwht], bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - return - } - } - default: - bts, err = msgp.Skip(bts) - if err != nil { - return - } - } - } - } - case "i": - z.i, bts, err = msgp.ReadIntBytes(bts) - if err != nil { - return - } - default: - bts, err = msgp.Skip(bts) - if err != nil { - return - } - } - } - o = bts - return -} - -// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z *bitmapContainerShortIterator) Msgsize() (s int) { - s = 1 + 4 - if z.ptr == nil { - s += msgp.NilSize - } else { - s += 1 + 12 + msgp.IntSize + 7 + msgp.ArrayHeaderSize + (len(z.ptr.bitmap) * (msgp.Uint64Size)) - } - s += 2 + msgp.IntSize - return -} diff --git a/vendor/github.com/RoaringBitmap/roaring/byte_input.go b/vendor/github.com/RoaringBitmap/roaring/byte_input.go deleted file mode 100644 index f7a98a1d..00000000 --- a/vendor/github.com/RoaringBitmap/roaring/byte_input.go +++ /dev/null @@ -1,161 +0,0 @@ -package roaring - -import ( - "encoding/binary" - "io" -) - -type byteInput interface { - // next returns a slice containing the next n bytes from the buffer, - // advancing the buffer as if the bytes had been returned by Read. - next(n int) ([]byte, error) - // readUInt32 reads uint32 with LittleEndian order - readUInt32() (uint32, error) - // readUInt16 reads uint16 with LittleEndian order - readUInt16() (uint16, error) - // getReadBytes returns read bytes - getReadBytes() int64 - // skipBytes skips exactly n bytes - skipBytes(n int) error -} - -func newByteInputFromReader(reader io.Reader) byteInput { - return &byteInputAdapter{ - r: reader, - readBytes: 0, - } -} - -func newByteInput(buf []byte) byteInput { - return &byteBuffer{ - buf: buf, - off: 0, - } -} - -type byteBuffer struct { - buf []byte - off int -} - -// next returns a slice containing the next n bytes from the reader -// If there are fewer bytes than the given n, io.ErrUnexpectedEOF will be returned -func (b *byteBuffer) next(n int) ([]byte, error) { - m := len(b.buf) - b.off - - if n > m { - return nil, io.ErrUnexpectedEOF - } - - data := b.buf[b.off : b.off+n] - b.off += n - - return data, nil -} - -// readUInt32 reads uint32 with LittleEndian order -func (b *byteBuffer) readUInt32() (uint32, error) { - if len(b.buf)-b.off < 4 { - return 0, io.ErrUnexpectedEOF - } - - v := binary.LittleEndian.Uint32(b.buf[b.off:]) - b.off += 4 - - return v, nil -} - -// readUInt16 reads uint16 with LittleEndian order -func (b *byteBuffer) readUInt16() (uint16, error) { - if len(b.buf)-b.off < 2 { - return 0, io.ErrUnexpectedEOF - } - - v := binary.LittleEndian.Uint16(b.buf[b.off:]) - b.off += 2 - - return v, nil -} - -// getReadBytes returns read bytes -func (b *byteBuffer) getReadBytes() int64 { - return int64(b.off) -} - -// skipBytes skips exactly n bytes -func (b *byteBuffer) skipBytes(n int) error { - m := len(b.buf) - b.off - - if n > m { - return io.ErrUnexpectedEOF - } - - b.off += n - - return nil -} - -// reset resets the given buffer with a new byte slice -func (b *byteBuffer) reset(buf []byte) { - b.buf = buf - b.off = 0 -} - -type byteInputAdapter struct { - r io.Reader - readBytes int -} - -// next returns a slice containing the next n bytes from the buffer, -// advancing the buffer as if the bytes had been returned by Read. -func (b *byteInputAdapter) next(n int) ([]byte, error) { - buf := make([]byte, n) - m, err := io.ReadAtLeast(b.r, buf, n) - b.readBytes += m - - if err != nil { - return nil, err - } - - return buf, nil -} - -// readUInt32 reads uint32 with LittleEndian order -func (b *byteInputAdapter) readUInt32() (uint32, error) { - buf, err := b.next(4) - - if err != nil { - return 0, err - } - - return binary.LittleEndian.Uint32(buf), nil -} - -// readUInt16 reads uint16 with LittleEndian order -func (b *byteInputAdapter) readUInt16() (uint16, error) { - buf, err := b.next(2) - - if err != nil { - return 0, err - } - - return binary.LittleEndian.Uint16(buf), nil -} - -// getReadBytes returns read bytes -func (b *byteInputAdapter) getReadBytes() int64 { - return int64(b.readBytes) -} - -// skipBytes skips exactly n bytes -func (b *byteInputAdapter) skipBytes(n int) error { - _, err := b.next(n) - - return err -} - -// reset resets the given buffer with a new stream -func (b *byteInputAdapter) reset(stream io.Reader) { - b.r = stream - b.readBytes = 0 -} diff --git a/vendor/github.com/RoaringBitmap/roaring/clz.go b/vendor/github.com/RoaringBitmap/roaring/clz.go index bcd80d32..ee0ebc6c 100644 --- a/vendor/github.com/RoaringBitmap/roaring/clz.go +++ b/vendor/github.com/RoaringBitmap/roaring/clz.go @@ -1,4 +1,6 @@ +//go:build go1.9 // +build go1.9 + // "go1.9", from Go version 1.9 onward // See https://golang.org/pkg/go/build/#hdr-Build_Constraints diff --git a/vendor/github.com/RoaringBitmap/roaring/clz_compat.go b/vendor/github.com/RoaringBitmap/roaring/clz_compat.go index eeef4de3..7ee16b4a 100644 --- a/vendor/github.com/RoaringBitmap/roaring/clz_compat.go +++ b/vendor/github.com/RoaringBitmap/roaring/clz_compat.go @@ -1,3 +1,4 @@ +//go:build !go1.9 // +build !go1.9 package roaring diff --git a/vendor/github.com/RoaringBitmap/roaring/ctz.go b/vendor/github.com/RoaringBitmap/roaring/ctz.go index e399ddde..fbcfe912 100644 --- a/vendor/github.com/RoaringBitmap/roaring/ctz.go +++ b/vendor/github.com/RoaringBitmap/roaring/ctz.go @@ -1,4 +1,6 @@ +//go:build go1.9 // +build go1.9 + // "go1.9", from Go version 1.9 onward // See https://golang.org/pkg/go/build/#hdr-Build_Constraints diff --git a/vendor/github.com/RoaringBitmap/roaring/ctz_compat.go b/vendor/github.com/RoaringBitmap/roaring/ctz_compat.go index 80220e6b..d01df825 100644 --- a/vendor/github.com/RoaringBitmap/roaring/ctz_compat.go +++ b/vendor/github.com/RoaringBitmap/roaring/ctz_compat.go @@ -1,3 +1,4 @@ +//go:build !go1.9 // +build !go1.9 package roaring diff --git a/vendor/github.com/RoaringBitmap/roaring/fastaggregation.go b/vendor/github.com/RoaringBitmap/roaring/fastaggregation.go index ae731b35..7d0a92fe 100644 --- a/vendor/github.com/RoaringBitmap/roaring/fastaggregation.go +++ b/vendor/github.com/RoaringBitmap/roaring/fastaggregation.go @@ -33,15 +33,6 @@ main: s2 = x2.highlowcontainer.getKeyAtIndex(pos2) } else { c1 := x1.highlowcontainer.getContainerAtIndex(pos1) - switch t := c1.(type) { - case *arrayContainer: - c1 = t.toBitmapContainer() - case *runContainer16: - if !t.isFull() { - c1 = t.toBitmapContainer() - } - } - answer.highlowcontainer.appendContainer(s1, c1.lazyOR(x2.highlowcontainer.getContainerAtIndex(pos2)), false) pos1++ pos2++ @@ -89,18 +80,7 @@ main: } s2 = x2.highlowcontainer.getKeyAtIndex(pos2) } else { - c1 := x1.highlowcontainer.getContainerAtIndex(pos1) - switch t := c1.(type) { - case *arrayContainer: - c1 = t.toBitmapContainer() - case *runContainer16: - if !t.isFull() { - c1 = t.toBitmapContainer() - } - case *bitmapContainer: - c1 = x1.highlowcontainer.getWritableContainerAtIndex(pos1) - } - + c1 := x1.highlowcontainer.getWritableContainerAtIndex(pos1) x1.highlowcontainer.containers[pos1] = c1.lazyIOR(x2.highlowcontainer.getContainerAtIndex(pos2)) x1.highlowcontainer.needCopyOnWrite[pos1] = false pos1++ @@ -141,6 +121,10 @@ func (x1 *Bitmap) repairAfterLazy() { // FastAnd computes the intersection between many bitmaps quickly // Compared to the And function, it can take many bitmaps as input, thus saving the trouble // of manually calling "And" many times. +// +// Performance hints: if you have very large and tiny bitmaps, +// it may be beneficial performance-wise to put a tiny bitmap +// in first position. func FastAnd(bitmaps ...*Bitmap) *Bitmap { if len(bitmaps) == 0 { return NewBitmap() @@ -301,9 +285,6 @@ func (x1 *Bitmap) AndAny(bitmaps ...*Bitmap) { tmpBitmap = newBitmapContainer() } tmpBitmap.resetTo(keyContainers[0]) - for _, c := range keyContainers[1:] { - tmpBitmap.ior(c) - } ored = tmpBitmap } else { if tmpArray == nil { @@ -311,15 +292,15 @@ func (x1 *Bitmap) AndAny(bitmaps ...*Bitmap) { } tmpArray.realloc(maxPossibleOr) tmpArray.resetTo(keyContainers[0]) - for _, c := range keyContainers[1:] { - tmpArray.ior(c) - } ored = tmpArray } + for _, c := range keyContainers[1:] { + ored = ored.ior(c) + } } result := x1.highlowcontainer.getWritableContainerAtIndex(basePos).iand(ored) - if result.getCardinality() > 0 { + if !result.isEmpty() { x1.highlowcontainer.replaceKeyAndContainerAtIndex(intersections, baseKey, result, false) intersections++ } diff --git a/vendor/github.com/RoaringBitmap/roaring/internal/byte_input.go b/vendor/github.com/RoaringBitmap/roaring/internal/byte_input.go new file mode 100644 index 00000000..3e5490a9 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/internal/byte_input.go @@ -0,0 +1,166 @@ +package internal + +import ( + "encoding/binary" + "io" +) + +// ByteInput typed interface around io.Reader or raw bytes +type ByteInput interface { + // Next returns a slice containing the next n bytes from the buffer, + // advancing the buffer as if the bytes had been returned by Read. + Next(n int) ([]byte, error) + // ReadUInt32 reads uint32 with LittleEndian order + ReadUInt32() (uint32, error) + // ReadUInt16 reads uint16 with LittleEndian order + ReadUInt16() (uint16, error) + // GetReadBytes returns read bytes + GetReadBytes() int64 + // SkipBytes skips exactly n bytes + SkipBytes(n int) error +} + +// NewByteInputFromReader creates reader wrapper +func NewByteInputFromReader(reader io.Reader) ByteInput { + return &ByteInputAdapter{ + r: reader, + readBytes: 0, + } +} + +// NewByteInput creates raw bytes wrapper +func NewByteInput(buf []byte) ByteInput { + return &ByteBuffer{ + buf: buf, + off: 0, + } +} + +// ByteBuffer raw bytes wrapper +type ByteBuffer struct { + buf []byte + off int +} + +// Next returns a slice containing the next n bytes from the reader +// If there are fewer bytes than the given n, io.ErrUnexpectedEOF will be returned +func (b *ByteBuffer) Next(n int) ([]byte, error) { + m := len(b.buf) - b.off + + if n > m { + return nil, io.ErrUnexpectedEOF + } + + data := b.buf[b.off : b.off+n] + b.off += n + + return data, nil +} + +// ReadUInt32 reads uint32 with LittleEndian order +func (b *ByteBuffer) ReadUInt32() (uint32, error) { + if len(b.buf)-b.off < 4 { + return 0, io.ErrUnexpectedEOF + } + + v := binary.LittleEndian.Uint32(b.buf[b.off:]) + b.off += 4 + + return v, nil +} + +// ReadUInt16 reads uint16 with LittleEndian order +func (b *ByteBuffer) ReadUInt16() (uint16, error) { + if len(b.buf)-b.off < 2 { + return 0, io.ErrUnexpectedEOF + } + + v := binary.LittleEndian.Uint16(b.buf[b.off:]) + b.off += 2 + + return v, nil +} + +// GetReadBytes returns read bytes +func (b *ByteBuffer) GetReadBytes() int64 { + return int64(b.off) +} + +// SkipBytes skips exactly n bytes +func (b *ByteBuffer) SkipBytes(n int) error { + m := len(b.buf) - b.off + + if n > m { + return io.ErrUnexpectedEOF + } + + b.off += n + + return nil +} + +// Reset resets the given buffer with a new byte slice +func (b *ByteBuffer) Reset(buf []byte) { + b.buf = buf + b.off = 0 +} + +// ByteInputAdapter reader wrapper +type ByteInputAdapter struct { + r io.Reader + readBytes int +} + +// Next returns a slice containing the next n bytes from the buffer, +// advancing the buffer as if the bytes had been returned by Read. +func (b *ByteInputAdapter) Next(n int) ([]byte, error) { + buf := make([]byte, n) + m, err := io.ReadAtLeast(b.r, buf, n) + b.readBytes += m + + if err != nil { + return nil, err + } + + return buf, nil +} + +// ReadUInt32 reads uint32 with LittleEndian order +func (b *ByteInputAdapter) ReadUInt32() (uint32, error) { + buf, err := b.Next(4) + + if err != nil { + return 0, err + } + + return binary.LittleEndian.Uint32(buf), nil +} + +// ReadUInt16 reads uint16 with LittleEndian order +func (b *ByteInputAdapter) ReadUInt16() (uint16, error) { + buf, err := b.Next(2) + + if err != nil { + return 0, err + } + + return binary.LittleEndian.Uint16(buf), nil +} + +// GetReadBytes returns read bytes +func (b *ByteInputAdapter) GetReadBytes() int64 { + return int64(b.readBytes) +} + +// SkipBytes skips exactly n bytes +func (b *ByteInputAdapter) SkipBytes(n int) error { + _, err := b.Next(n) + + return err +} + +// Reset resets the given buffer with a new stream +func (b *ByteInputAdapter) Reset(stream io.Reader) { + b.r = stream + b.readBytes = 0 +} diff --git a/vendor/github.com/RoaringBitmap/roaring/internal/pools.go b/vendor/github.com/RoaringBitmap/roaring/internal/pools.go new file mode 100644 index 00000000..d2583568 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/internal/pools.go @@ -0,0 +1,21 @@ +package internal + +import ( + "sync" +) + +var ( + // ByteInputAdapterPool shared pool + ByteInputAdapterPool = sync.Pool{ + New: func() interface{} { + return &ByteInputAdapter{} + }, + } + + // ByteBufferPool shared pool + ByteBufferPool = sync.Pool{ + New: func() interface{} { + return &ByteBuffer{} + }, + } +) diff --git a/vendor/github.com/RoaringBitmap/roaring/parallel.go b/vendor/github.com/RoaringBitmap/roaring/parallel.go index 2af1aed4..9208e3e3 100644 --- a/vendor/github.com/RoaringBitmap/roaring/parallel.go +++ b/vendor/github.com/RoaringBitmap/roaring/parallel.go @@ -166,7 +166,6 @@ func appenderRoutine(bitmapChan chan<- *Bitmap, resultChan <-chan keyedContainer make([]container, 0, expectedKeys), make([]bool, 0, expectedKeys), false, - nil, }, } for i := range keys { @@ -286,14 +285,14 @@ func ParAnd(parallelism int, bitmaps ...*Bitmap) *Bitmap { for input := range inputChan { c := input.containers[0].and(input.containers[1]) for _, next := range input.containers[2:] { - if c.getCardinality() == 0 { + if c.isEmpty() { break } c = c.iand(next) } // Send a nil explicitly if the result of the intersection is an empty container - if c.getCardinality() == 0 { + if c.isEmpty() { c = nil } @@ -355,10 +354,10 @@ func ParOr(parallelism int, bitmaps ...*Bitmap) *Bitmap { if lKey == MaxUint16 && hKey == 0 { return New() } else if len(bitmaps) == 1 { - return bitmaps[0] + return bitmaps[0].Clone() } - keyRange := hKey - lKey + 1 + keyRange := int(hKey) - int(lKey) + 1 if keyRange == 1 { // revert to FastOr. Since the key range is 0 // no container-level aggregation parallelism is achievable diff --git a/vendor/github.com/RoaringBitmap/roaring/popcnt.go b/vendor/github.com/RoaringBitmap/roaring/popcnt.go index 9d99508c..b4980aad 100644 --- a/vendor/github.com/RoaringBitmap/roaring/popcnt.go +++ b/vendor/github.com/RoaringBitmap/roaring/popcnt.go @@ -1,4 +1,6 @@ +//go:build go1.9 // +build go1.9 + // "go1.9", from Go version 1.9 onward // See https://golang.org/pkg/go/build/#hdr-Build_Constraints diff --git a/vendor/github.com/RoaringBitmap/roaring/popcnt_asm.go b/vendor/github.com/RoaringBitmap/roaring/popcnt_asm.go index 882d7f4e..ba2dac91 100644 --- a/vendor/github.com/RoaringBitmap/roaring/popcnt_asm.go +++ b/vendor/github.com/RoaringBitmap/roaring/popcnt_asm.go @@ -1,3 +1,4 @@ +//go:build amd64 && !appengine && !go1.9 // +build amd64,!appengine,!go1.9 package roaring diff --git a/vendor/github.com/RoaringBitmap/roaring/popcnt_compat.go b/vendor/github.com/RoaringBitmap/roaring/popcnt_compat.go index 7ae82d4c..5933e52f 100644 --- a/vendor/github.com/RoaringBitmap/roaring/popcnt_compat.go +++ b/vendor/github.com/RoaringBitmap/roaring/popcnt_compat.go @@ -1,3 +1,4 @@ +//go:build !go1.9 // +build !go1.9 package roaring diff --git a/vendor/github.com/RoaringBitmap/roaring/popcnt_generic.go b/vendor/github.com/RoaringBitmap/roaring/popcnt_generic.go index edf2083f..4ae6d5af 100644 --- a/vendor/github.com/RoaringBitmap/roaring/popcnt_generic.go +++ b/vendor/github.com/RoaringBitmap/roaring/popcnt_generic.go @@ -1,3 +1,4 @@ +//go:build !amd64 || appengine || go1.9 // +build !amd64 appengine go1.9 package roaring diff --git a/vendor/github.com/RoaringBitmap/roaring/roaring.go b/vendor/github.com/RoaringBitmap/roaring/roaring.go index 477bad18..7220da27 100644 --- a/vendor/github.com/RoaringBitmap/roaring/roaring.go +++ b/vendor/github.com/RoaringBitmap/roaring/roaring.go @@ -11,7 +11,8 @@ import ( "fmt" "io" "strconv" - "sync" + + "github.com/RoaringBitmap/roaring/internal" ) // Bitmap represents a compressed bitmap where you can add integers. @@ -52,27 +53,72 @@ func (rb *Bitmap) ToBytes() ([]byte, error) { return rb.highlowcontainer.toBytes() } -// Deprecated: WriteToMsgpack writes a msgpack2/snappy-streaming compressed serialized -// version of this bitmap to stream. The format is not -// compatible with the WriteTo() format, and is -// experimental: it may produce smaller on disk -// footprint and/or be faster to read, depending -// on your content. Currently only the Go roaring -// implementation supports this format. -func (rb *Bitmap) WriteToMsgpack(stream io.Writer) (int64, error) { - return 0, rb.highlowcontainer.writeToMsgpack(stream) +// Checksum computes a hash (currently FNV-1a) for a bitmap that is suitable for +// using bitmaps as elements in hash sets or as keys in hash maps, as well as +// generally quicker comparisons. +// The implementation is biased towards efficiency in little endian machines, so +// expect some extra CPU cycles and memory to be used if your machine is big endian. +// Likewise, don't use this to verify integrity unless you're certain you'll load +// the bitmap on a machine with the same endianess used to create it. +func (rb *Bitmap) Checksum() uint64 { + const ( + offset = 14695981039346656037 + prime = 1099511628211 + ) + + var bytes []byte + + hash := uint64(offset) + + bytes = uint16SliceAsByteSlice(rb.highlowcontainer.keys) + + for _, b := range bytes { + hash ^= uint64(b) + hash *= prime + } + + for _, c := range rb.highlowcontainer.containers { + // 0 separator + hash ^= 0 + hash *= prime + + switch c := c.(type) { + case *bitmapContainer: + bytes = uint64SliceAsByteSlice(c.bitmap) + case *arrayContainer: + bytes = uint16SliceAsByteSlice(c.content) + case *runContainer16: + bytes = interval16SliceAsByteSlice(c.iv) + default: + panic("invalid container type") + } + + if len(bytes) == 0 { + panic("empty containers are not supported") + } + + for _, b := range bytes { + hash ^= uint64(b) + hash *= prime + } + } + + return hash } // ReadFrom reads a serialized version of this bitmap from stream. // The format is compatible with other RoaringBitmap // implementations (Java, C) and is documented here: // https://github.com/RoaringBitmap/RoaringFormatSpec -func (rb *Bitmap) ReadFrom(reader io.Reader) (p int64, err error) { - stream := byteInputAdapterPool.Get().(*byteInputAdapter) - stream.reset(reader) +// Since io.Reader is regarded as a stream and cannot be read twice. +// So add cookieHeader to accept the 4-byte data that has been read in roaring64.ReadFrom. +// It is not necessary to pass cookieHeader when call roaring.ReadFrom to read the roaring32 data directly. +func (rb *Bitmap) ReadFrom(reader io.Reader, cookieHeader ...byte) (p int64, err error) { + stream := internal.ByteInputAdapterPool.Get().(*internal.ByteInputAdapter) + stream.Reset(reader) - p, err = rb.highlowcontainer.readFrom(stream) - byteInputAdapterPool.Put(stream) + p, err = rb.highlowcontainer.readFrom(stream, cookieHeader...) + internal.ByteInputAdapterPool.Put(stream) return } @@ -100,29 +146,15 @@ func (rb *Bitmap) ReadFrom(reader io.Reader) (p int64, err error) { // call CloneCopyOnWriteContainers on all such bitmaps. // func (rb *Bitmap) FromBuffer(buf []byte) (p int64, err error) { - stream := byteBufferPool.Get().(*byteBuffer) - stream.reset(buf) + stream := internal.ByteBufferPool.Get().(*internal.ByteBuffer) + stream.Reset(buf) p, err = rb.highlowcontainer.readFrom(stream) - byteBufferPool.Put(stream) + internal.ByteBufferPool.Put(stream) return } -var ( - byteBufferPool = sync.Pool{ - New: func() interface{} { - return &byteBuffer{} - }, - } - - byteInputAdapterPool = sync.Pool{ - New: func() interface{} { - return &byteInputAdapter{} - }, - } -) - // RunOptimize attempts to further compress the runs of consecutive values found in the bitmap func (rb *Bitmap) RunOptimize() { rb.highlowcontainer.runOptimize() @@ -133,14 +165,6 @@ func (rb *Bitmap) HasRunCompression() bool { return rb.highlowcontainer.hasRunCompression() } -// Deprecated: ReadFromMsgpack reads a msgpack2/snappy-streaming serialized -// version of this bitmap from stream. The format is -// expected is that written by the WriteToMsgpack() -// call; see additional notes there. -func (rb *Bitmap) ReadFromMsgpack(stream io.Reader) (int64, error) { - return 0, rb.highlowcontainer.readFromMsgpack(stream) -} - // MarshalBinary implements the encoding.BinaryMarshaler interface for the bitmap // (same as ToBytes) func (rb *Bitmap) MarshalBinary() ([]byte, error) { @@ -180,8 +204,7 @@ func (rb *Bitmap) ToArray() []uint32 { hs := uint32(rb.highlowcontainer.getKeyAtIndex(pos)) << 16 c := rb.highlowcontainer.getContainerAtIndex(pos) pos++ - c.fillLeastSignificant16bits(array, pos2, hs) - pos2 += c.getCardinality() + pos2 = c.fillLeastSignificant16bits(array, pos2, hs) } return array } @@ -248,6 +271,14 @@ type intIterator struct { hs uint32 iter shortPeekable highlowcontainer *roaringArray + + // These embedded iterators per container type help reduce load in the GC. + // This way, instead of making up-to 64k allocations per full iteration + // we get a single allocation and simply reinitialize the appropriate + // iterator and point to it in the generic `iter` member on each key bound. + shortIter shortIterator + runIter runIterator16 + bitmapIter bitmapContainerShortIterator } // HasNext returns true if there are more integers to iterate over @@ -257,8 +288,19 @@ func (ii *intIterator) HasNext() bool { func (ii *intIterator) init() { if ii.highlowcontainer.size() > ii.pos { - ii.iter = ii.highlowcontainer.getContainerAtIndex(ii.pos).getShortIterator() ii.hs = uint32(ii.highlowcontainer.getKeyAtIndex(ii.pos)) << 16 + c := ii.highlowcontainer.getContainerAtIndex(ii.pos) + switch t := c.(type) { + case *arrayContainer: + ii.shortIter = shortIterator{t.content, 0} + ii.iter = &ii.shortIter + case *runContainer16: + ii.runIter = runIterator16{rc: t, curIndex: 0, curPosInIndex: 0} + ii.iter = &ii.runIter + case *bitmapContainer: + ii.bitmapIter = bitmapContainerShortIterator{t, t.NextSetBit(0)} + ii.iter = &ii.bitmapIter + } } } @@ -279,14 +321,14 @@ func (ii *intIterator) PeekNext() uint32 { // AdvanceIfNeeded advances as long as the next value is smaller than minval func (ii *intIterator) AdvanceIfNeeded(minval uint32) { - to := minval >> 16 + to := minval & 0xffff0000 - for ii.HasNext() && (ii.hs>>16) < to { + for ii.HasNext() && ii.hs < to { ii.pos++ ii.init() } - if ii.HasNext() && (ii.hs>>16) == to { + if ii.HasNext() && ii.hs == to { ii.iter.advanceIfNeeded(lowbits(minval)) if !ii.iter.hasNext() { @@ -296,12 +338,17 @@ func (ii *intIterator) AdvanceIfNeeded(minval uint32) { } } -func newIntIterator(a *Bitmap) *intIterator { - p := new(intIterator) +// IntIterator is meant to allow you to iterate through the values of a bitmap, see Initialize(a *Bitmap) +type IntIterator = intIterator + + +// Initialize configures the existing iterator so that it can iterate through the values of +// the provided bitmap. +// The iteration results are undefined if the bitmap is modified (e.g., with Add or Remove). +func (p *intIterator) Initialize(a *Bitmap) { p.pos = 0 p.highlowcontainer = &a.highlowcontainer p.init() - return p } type intReverseIterator struct { @@ -309,6 +356,10 @@ type intReverseIterator struct { hs uint32 iter shortIterable highlowcontainer *roaringArray + + shortIter reverseIterator + runIter runReverseIterator16 + bitmapIter reverseBitmapContainerShortIterator } // HasNext returns true if there are more integers to iterate over @@ -318,8 +369,30 @@ func (ii *intReverseIterator) HasNext() bool { func (ii *intReverseIterator) init() { if ii.pos >= 0 { - ii.iter = ii.highlowcontainer.getContainerAtIndex(ii.pos).getReverseIterator() ii.hs = uint32(ii.highlowcontainer.getKeyAtIndex(ii.pos)) << 16 + c := ii.highlowcontainer.getContainerAtIndex(ii.pos) + switch t := c.(type) { + case *arrayContainer: + ii.shortIter = reverseIterator{t.content, len(t.content) - 1} + ii.iter = &ii.shortIter + case *runContainer16: + index := int(len(t.iv)) - 1 + pos := uint16(0) + + if index >= 0 { + pos = t.iv[index].length + } + + ii.runIter = runReverseIterator16{rc: t, curIndex: index, curPosInIndex: pos} + ii.iter = &ii.runIter + case *bitmapContainer: + pos := -1 + if t.cardinality > 0 { + pos = int(t.maximum()) + } + ii.bitmapIter = reverseBitmapContainerShortIterator{t, pos} + ii.iter = &ii.bitmapIter + } } else { ii.iter = nil } @@ -335,19 +408,23 @@ func (ii *intReverseIterator) Next() uint32 { return x } -func newIntReverseIterator(a *Bitmap) *intReverseIterator { - p := new(intReverseIterator) +// IntReverseIterator is meant to allow you to iterate through the values of a bitmap, see Initialize(a *Bitmap) +type IntReverseIterator = intReverseIterator + +// Initialize configures the existing iterator so that it can iterate through the values of +// the provided bitmap. +// The iteration results are undefined if the bitmap is modified (e.g., with Add or Remove). +func (p *intReverseIterator) Initialize(a *Bitmap) { p.highlowcontainer = &a.highlowcontainer p.pos = a.highlowcontainer.size() - 1 p.init() - return p } // ManyIntIterable allows you to iterate over the values in a Bitmap type ManyIntIterable interface { - // pass in a buffer to fill up with values, returns how many values were returned + // NextMany fills buf up with values, returns how many values were returned NextMany(buf []uint32) int - // pass in a buffer to fill up with 64 bit values, returns how many values were returned + // NextMany64 fills up buf with 64 bit values, uses hs as a mask (OR), returns how many values were returned NextMany64(hs uint64, buf []uint64) int } @@ -356,12 +433,27 @@ type manyIntIterator struct { hs uint32 iter manyIterable highlowcontainer *roaringArray + + shortIter shortIterator + runIter runIterator16 + bitmapIter bitmapContainerManyIterator } func (ii *manyIntIterator) init() { if ii.highlowcontainer.size() > ii.pos { - ii.iter = ii.highlowcontainer.getContainerAtIndex(ii.pos).getManyIterator() ii.hs = uint32(ii.highlowcontainer.getKeyAtIndex(ii.pos)) << 16 + c := ii.highlowcontainer.getContainerAtIndex(ii.pos) + switch t := c.(type) { + case *arrayContainer: + ii.shortIter = shortIterator{t.content, 0} + ii.iter = &ii.shortIter + case *runContainer16: + ii.runIter = runIterator16{rc: t, curIndex: 0, curPosInIndex: 0} + ii.iter = &ii.runIter + case *bitmapContainer: + ii.bitmapIter = bitmapContainerManyIterator{t, -1, 0} + ii.iter = &ii.bitmapIter + } } else { ii.iter = nil } @@ -403,12 +495,17 @@ func (ii *manyIntIterator) NextMany64(hs64 uint64, buf []uint64) int { return n } -func newManyIntIterator(a *Bitmap) *manyIntIterator { - p := new(manyIntIterator) + +// ManyIntIterator is meant to allow you to iterate through the values of a bitmap, see Initialize(a *Bitmap) +type ManyIntIterator = manyIntIterator + +// Initialize configures the existing iterator so that it can iterate through the values of +// the provided bitmap. +// The iteration results are undefined if the bitmap is modified (e.g., with Add or Remove). +func (p *manyIntIterator) Initialize(a *Bitmap) { p.pos = 0 p.highlowcontainer = &a.highlowcontainer p.init() - return p } // String creates a string representation of the Bitmap @@ -440,7 +537,7 @@ func (rb *Bitmap) String() string { // Iterate iterates over the bitmap, calling the given callback with each value in the bitmap. If the callback returns // false, the iteration is halted. // The iteration results are undefined if the bitmap is modified (e.g., with Add or Remove). -// There is no guarantee as to what order the values will be iterated +// There is no guarantee as to what order the values will be iterated. func (rb *Bitmap) Iterate(cb func(x uint32) bool) { for i := 0; i < rb.highlowcontainer.size(); i++ { hs := uint32(rb.highlowcontainer.getKeyAtIndex(i)) << 16 @@ -472,19 +569,25 @@ func (rb *Bitmap) Iterate(cb func(x uint32) bool) { // Iterator creates a new IntPeekable to iterate over the integers contained in the bitmap, in sorted order; // the iterator becomes invalid if the bitmap is modified (e.g., with Add or Remove). func (rb *Bitmap) Iterator() IntPeekable { - return newIntIterator(rb) + p := new(intIterator) + p.Initialize(rb) + return p } // ReverseIterator creates a new IntIterable to iterate over the integers contained in the bitmap, in sorted order; // the iterator becomes invalid if the bitmap is modified (e.g., with Add or Remove). func (rb *Bitmap) ReverseIterator() IntIterable { - return newIntReverseIterator(rb) + p := new(intReverseIterator) + p.Initialize(rb) + return p } // ManyIterator creates a new ManyIntIterable to iterate over the integers contained in the bitmap, in sorted order; // the iterator becomes invalid if the bitmap is modified (e.g., with Add or Remove). func (rb *Bitmap) ManyIterator() ManyIntIterable { - return newManyIntIterator(rb) + p := new(manyIntIterator) + p.Initialize(rb) + return p } // Clone creates a copy of the Bitmap @@ -496,11 +599,17 @@ func (rb *Bitmap) Clone() *Bitmap { // Minimum get the smallest value stored in this roaring bitmap, assumes that it is not empty func (rb *Bitmap) Minimum() uint32 { + if len(rb.highlowcontainer.containers) == 0 { + panic("Empty bitmap") + } return uint32(rb.highlowcontainer.containers[0].minimum()) | (uint32(rb.highlowcontainer.keys[0]) << 16) } // Maximum get the largest value stored in this roaring bitmap, assumes that it is not empty func (rb *Bitmap) Maximum() uint32 { + if len(rb.highlowcontainer.containers) == 0 { + panic("Empty bitmap") + } lastindex := len(rb.highlowcontainer.containers) - 1 return uint32(rb.highlowcontainer.containers[lastindex].maximum()) | (uint32(rb.highlowcontainer.keys[lastindex]) << 16) } @@ -544,34 +653,38 @@ func AddOffset64(x *Bitmap, offset int64) (answer *Bitmap) { containerOffset64 = offset >> 16 } - if containerOffset64 >= (1<<16) || containerOffset64 <= -(1<<16) { - return New() + answer = New() + + if containerOffset64 >= (1<<16) || containerOffset64 < -(1<<16) { + return answer } containerOffset := int32(containerOffset64) inOffset := (uint16)(offset - containerOffset64*(1<<16)) if inOffset == 0 { - answer = x.Clone() - for pos := 0; pos < answer.highlowcontainer.size(); pos++ { - key := int32(answer.highlowcontainer.getKeyAtIndex(pos)) + for pos := 0; pos < x.highlowcontainer.size(); pos++ { + key := int32(x.highlowcontainer.getKeyAtIndex(pos)) key += containerOffset if key >= 0 && key <= MaxUint16 { - answer.highlowcontainer.keys[pos] = uint16(key) + c := x.highlowcontainer.getContainerAtIndex(pos).clone() + answer.highlowcontainer.appendContainer(uint16(key), c, false) } } } else { - answer = New() - for pos := 0; pos < x.highlowcontainer.size(); pos++ { key := int32(x.highlowcontainer.getKeyAtIndex(pos)) key += containerOffset + if key+1 < 0 || key > MaxUint16 { + continue + } + c := x.highlowcontainer.getContainerAtIndex(pos) - offsetted := c.addOffset(inOffset) + lo, hi := c.addOffset(inOffset) - if offsetted[0].getCardinality() > 0 && (key >= 0 && key <= MaxUint16) { + if lo != nil && key >= 0 { curSize := answer.highlowcontainer.size() lastkey := int32(0) @@ -581,15 +694,15 @@ func AddOffset64(x *Bitmap, offset int64) (answer *Bitmap) { if curSize > 0 && lastkey == key { prev := answer.highlowcontainer.getContainerAtIndex(curSize - 1) - orrseult := prev.ior(offsetted[0]) - answer.highlowcontainer.setContainerAtIndex(curSize-1, orrseult) + orresult := prev.ior(lo) + answer.highlowcontainer.setContainerAtIndex(curSize-1, orresult) } else { - answer.highlowcontainer.appendContainer(uint16(key), offsetted[0], false) + answer.highlowcontainer.appendContainer(uint16(key), lo, false) } } - if offsetted[1].getCardinality() > 0 && ((key+1) >= 0 && (key+1) <= MaxUint16) { - answer.highlowcontainer.appendContainer(uint16(key+1), offsetted[1], false) + if hi != nil && key+1 <= MaxUint16 { + answer.highlowcontainer.appendContainer(uint16(key+1), hi, false) } } } @@ -659,13 +772,13 @@ func (rb *Bitmap) Remove(x uint32) { if i >= 0 { c := rb.highlowcontainer.getWritableContainerAtIndex(i).iremoveReturnMinimized(lowbits(x)) rb.highlowcontainer.setContainerAtIndex(i, c) - if rb.highlowcontainer.getContainerAtIndex(i).getCardinality() == 0 { + if rb.highlowcontainer.getContainerAtIndex(i).isEmpty() { rb.highlowcontainer.removeAtIndex(i) } } } -// CheckedRemove removes the integer x from the bitmap and return true if the integer was effectively remove (and false if the integer was not present) +// CheckedRemove removes the integer x from the bitmap and return true if the integer was effectively removed (and false if the integer was not present) func (rb *Bitmap) CheckedRemove(x uint32) bool { // TODO: add unit tests for this method hb := highbits(x) @@ -675,7 +788,7 @@ func (rb *Bitmap) CheckedRemove(x uint32) bool { oldcard := C.getCardinality() C = C.iremoveReturnMinimized(lowbits(x)) rb.highlowcontainer.setContainerAtIndex(i, C) - if rb.highlowcontainer.getContainerAtIndex(i).getCardinality() == 0 { + if rb.highlowcontainer.getContainerAtIndex(i).isEmpty() { rb.highlowcontainer.removeAtIndex(i) return true } @@ -723,15 +836,12 @@ func (rb *Bitmap) Rank(x uint32) uint64 { // the smallest element. Note that this function differs in convention from // the Rank function which returns 1 on the smallest value. func (rb *Bitmap) Select(x uint32) (uint32, error) { - if rb.GetCardinality() <= uint64(x) { - return 0, fmt.Errorf("can't find %dth integer in a bitmap with only %d items", x, rb.GetCardinality()) - } - remaining := x for i := 0; i < rb.highlowcontainer.size(); i++ { c := rb.highlowcontainer.getContainerAtIndex(i) - if remaining >= uint32(c.getCardinality()) { - remaining -= uint32(c.getCardinality()) + card := uint32(c.getCardinality()) + if remaining >= card { + remaining -= card } else { key := rb.highlowcontainer.getKeyAtIndex(i) return uint32(key)<<16 + uint32(c.selectInt(uint16(remaining))), nil @@ -758,7 +868,7 @@ main: c1 := rb.highlowcontainer.getWritableContainerAtIndex(pos1) c2 := x2.highlowcontainer.getContainerAtIndex(pos2) diff := c1.iand(c2) - if diff.getCardinality() > 0 { + if !diff.isEmpty() { rb.highlowcontainer.replaceKeyAndContainerAtIndex(intersectionsize, s1, diff, false) intersectionsize++ } @@ -889,6 +999,28 @@ main: return answer } +// IntersectsWithInterval checks whether a bitmap 'rb' and an open interval '[x,y)' intersect. +func (rb *Bitmap) IntersectsWithInterval(x, y uint64) bool { + if x >= y { + return false + } + if x > MaxUint32 { + return false + } + + it := intIterator{} + it.Initialize(rb) + it.AdvanceIfNeeded(uint32(x)) + if !it.HasNext() { + return false + } + if uint64(it.Next()) >= y { + return false + } + + return true +} + // Intersects checks whether two bitmap intersects, bitmaps are not modified func (rb *Bitmap) Intersects(x2 *Bitmap) bool { pos1 := 0 @@ -960,7 +1092,7 @@ func (rb *Bitmap) Xor(x2 *Bitmap) { } else { // TODO: couple be computed in-place for reduced memory usage c := rb.highlowcontainer.getContainerAtIndex(pos1).xor(x2.highlowcontainer.getContainerAtIndex(pos2)) - if c.getCardinality() > 0 { + if !c.isEmpty() { rb.highlowcontainer.setContainerAtIndex(pos1, c) pos1++ } else { @@ -1006,7 +1138,7 @@ main: } s2 = x2.highlowcontainer.getKeyAtIndex(pos2) } else { - rb.highlowcontainer.replaceKeyAndContainerAtIndex(pos1, s1, rb.highlowcontainer.getWritableContainerAtIndex(pos1).ior(x2.highlowcontainer.getContainerAtIndex(pos2)), false) + rb.highlowcontainer.replaceKeyAndContainerAtIndex(pos1, s1, rb.highlowcontainer.getUnionedWritableContainer(pos1, x2.highlowcontainer.getContainerAtIndex(pos2)), false) pos1++ pos2++ if (pos1 == length1) || (pos2 == length2) { @@ -1040,7 +1172,7 @@ main: c1 := rb.highlowcontainer.getWritableContainerAtIndex(pos1) c2 := x2.highlowcontainer.getContainerAtIndex(pos2) diff := c1.iandNot(c2) - if diff.getCardinality() > 0 { + if !diff.isEmpty() { rb.highlowcontainer.replaceKeyAndContainerAtIndex(intersectionsize, s1, diff, false) intersectionsize++ } @@ -1149,7 +1281,7 @@ main: C := x1.highlowcontainer.getContainerAtIndex(pos1) C = C.and(x2.highlowcontainer.getContainerAtIndex(pos2)) - if C.getCardinality() > 0 { + if !C.isEmpty() { answer.highlowcontainer.appendContainer(s1, C, false) } pos1++ @@ -1196,7 +1328,7 @@ func Xor(x1, x2 *Bitmap) *Bitmap { pos2++ } else { c := x1.highlowcontainer.getContainerAtIndex(pos1).xor(x2.highlowcontainer.getContainerAtIndex(pos2)) - if c.getCardinality() > 0 { + if !c.isEmpty() { answer.highlowcontainer.appendContainer(s1, c, false) } pos1++ @@ -1239,7 +1371,7 @@ main: c1 := x1.highlowcontainer.getContainerAtIndex(pos1) c2 := x2.highlowcontainer.getContainerAtIndex(pos2) diff := c1.andNot(c2) - if diff.getCardinality() > 0 { + if !diff.isEmpty() { answer.highlowcontainer.appendContainer(s1, diff, false) } pos1++ @@ -1329,7 +1461,7 @@ func (rb *Bitmap) Flip(rangeStart, rangeEnd uint64) { if i >= 0 { c := rb.highlowcontainer.getWritableContainerAtIndex(i).inot(int(containerStart), int(containerLast)+1) - if c.getCardinality() > 0 { + if !c.isEmpty() { rb.highlowcontainer.setContainerAtIndex(i, c) } else { rb.highlowcontainer.removeAtIndex(i) @@ -1410,7 +1542,7 @@ func (rb *Bitmap) RemoveRange(rangeStart, rangeEnd uint64) { return } c := rb.highlowcontainer.getWritableContainerAtIndex(i).iremoveRange(int(lbStart), int(lbLast+1)) - if c.getCardinality() > 0 { + if !c.isEmpty() { rb.highlowcontainer.setContainerAtIndex(i, c) } else { rb.highlowcontainer.removeAtIndex(i) @@ -1423,7 +1555,7 @@ func (rb *Bitmap) RemoveRange(rangeStart, rangeEnd uint64) { if ifirst >= 0 { if lbStart != 0 { c := rb.highlowcontainer.getWritableContainerAtIndex(ifirst).iremoveRange(int(lbStart), int(max+1)) - if c.getCardinality() > 0 { + if !c.isEmpty() { rb.highlowcontainer.setContainerAtIndex(ifirst, c) ifirst++ } @@ -1434,7 +1566,7 @@ func (rb *Bitmap) RemoveRange(rangeStart, rangeEnd uint64) { if ilast >= 0 { if lbLast != max { c := rb.highlowcontainer.getWritableContainerAtIndex(ilast).iremoveRange(int(0), int(lbLast+1)) - if c.getCardinality() > 0 { + if !c.isEmpty() { rb.highlowcontainer.setContainerAtIndex(ilast, c) } else { ilast++ @@ -1490,7 +1622,7 @@ func Flip(bm *Bitmap, rangeStart, rangeEnd uint64) *Bitmap { if i >= 0 { c := bm.highlowcontainer.getContainerAtIndex(i).not(int(containerStart), int(containerLast)+1) - if c.getCardinality() > 0 { + if !c.isEmpty() { answer.highlowcontainer.insertNewKeyValueAt(-j-1, uint16(hb), c) } @@ -1581,7 +1713,3 @@ func (rb *Bitmap) Stats() Statistics { } return stats } - -func (rb *Bitmap) FillLeastSignificant32bits(x []uint64, i uint64, mask uint64) { - rb.ManyIterator().NextMany64(mask, x[i:]) -} diff --git a/vendor/github.com/RoaringBitmap/roaring/roaringarray.go b/vendor/github.com/RoaringBitmap/roaring/roaringarray.go index 89275577..eeb3d313 100644 --- a/vendor/github.com/RoaringBitmap/roaring/roaringarray.go +++ b/vendor/github.com/RoaringBitmap/roaring/roaringarray.go @@ -4,16 +4,15 @@ import ( "bytes" "encoding/binary" "fmt" + "github.com/RoaringBitmap/roaring/internal" "io" - - snappy "github.com/glycerine/go-unsnap-stream" - "github.com/tinylib/msgp/msgp" ) -//go:generate msgp -unexported - type container interface { - addOffset(uint16) []container + // addOffset returns the (low, high) parts of the shifted container. + // Whenever one of them would be empty, nil will be returned instead to + // avoid unnecessary allocations. + addOffset(uint16) (container, container) clone() container and(container) container @@ -21,6 +20,7 @@ type container interface { iand(container) container // i stands for inplace andNot(container) container iandNot(container) container // i stands for inplace + isEmpty() bool getCardinality() int // rank returns the number of integers that are // smaller or equal to x. rank(infinity) would be getCardinality(). @@ -51,7 +51,7 @@ type container interface { // any of the implementations. equals(r container) bool - fillLeastSignificant16bits(array []uint32, i int, mask uint32) + fillLeastSignificant16bits(array []uint32, i int, mask uint32) int or(r container) container orCardinality(r container) int isFull() bool @@ -103,18 +103,6 @@ type roaringArray struct { containers []container `msg:"-"` // don't try to serialize directly. needCopyOnWrite []bool copyOnWrite bool - - // conserz is used at serialization time - // to serialize containers. Otherwise empty. - conserz []containerSerz -} - -// containerSerz facilitates serializing container (tricky to -// serialize because it is an interface) by providing a -// light wrapper with a type identifier. -type containerSerz struct { - t contype `msg:"t"` // type - r msgp.Raw `msg:"r"` // Raw msgpack of the actual container type } func newRoaringArray() *roaringArray { @@ -246,7 +234,6 @@ func (ra *roaringArray) resize(newsize int) { func (ra *roaringArray) clear() { ra.resize(0) ra.copyOnWrite = false - ra.conserz = nil } func (ra *roaringArray) clone() *roaringArray { @@ -328,6 +315,17 @@ func (ra *roaringArray) getFastContainerAtIndex(i int, needsWriteable bool) cont return c } +// getUnionedWritableContainer switches behavior for in-place Or +// depending on whether the container requires a copy on write. +// If it does using the non-inplace or() method leads to fewer allocations. +func (ra *roaringArray) getUnionedWritableContainer(pos int, other container) container { + if ra.needCopyOnWrite[pos] { + return ra.getContainerAtIndex(pos).or(other) + } + return ra.getContainerAtIndex(pos).ior(other) + +} + func (ra *roaringArray) getWritableContainerAtIndex(i int) container { if ra.needCopyOnWrite[i] { ra.containers[i] = ra.containers[i].clone() @@ -555,51 +553,58 @@ func (ra *roaringArray) toBytes() ([]byte, error) { return buf.Bytes(), err } -func (ra *roaringArray) readFrom(stream byteInput) (int64, error) { - cookie, err := stream.readUInt32() - - if err != nil { - return stream.getReadBytes(), fmt.Errorf("error in roaringArray.readFrom: could not read initial cookie: %s", err) +func (ra *roaringArray) readFrom(stream internal.ByteInput, cookieHeader ...byte) (int64, error) { + var cookie uint32 + var err error + if len(cookieHeader) > 0 && len(cookieHeader) != 4 { + return int64(len(cookieHeader)), fmt.Errorf("error in roaringArray.readFrom: could not read initial cookie: incorrect size of cookie header") + } + if len(cookieHeader) == 4 { + cookie = binary.LittleEndian.Uint32(cookieHeader) + } else { + cookie, err = stream.ReadUInt32() + if err != nil { + return stream.GetReadBytes(), fmt.Errorf("error in roaringArray.readFrom: could not read initial cookie: %s", err) + } } var size uint32 var isRunBitmap []byte if cookie&0x0000FFFF == serialCookie { - size = uint32(uint16(cookie>>16) + 1) + size = uint32(cookie>>16 + 1) // create is-run-container bitmap isRunBitmapSize := (int(size) + 7) / 8 - isRunBitmap, err = stream.next(isRunBitmapSize) + isRunBitmap, err = stream.Next(isRunBitmapSize) if err != nil { - return stream.getReadBytes(), fmt.Errorf("malformed bitmap, failed to read is-run bitmap, got: %s", err) + return stream.GetReadBytes(), fmt.Errorf("malformed bitmap, failed to read is-run bitmap, got: %s", err) } } else if cookie == serialCookieNoRunContainer { - size, err = stream.readUInt32() - + size, err = stream.ReadUInt32() if err != nil { - return stream.getReadBytes(), fmt.Errorf("malformed bitmap, failed to read a bitmap size: %s", err) + return stream.GetReadBytes(), fmt.Errorf("malformed bitmap, failed to read a bitmap size: %s", err) } } else { - return stream.getReadBytes(), fmt.Errorf("error in roaringArray.readFrom: did not find expected serialCookie in header") + return stream.GetReadBytes(), fmt.Errorf("error in roaringArray.readFrom: did not find expected serialCookie in header") } if size > (1 << 16) { - return stream.getReadBytes(), fmt.Errorf("it is logically impossible to have more than (1<<16) containers") + return stream.GetReadBytes(), fmt.Errorf("it is logically impossible to have more than (1<<16) containers") } // descriptive header - buf, err := stream.next(2 * 2 * int(size)) + buf, err := stream.Next(2 * 2 * int(size)) if err != nil { - return stream.getReadBytes(), fmt.Errorf("failed to read descriptive header: %s", err) + return stream.GetReadBytes(), fmt.Errorf("failed to read descriptive header: %s", err) } keycard := byteSliceAsUint16Slice(buf) if isRunBitmap == nil || size >= noOffsetThreshold { - if err := stream.skipBytes(int(size) * 4); err != nil { - return stream.getReadBytes(), fmt.Errorf("failed to skip bytes: %s", err) + if err := stream.SkipBytes(int(size) * 4); err != nil { + return stream.GetReadBytes(), fmt.Errorf("failed to skip bytes: %s", err) } } @@ -630,30 +635,29 @@ func (ra *roaringArray) readFrom(stream byteInput) (int64, error) { if isRunBitmap != nil && isRunBitmap[i/8]&(1<<(i%8)) != 0 { // run container - nr, err := stream.readUInt16() + nr, err := stream.ReadUInt16() if err != nil { return 0, fmt.Errorf("failed to read runtime container size: %s", err) } - buf, err := stream.next(int(nr) * 4) + buf, err := stream.Next(int(nr) * 4) if err != nil { - return stream.getReadBytes(), fmt.Errorf("failed to read runtime container content: %s", err) + return stream.GetReadBytes(), fmt.Errorf("failed to read runtime container content: %s", err) } nb := runContainer16{ - iv: byteSliceAsInterval16Slice(buf), - card: int64(card), + iv: byteSliceAsInterval16Slice(buf), } ra.containers[i] = &nb } else if card > arrayDefaultMaxSize { // bitmap container - buf, err := stream.next(arrayDefaultMaxSize * 2) + buf, err := stream.Next(arrayDefaultMaxSize * 2) if err != nil { - return stream.getReadBytes(), fmt.Errorf("failed to read bitmap container: %s", err) + return stream.GetReadBytes(), fmt.Errorf("failed to read bitmap container: %s", err) } nb := bitmapContainer{ @@ -664,10 +668,10 @@ func (ra *roaringArray) readFrom(stream byteInput) (int64, error) { ra.containers[i] = &nb } else { // array container - buf, err := stream.next(card * 2) + buf, err := stream.Next(card * 2) if err != nil { - return stream.getReadBytes(), fmt.Errorf("failed to read array container: %s", err) + return stream.GetReadBytes(), fmt.Errorf("failed to read array container: %s", err) } nb := arrayContainer{ @@ -678,7 +682,7 @@ func (ra *roaringArray) readFrom(stream byteInput) (int64, error) { } } - return stream.getReadBytes(), nil + return stream.GetReadBytes(), nil } func (ra *roaringArray) hasRunCompression() bool { @@ -691,84 +695,6 @@ func (ra *roaringArray) hasRunCompression() bool { return false } -func (ra *roaringArray) writeToMsgpack(stream io.Writer) error { - - ra.conserz = make([]containerSerz, len(ra.containers)) - for i, v := range ra.containers { - switch cn := v.(type) { - case *bitmapContainer: - bts, err := cn.MarshalMsg(nil) - if err != nil { - return err - } - ra.conserz[i].t = bitmapContype - ra.conserz[i].r = bts - case *arrayContainer: - bts, err := cn.MarshalMsg(nil) - if err != nil { - return err - } - ra.conserz[i].t = arrayContype - ra.conserz[i].r = bts - case *runContainer16: - bts, err := cn.MarshalMsg(nil) - if err != nil { - return err - } - ra.conserz[i].t = run16Contype - ra.conserz[i].r = bts - default: - panic(fmt.Errorf("Unrecognized container implementation: %T", cn)) - } - } - w := snappy.NewWriter(stream) - err := msgp.Encode(w, ra) - ra.conserz = nil - return err -} - -func (ra *roaringArray) readFromMsgpack(stream io.Reader) error { - r := snappy.NewReader(stream) - err := msgp.Decode(r, ra) - if err != nil { - return err - } - - if len(ra.containers) != len(ra.keys) { - ra.containers = make([]container, len(ra.keys)) - } - - for i, v := range ra.conserz { - switch v.t { - case bitmapContype: - c := &bitmapContainer{} - _, err = c.UnmarshalMsg(v.r) - if err != nil { - return err - } - ra.containers[i] = c - case arrayContype: - c := &arrayContainer{} - _, err = c.UnmarshalMsg(v.r) - if err != nil { - return err - } - ra.containers[i] = c - case run16Contype: - c := &runContainer16{} - _, err = c.UnmarshalMsg(v.r) - if err != nil { - return err - } - ra.containers[i] = c - default: - return fmt.Errorf("unrecognized contype serialization code: '%v'", v.t) - } - } - ra.conserz = nil - return nil -} - func (ra *roaringArray) advanceUntil(min uint16, pos int) int { lower := pos + 1 diff --git a/vendor/github.com/RoaringBitmap/roaring/roaringarray_gen.go b/vendor/github.com/RoaringBitmap/roaring/roaringarray_gen.go deleted file mode 100644 index dcd71875..00000000 --- a/vendor/github.com/RoaringBitmap/roaring/roaringarray_gen.go +++ /dev/null @@ -1,529 +0,0 @@ -package roaring - -// NOTE: THIS FILE WAS PRODUCED BY THE -// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp) -// DO NOT EDIT - -import ( - "github.com/tinylib/msgp/msgp" -) - -// Deprecated: DecodeMsg implements msgp.Decodable -func (z *containerSerz) DecodeMsg(dc *msgp.Reader) (err error) { - var field []byte - _ = field - var zxvk uint32 - zxvk, err = dc.ReadMapHeader() - if err != nil { - return - } - for zxvk > 0 { - zxvk-- - field, err = dc.ReadMapKeyPtr() - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "t": - { - var zbzg uint8 - zbzg, err = dc.ReadUint8() - z.t = contype(zbzg) - } - if err != nil { - return - } - case "r": - err = z.r.DecodeMsg(dc) - if err != nil { - return - } - default: - err = dc.Skip() - if err != nil { - return - } - } - } - return -} - -// Deprecated: EncodeMsg implements msgp.Encodable -func (z *containerSerz) EncodeMsg(en *msgp.Writer) (err error) { - // map header, size 2 - // write "t" - err = en.Append(0x82, 0xa1, 0x74) - if err != nil { - return err - } - err = en.WriteUint8(uint8(z.t)) - if err != nil { - return - } - // write "r" - err = en.Append(0xa1, 0x72) - if err != nil { - return err - } - err = z.r.EncodeMsg(en) - if err != nil { - return - } - return -} - -// Deprecated: MarshalMsg implements msgp.Marshaler -func (z *containerSerz) MarshalMsg(b []byte) (o []byte, err error) { - o = msgp.Require(b, z.Msgsize()) - // map header, size 2 - // string "t" - o = append(o, 0x82, 0xa1, 0x74) - o = msgp.AppendUint8(o, uint8(z.t)) - // string "r" - o = append(o, 0xa1, 0x72) - o, err = z.r.MarshalMsg(o) - if err != nil { - return - } - return -} - -// Deprecated: UnmarshalMsg implements msgp.Unmarshaler -func (z *containerSerz) UnmarshalMsg(bts []byte) (o []byte, err error) { - var field []byte - _ = field - var zbai uint32 - zbai, bts, err = msgp.ReadMapHeaderBytes(bts) - if err != nil { - return - } - for zbai > 0 { - zbai-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "t": - { - var zcmr uint8 - zcmr, bts, err = msgp.ReadUint8Bytes(bts) - z.t = contype(zcmr) - } - if err != nil { - return - } - case "r": - bts, err = z.r.UnmarshalMsg(bts) - if err != nil { - return - } - default: - bts, err = msgp.Skip(bts) - if err != nil { - return - } - } - } - o = bts - return -} - -// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z *containerSerz) Msgsize() (s int) { - s = 1 + 2 + msgp.Uint8Size + 2 + z.r.Msgsize() - return -} - -// Deprecated: DecodeMsg implements msgp.Decodable -func (z *contype) DecodeMsg(dc *msgp.Reader) (err error) { - { - var zajw uint8 - zajw, err = dc.ReadUint8() - (*z) = contype(zajw) - } - if err != nil { - return - } - return -} - -// Deprecated: EncodeMsg implements msgp.Encodable -func (z contype) EncodeMsg(en *msgp.Writer) (err error) { - err = en.WriteUint8(uint8(z)) - if err != nil { - return - } - return -} - -// Deprecated: MarshalMsg implements msgp.Marshaler -func (z contype) MarshalMsg(b []byte) (o []byte, err error) { - o = msgp.Require(b, z.Msgsize()) - o = msgp.AppendUint8(o, uint8(z)) - return -} - -// Deprecated: UnmarshalMsg implements msgp.Unmarshaler -func (z *contype) UnmarshalMsg(bts []byte) (o []byte, err error) { - { - var zwht uint8 - zwht, bts, err = msgp.ReadUint8Bytes(bts) - (*z) = contype(zwht) - } - if err != nil { - return - } - o = bts - return -} - -// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z contype) Msgsize() (s int) { - s = msgp.Uint8Size - return -} - -// Deprecated: DecodeMsg implements msgp.Decodable -func (z *roaringArray) DecodeMsg(dc *msgp.Reader) (err error) { - var field []byte - _ = field - var zlqf uint32 - zlqf, err = dc.ReadMapHeader() - if err != nil { - return - } - for zlqf > 0 { - zlqf-- - field, err = dc.ReadMapKeyPtr() - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "keys": - var zdaf uint32 - zdaf, err = dc.ReadArrayHeader() - if err != nil { - return - } - if cap(z.keys) >= int(zdaf) { - z.keys = (z.keys)[:zdaf] - } else { - z.keys = make([]uint16, zdaf) - } - for zhct := range z.keys { - z.keys[zhct], err = dc.ReadUint16() - if err != nil { - return - } - } - case "needCopyOnWrite": - var zpks uint32 - zpks, err = dc.ReadArrayHeader() - if err != nil { - return - } - if cap(z.needCopyOnWrite) >= int(zpks) { - z.needCopyOnWrite = (z.needCopyOnWrite)[:zpks] - } else { - z.needCopyOnWrite = make([]bool, zpks) - } - for zcua := range z.needCopyOnWrite { - z.needCopyOnWrite[zcua], err = dc.ReadBool() - if err != nil { - return - } - } - case "copyOnWrite": - z.copyOnWrite, err = dc.ReadBool() - if err != nil { - return - } - case "conserz": - var zjfb uint32 - zjfb, err = dc.ReadArrayHeader() - if err != nil { - return - } - if cap(z.conserz) >= int(zjfb) { - z.conserz = (z.conserz)[:zjfb] - } else { - z.conserz = make([]containerSerz, zjfb) - } - for zxhx := range z.conserz { - var zcxo uint32 - zcxo, err = dc.ReadMapHeader() - if err != nil { - return - } - for zcxo > 0 { - zcxo-- - field, err = dc.ReadMapKeyPtr() - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "t": - { - var zeff uint8 - zeff, err = dc.ReadUint8() - z.conserz[zxhx].t = contype(zeff) - } - if err != nil { - return - } - case "r": - err = z.conserz[zxhx].r.DecodeMsg(dc) - if err != nil { - return - } - default: - err = dc.Skip() - if err != nil { - return - } - } - } - } - default: - err = dc.Skip() - if err != nil { - return - } - } - } - return -} - -// Deprecated: EncodeMsg implements msgp.Encodable -func (z *roaringArray) EncodeMsg(en *msgp.Writer) (err error) { - // map header, size 4 - // write "keys" - err = en.Append(0x84, 0xa4, 0x6b, 0x65, 0x79, 0x73) - if err != nil { - return err - } - err = en.WriteArrayHeader(uint32(len(z.keys))) - if err != nil { - return - } - for zhct := range z.keys { - err = en.WriteUint16(z.keys[zhct]) - if err != nil { - return - } - } - // write "needCopyOnWrite" - err = en.Append(0xaf, 0x6e, 0x65, 0x65, 0x64, 0x43, 0x6f, 0x70, 0x79, 0x4f, 0x6e, 0x57, 0x72, 0x69, 0x74, 0x65) - if err != nil { - return err - } - err = en.WriteArrayHeader(uint32(len(z.needCopyOnWrite))) - if err != nil { - return - } - for zcua := range z.needCopyOnWrite { - err = en.WriteBool(z.needCopyOnWrite[zcua]) - if err != nil { - return - } - } - // write "copyOnWrite" - err = en.Append(0xab, 0x63, 0x6f, 0x70, 0x79, 0x4f, 0x6e, 0x57, 0x72, 0x69, 0x74, 0x65) - if err != nil { - return err - } - err = en.WriteBool(z.copyOnWrite) - if err != nil { - return - } - // write "conserz" - err = en.Append(0xa7, 0x63, 0x6f, 0x6e, 0x73, 0x65, 0x72, 0x7a) - if err != nil { - return err - } - err = en.WriteArrayHeader(uint32(len(z.conserz))) - if err != nil { - return - } - for zxhx := range z.conserz { - // map header, size 2 - // write "t" - err = en.Append(0x82, 0xa1, 0x74) - if err != nil { - return err - } - err = en.WriteUint8(uint8(z.conserz[zxhx].t)) - if err != nil { - return - } - // write "r" - err = en.Append(0xa1, 0x72) - if err != nil { - return err - } - err = z.conserz[zxhx].r.EncodeMsg(en) - if err != nil { - return - } - } - return -} - -// Deprecated: MarshalMsg implements msgp.Marshaler -func (z *roaringArray) MarshalMsg(b []byte) (o []byte, err error) { - o = msgp.Require(b, z.Msgsize()) - // map header, size 4 - // string "keys" - o = append(o, 0x84, 0xa4, 0x6b, 0x65, 0x79, 0x73) - o = msgp.AppendArrayHeader(o, uint32(len(z.keys))) - for zhct := range z.keys { - o = msgp.AppendUint16(o, z.keys[zhct]) - } - // string "needCopyOnWrite" - o = append(o, 0xaf, 0x6e, 0x65, 0x65, 0x64, 0x43, 0x6f, 0x70, 0x79, 0x4f, 0x6e, 0x57, 0x72, 0x69, 0x74, 0x65) - o = msgp.AppendArrayHeader(o, uint32(len(z.needCopyOnWrite))) - for zcua := range z.needCopyOnWrite { - o = msgp.AppendBool(o, z.needCopyOnWrite[zcua]) - } - // string "copyOnWrite" - o = append(o, 0xab, 0x63, 0x6f, 0x70, 0x79, 0x4f, 0x6e, 0x57, 0x72, 0x69, 0x74, 0x65) - o = msgp.AppendBool(o, z.copyOnWrite) - // string "conserz" - o = append(o, 0xa7, 0x63, 0x6f, 0x6e, 0x73, 0x65, 0x72, 0x7a) - o = msgp.AppendArrayHeader(o, uint32(len(z.conserz))) - for zxhx := range z.conserz { - // map header, size 2 - // string "t" - o = append(o, 0x82, 0xa1, 0x74) - o = msgp.AppendUint8(o, uint8(z.conserz[zxhx].t)) - // string "r" - o = append(o, 0xa1, 0x72) - o, err = z.conserz[zxhx].r.MarshalMsg(o) - if err != nil { - return - } - } - return -} - -// Deprecated: UnmarshalMsg implements msgp.Unmarshaler -func (z *roaringArray) UnmarshalMsg(bts []byte) (o []byte, err error) { - var field []byte - _ = field - var zrsw uint32 - zrsw, bts, err = msgp.ReadMapHeaderBytes(bts) - if err != nil { - return - } - for zrsw > 0 { - zrsw-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "keys": - var zxpk uint32 - zxpk, bts, err = msgp.ReadArrayHeaderBytes(bts) - if err != nil { - return - } - if cap(z.keys) >= int(zxpk) { - z.keys = (z.keys)[:zxpk] - } else { - z.keys = make([]uint16, zxpk) - } - for zhct := range z.keys { - z.keys[zhct], bts, err = msgp.ReadUint16Bytes(bts) - if err != nil { - return - } - } - case "needCopyOnWrite": - var zdnj uint32 - zdnj, bts, err = msgp.ReadArrayHeaderBytes(bts) - if err != nil { - return - } - if cap(z.needCopyOnWrite) >= int(zdnj) { - z.needCopyOnWrite = (z.needCopyOnWrite)[:zdnj] - } else { - z.needCopyOnWrite = make([]bool, zdnj) - } - for zcua := range z.needCopyOnWrite { - z.needCopyOnWrite[zcua], bts, err = msgp.ReadBoolBytes(bts) - if err != nil { - return - } - } - case "copyOnWrite": - z.copyOnWrite, bts, err = msgp.ReadBoolBytes(bts) - if err != nil { - return - } - case "conserz": - var zobc uint32 - zobc, bts, err = msgp.ReadArrayHeaderBytes(bts) - if err != nil { - return - } - if cap(z.conserz) >= int(zobc) { - z.conserz = (z.conserz)[:zobc] - } else { - z.conserz = make([]containerSerz, zobc) - } - for zxhx := range z.conserz { - var zsnv uint32 - zsnv, bts, err = msgp.ReadMapHeaderBytes(bts) - if err != nil { - return - } - for zsnv > 0 { - zsnv-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "t": - { - var zkgt uint8 - zkgt, bts, err = msgp.ReadUint8Bytes(bts) - z.conserz[zxhx].t = contype(zkgt) - } - if err != nil { - return - } - case "r": - bts, err = z.conserz[zxhx].r.UnmarshalMsg(bts) - if err != nil { - return - } - default: - bts, err = msgp.Skip(bts) - if err != nil { - return - } - } - } - } - default: - bts, err = msgp.Skip(bts) - if err != nil { - return - } - } - } - o = bts - return -} - -// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z *roaringArray) Msgsize() (s int) { - s = 1 + 5 + msgp.ArrayHeaderSize + (len(z.keys) * (msgp.Uint16Size)) + 16 + msgp.ArrayHeaderSize + (len(z.needCopyOnWrite) * (msgp.BoolSize)) + 12 + msgp.BoolSize + 8 + msgp.ArrayHeaderSize - for zxhx := range z.conserz { - s += 1 + 2 + msgp.Uint8Size + 2 + z.conserz[zxhx].r.Msgsize() - } - return -} diff --git a/vendor/github.com/RoaringBitmap/roaring/runcontainer.go b/vendor/github.com/RoaringBitmap/roaring/runcontainer.go index fc1f456f..4ce48a29 100644 --- a/vendor/github.com/RoaringBitmap/roaring/runcontainer.go +++ b/vendor/github.com/RoaringBitmap/roaring/runcontainer.go @@ -44,16 +44,10 @@ import ( "unsafe" ) -//go:generate msgp -unexported - // runContainer16 does run-length encoding of sets of // uint16 integers. type runContainer16 struct { - iv []interval16 - card int64 - - // avoid allocation during search - myOpts searchOptions `msg:"-"` + iv []interval16 } // interval16 is the internal to runContainer16 @@ -76,8 +70,8 @@ func newInterval16Range(start, last uint16) interval16 { } // runlen returns the count of integers in the interval. -func (iv interval16) runlen() int64 { - return int64(iv.length) + 1 +func (iv interval16) runlen() int { + return int(iv.length) + 1 } func (iv interval16) last() uint16 { @@ -120,8 +114,6 @@ func (p uint16Slice) Less(i, j int) bool { return p[i] < p[j] } // Swap swaps elements i and j. func (p uint16Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -//msgp:ignore addHelper - // addHelper helps build a runContainer16. type addHelper16 struct { runstart uint16 @@ -201,7 +193,6 @@ func newRunContainer16FromVals(alreadySorted bool, vals ...uint16) *runContainer ah.storeIval(ah.runstart, ah.runlen) } rc.iv = ah.m - rc.card = int64(ah.actuallyAdded) return rc } @@ -291,7 +282,6 @@ func newRunContainer16FromArray(arr *arrayContainer) *runContainer16 { ah.storeIval(ah.runstart, ah.runlen) } rc.iv = ah.m - rc.card = int64(ah.actuallyAdded) return rc } @@ -308,7 +298,6 @@ func (rc *runContainer16) set(alreadySorted bool, vals ...uint16) { rc2 := newRunContainer16FromVals(alreadySorted, vals...) un := rc.union(rc2) rc.iv = un.iv - rc.card = 0 } // canMerge returns true iff the intervals @@ -316,10 +305,10 @@ func (rc *runContainer16) set(alreadySorted bool, vals ...uint16) { // contiguous and so can be merged into // a single interval. func canMerge16(a, b interval16) bool { - if int64(a.last())+1 < int64(b.start) { + if int(a.last())+1 < int(b.start) { return false } - return int64(b.last())+1 >= int64(a.start) + return int(b.last())+1 >= int(a.start) } // haveOverlap differs from canMerge in that @@ -328,10 +317,10 @@ func canMerge16(a, b interval16) bool { // it would be the empty set, and we return // false). func haveOverlap16(a, b interval16) bool { - if int64(a.last())+1 <= int64(b.start) { + if int(a.last())+1 <= int(b.start) { return false } - return int64(b.last())+1 > int64(a.start) + return int(b.last())+1 > int(a.start) } // mergeInterval16s joins a and b into a @@ -392,11 +381,11 @@ func (rc *runContainer16) union(b *runContainer16) *runContainer16 { var m []interval16 - alim := int64(len(rc.iv)) - blim := int64(len(b.iv)) + alim := int(len(rc.iv)) + blim := int(len(b.iv)) - var na int64 // next from a - var nb int64 // next from b + var na int // next from a + var nb int // next from b // merged holds the current merge output, which might // get additional merges before being appended to m. @@ -416,12 +405,12 @@ func (rc *runContainer16) union(b *runContainer16) *runContainer16 { mergedUpdated := false if canMerge16(cura, merged) { merged = mergeInterval16s(cura, merged) - na = rc.indexOfIntervalAtOrAfter(int64(merged.last())+1, na+1) + na = rc.indexOfIntervalAtOrAfter(int(merged.last())+1, na+1) mergedUpdated = true } if canMerge16(curb, merged) { merged = mergeInterval16s(curb, merged) - nb = b.indexOfIntervalAtOrAfter(int64(merged.last())+1, nb+1) + nb = b.indexOfIntervalAtOrAfter(int(merged.last())+1, nb+1) mergedUpdated = true } if !mergedUpdated { @@ -444,8 +433,8 @@ func (rc *runContainer16) union(b *runContainer16) *runContainer16 { } else { merged = mergeInterval16s(cura, curb) mergedUsed = true - na = rc.indexOfIntervalAtOrAfter(int64(merged.last())+1, na+1) - nb = b.indexOfIntervalAtOrAfter(int64(merged.last())+1, nb+1) + na = rc.indexOfIntervalAtOrAfter(int(merged.last())+1, na+1) + nb = b.indexOfIntervalAtOrAfter(int(merged.last())+1, nb+1) } } } @@ -464,7 +453,7 @@ func (rc *runContainer16) union(b *runContainer16) *runContainer16 { cura = rc.iv[na] if canMerge16(cura, merged) { merged = mergeInterval16s(cura, merged) - na = rc.indexOfIntervalAtOrAfter(int64(merged.last())+1, na+1) + na = rc.indexOfIntervalAtOrAfter(int(merged.last())+1, na+1) } else { break aAdds } @@ -478,7 +467,7 @@ func (rc *runContainer16) union(b *runContainer16) *runContainer16 { curb = b.iv[nb] if canMerge16(curb, merged) { merged = mergeInterval16s(curb, merged) - nb = b.indexOfIntervalAtOrAfter(int64(merged.last())+1, nb+1) + nb = b.indexOfIntervalAtOrAfter(int(merged.last())+1, nb+1) } else { break bAdds } @@ -500,17 +489,17 @@ func (rc *runContainer16) union(b *runContainer16) *runContainer16 { } // unionCardinality returns the cardinality of the merger of two runContainer16s, the union of rc and b. -func (rc *runContainer16) unionCardinality(b *runContainer16) uint64 { +func (rc *runContainer16) unionCardinality(b *runContainer16) uint { // rc is also known as 'a' here, but golint insisted we // call it rc for consistency with the rest of the methods. - answer := uint64(0) + answer := uint(0) - alim := int64(len(rc.iv)) - blim := int64(len(b.iv)) + alim := int(len(rc.iv)) + blim := int(len(b.iv)) - var na int64 // next from a - var nb int64 // next from b + var na int // next from a + var nb int // next from b // merged holds the current merge output, which might // get additional merges before being appended to m. @@ -530,18 +519,18 @@ func (rc *runContainer16) unionCardinality(b *runContainer16) uint64 { mergedUpdated := false if canMerge16(cura, merged) { merged = mergeInterval16s(cura, merged) - na = rc.indexOfIntervalAtOrAfter(int64(merged.last())+1, na+1) + na = rc.indexOfIntervalAtOrAfter(int(merged.last())+1, na+1) mergedUpdated = true } if canMerge16(curb, merged) { merged = mergeInterval16s(curb, merged) - nb = b.indexOfIntervalAtOrAfter(int64(merged.last())+1, nb+1) + nb = b.indexOfIntervalAtOrAfter(int(merged.last())+1, nb+1) mergedUpdated = true } if !mergedUpdated { // we know that merged is disjoint from cura and curb //m = append(m, merged) - answer += uint64(merged.last()) - uint64(merged.start) + 1 + answer += uint(merged.last()) - uint(merged.start) + 1 mergedUsed = false } continue @@ -550,19 +539,19 @@ func (rc *runContainer16) unionCardinality(b *runContainer16) uint64 { // !mergedUsed if !canMerge16(cura, curb) { if cura.start < curb.start { - answer += uint64(cura.last()) - uint64(cura.start) + 1 + answer += uint(cura.last()) - uint(cura.start) + 1 //m = append(m, cura) na++ } else { - answer += uint64(curb.last()) - uint64(curb.start) + 1 + answer += uint(curb.last()) - uint(curb.start) + 1 //m = append(m, curb) nb++ } } else { merged = mergeInterval16s(cura, curb) mergedUsed = true - na = rc.indexOfIntervalAtOrAfter(int64(merged.last())+1, na+1) - nb = b.indexOfIntervalAtOrAfter(int64(merged.last())+1, nb+1) + na = rc.indexOfIntervalAtOrAfter(int(merged.last())+1, na+1) + nb = b.indexOfIntervalAtOrAfter(int(merged.last())+1, nb+1) } } } @@ -581,7 +570,7 @@ func (rc *runContainer16) unionCardinality(b *runContainer16) uint64 { cura = rc.iv[na] if canMerge16(cura, merged) { merged = mergeInterval16s(cura, merged) - na = rc.indexOfIntervalAtOrAfter(int64(merged.last())+1, na+1) + na = rc.indexOfIntervalAtOrAfter(int(merged.last())+1, na+1) } else { break aAdds } @@ -595,7 +584,7 @@ func (rc *runContainer16) unionCardinality(b *runContainer16) uint64 { curb = b.iv[nb] if canMerge16(curb, merged) { merged = mergeInterval16s(curb, merged) - nb = b.indexOfIntervalAtOrAfter(int64(merged.last())+1, nb+1) + nb = b.indexOfIntervalAtOrAfter(int(merged.last())+1, nb+1) } else { break bAdds } @@ -604,23 +593,20 @@ func (rc *runContainer16) unionCardinality(b *runContainer16) uint64 { } //m = append(m, merged) - answer += uint64(merged.last()) - uint64(merged.start) + 1 + answer += uint(merged.last()) - uint(merged.start) + 1 } for _, r := range rc.iv[na:] { - answer += uint64(r.last()) - uint64(r.start) + 1 + answer += uint(r.last()) - uint(r.start) + 1 } for _, r := range b.iv[nb:] { - answer += uint64(r.last()) - uint64(r.start) + 1 + answer += uint(r.last()) - uint(r.start) + 1 } return answer } // indexOfIntervalAtOrAfter is a helper for union. -func (rc *runContainer16) indexOfIntervalAtOrAfter(key int64, startIndex int64) int64 { - rc.myOpts.startIndex = startIndex - rc.myOpts.endxIndex = 0 - - w, already, _ := rc.search(key, &rc.myOpts) +func (rc *runContainer16) indexOfIntervalAtOrAfter(key int, startIndex int) int { + w, already, _ := rc.searchRange(key, startIndex, 0) if already { return w } @@ -632,8 +618,8 @@ func (rc *runContainer16) indexOfIntervalAtOrAfter(key int64, startIndex int64) func (rc *runContainer16) intersect(b *runContainer16) *runContainer16 { a := rc - numa := int64(len(a.iv)) - numb := int64(len(b.iv)) + numa := int(len(a.iv)) + numb := int(len(b.iv)) res := &runContainer16{} if numa == 0 || numb == 0 { return res @@ -647,21 +633,21 @@ func (rc *runContainer16) intersect(b *runContainer16) *runContainer16 { var output []interval16 - var acuri int64 - var bcuri int64 + var acuri int + var bcuri int - astart := int64(a.iv[acuri].start) - bstart := int64(b.iv[bcuri].start) + astart := int(a.iv[acuri].start) + bstart := int(b.iv[bcuri].start) var intersection interval16 - var leftoverstart int64 + var leftoverstart int var isOverlap, isLeftoverA, isLeftoverB bool var done bool toploop: for acuri < numa && bcuri < numb { isOverlap, isLeftoverA, isLeftoverB, leftoverstart, intersection = - intersectWithLeftover16(astart, int64(a.iv[acuri].last()), bstart, int64(b.iv[bcuri].last())) + intersectWithLeftover16(astart, int(a.iv[acuri].last()), bstart, int(b.iv[bcuri].last())) if !isOverlap { switch { @@ -670,17 +656,14 @@ toploop: if done { break toploop } - astart = int64(a.iv[acuri].start) + astart = int(a.iv[acuri].start) case astart > bstart: bcuri, done = b.findNextIntervalThatIntersectsStartingFrom(bcuri+1, astart) if done { break toploop } - bstart = int64(b.iv[bcuri].start) - - //default: - // panic("impossible that astart == bstart, since !isOverlap") + bstart = int(b.iv[bcuri].start) } } else { @@ -695,7 +678,7 @@ toploop: if bcuri >= numb { break toploop } - bstart = int64(b.iv[bcuri].start) + bstart = int(b.iv[bcuri].start) case isLeftoverB: // note that we change bstart without advancing bcuri, // since we need to capture any 2ndary intersections with b.iv[bcuri] @@ -704,27 +687,23 @@ toploop: if acuri >= numa { break toploop } - astart = int64(a.iv[acuri].start) + astart = int(a.iv[acuri].start) default: // neither had leftover, both completely consumed - // optionally, assert for sanity: - //if a.iv[acuri].endx != b.iv[bcuri].endx { - // panic("huh? should only be possible that endx agree now!") - //} // advance to next a interval acuri++ if acuri >= numa { break toploop } - astart = int64(a.iv[acuri].start) + astart = int(a.iv[acuri].start) // advance to next b interval bcuri++ if bcuri >= numb { break toploop } - bstart = int64(b.iv[bcuri].start) + bstart = int(b.iv[bcuri].start) } } } // end for toploop @@ -739,12 +718,12 @@ toploop: // intersectCardinality returns the cardinality of the // intersection of rc (also known as 'a') and b. -func (rc *runContainer16) intersectCardinality(b *runContainer16) int64 { - answer := int64(0) +func (rc *runContainer16) intersectCardinality(b *runContainer16) int { + answer := int(0) a := rc - numa := int64(len(a.iv)) - numb := int64(len(b.iv)) + numa := int(len(a.iv)) + numb := int(len(b.iv)) if numa == 0 || numb == 0 { return 0 } @@ -755,14 +734,14 @@ func (rc *runContainer16) intersectCardinality(b *runContainer16) int64 { } } - var acuri int64 - var bcuri int64 + var acuri int + var bcuri int - astart := int64(a.iv[acuri].start) - bstart := int64(b.iv[bcuri].start) + astart := int(a.iv[acuri].start) + bstart := int(b.iv[bcuri].start) var intersection interval16 - var leftoverstart int64 + var leftoverstart int var isOverlap, isLeftoverA, isLeftoverB bool var done bool pass := 0 @@ -771,7 +750,7 @@ toploop: pass++ isOverlap, isLeftoverA, isLeftoverB, leftoverstart, intersection = - intersectWithLeftover16(astart, int64(a.iv[acuri].last()), bstart, int64(b.iv[bcuri].last())) + intersectWithLeftover16(astart, int(a.iv[acuri].last()), bstart, int(b.iv[bcuri].last())) if !isOverlap { switch { @@ -780,22 +759,19 @@ toploop: if done { break toploop } - astart = int64(a.iv[acuri].start) + astart = int(a.iv[acuri].start) case astart > bstart: bcuri, done = b.findNextIntervalThatIntersectsStartingFrom(bcuri+1, astart) if done { break toploop } - bstart = int64(b.iv[bcuri].start) - - //default: - // panic("impossible that astart == bstart, since !isOverlap") + bstart = int(b.iv[bcuri].start) } } else { // isOverlap - answer += int64(intersection.last()) - int64(intersection.start) + 1 + answer += int(intersection.last()) - int(intersection.start) + 1 switch { case isLeftoverA: // note that we change astart without advancing acuri, @@ -805,7 +781,7 @@ toploop: if bcuri >= numb { break toploop } - bstart = int64(b.iv[bcuri].start) + bstart = int(b.iv[bcuri].start) case isLeftoverB: // note that we change bstart without advancing bcuri, // since we need to capture any 2ndary intersections with b.iv[bcuri] @@ -814,27 +790,23 @@ toploop: if acuri >= numa { break toploop } - astart = int64(a.iv[acuri].start) + astart = int(a.iv[acuri].start) default: // neither had leftover, both completely consumed - // optionally, assert for sanity: - //if a.iv[acuri].endx != b.iv[bcuri].endx { - // panic("huh? should only be possible that endx agree now!") - //} // advance to next a interval acuri++ if acuri >= numa { break toploop } - astart = int64(a.iv[acuri].start) + astart = int(a.iv[acuri].start) // advance to next b interval bcuri++ if bcuri >= numb { break toploop } - bstart = int64(b.iv[bcuri].start) + bstart = int(b.iv[bcuri].start) } } } // end for toploop @@ -844,7 +816,7 @@ toploop: // get returns true iff key is in the container. func (rc *runContainer16) contains(key uint16) bool { - _, in, _ := rc.search(int64(key), nil) + _, in, _ := rc.search(int(key)) return in } @@ -853,22 +825,7 @@ func (rc *runContainer16) numIntervals() int { return len(rc.iv) } -// searchOptions allows us to accelerate search with -// prior knowledge of (mostly lower) bounds. This is used by Union -// and Intersect. -type searchOptions struct { - // start here instead of at 0 - startIndex int64 - - // upper bound instead of len(rc.iv); - // endxIndex == 0 means ignore the bound and use - // endxIndex == n ==len(rc.iv) which is also - // naturally the default for search() - // when opt = nil. - endxIndex int64 -} - -// search returns alreadyPresent to indicate if the +// searchRange returns alreadyPresent to indicate if the // key is already in one of our interval16s. // // If key is alreadyPresent, then whichInterval16 tells @@ -892,24 +849,16 @@ type searchOptions struct { // // runContainer16.search always returns whichInterval16 < len(rc.iv). // -// If not nil, opts can be used to further restrict -// the search space. +// The search space is from startIndex to endxIndex. If endxIndex is set to zero, then there +// no upper bound. // -func (rc *runContainer16) search(key int64, opts *searchOptions) (whichInterval16 int64, alreadyPresent bool, numCompares int) { - n := int64(len(rc.iv)) +func (rc *runContainer16) searchRange(key int, startIndex int, endxIndex int) (whichInterval16 int, alreadyPresent bool, numCompares int) { + n := int(len(rc.iv)) if n == 0 { return -1, false, 0 } - - startIndex := int64(0) - endxIndex := n - if opts != nil { - startIndex = opts.startIndex - - // let endxIndex == 0 mean no effect - if opts.endxIndex > 0 { - endxIndex = opts.endxIndex - } + if endxIndex == 0 { + endxIndex = n } // sort.Search returns the smallest index i @@ -927,7 +876,7 @@ func (rc *runContainer16) search(key int64, opts *searchOptions) (whichInterval1 h := i + (j-i)/2 // avoid overflow when computing h as the bisector // i <= h < j numCompares++ - if !(key < int64(rc.iv[h].start)) { + if !(key < int(rc.iv[h].start)) { i = h + 1 } else { j = h @@ -947,7 +896,7 @@ func (rc *runContainer16) search(key int64, opts *searchOptions) (whichInterval1 if below == n { // all falses => key is >= start of all interval16s // ... so does it belong to the last interval16? - if key < int64(rc.iv[n-1].last())+1 { + if key < int(rc.iv[n-1].last())+1 { // yes, it belongs to the last interval16 alreadyPresent = true return @@ -968,7 +917,7 @@ func (rc *runContainer16) search(key int64, opts *searchOptions) (whichInterval1 // key is < rc.iv[below].start // is key in below-1 interval16? - if key >= int64(rc.iv[below-1].start) && key < int64(rc.iv[below-1].last())+1 { + if key >= int(rc.iv[below-1].start) && key < int(rc.iv[below-1].last())+1 { // yes, it is. key is in below-1 interval16. alreadyPresent = true return @@ -979,28 +928,55 @@ func (rc *runContainer16) search(key int64, opts *searchOptions) (whichInterval1 return } -// cardinality returns the count of the integers stored in the -// runContainer16. -func (rc *runContainer16) cardinality() int64 { - if len(rc.iv) == 0 { - rc.card = 0 - return 0 - } - if rc.card > 0 { - return rc.card // already cached - } +// search returns alreadyPresent to indicate if the +// key is already in one of our interval16s. +// +// If key is alreadyPresent, then whichInterval16 tells +// you where. +// +// If key is not already present, then whichInterval16 is +// set as follows: +// +// a) whichInterval16 == len(rc.iv)-1 if key is beyond our +// last interval16 in rc.iv; +// +// b) whichInterval16 == -1 if key is before our first +// interval16 in rc.iv; +// +// c) whichInterval16 is set to the minimum index of rc.iv +// which comes strictly before the key; +// so rc.iv[whichInterval16].last < key, +// and if whichInterval16+1 exists, then key < rc.iv[whichInterval16+1].start +// (Note that whichInterval16+1 won't exist when +// whichInterval16 is the last interval.) +// +// runContainer16.search always returns whichInterval16 < len(rc.iv). +// +func (rc *runContainer16) search(key int) (whichInterval16 int, alreadyPresent bool, numCompares int) { + return rc.searchRange(key, 0, 0) +} + +// getCardinality returns the count of the integers stored in the +// runContainer16. The running complexity depends on the size +// of the container. +func (rc *runContainer16) getCardinality() int { // have to compute it - var n int64 + n := 0 for _, p := range rc.iv { n += p.runlen() } - rc.card = n // cache it return n } +// isEmpty returns true if the container is empty. +// It runs in constant time. +func (rc *runContainer16) isEmpty() bool { + return len(rc.iv) == 0 +} + // AsSlice decompresses the contents into a []uint16 slice. func (rc *runContainer16) AsSlice() []uint16 { - s := make([]uint16, rc.cardinality()) + s := make([]uint16, rc.getCardinality()) j := 0 for _, p := range rc.iv { for i := p.start; i <= p.last(); i++ { @@ -1070,19 +1046,15 @@ func (rc *runContainer16) Add(k uint16) (wasNew bool) { // but note that some unit tests use this method to build up test // runcontainers without calling runOptimize - k64 := int64(k) + k64 := int(k) - index, present, _ := rc.search(k64, nil) + index, present, _ := rc.search(k64) if present { return // already there } wasNew = true - // increment card if it is cached already - if rc.card > 0 { - rc.card++ - } - n := int64(len(rc.iv)) + n := int(len(rc.iv)) if index == -1 { // we may need to extend the first run if n > 0 { @@ -1099,7 +1071,7 @@ func (rc *runContainer16) Add(k uint16) (wasNew bool) { // are we off the end? handle both index == n and index == n-1: if index >= n-1 { - if int64(rc.iv[n-1].last())+1 == k64 { + if int(rc.iv[n-1].last())+1 == k64 { rc.iv[n-1].length++ return } @@ -1118,7 +1090,7 @@ func (rc *runContainer16) Add(k uint16) (wasNew bool) { right := index + 1 // are we fusing left and right by adding k? - if int64(rc.iv[left].last())+1 == k64 && int64(rc.iv[right].start) == k64+1 { + if int(rc.iv[left].last())+1 == k64 && int(rc.iv[right].start) == k64+1 { // fuse into left rc.iv[left].length = rc.iv[right].last() - rc.iv[left].start // remove redundant right @@ -1127,14 +1099,14 @@ func (rc *runContainer16) Add(k uint16) (wasNew bool) { } // are we an addition to left? - if int64(rc.iv[left].last())+1 == k64 { + if int(rc.iv[left].last())+1 == k64 { // yes rc.iv[left].length++ return } // are we an addition to right? - if int64(rc.iv[right].start) == k64+1 { + if int(rc.iv[right].start) == k64+1 { // yes rc.iv[right].start = k rc.iv[right].length++ @@ -1147,13 +1119,11 @@ func (rc *runContainer16) Add(k uint16) (wasNew bool) { return } -//msgp:ignore runIterator - // runIterator16 advice: you must call hasNext() // before calling next()/peekNext() to insure there are contents. type runIterator16 struct { rc *runContainer16 - curIndex int64 + curIndex int curPosInIndex uint16 } @@ -1178,8 +1148,8 @@ func (rc *runContainer16) iterate(cb func(x uint16) bool) bool { // returns true when there is at least one more value // available in the iteration sequence. func (ri *runIterator16) hasNext() bool { - return int64(len(ri.rc.iv)) > ri.curIndex+1 || - (int64(len(ri.rc.iv)) == ri.curIndex+1 && ri.rc.iv[ri.curIndex].length >= ri.curPosInIndex) + return int(len(ri.rc.iv)) > ri.curIndex+1 || + (int(len(ri.rc.iv)) == ri.curIndex+1 && ri.rc.iv[ri.curIndex].length >= ri.curPosInIndex) } // next returns the next value in the iteration sequence. @@ -1207,13 +1177,8 @@ func (ri *runIterator16) advanceIfNeeded(minval uint16) { return } - opt := &searchOptions{ - startIndex: ri.curIndex, - endxIndex: int64(len(ri.rc.iv)), - } - // interval cannot be -1 because of minval > peekNext - interval, isPresent, _ := ri.rc.search(int64(minval), opt) + interval, isPresent, _ := ri.rc.searchRange(int(minval), ri.curIndex, int(len(ri.rc.iv))) // if the minval is present, set the curPosIndex at the right position if isPresent { @@ -1231,13 +1196,13 @@ func (ri *runIterator16) advanceIfNeeded(minval uint16) { // before calling next() to insure there are contents. type runReverseIterator16 struct { rc *runContainer16 - curIndex int64 // index into rc.iv + curIndex int // index into rc.iv curPosInIndex uint16 // offset in rc.iv[curIndex] } // newRunReverseIterator16 returns a new empty run iterator. func (rc *runContainer16) newRunReverseIterator16() *runReverseIterator16 { - index := int64(len(rc.iv)) - 1 + index := int(len(rc.iv)) - 1 pos := uint16(0) if index >= 0 { @@ -1310,7 +1275,7 @@ func (ri *runIterator16) nextMany(hs uint32, buf []uint32) int { ri.curPosInIndex = 0 ri.curIndex++ - if ri.curIndex == int64(len(ri.rc.iv)) { + if ri.curIndex == int(len(ri.rc.iv)) { break } } else { @@ -1351,7 +1316,7 @@ func (ri *runIterator16) nextMany64(hs uint64, buf []uint64) int { ri.curPosInIndex = 0 ri.curIndex++ - if ri.curIndex == int64(len(ri.rc.iv)) { + if ri.curIndex == int(len(ri.rc.iv)) { break } } else { @@ -1365,8 +1330,8 @@ func (ri *runIterator16) nextMany64(hs uint64, buf []uint64) int { // remove removes key from the container. func (rc *runContainer16) removeKey(key uint16) (wasPresent bool) { - var index int64 - index, wasPresent, _ = rc.search(int64(key), nil) + var index int + index, wasPresent, _ = rc.search(int(key)) if !wasPresent { return // already removed, nothing to do. } @@ -1377,15 +1342,14 @@ func (rc *runContainer16) removeKey(key uint16) (wasPresent bool) { // internal helper functions -func (rc *runContainer16) deleteAt(curIndex *int64, curPosInIndex *uint16) { - rc.card-- +func (rc *runContainer16) deleteAt(curIndex *int, curPosInIndex *uint16) { ci := *curIndex pos := *curPosInIndex // are we first, last, or in the middle of our interval16? switch { case pos == 0: - if int64(rc.iv[ci].length) == 0 { + if int(rc.iv[ci].length) == 0 { // our interval disappears rc.iv = append(rc.iv[:ci], rc.iv[ci+1:]...) // curIndex stays the same, since the delete did @@ -1406,8 +1370,8 @@ func (rc *runContainer16) deleteAt(curIndex *int64, curPosInIndex *uint16) { // split into two, adding an interval16 new0 := newInterval16Range(rc.iv[ci].start, rc.iv[ci].start+*curPosInIndex-1) - new1start := int64(rc.iv[ci].start+*curPosInIndex) + 1 - if new1start > int64(MaxUint16) { + new1start := int(rc.iv[ci].start+*curPosInIndex) + 1 + if new1start > int(MaxUint16) { panic("overflow?!?!") } new1 := newInterval16Range(uint16(new1start), rc.iv[ci].last()) @@ -1420,14 +1384,14 @@ func (rc *runContainer16) deleteAt(curIndex *int64, curPosInIndex *uint16) { } -func have4Overlap16(astart, alast, bstart, blast int64) bool { +func have4Overlap16(astart, alast, bstart, blast int) bool { if alast+1 <= bstart { return false } return blast+1 > astart } -func intersectWithLeftover16(astart, alast, bstart, blast int64) (isOverlap, isLeftoverA, isLeftoverB bool, leftoverstart int64, intersection interval16) { +func intersectWithLeftover16(astart, alast, bstart, blast int) (isOverlap, isLeftoverA, isLeftoverB bool, leftoverstart int, intersection interval16) { if !have4Overlap16(astart, alast, bstart, blast) { return } @@ -1457,17 +1421,13 @@ func intersectWithLeftover16(astart, alast, bstart, blast int64) (isOverlap, isL return } -func (rc *runContainer16) findNextIntervalThatIntersectsStartingFrom(startIndex int64, key int64) (index int64, done bool) { - - rc.myOpts.startIndex = startIndex - rc.myOpts.endxIndex = 0 - - w, _, _ := rc.search(key, &rc.myOpts) +func (rc *runContainer16) findNextIntervalThatIntersectsStartingFrom(startIndex int, key int) (index int, done bool) { + w, _, _ := rc.searchRange(key, startIndex, 0) // rc.search always returns w < len(rc.iv) if w < startIndex { // not found and comes before lower bound startIndex, // so just use the lower bound. - if startIndex == int64(len(rc.iv)) { + if startIndex == int(len(rc.iv)) { // also this bump up means that we are done return startIndex, true } @@ -1485,25 +1445,6 @@ func sliceToString16(m []interval16) string { return s } -// selectInt16 returns the j-th value in the container. -// We panic of j is out of bounds. -func (rc *runContainer16) selectInt16(j uint16) int { - n := rc.cardinality() - if int64(j) > n { - panic(fmt.Sprintf("Cannot select %v since Cardinality is %v", j, n)) - } - - var offset int64 - for k := range rc.iv { - nextOffset := offset + rc.iv[k].runlen() - if nextOffset > int64(j) { - return int(int64(rc.iv[k].start) + (int64(j) - offset)) - } - offset = nextOffset - } - panic(fmt.Sprintf("Cannot select %v since Cardinality is %v", j, n)) -} - // helper for invert func (rc *runContainer16) invertlastInterval(origin uint16, lastIdx int) []interval16 { cur := rc.iv[lastIdx] @@ -1535,7 +1476,7 @@ func (rc *runContainer16) invert() *runContainer16 { case 1: return &runContainer16{iv: rc.invertlastInterval(0, 0)} } - var invstart int64 + var invstart int ult := ni - 1 for i, cur := range rc.iv { if i == ult { @@ -1554,7 +1495,7 @@ func (rc *runContainer16) invert() *runContainer16 { if cur.start > 0 { m = append(m, newInterval16Range(uint16(invstart), cur.start-1)) } - invstart = int64(cur.last() + 1) + invstart = int(cur.last() + 1) } return &runContainer16{iv: m} } @@ -1567,7 +1508,7 @@ func (iv interval16) isSuperSetOf(b interval16) bool { return iv.start <= b.start && b.last() <= iv.last() } -func (iv interval16) subtractInterval(del interval16) (left []interval16, delcount int64) { +func (iv interval16) subtractInterval(del interval16) (left []interval16, delcount int) { isect, isEmpty := intersectInterval16s(iv, del) if isEmpty { @@ -1592,7 +1533,7 @@ func (iv interval16) subtractInterval(del interval16) (left []interval16, delcou func (rc *runContainer16) isubtract(del interval16) { origiv := make([]interval16, len(rc.iv)) copy(origiv, rc.iv) - n := int64(len(rc.iv)) + n := int(len(rc.iv)) if n == 0 { return // already done. } @@ -1603,9 +1544,8 @@ func (rc *runContainer16) isubtract(del interval16) { } // INVAR there is some intersection between rc and del - istart, startAlready, _ := rc.search(int64(del.start), nil) - ilast, lastAlready, _ := rc.search(int64(del.last()), nil) - rc.card = -1 + istart, startAlready, _ := rc.search(int(del.start)) + ilast, lastAlready, _ := rc.search(int(del.last())) if istart == -1 { if ilast == n-1 && !lastAlready { rc.iv = nil @@ -1620,8 +1560,8 @@ func (rc *runContainer16) isubtract(del interval16) { // would overwrite values in iv b/c res0 can have len 2. so // write to origiv instead. lost := 1 + ilast - istart - changeSize := int64(len(res0)) - lost - newSize := int64(len(rc.iv)) + changeSize + changeSize := int(len(res0)) - lost + newSize := int(len(rc.iv)) + changeSize // rc.iv = append(pre, caboose...) // return @@ -1629,19 +1569,19 @@ func (rc *runContainer16) isubtract(del interval16) { if ilast != istart { res1, _ := rc.iv[ilast].subtractInterval(del) res0 = append(res0, res1...) - changeSize = int64(len(res0)) - lost - newSize = int64(len(rc.iv)) + changeSize + changeSize = int(len(res0)) - lost + newSize = int(len(rc.iv)) + changeSize } switch { case changeSize < 0: // shrink - copy(rc.iv[istart+int64(len(res0)):], rc.iv[ilast+1:]) - copy(rc.iv[istart:istart+int64(len(res0))], res0) + copy(rc.iv[istart+int(len(res0)):], rc.iv[ilast+1:]) + copy(rc.iv[istart:istart+int(len(res0))], res0) rc.iv = rc.iv[:newSize] return case changeSize == 0: // stay the same - copy(rc.iv[istart:istart+int64(len(res0))], res0) + copy(rc.iv[istart:istart+int(len(res0))], res0) return default: // changeSize > 0 is only possible when ilast == istart. @@ -1698,7 +1638,7 @@ func (rc *runContainer16) isubtract(del interval16) { // INVAR: ilast < n-1 lost := ilast - istart changeSize := -lost - newSize := int64(len(rc.iv)) + changeSize + newSize := int(len(rc.iv)) + changeSize if changeSize != 0 { copy(rc.iv[ilast+1+changeSize:], rc.iv[ilast+1:]) } @@ -1715,8 +1655,8 @@ func (rc *runContainer16) isubtract(del interval16) { rc.iv[istart] = res0[0] } lost := 1 + (ilast - istart) - changeSize := int64(len(res0)) - lost - newSize := int64(len(rc.iv)) + changeSize + changeSize := int(len(res0)) - lost + newSize := int(len(rc.iv)) + changeSize if changeSize != 0 { copy(rc.iv[ilast+1+changeSize:], rc.iv[ilast+1:]) } @@ -1727,8 +1667,8 @@ func (rc *runContainer16) isubtract(del interval16) { // we can only shrink or stay the same size res1, _ := rc.iv[ilast].subtractInterval(del) lost := ilast - istart - changeSize := int64(len(res1)) - lost - newSize := int64(len(rc.iv)) + changeSize + changeSize := int(len(res1)) - lost + newSize := int(len(rc.iv)) + changeSize if changeSize != 0 { // move the tail first to make room for res1 copy(rc.iv[ilast+1+changeSize:], rc.iv[ilast+1:]) @@ -1932,8 +1872,6 @@ func (rc *runContainer16) iand(a container) container { } func (rc *runContainer16) inplaceIntersect(rc2 *runContainer16) container { - // TODO: optimize by doing less allocation, possibly? - // sect will be new sect := rc.intersect(rc2) *rc = *sect return rc @@ -1987,17 +1925,18 @@ func (rc *runContainer16) andNot(a container) container { panic("unsupported container type") } -func (rc *runContainer16) fillLeastSignificant16bits(x []uint32, i int, mask uint32) { - k := 0 - var val int64 +func (rc *runContainer16) fillLeastSignificant16bits(x []uint32, i int, mask uint32) int { + k := i + var val int for _, p := range rc.iv { n := p.runlen() - for j := int64(0); j < n; j++ { - val = int64(p.start) + j - x[k+i] = uint32(val) | mask + for j := int(0); j < n; j++ { + val = int(p.start) + j + x[k] = uint32(val) | mask k++ } } + return k } func (rc *runContainer16) getShortIterator() shortPeekable { @@ -2016,8 +1955,11 @@ func (rc *runContainer16) getManyIterator() manyIterable { // is still abe to express 2^16 because it is an int not an uint16. func (rc *runContainer16) iaddRange(firstOfRange, endx int) container { - if firstOfRange >= endx { - panic(fmt.Sprintf("invalid %v = endx >= firstOfRange", endx)) + if firstOfRange > endx { + panic(fmt.Sprintf("invalid %v = endx > firstOfRange", endx)) + } + if firstOfRange == endx { + return rc } addme := newRunContainer16TakeOwnership([]interval16{ { @@ -2031,10 +1973,13 @@ func (rc *runContainer16) iaddRange(firstOfRange, endx int) container { // remove the values in the range [firstOfRange,endx) func (rc *runContainer16) iremoveRange(firstOfRange, endx int) container { - if firstOfRange >= endx { + if firstOfRange > endx { panic(fmt.Sprintf("request to iremove empty set [%v, %v),"+ " nothing to do.", firstOfRange, endx)) - //return rc + } + // empty removal + if firstOfRange == endx { + return rc } x := newInterval16Range(uint16(firstOfRange), uint16(endx-1)) rc.isubtract(x) @@ -2043,8 +1988,8 @@ func (rc *runContainer16) iremoveRange(firstOfRange, endx int) container { // not flip the values in the range [firstOfRange,endx) func (rc *runContainer16) not(firstOfRange, endx int) container { - if firstOfRange >= endx { - panic(fmt.Sprintf("invalid %v = endx >= firstOfRange = %v", endx, firstOfRange)) + if firstOfRange > endx { + panic(fmt.Sprintf("invalid %v = endx > firstOfRange = %v", endx, firstOfRange)) } return rc.Not(firstOfRange, endx) @@ -2064,8 +2009,8 @@ func (rc *runContainer16) not(firstOfRange, endx int) container { // func (rc *runContainer16) Not(firstOfRange, endx int) *runContainer16 { - if firstOfRange >= endx { - panic(fmt.Sprintf("invalid %v = endx >= firstOfRange == %v", endx, firstOfRange)) + if firstOfRange > endx { + panic(fmt.Sprintf("invalid %v = endx > firstOfRange == %v", endx, firstOfRange)) } if firstOfRange >= endx { @@ -2203,9 +2148,21 @@ func (rc *runContainer16) orBitmapContainerCardinality(bc *bitmapContainer) int // orArray finds the union of rc and ac. func (rc *runContainer16) orArray(ac *arrayContainer) container { - bc1 := newBitmapContainerFromRun(rc) - bc2 := ac.toBitmapContainer() - return bc1.orBitmap(bc2) + if ac.isEmpty() { + return rc.clone() + } + if rc.isEmpty() { + return ac.clone() + } + intervals, cardMinusOne := runArrayUnionToRuns(rc, ac) + result := newRunContainer16TakeOwnership(intervals) + if len(intervals) >= 2048 && cardMinusOne >= arrayDefaultMaxSize { + return newBitmapContainerFromRun(result) + } + if len(intervals)*2 > 1+int(cardMinusOne) { + return result.toArrayContainer() + } + return result } // orArray finds the union of rc and ac. @@ -2230,8 +2187,8 @@ func (rc *runContainer16) ior(a container) container { func (rc *runContainer16) inplaceUnion(rc2 *runContainer16) container { for _, p := range rc2.iv { - last := int64(p.last()) - for i := int64(p.start); i <= last; i++ { + last := int(p.last()) + for i := int(p.start); i <= last; i++ { rc.Add(uint16(i)) } } @@ -2248,13 +2205,88 @@ func (rc *runContainer16) iorBitmapContainer(bc *bitmapContainer) container { } func (rc *runContainer16) iorArray(ac *arrayContainer) container { - it := ac.getShortIterator() - for it.hasNext() { - rc.Add(it.next()) + if rc.isEmpty() { + return ac.clone() + } + if ac.isEmpty() { + return rc + } + var cardMinusOne uint16 + //TODO: perform the union algorithm in-place using rc.iv + // this can be done with methods like the in-place array container union + // but maybe lazily moving the remaining elements back. + rc.iv, cardMinusOne = runArrayUnionToRuns(rc, ac) + if len(rc.iv) >= 2048 && cardMinusOne >= arrayDefaultMaxSize { + return newBitmapContainerFromRun(rc) + } + if len(rc.iv)*2 > 1+int(cardMinusOne) { + return rc.toArrayContainer() } return rc } +func runArrayUnionToRuns(rc *runContainer16, ac *arrayContainer) ([]interval16, uint16) { + pos1 := 0 + pos2 := 0 + length1 := len(ac.content) + length2 := len(rc.iv) + target := make([]interval16, 0, len(rc.iv)) + // have to find the first range + // options are + // 1. from array container + // 2. from run container + var previousInterval interval16 + var cardMinusOne uint16 + if ac.content[0] < rc.iv[0].start { + previousInterval.start = ac.content[0] + previousInterval.length = 0 + pos1++ + } else { + previousInterval.start = rc.iv[0].start + previousInterval.length = rc.iv[0].length + pos2++ + } + + for pos1 < length1 || pos2 < length2 { + if pos1 < length1 { + s1 := ac.content[pos1] + if s1 <= previousInterval.start+previousInterval.length { + pos1++ + continue + } + if previousInterval.last() < MaxUint16 && previousInterval.last()+1 == s1 { + previousInterval.length++ + pos1++ + continue + } + } + if pos2 < length2 { + range2 := rc.iv[pos2] + if range2.start <= previousInterval.last() || range2.start > 0 && range2.start-1 == previousInterval.last() { + pos2++ + if previousInterval.last() < range2.last() { + previousInterval.length = range2.last() - previousInterval.start + } + continue + } + } + cardMinusOne += previousInterval.length + 1 + target = append(target, previousInterval) + if pos2 == length2 || pos1 < length1 && ac.content[pos1] < rc.iv[pos2].start { + previousInterval.start = ac.content[pos1] + previousInterval.length = 0 + pos1++ + } else { + previousInterval = rc.iv[pos2] + pos2++ + } + } + cardMinusOne += previousInterval.length + target = append(target, previousInterval) + + return target, cardMinusOne +} + // lazyIOR is described (not yet implemented) in // this nice note from @lemire on // https://github.com/RoaringBitmap/roaring/pull/70#issuecomment-263613737 @@ -2310,9 +2342,9 @@ func (rc *runContainer16) lazyOR(a container) container { } func (rc *runContainer16) intersects(a container) bool { - // TODO: optimize by doing inplace/less allocation, possibly? + // TODO: optimize by doing inplace/less allocation isect := rc.and(a) - return isect.getCardinality() > 0 + return !isect.isEmpty() } func (rc *runContainer16) xor(a container) container { @@ -2341,44 +2373,51 @@ func (rc *runContainer16) iandNot(a container) container { // flip the values in the range [firstOfRange,endx) func (rc *runContainer16) inot(firstOfRange, endx int) container { - if firstOfRange >= endx { - panic(fmt.Sprintf("invalid %v = endx >= firstOfRange = %v", endx, firstOfRange)) + if firstOfRange > endx { + panic(fmt.Sprintf("invalid %v = endx > firstOfRange = %v", endx, firstOfRange)) + } + if firstOfRange > endx { + return rc } // TODO: minimize copies, do it all inplace; not() makes a copy. rc = rc.Not(firstOfRange, endx) return rc } -func (rc *runContainer16) getCardinality() int { - return int(rc.cardinality()) -} - func (rc *runContainer16) rank(x uint16) int { - n := int64(len(rc.iv)) - xx := int64(x) - w, already, _ := rc.search(xx, nil) + n := int(len(rc.iv)) + xx := int(x) + w, already, _ := rc.search(xx) if w < 0 { return 0 } if !already && w == n-1 { return rc.getCardinality() } - var rnk int64 + var rnk int if !already { - for i := int64(0); i <= w; i++ { + for i := int(0); i <= w; i++ { rnk += rc.iv[i].runlen() } return int(rnk) } - for i := int64(0); i < w; i++ { + for i := int(0); i < w; i++ { rnk += rc.iv[i].runlen() } - rnk += int64(x-rc.iv[w].start) + 1 + rnk += int(x-rc.iv[w].start) + 1 return int(rnk) } func (rc *runContainer16) selectInt(x uint16) int { - return rc.selectInt16(x) + var offset int + for k := range rc.iv { + nextOffset := offset + rc.iv[k].runlen() + if nextOffset > int(x) { + return int(int(rc.iv[k].start) + (int(x) - offset)) + } + offset = nextOffset + } + panic("cannot select x") } func (rc *runContainer16) andNotRunContainer16(b *runContainer16) container { @@ -2456,11 +2495,9 @@ func (rc *runContainer16) xorBitmap(bc *bitmapContainer) container { // convert to bitmap or array *if needed* func (rc *runContainer16) toEfficientContainer() container { - - // runContainer16SerializedSizeInBytes(numRuns) sizeAsRunContainer := rc.getSizeInBytes() sizeAsBitmapContainer := bitmapContainerSizeInBytes() - card := int(rc.cardinality()) + card := rc.getCardinality() sizeAsArrayContainer := arrayContainerSizeInBytes(card) if sizeAsRunContainer <= minOfInt(sizeAsBitmapContainer, sizeAsArrayContainer) { return rc @@ -2545,9 +2582,27 @@ func (rc *runContainer16) serializedSizeInBytes() int { return 2 + len(rc.iv)*4 } -func (rc *runContainer16) addOffset(x uint16) []container { - low := newRunContainer16() - high := newRunContainer16() +func (rc *runContainer16) addOffset(x uint16) (container, container) { + var low, high *runContainer16 + + if len(rc.iv) == 0 { + return nil, nil + } + + first := uint32(rc.iv[0].start) + uint32(x) + if highbits(first) == 0 { + // Some elements will fall into low part, allocate a container. + // Checking the first one is enough because they are ordered. + low = newRunContainer16() + } + last := uint32(rc.iv[len(rc.iv)-1].start) + last += uint32(rc.iv[len(rc.iv)-1].length) + last += uint32(x) + if highbits(last) > 0 { + // Some elements will fall into high part, allocate a container. + // Checking the last one is enough because they are ordered. + high = newRunContainer16() + } for _, iv := range rc.iv { val := int(iv.start) + int(x) @@ -2563,5 +2618,14 @@ func (rc *runContainer16) addOffset(x uint16) []container { high.iv = append(high.iv, interval16{uint16(val & 0xffff), iv.length}) } } - return []container{low, high} + + // Ensure proper nil interface. + if low == nil { + return nil, high + } + if high == nil { + return low, nil + } + + return low, high } diff --git a/vendor/github.com/RoaringBitmap/roaring/runcontainer_gen.go b/vendor/github.com/RoaringBitmap/roaring/runcontainer_gen.go deleted file mode 100644 index 84537d08..00000000 --- a/vendor/github.com/RoaringBitmap/roaring/runcontainer_gen.go +++ /dev/null @@ -1,1104 +0,0 @@ -package roaring - -// NOTE: THIS FILE WAS PRODUCED BY THE -// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp) -// DO NOT EDIT - -import "github.com/tinylib/msgp/msgp" - -// Deprecated: DecodeMsg implements msgp.Decodable -func (z *addHelper16) DecodeMsg(dc *msgp.Reader) (err error) { - var field []byte - _ = field - var zbai uint32 - zbai, err = dc.ReadMapHeader() - if err != nil { - return - } - for zbai > 0 { - zbai-- - field, err = dc.ReadMapKeyPtr() - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "runstart": - z.runstart, err = dc.ReadUint16() - if err != nil { - return - } - case "runlen": - z.runlen, err = dc.ReadUint16() - if err != nil { - return - } - case "actuallyAdded": - z.actuallyAdded, err = dc.ReadUint16() - if err != nil { - return - } - case "m": - var zcmr uint32 - zcmr, err = dc.ReadArrayHeader() - if err != nil { - return - } - if cap(z.m) >= int(zcmr) { - z.m = (z.m)[:zcmr] - } else { - z.m = make([]interval16, zcmr) - } - for zxvk := range z.m { - var zajw uint32 - zajw, err = dc.ReadMapHeader() - if err != nil { - return - } - for zajw > 0 { - zajw-- - field, err = dc.ReadMapKeyPtr() - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "start": - z.m[zxvk].start, err = dc.ReadUint16() - if err != nil { - return - } - case "last": - z.m[zxvk].length, err = dc.ReadUint16() - z.m[zxvk].length -= z.m[zxvk].start - if err != nil { - return - } - default: - err = dc.Skip() - if err != nil { - return - } - } - } - } - case "rc": - if dc.IsNil() { - err = dc.ReadNil() - if err != nil { - return - } - z.rc = nil - } else { - if z.rc == nil { - z.rc = new(runContainer16) - } - var zwht uint32 - zwht, err = dc.ReadMapHeader() - if err != nil { - return - } - for zwht > 0 { - zwht-- - field, err = dc.ReadMapKeyPtr() - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "iv": - var zhct uint32 - zhct, err = dc.ReadArrayHeader() - if err != nil { - return - } - if cap(z.rc.iv) >= int(zhct) { - z.rc.iv = (z.rc.iv)[:zhct] - } else { - z.rc.iv = make([]interval16, zhct) - } - for zbzg := range z.rc.iv { - var zcua uint32 - zcua, err = dc.ReadMapHeader() - if err != nil { - return - } - for zcua > 0 { - zcua-- - field, err = dc.ReadMapKeyPtr() - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "start": - z.rc.iv[zbzg].start, err = dc.ReadUint16() - if err != nil { - return - } - case "last": - z.rc.iv[zbzg].length, err = dc.ReadUint16() - z.rc.iv[zbzg].length -= z.rc.iv[zbzg].start - if err != nil { - return - } - default: - err = dc.Skip() - if err != nil { - return - } - } - } - } - case "card": - z.rc.card, err = dc.ReadInt64() - if err != nil { - return - } - default: - err = dc.Skip() - if err != nil { - return - } - } - } - } - default: - err = dc.Skip() - if err != nil { - return - } - } - } - return -} - -// Deprecated: EncodeMsg implements msgp.Encodable -func (z *addHelper16) EncodeMsg(en *msgp.Writer) (err error) { - // map header, size 5 - // write "runstart" - err = en.Append(0x85, 0xa8, 0x72, 0x75, 0x6e, 0x73, 0x74, 0x61, 0x72, 0x74) - if err != nil { - return err - } - err = en.WriteUint16(z.runstart) - if err != nil { - return - } - // write "runlen" - err = en.Append(0xa6, 0x72, 0x75, 0x6e, 0x6c, 0x65, 0x6e) - if err != nil { - return err - } - err = en.WriteUint16(z.runlen) - if err != nil { - return - } - // write "actuallyAdded" - err = en.Append(0xad, 0x61, 0x63, 0x74, 0x75, 0x61, 0x6c, 0x6c, 0x79, 0x41, 0x64, 0x64, 0x65, 0x64) - if err != nil { - return err - } - err = en.WriteUint16(z.actuallyAdded) - if err != nil { - return - } - // write "m" - err = en.Append(0xa1, 0x6d) - if err != nil { - return err - } - err = en.WriteArrayHeader(uint32(len(z.m))) - if err != nil { - return - } - for zxvk := range z.m { - // map header, size 2 - // write "start" - err = en.Append(0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74) - if err != nil { - return err - } - err = en.WriteUint16(z.m[zxvk].start) - if err != nil { - return - } - // write "last" - err = en.Append(0xa4, 0x6c, 0x61, 0x73, 0x74) - if err != nil { - return err - } - err = en.WriteUint16(z.m[zxvk].last()) - if err != nil { - return - } - } - // write "rc" - err = en.Append(0xa2, 0x72, 0x63) - if err != nil { - return err - } - if z.rc == nil { - err = en.WriteNil() - if err != nil { - return - } - } else { - // map header, size 2 - // write "iv" - err = en.Append(0x82, 0xa2, 0x69, 0x76) - if err != nil { - return err - } - err = en.WriteArrayHeader(uint32(len(z.rc.iv))) - if err != nil { - return - } - for zbzg := range z.rc.iv { - // map header, size 2 - // write "start" - err = en.Append(0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74) - if err != nil { - return err - } - err = en.WriteUint16(z.rc.iv[zbzg].start) - if err != nil { - return - } - // write "last" - err = en.Append(0xa4, 0x6c, 0x61, 0x73, 0x74) - if err != nil { - return err - } - err = en.WriteUint16(z.rc.iv[zbzg].last()) - if err != nil { - return - } - } - // write "card" - err = en.Append(0xa4, 0x63, 0x61, 0x72, 0x64) - if err != nil { - return err - } - err = en.WriteInt64(z.rc.card) - if err != nil { - return - } - } - return -} - -// Deprecated: MarshalMsg implements msgp.Marshaler -func (z *addHelper16) MarshalMsg(b []byte) (o []byte, err error) { - o = msgp.Require(b, z.Msgsize()) - // map header, size 5 - // string "runstart" - o = append(o, 0x85, 0xa8, 0x72, 0x75, 0x6e, 0x73, 0x74, 0x61, 0x72, 0x74) - o = msgp.AppendUint16(o, z.runstart) - // string "runlen" - o = append(o, 0xa6, 0x72, 0x75, 0x6e, 0x6c, 0x65, 0x6e) - o = msgp.AppendUint16(o, z.runlen) - // string "actuallyAdded" - o = append(o, 0xad, 0x61, 0x63, 0x74, 0x75, 0x61, 0x6c, 0x6c, 0x79, 0x41, 0x64, 0x64, 0x65, 0x64) - o = msgp.AppendUint16(o, z.actuallyAdded) - // string "m" - o = append(o, 0xa1, 0x6d) - o = msgp.AppendArrayHeader(o, uint32(len(z.m))) - for zxvk := range z.m { - // map header, size 2 - // string "start" - o = append(o, 0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74) - o = msgp.AppendUint16(o, z.m[zxvk].start) - // string "last" - o = append(o, 0xa4, 0x6c, 0x61, 0x73, 0x74) - o = msgp.AppendUint16(o, z.m[zxvk].last()) - } - // string "rc" - o = append(o, 0xa2, 0x72, 0x63) - if z.rc == nil { - o = msgp.AppendNil(o) - } else { - // map header, size 2 - // string "iv" - o = append(o, 0x82, 0xa2, 0x69, 0x76) - o = msgp.AppendArrayHeader(o, uint32(len(z.rc.iv))) - for zbzg := range z.rc.iv { - // map header, size 2 - // string "start" - o = append(o, 0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74) - o = msgp.AppendUint16(o, z.rc.iv[zbzg].start) - // string "last" - o = append(o, 0xa4, 0x6c, 0x61, 0x73, 0x74) - o = msgp.AppendUint16(o, z.rc.iv[zbzg].last()) - } - // string "card" - o = append(o, 0xa4, 0x63, 0x61, 0x72, 0x64) - o = msgp.AppendInt64(o, z.rc.card) - } - return -} - -// Deprecated: UnmarshalMsg implements msgp.Unmarshaler -func (z *addHelper16) UnmarshalMsg(bts []byte) (o []byte, err error) { - var field []byte - _ = field - var zxhx uint32 - zxhx, bts, err = msgp.ReadMapHeaderBytes(bts) - if err != nil { - return - } - for zxhx > 0 { - zxhx-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "runstart": - z.runstart, bts, err = msgp.ReadUint16Bytes(bts) - if err != nil { - return - } - case "runlen": - z.runlen, bts, err = msgp.ReadUint16Bytes(bts) - if err != nil { - return - } - case "actuallyAdded": - z.actuallyAdded, bts, err = msgp.ReadUint16Bytes(bts) - if err != nil { - return - } - case "m": - var zlqf uint32 - zlqf, bts, err = msgp.ReadArrayHeaderBytes(bts) - if err != nil { - return - } - if cap(z.m) >= int(zlqf) { - z.m = (z.m)[:zlqf] - } else { - z.m = make([]interval16, zlqf) - } - for zxvk := range z.m { - var zdaf uint32 - zdaf, bts, err = msgp.ReadMapHeaderBytes(bts) - if err != nil { - return - } - for zdaf > 0 { - zdaf-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "start": - z.m[zxvk].start, bts, err = msgp.ReadUint16Bytes(bts) - if err != nil { - return - } - case "last": - z.m[zxvk].length, bts, err = msgp.ReadUint16Bytes(bts) - z.m[zxvk].length -= z.m[zxvk].start - if err != nil { - return - } - default: - bts, err = msgp.Skip(bts) - if err != nil { - return - } - } - } - } - case "rc": - if msgp.IsNil(bts) { - bts, err = msgp.ReadNilBytes(bts) - if err != nil { - return - } - z.rc = nil - } else { - if z.rc == nil { - z.rc = new(runContainer16) - } - var zpks uint32 - zpks, bts, err = msgp.ReadMapHeaderBytes(bts) - if err != nil { - return - } - for zpks > 0 { - zpks-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "iv": - var zjfb uint32 - zjfb, bts, err = msgp.ReadArrayHeaderBytes(bts) - if err != nil { - return - } - if cap(z.rc.iv) >= int(zjfb) { - z.rc.iv = (z.rc.iv)[:zjfb] - } else { - z.rc.iv = make([]interval16, zjfb) - } - for zbzg := range z.rc.iv { - var zcxo uint32 - zcxo, bts, err = msgp.ReadMapHeaderBytes(bts) - if err != nil { - return - } - for zcxo > 0 { - zcxo-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "start": - z.rc.iv[zbzg].start, bts, err = msgp.ReadUint16Bytes(bts) - if err != nil { - return - } - case "last": - z.rc.iv[zbzg].length, bts, err = msgp.ReadUint16Bytes(bts) - z.rc.iv[zbzg].length -= z.rc.iv[zbzg].start - if err != nil { - return - } - default: - bts, err = msgp.Skip(bts) - if err != nil { - return - } - } - } - } - case "card": - z.rc.card, bts, err = msgp.ReadInt64Bytes(bts) - if err != nil { - return - } - default: - bts, err = msgp.Skip(bts) - if err != nil { - return - } - } - } - } - default: - bts, err = msgp.Skip(bts) - if err != nil { - return - } - } - } - o = bts - return -} - -// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z *addHelper16) Msgsize() (s int) { - s = 1 + 9 + msgp.Uint16Size + 7 + msgp.Uint16Size + 14 + msgp.Uint16Size + 2 + msgp.ArrayHeaderSize + (len(z.m) * (12 + msgp.Uint16Size + msgp.Uint16Size)) + 3 - if z.rc == nil { - s += msgp.NilSize - } else { - s += 1 + 3 + msgp.ArrayHeaderSize + (len(z.rc.iv) * (12 + msgp.Uint16Size + msgp.Uint16Size)) + 5 + msgp.Int64Size - } - return -} - -// Deprecated: DecodeMsg implements msgp.Decodable -func (z *interval16) DecodeMsg(dc *msgp.Reader) (err error) { - var field []byte - _ = field - var zeff uint32 - zeff, err = dc.ReadMapHeader() - if err != nil { - return - } - for zeff > 0 { - zeff-- - field, err = dc.ReadMapKeyPtr() - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "start": - z.start, err = dc.ReadUint16() - if err != nil { - return - } - case "last": - z.length, err = dc.ReadUint16() - z.length = -z.start - if err != nil { - return - } - default: - err = dc.Skip() - if err != nil { - return - } - } - } - return -} - -// Deprecated: EncodeMsg implements msgp.Encodable -func (z interval16) EncodeMsg(en *msgp.Writer) (err error) { - // map header, size 2 - // write "start" - err = en.Append(0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74) - if err != nil { - return err - } - err = en.WriteUint16(z.start) - if err != nil { - return - } - // write "last" - err = en.Append(0xa4, 0x6c, 0x61, 0x73, 0x74) - if err != nil { - return err - } - err = en.WriteUint16(z.last()) - if err != nil { - return - } - return -} - -// Deprecated: MarshalMsg implements msgp.Marshaler -func (z interval16) MarshalMsg(b []byte) (o []byte, err error) { - o = msgp.Require(b, z.Msgsize()) - // map header, size 2 - // string "start" - o = append(o, 0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74) - o = msgp.AppendUint16(o, z.start) - // string "last" - o = append(o, 0xa4, 0x6c, 0x61, 0x73, 0x74) - o = msgp.AppendUint16(o, z.last()) - return -} - -// Deprecated: UnmarshalMsg implements msgp.Unmarshaler -func (z *interval16) UnmarshalMsg(bts []byte) (o []byte, err error) { - var field []byte - _ = field - var zrsw uint32 - zrsw, bts, err = msgp.ReadMapHeaderBytes(bts) - if err != nil { - return - } - for zrsw > 0 { - zrsw-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "start": - z.start, bts, err = msgp.ReadUint16Bytes(bts) - if err != nil { - return - } - case "last": - z.length, bts, err = msgp.ReadUint16Bytes(bts) - z.length -= z.start - if err != nil { - return - } - default: - bts, err = msgp.Skip(bts) - if err != nil { - return - } - } - } - o = bts - return -} - -// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z interval16) Msgsize() (s int) { - s = 1 + 6 + msgp.Uint16Size + 5 + msgp.Uint16Size - return -} - -// Deprecated: DecodeMsg implements msgp.Decodable -func (z *runContainer16) DecodeMsg(dc *msgp.Reader) (err error) { - var field []byte - _ = field - var zdnj uint32 - zdnj, err = dc.ReadMapHeader() - if err != nil { - return - } - for zdnj > 0 { - zdnj-- - field, err = dc.ReadMapKeyPtr() - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "iv": - var zobc uint32 - zobc, err = dc.ReadArrayHeader() - if err != nil { - return - } - if cap(z.iv) >= int(zobc) { - z.iv = (z.iv)[:zobc] - } else { - z.iv = make([]interval16, zobc) - } - for zxpk := range z.iv { - var zsnv uint32 - zsnv, err = dc.ReadMapHeader() - if err != nil { - return - } - for zsnv > 0 { - zsnv-- - field, err = dc.ReadMapKeyPtr() - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "start": - z.iv[zxpk].start, err = dc.ReadUint16() - if err != nil { - return - } - case "last": - z.iv[zxpk].length, err = dc.ReadUint16() - z.iv[zxpk].length -= z.iv[zxpk].start - if err != nil { - return - } - default: - err = dc.Skip() - if err != nil { - return - } - } - } - } - case "card": - z.card, err = dc.ReadInt64() - if err != nil { - return - } - default: - err = dc.Skip() - if err != nil { - return - } - } - } - return -} - -// Deprecated: EncodeMsg implements msgp.Encodable -func (z *runContainer16) EncodeMsg(en *msgp.Writer) (err error) { - // map header, size 2 - // write "iv" - err = en.Append(0x82, 0xa2, 0x69, 0x76) - if err != nil { - return err - } - err = en.WriteArrayHeader(uint32(len(z.iv))) - if err != nil { - return - } - for zxpk := range z.iv { - // map header, size 2 - // write "start" - err = en.Append(0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74) - if err != nil { - return err - } - err = en.WriteUint16(z.iv[zxpk].start) - if err != nil { - return - } - // write "last" - err = en.Append(0xa4, 0x6c, 0x61, 0x73, 0x74) - if err != nil { - return err - } - err = en.WriteUint16(z.iv[zxpk].last()) - if err != nil { - return - } - } - // write "card" - err = en.Append(0xa4, 0x63, 0x61, 0x72, 0x64) - if err != nil { - return err - } - err = en.WriteInt64(z.card) - if err != nil { - return - } - return -} - -// Deprecated: MarshalMsg implements msgp.Marshaler -func (z *runContainer16) MarshalMsg(b []byte) (o []byte, err error) { - o = msgp.Require(b, z.Msgsize()) - // map header, size 2 - // string "iv" - o = append(o, 0x82, 0xa2, 0x69, 0x76) - o = msgp.AppendArrayHeader(o, uint32(len(z.iv))) - for zxpk := range z.iv { - // map header, size 2 - // string "start" - o = append(o, 0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74) - o = msgp.AppendUint16(o, z.iv[zxpk].start) - // string "last" - o = append(o, 0xa4, 0x6c, 0x61, 0x73, 0x74) - o = msgp.AppendUint16(o, z.iv[zxpk].last()) - } - // string "card" - o = append(o, 0xa4, 0x63, 0x61, 0x72, 0x64) - o = msgp.AppendInt64(o, z.card) - return -} - -// Deprecated: UnmarshalMsg implements msgp.Unmarshaler -func (z *runContainer16) UnmarshalMsg(bts []byte) (o []byte, err error) { - var field []byte - _ = field - var zkgt uint32 - zkgt, bts, err = msgp.ReadMapHeaderBytes(bts) - if err != nil { - return - } - for zkgt > 0 { - zkgt-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "iv": - var zema uint32 - zema, bts, err = msgp.ReadArrayHeaderBytes(bts) - if err != nil { - return - } - if cap(z.iv) >= int(zema) { - z.iv = (z.iv)[:zema] - } else { - z.iv = make([]interval16, zema) - } - for zxpk := range z.iv { - var zpez uint32 - zpez, bts, err = msgp.ReadMapHeaderBytes(bts) - if err != nil { - return - } - for zpez > 0 { - zpez-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "start": - z.iv[zxpk].start, bts, err = msgp.ReadUint16Bytes(bts) - if err != nil { - return - } - case "last": - z.iv[zxpk].length, bts, err = msgp.ReadUint16Bytes(bts) - z.iv[zxpk].length -= z.iv[zxpk].start - if err != nil { - return - } - default: - bts, err = msgp.Skip(bts) - if err != nil { - return - } - } - } - } - case "card": - z.card, bts, err = msgp.ReadInt64Bytes(bts) - if err != nil { - return - } - default: - bts, err = msgp.Skip(bts) - if err != nil { - return - } - } - } - o = bts - return -} - -// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z *runContainer16) Msgsize() (s int) { - s = 1 + 3 + msgp.ArrayHeaderSize + (len(z.iv) * (12 + msgp.Uint16Size + msgp.Uint16Size)) + 5 + msgp.Int64Size - return -} - -// Deprecated: DecodeMsg implements msgp.Decodable -func (z *runIterator16) DecodeMsg(dc *msgp.Reader) (err error) { - var field []byte - _ = field - var zqke uint32 - zqke, err = dc.ReadMapHeader() - if err != nil { - return - } - for zqke > 0 { - zqke-- - field, err = dc.ReadMapKeyPtr() - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "rc": - if dc.IsNil() { - err = dc.ReadNil() - if err != nil { - return - } - z.rc = nil - } else { - if z.rc == nil { - z.rc = new(runContainer16) - } - err = z.rc.DecodeMsg(dc) - if err != nil { - return - } - } - case "curIndex": - z.curIndex, err = dc.ReadInt64() - if err != nil { - return - } - case "curPosInIndex": - z.curPosInIndex, err = dc.ReadUint16() - if err != nil { - return - } - default: - err = dc.Skip() - if err != nil { - return - } - } - } - return -} - -// Deprecated: EncodeMsg implements msgp.Encodable -func (z *runIterator16) EncodeMsg(en *msgp.Writer) (err error) { - // map header, size 3 - // write "rc" - err = en.Append(0x83, 0xa2, 0x72, 0x63) - if err != nil { - return err - } - if z.rc == nil { - err = en.WriteNil() - if err != nil { - return - } - } else { - err = z.rc.EncodeMsg(en) - if err != nil { - return - } - } - // write "curIndex" - err = en.Append(0xa8, 0x63, 0x75, 0x72, 0x49, 0x6e, 0x64, 0x65, 0x78) - if err != nil { - return err - } - err = en.WriteInt64(z.curIndex) - if err != nil { - return - } - // write "curPosInIndex" - err = en.Append(0xad, 0x63, 0x75, 0x72, 0x50, 0x6f, 0x73, 0x49, 0x6e, 0x49, 0x6e, 0x64, 0x65, 0x78) - if err != nil { - return err - } - err = en.WriteUint16(z.curPosInIndex) - if err != nil { - return - } - return -} - -// Deprecated: MarshalMsg implements msgp.Marshaler -func (z *runIterator16) MarshalMsg(b []byte) (o []byte, err error) { - o = msgp.Require(b, z.Msgsize()) - // map header, size 3 - // string "rc" - o = append(o, 0x83, 0xa2, 0x72, 0x63) - if z.rc == nil { - o = msgp.AppendNil(o) - } else { - o, err = z.rc.MarshalMsg(o) - if err != nil { - return - } - } - // string "curIndex" - o = append(o, 0xa8, 0x63, 0x75, 0x72, 0x49, 0x6e, 0x64, 0x65, 0x78) - o = msgp.AppendInt64(o, z.curIndex) - // string "curPosInIndex" - o = append(o, 0xad, 0x63, 0x75, 0x72, 0x50, 0x6f, 0x73, 0x49, 0x6e, 0x49, 0x6e, 0x64, 0x65, 0x78) - o = msgp.AppendUint16(o, z.curPosInIndex) - return -} - -// Deprecated: UnmarshalMsg implements msgp.Unmarshaler -func (z *runIterator16) UnmarshalMsg(bts []byte) (o []byte, err error) { - var field []byte - _ = field - var zqyh uint32 - zqyh, bts, err = msgp.ReadMapHeaderBytes(bts) - if err != nil { - return - } - for zqyh > 0 { - zqyh-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - return - } - switch msgp.UnsafeString(field) { - case "rc": - if msgp.IsNil(bts) { - bts, err = msgp.ReadNilBytes(bts) - if err != nil { - return - } - z.rc = nil - } else { - if z.rc == nil { - z.rc = new(runContainer16) - } - bts, err = z.rc.UnmarshalMsg(bts) - if err != nil { - return - } - } - case "curIndex": - z.curIndex, bts, err = msgp.ReadInt64Bytes(bts) - if err != nil { - return - } - case "curPosInIndex": - z.curPosInIndex, bts, err = msgp.ReadUint16Bytes(bts) - if err != nil { - return - } - default: - bts, err = msgp.Skip(bts) - if err != nil { - return - } - } - } - o = bts - return -} - -// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z *runIterator16) Msgsize() (s int) { - s = 1 + 3 - if z.rc == nil { - s += msgp.NilSize - } else { - s += z.rc.Msgsize() - } - s += 9 + msgp.Int64Size + 14 + msgp.Uint16Size - return -} - -// Deprecated: DecodeMsg implements msgp.Decodable -func (z *uint16Slice) DecodeMsg(dc *msgp.Reader) (err error) { - var zjpj uint32 - zjpj, err = dc.ReadArrayHeader() - if err != nil { - return - } - if cap((*z)) >= int(zjpj) { - (*z) = (*z)[:zjpj] - } else { - (*z) = make(uint16Slice, zjpj) - } - for zywj := range *z { - (*z)[zywj], err = dc.ReadUint16() - if err != nil { - return - } - } - return -} - -// Deprecated: EncodeMsg implements msgp.Encodable -func (z uint16Slice) EncodeMsg(en *msgp.Writer) (err error) { - err = en.WriteArrayHeader(uint32(len(z))) - if err != nil { - return - } - for zzpf := range z { - err = en.WriteUint16(z[zzpf]) - if err != nil { - return - } - } - return -} - -// Deprecated: MarshalMsg implements msgp.Marshaler -func (z uint16Slice) MarshalMsg(b []byte) (o []byte, err error) { - o = msgp.Require(b, z.Msgsize()) - o = msgp.AppendArrayHeader(o, uint32(len(z))) - for zzpf := range z { - o = msgp.AppendUint16(o, z[zzpf]) - } - return -} - -// Deprecated: UnmarshalMsg implements msgp.Unmarshaler -func (z *uint16Slice) UnmarshalMsg(bts []byte) (o []byte, err error) { - var zgmo uint32 - zgmo, bts, err = msgp.ReadArrayHeaderBytes(bts) - if err != nil { - return - } - if cap((*z)) >= int(zgmo) { - (*z) = (*z)[:zgmo] - } else { - (*z) = make(uint16Slice, zgmo) - } - for zrfe := range *z { - (*z)[zrfe], bts, err = msgp.ReadUint16Bytes(bts) - if err != nil { - return - } - } - o = bts - return -} - -// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z uint16Slice) Msgsize() (s int) { - s = msgp.ArrayHeaderSize + (len(z) * (msgp.Uint16Size)) - return -} diff --git a/vendor/github.com/RoaringBitmap/roaring/serialization.go b/vendor/github.com/RoaringBitmap/roaring/serialization.go index 7b7ed29b..70e3bbcc 100644 --- a/vendor/github.com/RoaringBitmap/roaring/serialization.go +++ b/vendor/github.com/RoaringBitmap/roaring/serialization.go @@ -3,8 +3,6 @@ package roaring import ( "encoding/binary" "io" - - "github.com/tinylib/msgp/msgp" ) // writeTo for runContainer16 follows this @@ -19,16 +17,3 @@ func (b *runContainer16) writeTo(stream io.Writer) (int, error) { } return stream.Write(buf) } - -func (b *runContainer16) writeToMsgpack(stream io.Writer) (int, error) { - bts, err := b.MarshalMsg(nil) - if err != nil { - return 0, err - } - return stream.Write(bts) -} - -func (b *runContainer16) readFromMsgpack(stream io.Reader) (int, error) { - err := msgp.Decode(stream, b) - return 0, err -} diff --git a/vendor/github.com/RoaringBitmap/roaring/serialization_generic.go b/vendor/github.com/RoaringBitmap/roaring/serialization_generic.go index 4b9d9e3d..7e1f1802 100644 --- a/vendor/github.com/RoaringBitmap/roaring/serialization_generic.go +++ b/vendor/github.com/RoaringBitmap/roaring/serialization_generic.go @@ -1,4 +1,5 @@ -// +build !amd64,!386 appengine +//go:build (!amd64 && !386 && !arm && !arm64 && !ppc64le && !mipsle && !mips64le && !mips64p32le && !wasm) || appengine +// +build !amd64,!386,!arm,!arm64,!ppc64le,!mipsle,!mips64le,!mips64p32le,!wasm appengine package roaring @@ -84,6 +85,17 @@ func uint16SliceAsByteSlice(slice []uint16) []byte { return by } +func interval16SliceAsByteSlice(slice []interval16) []byte { + by := make([]byte, len(slice)*4) + + for i, v := range slice { + binary.LittleEndian.PutUint16(by[i*2:], v.start) + binary.LittleEndian.PutUint16(by[i*2+2:], v.length) + } + + return by +} + func byteSliceAsUint16Slice(slice []byte) []uint16 { if len(slice)%2 != 0 { panic("Slice size should be divisible by 2") diff --git a/vendor/github.com/RoaringBitmap/roaring/serialization_littleendian.go b/vendor/github.com/RoaringBitmap/roaring/serialization_littleendian.go index 818a06c8..2e4ea595 100644 --- a/vendor/github.com/RoaringBitmap/roaring/serialization_littleendian.go +++ b/vendor/github.com/RoaringBitmap/roaring/serialization_littleendian.go @@ -1,8 +1,10 @@ -// +build 386 amd64,!appengine +//go:build (386 && !appengine) || (amd64 && !appengine) || (arm && !appengine) || (arm64 && !appengine) || (ppc64le && !appengine) || (mipsle && !appengine) || (mips64le && !appengine) || (mips64p32le && !appengine) || (wasm && !appengine) +// +build 386,!appengine amd64,!appengine arm,!appengine arm64,!appengine ppc64le,!appengine mipsle,!appengine mips64le,!appengine mips64p32le,!appengine wasm,!appengine package roaring import ( + "encoding/binary" "errors" "io" "reflect" @@ -55,6 +57,22 @@ func uint16SliceAsByteSlice(slice []uint16) []byte { return result } +func interval16SliceAsByteSlice(slice []interval16) []byte { + // make a new slice header + header := *(*reflect.SliceHeader)(unsafe.Pointer(&slice)) + + // update its capacity and length + header.Len *= 4 + header.Cap *= 4 + + // instantiate result and use KeepAlive so data isn't unmapped. + result := *(*[]byte)(unsafe.Pointer(&header)) + runtime.KeepAlive(&slice) + + // return it + return result +} + func (bc *bitmapContainer) asLittleEndianByteSlice() []byte { return uint64SliceAsByteSlice(bc.bitmap) } @@ -132,3 +150,503 @@ func byteSliceAsInterval16Slice(slice []byte) (result []interval16) { // return result return } + +func byteSliceAsContainerSlice(slice []byte) (result []container) { + var c container + containerSize := int(unsafe.Sizeof(c)) + + if len(slice)%containerSize != 0 { + panic("Slice size should be divisible by unsafe.Sizeof(container)") + } + // reference: https://go101.org/article/unsafe.html + + // make a new slice header + bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice)) + rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result)) + + // transfer the data from the given slice to a new variable (our result) + rHeader.Data = bHeader.Data + rHeader.Len = bHeader.Len / containerSize + rHeader.Cap = bHeader.Cap / containerSize + + // instantiate result and use KeepAlive so data isn't unmapped. + runtime.KeepAlive(&slice) // it is still crucial, GC can free it) + + // return result + return +} + +func byteSliceAsBitsetSlice(slice []byte) (result []bitmapContainer) { + bitsetSize := int(unsafe.Sizeof(bitmapContainer{})) + if len(slice)%bitsetSize != 0 { + panic("Slice size should be divisible by unsafe.Sizeof(bitmapContainer)") + } + // reference: https://go101.org/article/unsafe.html + + // make a new slice header + bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice)) + rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result)) + + // transfer the data from the given slice to a new variable (our result) + rHeader.Data = bHeader.Data + rHeader.Len = bHeader.Len / bitsetSize + rHeader.Cap = bHeader.Cap / bitsetSize + + // instantiate result and use KeepAlive so data isn't unmapped. + runtime.KeepAlive(&slice) // it is still crucial, GC can free it) + + // return result + return +} + +func byteSliceAsArraySlice(slice []byte) (result []arrayContainer) { + arraySize := int(unsafe.Sizeof(arrayContainer{})) + if len(slice)%arraySize != 0 { + panic("Slice size should be divisible by unsafe.Sizeof(arrayContainer)") + } + // reference: https://go101.org/article/unsafe.html + + // make a new slice header + bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice)) + rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result)) + + // transfer the data from the given slice to a new variable (our result) + rHeader.Data = bHeader.Data + rHeader.Len = bHeader.Len / arraySize + rHeader.Cap = bHeader.Cap / arraySize + + // instantiate result and use KeepAlive so data isn't unmapped. + runtime.KeepAlive(&slice) // it is still crucial, GC can free it) + + // return result + return +} + +func byteSliceAsRun16Slice(slice []byte) (result []runContainer16) { + run16Size := int(unsafe.Sizeof(runContainer16{})) + if len(slice)%run16Size != 0 { + panic("Slice size should be divisible by unsafe.Sizeof(runContainer16)") + } + // reference: https://go101.org/article/unsafe.html + + // make a new slice header + bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice)) + rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result)) + + // transfer the data from the given slice to a new variable (our result) + rHeader.Data = bHeader.Data + rHeader.Len = bHeader.Len / run16Size + rHeader.Cap = bHeader.Cap / run16Size + + // instantiate result and use KeepAlive so data isn't unmapped. + runtime.KeepAlive(&slice) // it is still crucial, GC can free it) + + // return result + return +} + +func byteSliceAsBoolSlice(slice []byte) (result []bool) { + boolSize := int(unsafe.Sizeof(true)) + if len(slice)%boolSize != 0 { + panic("Slice size should be divisible by unsafe.Sizeof(bool)") + } + // reference: https://go101.org/article/unsafe.html + + // make a new slice header + bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice)) + rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result)) + + // transfer the data from the given slice to a new variable (our result) + rHeader.Data = bHeader.Data + rHeader.Len = bHeader.Len / boolSize + rHeader.Cap = bHeader.Cap / boolSize + + // instantiate result and use KeepAlive so data isn't unmapped. + runtime.KeepAlive(&slice) // it is still crucial, GC can free it) + + // return result + return +} + +// FrozenView creates a static view of a serialized bitmap stored in buf. +// It uses CRoaring's frozen bitmap format. +// +// The format specification is available here: +// https://github.com/RoaringBitmap/CRoaring/blob/2c867e9f9c9e2a3a7032791f94c4c7ae3013f6e0/src/roaring.c#L2756-L2783 +// +// The provided byte array (buf) is expected to be a constant. +// The function makes the best effort attempt not to copy data. +// Only little endian is supported. The function will err if it detects a big +// endian serialized file. +// You should take care not to modify buff as it will likely result in +// unexpected program behavior. +// If said buffer comes from a memory map, it's advisable to give it read +// only permissions, either at creation or by calling Mprotect from the +// golang.org/x/sys/unix package. +// +// Resulting bitmaps are effectively immutable in the following sense: +// a copy-on-write marker is used so that when you modify the resulting +// bitmap, copies of selected data (containers) are made. +// You should *not* change the copy-on-write status of the resulting +// bitmaps (SetCopyOnWrite). +// +// If buf becomes unavailable, then a bitmap created with +// FromBuffer would be effectively broken. Furthermore, any +// bitmap derived from this bitmap (e.g., via Or, And) might +// also be broken. Thus, before making buf unavailable, you should +// call CloneCopyOnWriteContainers on all such bitmaps. +// +func (rb *Bitmap) FrozenView(buf []byte) error { + return rb.highlowcontainer.frozenView(buf) +} + +/* Verbatim specification from CRoaring. + * + * FROZEN SERIALIZATION FORMAT DESCRIPTION + * + * -- (beginning must be aligned by 32 bytes) -- + * uint64_t[BITSET_CONTAINER_SIZE_IN_WORDS * num_bitset_containers] + * rle16_t[total number of rle elements in all run containers] + * uint16_t[total number of array elements in all array containers] + * uint16_t[num_containers] + * uint16_t[num_containers] + * uint8_t[num_containers] + *
uint32_t + * + *
is a 4-byte value which is a bit union of FROZEN_COOKIE (15 bits) + * and the number of containers (17 bits). + * + * stores number of elements for every container. + * Its meaning depends on container type. + * For array and bitset containers, this value is the container cardinality minus one. + * For run container, it is the number of rle_t elements (n_runs). + * + * ,, are flat arrays of elements of + * all containers of respective type. + * + * <*_data> and are kept close together because they are not accessed + * during deserilization. This may reduce IO in case of large mmaped bitmaps. + * All members have their native alignments during deserilization except
, + * which is not guaranteed to be aligned by 4 bytes. + */ +const FROZEN_COOKIE = 13766 + +var ( + FrozenBitmapInvalidCookie = errors.New("header does not contain the FROZEN_COOKIE") + FrozenBitmapBigEndian = errors.New("loading big endian frozen bitmaps is not supported") + FrozenBitmapIncomplete = errors.New("input buffer too small to contain a frozen bitmap") + FrozenBitmapOverpopulated = errors.New("too many containers") + FrozenBitmapUnexpectedData = errors.New("spurious data in input") + FrozenBitmapInvalidTypecode = errors.New("unrecognized typecode") + FrozenBitmapBufferTooSmall = errors.New("buffer too small") +) + +func (ra *roaringArray) frozenView(buf []byte) error { + if len(buf) < 4 { + return FrozenBitmapIncomplete + } + + headerBE := binary.BigEndian.Uint32(buf[len(buf)-4:]) + if headerBE&0x7fff == FROZEN_COOKIE { + return FrozenBitmapBigEndian + } + + header := binary.LittleEndian.Uint32(buf[len(buf)-4:]) + buf = buf[:len(buf)-4] + + if header&0x7fff != FROZEN_COOKIE { + return FrozenBitmapInvalidCookie + } + + nCont := int(header >> 15) + if nCont > (1 << 16) { + return FrozenBitmapOverpopulated + } + + // 1 byte per type, 2 bytes per key, 2 bytes per count. + if len(buf) < 5*nCont { + return FrozenBitmapIncomplete + } + + types := buf[len(buf)-nCont:] + buf = buf[:len(buf)-nCont] + + counts := byteSliceAsUint16Slice(buf[len(buf)-2*nCont:]) + buf = buf[:len(buf)-2*nCont] + + keys := byteSliceAsUint16Slice(buf[len(buf)-2*nCont:]) + buf = buf[:len(buf)-2*nCont] + + nBitmap, nArray, nRun := 0, 0, 0 + nArrayEl, nRunEl := 0, 0 + for i, t := range types { + switch t { + case 1: + nBitmap++ + case 2: + nArray++ + nArrayEl += int(counts[i]) + 1 + case 3: + nRun++ + nRunEl += int(counts[i]) + default: + return FrozenBitmapInvalidTypecode + } + } + + if len(buf) < (1<<13)*nBitmap+4*nRunEl+2*nArrayEl { + return FrozenBitmapIncomplete + } + + bitsetsArena := byteSliceAsUint64Slice(buf[:(1<<13)*nBitmap]) + buf = buf[(1<<13)*nBitmap:] + + runsArena := byteSliceAsInterval16Slice(buf[:4*nRunEl]) + buf = buf[4*nRunEl:] + + arraysArena := byteSliceAsUint16Slice(buf[:2*nArrayEl]) + buf = buf[2*nArrayEl:] + + if len(buf) != 0 { + return FrozenBitmapUnexpectedData + } + + var c container + containersSz := int(unsafe.Sizeof(c))*nCont + bitsetsSz := int(unsafe.Sizeof(bitmapContainer{}))*nBitmap + arraysSz := int(unsafe.Sizeof(arrayContainer{}))*nArray + runsSz := int(unsafe.Sizeof(runContainer16{}))*nRun + needCOWSz := int(unsafe.Sizeof(true))*nCont + + bitmapArenaSz := containersSz + bitsetsSz + arraysSz + runsSz + needCOWSz + bitmapArena := make([]byte, bitmapArenaSz) + + containers := byteSliceAsContainerSlice(bitmapArena[:containersSz]) + bitmapArena = bitmapArena[containersSz:] + + bitsets := byteSliceAsBitsetSlice(bitmapArena[:bitsetsSz]) + bitmapArena = bitmapArena[bitsetsSz:] + + arrays := byteSliceAsArraySlice(bitmapArena[:arraysSz]) + bitmapArena = bitmapArena[arraysSz:] + + runs := byteSliceAsRun16Slice(bitmapArena[:runsSz]) + bitmapArena = bitmapArena[runsSz:] + + needCOW := byteSliceAsBoolSlice(bitmapArena) + + iBitset, iArray, iRun := 0, 0, 0 + for i, t := range types { + needCOW[i] = true + + switch t { + case 1: + containers[i] = &bitsets[iBitset] + bitsets[iBitset].cardinality = int(counts[i]) + 1 + bitsets[iBitset].bitmap = bitsetsArena[:1024] + bitsetsArena = bitsetsArena[1024:] + iBitset++ + case 2: + containers[i] = &arrays[iArray] + sz := int(counts[i]) + 1 + arrays[iArray].content = arraysArena[:sz] + arraysArena = arraysArena[sz:] + iArray++ + case 3: + containers[i] = &runs[iRun] + runs[iRun].iv = runsArena[:counts[i]] + runsArena = runsArena[counts[i]:] + iRun++ + } + } + + // Not consuming the full input is a bug. + if iBitset != nBitmap || len(bitsetsArena) != 0 || + iArray != nArray || len(arraysArena) != 0 || + iRun != nRun || len(runsArena) != 0 { + panic("we missed something") + } + + ra.keys = keys + ra.containers = containers + ra.needCopyOnWrite = needCOW + ra.copyOnWrite = true + + return nil +} + +func (bm *Bitmap) GetFrozenSizeInBytes() uint64 { + nBits, nArrayEl, nRunEl := uint64(0), uint64(0), uint64(0) + for _, c := range bm.highlowcontainer.containers { + switch v := c.(type) { + case *bitmapContainer: + nBits++ + case *arrayContainer: + nArrayEl += uint64(len(v.content)) + case *runContainer16: + nRunEl += uint64(len(v.iv)) + } + } + return 4 + 5*uint64(len(bm.highlowcontainer.containers)) + + (nBits << 13) + 2*nArrayEl + 4*nRunEl +} + +func (bm *Bitmap) Freeze() ([]byte, error) { + sz := bm.GetFrozenSizeInBytes() + buf := make([]byte, sz) + _, err := bm.FreezeTo(buf) + return buf, err +} + +func (bm *Bitmap) FreezeTo(buf []byte) (int, error) { + containers := bm.highlowcontainer.containers + nCont := len(containers) + + nBits, nArrayEl, nRunEl := 0, 0, 0 + for _, c := range containers { + switch v := c.(type) { + case *bitmapContainer: + nBits++ + case *arrayContainer: + nArrayEl += len(v.content) + case *runContainer16: + nRunEl += len(v.iv) + } + } + + serialSize := 4 + 5*nCont + (1<<13)*nBits + 4*nRunEl + 2*nArrayEl + if len(buf) < serialSize { + return 0, FrozenBitmapBufferTooSmall + } + + bitsArena := byteSliceAsUint64Slice(buf[:(1<<13)*nBits]) + buf = buf[(1<<13)*nBits:] + + runsArena := byteSliceAsInterval16Slice(buf[:4*nRunEl]) + buf = buf[4*nRunEl:] + + arraysArena := byteSliceAsUint16Slice(buf[:2*nArrayEl]) + buf = buf[2*nArrayEl:] + + keys := byteSliceAsUint16Slice(buf[:2*nCont]) + buf = buf[2*nCont:] + + counts := byteSliceAsUint16Slice(buf[:2*nCont]) + buf = buf[2*nCont:] + + types := buf[:nCont] + buf = buf[nCont:] + + header := uint32(FROZEN_COOKIE | (nCont << 15)) + binary.LittleEndian.PutUint32(buf[:4], header) + + copy(keys, bm.highlowcontainer.keys[:]) + + for i, c := range containers { + switch v := c.(type) { + case *bitmapContainer: + copy(bitsArena, v.bitmap) + bitsArena = bitsArena[1024:] + counts[i] = uint16(v.cardinality - 1) + types[i] = 1 + case *arrayContainer: + copy(arraysArena, v.content) + arraysArena = arraysArena[len(v.content):] + elems := len(v.content) + counts[i] = uint16(elems - 1) + types[i] = 2 + case *runContainer16: + copy(runsArena, v.iv) + runs := len(v.iv) + runsArena = runsArena[runs:] + counts[i] = uint16(runs) + types[i] = 3 + } + } + + return serialSize, nil +} + +func (bm *Bitmap) WriteFrozenTo(wr io.Writer) (int, error) { + // FIXME: this is a naive version that iterates 4 times through the + // containers and allocates 3*len(containers) bytes; it's quite likely + // it can be done more efficiently. + containers := bm.highlowcontainer.containers + written := 0 + + for _, c := range containers { + c, ok := c.(*bitmapContainer) + if !ok { + continue + } + n, err := wr.Write(uint64SliceAsByteSlice(c.bitmap)) + written += n + if err != nil { + return written, err + } + } + + for _, c := range containers { + c, ok := c.(*runContainer16) + if !ok { + continue + } + n, err := wr.Write(interval16SliceAsByteSlice(c.iv)) + written += n + if err != nil { + return written, err + } + } + + for _, c := range containers { + c, ok := c.(*arrayContainer) + if !ok { + continue + } + n, err := wr.Write(uint16SliceAsByteSlice(c.content)) + written += n + if err != nil { + return written, err + } + } + + n, err := wr.Write(uint16SliceAsByteSlice(bm.highlowcontainer.keys)) + written += n + if err != nil { + return written, err + } + + countTypeBuf := make([]byte, 3*len(containers)) + counts := byteSliceAsUint16Slice(countTypeBuf[:2*len(containers)]) + types := countTypeBuf[2*len(containers):] + + for i, c := range containers { + switch c := c.(type) { + case *bitmapContainer: + counts[i] = uint16(c.cardinality - 1) + types[i] = 1 + case *arrayContainer: + elems := len(c.content) + counts[i] = uint16(elems - 1) + types[i] = 2 + case *runContainer16: + runs := len(c.iv) + counts[i] = uint16(runs) + types[i] = 3 + } + } + + n, err = wr.Write(countTypeBuf) + written += n + if err != nil { + return written, err + } + + header := uint32(FROZEN_COOKIE | (len(containers) << 15)) + if err := binary.Write(wr, binary.LittleEndian, header); err != nil { + return written, err + } + written += 4 + + return written, nil +} diff --git a/vendor/github.com/RoaringBitmap/roaring/serializationfuzz.go b/vendor/github.com/RoaringBitmap/roaring/serializationfuzz.go index 5eaa2220..c7fed02b 100644 --- a/vendor/github.com/RoaringBitmap/roaring/serializationfuzz.go +++ b/vendor/github.com/RoaringBitmap/roaring/serializationfuzz.go @@ -1,3 +1,4 @@ +//go:build gofuzz // +build gofuzz package roaring diff --git a/vendor/github.com/RoaringBitmap/roaring/setutil.go b/vendor/github.com/RoaringBitmap/roaring/setutil.go index 2fe81514..663c4fa3 100644 --- a/vendor/github.com/RoaringBitmap/roaring/setutil.go +++ b/vendor/github.com/RoaringBitmap/roaring/setutil.go @@ -135,66 +135,6 @@ func exclusiveUnion2by2(set1 []uint16, set2 []uint16, buffer []uint16) int { return pos } -func union2by2(set1 []uint16, set2 []uint16, buffer []uint16) int { - pos := 0 - k1 := 0 - k2 := 0 - if 0 == len(set2) { - buffer = buffer[:len(set1)] - copy(buffer, set1[:]) - return len(set1) - } - if 0 == len(set1) { - buffer = buffer[:len(set2)] - copy(buffer, set2[:]) - return len(set2) - } - s1 := set1[k1] - s2 := set2[k2] - buffer = buffer[:cap(buffer)] - for { - if s1 < s2 { - buffer[pos] = s1 - pos++ - k1++ - if k1 >= len(set1) { - copy(buffer[pos:], set2[k2:]) - pos += len(set2) - k2 - break - } - s1 = set1[k1] - } else if s1 == s2 { - buffer[pos] = s1 - pos++ - k1++ - k2++ - if k1 >= len(set1) { - copy(buffer[pos:], set2[k2:]) - pos += len(set2) - k2 - break - } - if k2 >= len(set2) { - copy(buffer[pos:], set1[k1:]) - pos += len(set1) - k1 - break - } - s1 = set1[k1] - s2 = set2[k2] - } else { // if (set1[k1]>set2[k2]) - buffer[pos] = s2 - pos++ - k2++ - if k2 >= len(set2) { - copy(buffer[pos:], set1[k1:]) - pos += len(set1) - k1 - break - } - s2 = set2[k2] - } - } - return pos -} - func union2by2Cardinality(set1 []uint16, set2 []uint16) int { pos := 0 k1 := 0 diff --git a/vendor/github.com/RoaringBitmap/roaring/setutil_arm64.go b/vendor/github.com/RoaringBitmap/roaring/setutil_arm64.go new file mode 100644 index 00000000..3e089650 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/setutil_arm64.go @@ -0,0 +1,7 @@ +//go:build arm64 && !gccgo && !appengine +// +build arm64,!gccgo,!appengine + +package roaring + +//go:noescape +func union2by2(set1 []uint16, set2 []uint16, buffer []uint16) (size int) diff --git a/vendor/github.com/RoaringBitmap/roaring/setutil_arm64.s b/vendor/github.com/RoaringBitmap/roaring/setutil_arm64.s new file mode 100644 index 00000000..e4f0f204 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/setutil_arm64.s @@ -0,0 +1,132 @@ +// +build arm64,!gccgo,!appengine + +#include "textflag.h" + + +// This implements union2by2 using golang's version of arm64 assembly +// The algorithm is very similar to the generic one, +// but makes better use of arm64 features so is notably faster. +// The basic algorithm structure is as follows: +// 1. If either set is empty, copy the other set into the buffer and return the length +// 2. Otherwise, load the first element of each set into a variable (s1 and s2). +// 3. a. Compare the values of s1 and s2. + // b. add the smaller one to the buffer. + // c. perform a bounds check before incrementing. + // If one set is finished, copy the rest of the other set over. + // d. update s1 and or s2 to the next value, continue loop. + // + // Past the fact of the algorithm, this code makes use of several arm64 features + // Condition Codes: + // arm64's CMP operation sets 4 bits that can be used for branching, + // rather than just true or false. + // As a consequence, a single comparison gives enough information to distinguish the three cases + // + // Post-increment pointers after load/store: + // Instructions like `MOVHU.P 2(R0), R6` + // increment the register by a specified amount, in this example 2. + // Because uint16's are exactly 2 bytes and the length of the slices + // is part of the slice header, + // there is no need to separately track the index into the slice. + // Instead, the code can calculate the final read value and compare against that, + // using the post-increment reads to move the pointers along. + // + // TODO: CALL out to memmove once the list is exhausted. + // Right now it moves the necessary shorts so that the remaining count + // is a multiple of 4 and then copies 64 bits at a time. + +TEXT ·union2by2(SB), NOSPLIT, $0-80 + // R0, R1, and R2 for the pointers to the three slices + MOVD set1+0(FP), R0 + MOVD set2+24(FP), R1 + MOVD buffer+48(FP), R2 + + //R3 and R4 will be the values at which we will have finished reading set1 and set2. + // R3 should be R0 + 2 * set1_len+8(FP) + MOVD set1_len+8(FP), R3 + MOVD set2_len+32(FP), R4 + + ADD R3<<1, R0, R3 + ADD R4<<1, R1, R4 + + + //Rather than counting the number of elements added separately + //Save the starting register of buffer. + MOVD buffer+48(FP), R5 + + // set1 is empty, just flush set2 + CMP R0, R3 + BEQ flush_right + + // set2 is empty, just flush set1 + CMP R1, R4 + BEQ flush_left + + // R6, R7 are the working space for s1 and s2 + MOVD ZR, R6 + MOVD ZR, R7 + + MOVHU.P 2(R0), R6 + MOVHU.P 2(R1), R7 +loop: + + CMP R6, R7 + BEQ pop_both // R6 == R7 + BLS pop_right // R6 > R7 +//pop_left: // R6 < R7 + MOVHU.P R6, 2(R2) + CMP R0, R3 + BEQ pop_then_flush_right + MOVHU.P 2(R0), R6 + JMP loop +pop_both: + MOVHU.P R6, 2(R2) //could also use R7, since they are equal + CMP R0, R3 + BEQ flush_right + CMP R1, R4 + BEQ flush_left + MOVHU.P 2(R0), R6 + MOVHU.P 2(R1), R7 + JMP loop +pop_right: + MOVHU.P R7, 2(R2) + CMP R1, R4 + BEQ pop_then_flush_left + MOVHU.P 2(R1), R7 + JMP loop + +pop_then_flush_right: + MOVHU.P R7, 2(R2) +flush_right: + MOVD R1, R0 + MOVD R4, R3 + JMP flush_left +pop_then_flush_left: + MOVHU.P R6, 2(R2) +flush_left: + CMP R0, R3 + BEQ return + //figure out how many bytes to slough off. Must be a multiple of two + SUB R0, R3, R4 + ANDS $6, R4 + BEQ long_flush //handles the 0 mod 8 case + SUBS $4, R4, R4 // since possible values are 2, 4, 6, this splits evenly + BLT pop_single // exactly the 2 case + MOVW.P 4(R0), R6 + MOVW.P R6, 4(R2) + BEQ long_flush // we're now aligned by 64 bits, as R4==4, otherwise 2 more +pop_single: + MOVHU.P 2(R0), R6 + MOVHU.P R6, 2(R2) +long_flush: + // at this point we know R3 - R0 is a multiple of 8. + CMP R0, R3 + BEQ return + MOVD.P 8(R0), R6 + MOVD.P R6, 8(R2) + JMP long_flush +return: + // number of shorts written is (R5 - R2) >> 1 + SUB R5, R2 + LSR $1, R2, R2 + MOVD R2, size+72(FP) + RET diff --git a/vendor/github.com/RoaringBitmap/roaring/setutil_generic.go b/vendor/github.com/RoaringBitmap/roaring/setutil_generic.go new file mode 100644 index 00000000..4755fd54 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/setutil_generic.go @@ -0,0 +1,64 @@ +//go:build !arm64 || gccgo || appengine +// +build !arm64 gccgo appengine + +package roaring + +func union2by2(set1 []uint16, set2 []uint16, buffer []uint16) int { + pos := 0 + k1 := 0 + k2 := 0 + if 0 == len(set2) { + buffer = buffer[:len(set1)] + copy(buffer, set1[:]) + return len(set1) + } + if 0 == len(set1) { + buffer = buffer[:len(set2)] + copy(buffer, set2[:]) + return len(set2) + } + s1 := set1[k1] + s2 := set2[k2] + buffer = buffer[:cap(buffer)] + for { + if s1 < s2 { + buffer[pos] = s1 + pos++ + k1++ + if k1 >= len(set1) { + copy(buffer[pos:], set2[k2:]) + pos += len(set2) - k2 + break + } + s1 = set1[k1] + } else if s1 == s2 { + buffer[pos] = s1 + pos++ + k1++ + k2++ + if k1 >= len(set1) { + copy(buffer[pos:], set2[k2:]) + pos += len(set2) - k2 + break + } + if k2 >= len(set2) { + copy(buffer[pos:], set1[k1:]) + pos += len(set1) - k1 + break + } + s1 = set1[k1] + s2 = set2[k2] + } else { // if (set1[k1]>set2[k2]) + buffer[pos] = s2 + pos++ + k2++ + if k2 >= len(set2) { + copy(buffer[pos:], set1[k1:]) + pos += len(set1) - k1 + break + } + s2 = set2[k2] + } + } + return pos +} diff --git a/vendor/github.com/RoaringBitmap/roaring/smat.go b/vendor/github.com/RoaringBitmap/roaring/smat.go index 9da47563..c52c5f07 100644 --- a/vendor/github.com/RoaringBitmap/roaring/smat.go +++ b/vendor/github.com/RoaringBitmap/roaring/smat.go @@ -1,3 +1,4 @@ +//go:build gofuzz // +build gofuzz /* @@ -62,8 +63,8 @@ import ( "fmt" "sort" + "github.com/bits-and-blooms/bitset" "github.com/mschoch/smat" - "github.com/willf/bitset" ) // fuzz test using state machine driven by byte stream. diff --git a/vendor/github.com/armon/go-metrics/metrics.go b/vendor/github.com/armon/go-metrics/metrics.go index 6753b13b..36642a42 100644 --- a/vendor/github.com/armon/go-metrics/metrics.go +++ b/vendor/github.com/armon/go-metrics/metrics.go @@ -5,7 +5,7 @@ import ( "strings" "time" - "github.com/hashicorp/go-immutable-radix" + iradix "github.com/hashicorp/go-immutable-radix" ) type Label struct { @@ -172,6 +172,12 @@ func (m *Metrics) UpdateFilterAndLabels(allow, block, allowedLabels, blockedLabe } } +func (m *Metrics) Shutdown() { + if ss, ok := m.sink.(ShutdownSink); ok { + ss.Shutdown() + } +} + // labelIsAllowed return true if a should be included in metric // the caller should lock m.filterLock while calling this method func (m *Metrics) labelIsAllowed(label *Label) bool { diff --git a/vendor/github.com/armon/go-metrics/sink.go b/vendor/github.com/armon/go-metrics/sink.go index 0b7d6e4b..6f4108ff 100644 --- a/vendor/github.com/armon/go-metrics/sink.go +++ b/vendor/github.com/armon/go-metrics/sink.go @@ -24,6 +24,15 @@ type MetricSink interface { AddSampleWithLabels(key []string, val float32, labels []Label) } +type ShutdownSink interface { + MetricSink + + // Shutdown the metric sink, flush metrics to storage, and cleanup resources. + // Called immediately prior to application exit. Implementations must block + // until metrics are flushed to storage. + Shutdown() +} + // BlackholeSink is used to just blackhole messages type BlackholeSink struct{} @@ -74,6 +83,14 @@ func (fh FanoutSink) AddSampleWithLabels(key []string, val float32, labels []Lab } } +func (fh FanoutSink) Shutdown() { + for _, s := range fh { + if ss, ok := s.(ShutdownSink); ok { + ss.Shutdown() + } + } +} + // sinkURLFactoryFunc is an generic interface around the *SinkFromURL() function provided // by each sink type type sinkURLFactoryFunc func(*url.URL) (MetricSink, error) diff --git a/vendor/github.com/armon/go-metrics/start.go b/vendor/github.com/armon/go-metrics/start.go index 6aa0bd38..38976f8d 100644 --- a/vendor/github.com/armon/go-metrics/start.go +++ b/vendor/github.com/armon/go-metrics/start.go @@ -144,3 +144,15 @@ func UpdateFilter(allow, block []string) { func UpdateFilterAndLabels(allow, block, allowedLabels, blockedLabels []string) { globalMetrics.Load().(*Metrics).UpdateFilterAndLabels(allow, block, allowedLabels, blockedLabels) } + +// Shutdown disables metric collection, then blocks while attempting to flush metrics to storage. +// WARNING: Not all MetricSink backends support this functionality, and calling this will cause them to leak resources. +// This is intended for use immediately prior to application exit. +func Shutdown() { + m := globalMetrics.Load().(*Metrics) + // Swap whatever MetricSink is currently active with a BlackholeSink. Callers must not have a + // reason to expect that calls to the library will successfully collect metrics after Shutdown + // has been called. + globalMetrics.Store(&Metrics{sink: &BlackholeSink{}}) + m.Shutdown() +} diff --git a/vendor/github.com/willf/bitset/.gitignore b/vendor/github.com/bits-and-blooms/bitset/.gitignore similarity index 100% rename from vendor/github.com/willf/bitset/.gitignore rename to vendor/github.com/bits-and-blooms/bitset/.gitignore diff --git a/vendor/github.com/willf/bitset/.travis.yml b/vendor/github.com/bits-and-blooms/bitset/.travis.yml similarity index 100% rename from vendor/github.com/willf/bitset/.travis.yml rename to vendor/github.com/bits-and-blooms/bitset/.travis.yml diff --git a/vendor/github.com/willf/bitset/LICENSE b/vendor/github.com/bits-and-blooms/bitset/LICENSE similarity index 100% rename from vendor/github.com/willf/bitset/LICENSE rename to vendor/github.com/bits-and-blooms/bitset/LICENSE diff --git a/vendor/github.com/bits-and-blooms/bitset/README.md b/vendor/github.com/bits-and-blooms/bitset/README.md new file mode 100644 index 00000000..97e83071 --- /dev/null +++ b/vendor/github.com/bits-and-blooms/bitset/README.md @@ -0,0 +1,93 @@ +# bitset + +*Go language library to map between non-negative integers and boolean values* + +[![Test](https://github.com/bits-and-blooms/bitset/workflows/Test/badge.svg)](https://github.com/willf/bitset/actions?query=workflow%3ATest) +[![Go Report Card](https://goreportcard.com/badge/github.com/willf/bitset)](https://goreportcard.com/report/github.com/willf/bitset) +[![PkgGoDev](https://pkg.go.dev/badge/github.com/bits-and-blooms/bitset?tab=doc)](https://pkg.go.dev/github.com/bits-and-blooms/bitset?tab=doc) + + +## Description + +Package bitset implements bitsets, a mapping between non-negative integers and boolean values. +It should be more efficient than map[uint] bool. + +It provides methods for setting, clearing, flipping, and testing individual integers. + +But it also provides set intersection, union, difference, complement, and symmetric operations, as well as tests to check whether any, all, or no bits are set, and querying a bitset's current length and number of positive bits. + +BitSets are expanded to the size of the largest set bit; the memory allocation is approximately Max bits, where Max is the largest set bit. BitSets are never shrunk. On creation, a hint can be given for the number of bits that will be used. + +Many of the methods, including Set, Clear, and Flip, return a BitSet pointer, which allows for chaining. + +### Example use: + +```go +package main + +import ( + "fmt" + "math/rand" + + "github.com/bits-and-blooms/bitset" +) + +func main() { + fmt.Printf("Hello from BitSet!\n") + var b bitset.BitSet + // play some Go Fish + for i := 0; i < 100; i++ { + card1 := uint(rand.Intn(52)) + card2 := uint(rand.Intn(52)) + b.Set(card1) + if b.Test(card2) { + fmt.Println("Go Fish!") + } + b.Clear(card1) + } + + // Chaining + b.Set(10).Set(11) + + for i, e := b.NextSet(0); e; i, e = b.NextSet(i + 1) { + fmt.Println("The following bit is set:", i) + } + if b.Intersection(bitset.New(100).Set(10)).Count() == 1 { + fmt.Println("Intersection works.") + } else { + fmt.Println("Intersection doesn't work???") + } +} +``` + +As an alternative to BitSets, one should check out the 'big' package, which provides a (less set-theoretical) view of bitsets. + +Package documentation is at: https://pkg.go.dev/github.com/bits-and-blooms/bitset?tab=doc + +## Memory Usage + +The memory usage of a bitset using N bits is at least N/8 bytes. The number of bits in a bitset is at least as large as one plus the greatest bit index you have accessed. Thus it is possible to run out of memory while using a bitset. If you have lots of bits, you might prefer compressed bitsets, like the [Roaring bitmaps](http://roaringbitmap.org) and its [Go implementation](https://github.com/RoaringBitmap/roaring). + +## Implementation Note + +Go 1.9 introduced a native `math/bits` library. We provide backward compatibility to Go 1.7, which might be removed. + +It is possible that a later version will match the `math/bits` return signature for counts (which is `int`, rather than our library's `unit64`). If so, the version will be bumped. + +## Installation + +```bash +go get github.com/bits-and-blooms/bitset +``` + +## Contributing + +If you wish to contribute to this project, please branch and issue a pull request against master ("[GitHub Flow](https://guides.github.com/introduction/flow/)") + +## Running all tests + +Before committing the code, please check if it passes tests, has adequate coverage, etc. +```bash +go test +go test -cover +``` diff --git a/vendor/github.com/willf/bitset/azure-pipelines.yml b/vendor/github.com/bits-and-blooms/bitset/azure-pipelines.yml similarity index 100% rename from vendor/github.com/willf/bitset/azure-pipelines.yml rename to vendor/github.com/bits-and-blooms/bitset/azure-pipelines.yml diff --git a/vendor/github.com/willf/bitset/bitset.go b/vendor/github.com/bits-and-blooms/bitset/bitset.go similarity index 97% rename from vendor/github.com/willf/bitset/bitset.go rename to vendor/github.com/bits-and-blooms/bitset/bitset.go index 21e889da..d688806a 100644 --- a/vendor/github.com/willf/bitset/bitset.go +++ b/vendor/github.com/bits-and-blooms/bitset/bitset.go @@ -209,6 +209,27 @@ func (b *BitSet) Flip(i uint) *BitSet { return b } +// FlipRange bit in [start, end). +// If end>= Cap(), this function will panic. +// Warning: using a very large value for 'end' +// may lead to a memory shortage and a panic: the caller is responsible +// for providing sensible parameters in line with their memory capacity. +func (b *BitSet) FlipRange(start, end uint) *BitSet { + if start >= end { + return b + } + + b.extendSetMaybe(end - 1) + var startWord uint = start >> log2WordSize + var endWord uint = end >> log2WordSize + b.set[startWord] ^= ^(^uint64(0) << (start & (wordSize - 1))) + for i := startWord; i < endWord; i++ { + b.set[i] = ^b.set[i] + } + b.set[endWord] ^= ^uint64(0) >> (-end & (wordSize - 1)) + return b +} + // Shrink shrinks BitSet so that the provided value is the last possible // set value. It clears all bits > the provided index and reduces the size // and length of the set. @@ -519,7 +540,7 @@ func (b *BitSet) Copy(c *BitSet) (count uint) { } // Count (number of set bits). -// Also known as "popcount" or "popularity count". +// Also known as "popcount" or "population count". func (b *BitSet) Count() uint { if b != nil && b.set != nil { return uint(popcntSlice(b.set)) diff --git a/vendor/github.com/willf/bitset/popcnt.go b/vendor/github.com/bits-and-blooms/bitset/popcnt.go similarity index 100% rename from vendor/github.com/willf/bitset/popcnt.go rename to vendor/github.com/bits-and-blooms/bitset/popcnt.go diff --git a/vendor/github.com/willf/bitset/popcnt_19.go b/vendor/github.com/bits-and-blooms/bitset/popcnt_19.go similarity index 100% rename from vendor/github.com/willf/bitset/popcnt_19.go rename to vendor/github.com/bits-and-blooms/bitset/popcnt_19.go diff --git a/vendor/github.com/willf/bitset/popcnt_amd64.go b/vendor/github.com/bits-and-blooms/bitset/popcnt_amd64.go similarity index 100% rename from vendor/github.com/willf/bitset/popcnt_amd64.go rename to vendor/github.com/bits-and-blooms/bitset/popcnt_amd64.go diff --git a/vendor/github.com/willf/bitset/popcnt_amd64.s b/vendor/github.com/bits-and-blooms/bitset/popcnt_amd64.s similarity index 100% rename from vendor/github.com/willf/bitset/popcnt_amd64.s rename to vendor/github.com/bits-and-blooms/bitset/popcnt_amd64.s diff --git a/vendor/github.com/willf/bitset/popcnt_generic.go b/vendor/github.com/bits-and-blooms/bitset/popcnt_generic.go similarity index 100% rename from vendor/github.com/willf/bitset/popcnt_generic.go rename to vendor/github.com/bits-and-blooms/bitset/popcnt_generic.go diff --git a/vendor/github.com/willf/bitset/trailing_zeros_18.go b/vendor/github.com/bits-and-blooms/bitset/trailing_zeros_18.go similarity index 100% rename from vendor/github.com/willf/bitset/trailing_zeros_18.go rename to vendor/github.com/bits-and-blooms/bitset/trailing_zeros_18.go diff --git a/vendor/github.com/willf/bitset/trailing_zeros_19.go b/vendor/github.com/bits-and-blooms/bitset/trailing_zeros_19.go similarity index 100% rename from vendor/github.com/willf/bitset/trailing_zeros_19.go rename to vendor/github.com/bits-and-blooms/bitset/trailing_zeros_19.go diff --git a/vendor/github.com/blevesearch/bleve/.gitignore b/vendor/github.com/blevesearch/bleve/.gitignore deleted file mode 100644 index ab7a1e21..00000000 --- a/vendor/github.com/blevesearch/bleve/.gitignore +++ /dev/null @@ -1,19 +0,0 @@ -#* -*.sublime-* -*~ -.#* -.project -.settings -**/.idea/ -**/*.iml -.DS_Store -query_string.y.go.tmp -/analysis/token_filters/cld2/cld2-read-only -/analysis/token_filters/cld2/libcld2_full.a -/cmd/bleve/bleve -vendor/** -!vendor/manifest -/y.output -/search/query/y.output -*.test -tags diff --git a/vendor/github.com/blevesearch/bleve/README.md b/vendor/github.com/blevesearch/bleve/README.md deleted file mode 100644 index eff0be97..00000000 --- a/vendor/github.com/blevesearch/bleve/README.md +++ /dev/null @@ -1,70 +0,0 @@ -# ![bleve](docs/bleve.png) bleve - -[![Tests](https://github.com/blevesearch/bleve/workflows/Tests/badge.svg?branch=master&event=push)](https://github.com/blevesearch/bleve/actions?query=workflow%3ATests+event%3Apush+branch%3Amaster) -[![Coverage Status](https://coveralls.io/repos/github/blevesearch/bleve/badge.svg?branch=master)](https://coveralls.io/github/blevesearch/bleve?branch=master) -[![GoDoc](https://godoc.org/github.com/blevesearch/bleve?status.svg)](https://godoc.org/github.com/blevesearch/bleve) -[![Join the chat at https://gitter.im/blevesearch/bleve](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/blevesearch/bleve?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) -[![codebeat](https://codebeat.co/badges/38a7cbc9-9cf5-41c0-a315-0746178230f4)](https://codebeat.co/projects/github-com-blevesearch-bleve) -[![Go Report Card](https://goreportcard.com/badge/blevesearch/bleve)](https://goreportcard.com/report/blevesearch/bleve) -[![Sourcegraph](https://sourcegraph.com/github.com/blevesearch/bleve/-/badge.svg)](https://sourcegraph.com/github.com/blevesearch/bleve?badge) -[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) - -modern text indexing in go - [blevesearch.com](http://www.blevesearch.com/) - -Try out bleve live by [searching the bleve website](http://www.blevesearch.com/search/?q=bleve). - -## Features - -* Index any go data structure (including JSON) -* Intelligent defaults backed up by powerful configuration -* Supported field types: - * Text, Numeric, Date -* Supported query types: - * Term, Phrase, Match, Match Phrase, Prefix - * Conjunction, Disjunction, Boolean - * Numeric Range, Date Range - * Simple query [syntax](http://www.blevesearch.com/docs/Query-String-Query/) for human entry -* tf-idf Scoring -* Search result match highlighting -* Supports Aggregating Facets: - * Terms Facet - * Numeric Range Facet - * Date Range Facet - -## Discussion - -Discuss usage and development of bleve in the [google group](https://groups.google.com/forum/#!forum/bleve). - -## Indexing - -```go -message := struct{ - Id string - From string - Body string -}{ - Id: "example", - From: "marty.schoch@gmail.com", - Body: "bleve indexing is easy", -} - -mapping := bleve.NewIndexMapping() -index, err := bleve.New("example.bleve", mapping) -if err != nil { - panic(err) -} -index.Index(message.Id, message) -``` - -## Querying - -```go -index, _ := bleve.Open("example.bleve") -query := bleve.NewQueryStringQuery("bleve") -searchRequest := bleve.NewSearchRequest(query) -searchResult, _ := index.Search(searchRequest) -``` - -## License - -Apache License Version 2.0 diff --git a/vendor/github.com/blevesearch/bleve/analysis/freq.go b/vendor/github.com/blevesearch/bleve/analysis/freq.go deleted file mode 100644 index 198c149b..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/freq.go +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package analysis - -import ( - "reflect" - - "github.com/blevesearch/bleve/size" -) - -var reflectStaticSizeTokenLocation int -var reflectStaticSizeTokenFreq int - -func init() { - var tl TokenLocation - reflectStaticSizeTokenLocation = int(reflect.TypeOf(tl).Size()) - var tf TokenFreq - reflectStaticSizeTokenFreq = int(reflect.TypeOf(tf).Size()) -} - -// TokenLocation represents one occurrence of a term at a particular location in -// a field. Start, End and Position have the same meaning as in analysis.Token. -// Field and ArrayPositions identify the field value in the source document. -// See document.Field for details. -type TokenLocation struct { - Field string - ArrayPositions []uint64 - Start int - End int - Position int -} - -func (tl *TokenLocation) Size() int { - rv := reflectStaticSizeTokenLocation - rv += len(tl.ArrayPositions) * size.SizeOfUint64 - return rv -} - -// TokenFreq represents all the occurrences of a term in all fields of a -// document. -type TokenFreq struct { - Term []byte - Locations []*TokenLocation - frequency int -} - -func (tf *TokenFreq) Size() int { - rv := reflectStaticSizeTokenFreq - rv += len(tf.Term) - for _, loc := range tf.Locations { - rv += loc.Size() - } - return rv -} - -func (tf *TokenFreq) Frequency() int { - return tf.frequency -} - -// TokenFrequencies maps document terms to their combined frequencies from all -// fields. -type TokenFrequencies map[string]*TokenFreq - -func (tfs TokenFrequencies) Size() int { - rv := size.SizeOfMap - rv += len(tfs) * (size.SizeOfString + size.SizeOfPtr) - for k, v := range tfs { - rv += len(k) - rv += v.Size() - } - return rv -} - -func (tfs TokenFrequencies) MergeAll(remoteField string, other TokenFrequencies) { - // walk the new token frequencies - for tfk, tf := range other { - // set the remoteField value in incoming token freqs - for _, l := range tf.Locations { - l.Field = remoteField - } - existingTf, exists := tfs[tfk] - if exists { - existingTf.Locations = append(existingTf.Locations, tf.Locations...) - existingTf.frequency = existingTf.frequency + tf.frequency - } else { - tfs[tfk] = &TokenFreq{ - Term: tf.Term, - frequency: tf.frequency, - Locations: make([]*TokenLocation, len(tf.Locations)), - } - copy(tfs[tfk].Locations, tf.Locations) - } - } -} - -func TokenFrequency(tokens TokenStream, arrayPositions []uint64, includeTermVectors bool) TokenFrequencies { - rv := make(map[string]*TokenFreq, len(tokens)) - - if includeTermVectors { - tls := make([]TokenLocation, len(tokens)) - tlNext := 0 - - for _, token := range tokens { - tls[tlNext] = TokenLocation{ - ArrayPositions: arrayPositions, - Start: token.Start, - End: token.End, - Position: token.Position, - } - - curr, ok := rv[string(token.Term)] - if ok { - curr.Locations = append(curr.Locations, &tls[tlNext]) - curr.frequency++ - } else { - rv[string(token.Term)] = &TokenFreq{ - Term: token.Term, - Locations: []*TokenLocation{&tls[tlNext]}, - frequency: 1, - } - } - - tlNext++ - } - } else { - for _, token := range tokens { - curr, exists := rv[string(token.Term)] - if exists { - curr.frequency++ - } else { - rv[string(token.Term)] = &TokenFreq{ - Term: token.Term, - frequency: 1, - } - } - } - } - - return rv -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/tokenizer/unicode/unicode.go b/vendor/github.com/blevesearch/bleve/analysis/tokenizer/unicode/unicode.go deleted file mode 100644 index 39e38b45..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/tokenizer/unicode/unicode.go +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package unicode - -import ( - "github.com/blevesearch/segment" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const Name = "unicode" - -type UnicodeTokenizer struct { -} - -func NewUnicodeTokenizer() *UnicodeTokenizer { - return &UnicodeTokenizer{} -} - -func (rt *UnicodeTokenizer) Tokenize(input []byte) analysis.TokenStream { - rvx := make([]analysis.TokenStream, 0, 10) // When rv gets full, append to rvx. - rv := make(analysis.TokenStream, 0, 1) - - ta := []analysis.Token(nil) - taNext := 0 - - segmenter := segment.NewWordSegmenterDirect(input) - start := 0 - pos := 1 - - guessRemaining := func(end int) int { - avgSegmentLen := end / (len(rv) + 1) - if avgSegmentLen < 1 { - avgSegmentLen = 1 - } - - remainingLen := len(input) - end - - return remainingLen / avgSegmentLen - } - - for segmenter.Segment() { - segmentBytes := segmenter.Bytes() - end := start + len(segmentBytes) - if segmenter.Type() != segment.None { - if taNext >= len(ta) { - remainingSegments := guessRemaining(end) - if remainingSegments > 1000 { - remainingSegments = 1000 - } - if remainingSegments < 1 { - remainingSegments = 1 - } - - ta = make([]analysis.Token, remainingSegments) - taNext = 0 - } - - token := &ta[taNext] - taNext++ - - token.Term = segmentBytes - token.Start = start - token.End = end - token.Position = pos - token.Type = convertType(segmenter.Type()) - - if len(rv) >= cap(rv) { // When rv is full, save it into rvx. - rvx = append(rvx, rv) - - rvCap := cap(rv) * 2 - if rvCap > 256 { - rvCap = 256 - } - - rv = make(analysis.TokenStream, 0, rvCap) // Next rv cap is bigger. - } - - rv = append(rv, token) - pos++ - } - start = end - } - - if len(rvx) > 0 { - n := len(rv) - for _, r := range rvx { - n += len(r) - } - rall := make(analysis.TokenStream, 0, n) - for _, r := range rvx { - rall = append(rall, r...) - } - return append(rall, rv...) - } - - return rv -} - -func UnicodeTokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error) { - return NewUnicodeTokenizer(), nil -} - -func init() { - registry.RegisterTokenizer(Name, UnicodeTokenizerConstructor) -} - -func convertType(segmentWordType int) analysis.TokenType { - switch segmentWordType { - case segment.Ideo: - return analysis.Ideographic - case segment.Kana: - return analysis.Ideographic - case segment.Number: - return analysis.Numeric - } - return analysis.AlphaNumeric -} diff --git a/vendor/github.com/blevesearch/bleve/builder.go b/vendor/github.com/blevesearch/bleve/builder.go deleted file mode 100644 index de00c97b..00000000 --- a/vendor/github.com/blevesearch/bleve/builder.go +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright (c) 2019 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bleve - -import ( - "encoding/json" - "fmt" - - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch" - "github.com/blevesearch/bleve/mapping" -) - -type builderImpl struct { - b index.IndexBuilder - m mapping.IndexMapping -} - -func (b *builderImpl) Index(id string, data interface{}) error { - if id == "" { - return ErrorEmptyID - } - - doc := document.NewDocument(id) - err := b.m.MapDocument(doc, data) - if err != nil { - return err - } - err = b.b.Index(doc) - return err -} - -func (b *builderImpl) Close() error { - return b.b.Close() -} - -func newBuilder(path string, mapping mapping.IndexMapping, config map[string]interface{}) (Builder, error) { - if path == "" { - return nil, fmt.Errorf("builder requires path") - } - - err := mapping.Validate() - if err != nil { - return nil, err - } - - if config == nil { - config = map[string]interface{}{} - } - - // the builder does not have an API to interact with internal storage - // however we can pass k/v pairs through the config - mappingBytes, err := json.Marshal(mapping) - if err != nil { - return nil, err - } - config["internal"] = map[string][]byte{ - string(mappingInternalKey): mappingBytes, - } - - // do not use real config, as these are options for the builder, - // not the resulting index - meta := newIndexMeta(scorch.Name, scorch.Name, map[string]interface{}{}) - err = meta.Save(path) - if err != nil { - return nil, err - } - - config["path"] = indexStorePath(path) - - b, err := scorch.NewBuilder(config) - if err != nil { - return nil, err - } - rv := &builderImpl{ - b: b, - m: mapping, - } - - return rv, nil -} diff --git a/vendor/github.com/blevesearch/bleve/config.go b/vendor/github.com/blevesearch/bleve/config.go deleted file mode 100644 index 99f2e081..00000000 --- a/vendor/github.com/blevesearch/bleve/config.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bleve - -import ( - "expvar" - "io/ioutil" - "log" - "time" - - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/store/gtreap" - "github.com/blevesearch/bleve/index/upsidedown" - "github.com/blevesearch/bleve/registry" - "github.com/blevesearch/bleve/search/highlight/highlighter/html" - - // force import of scorch so its accessible by default - _ "github.com/blevesearch/bleve/index/scorch" -) - -var bleveExpVar = expvar.NewMap("bleve") - -type configuration struct { - Cache *registry.Cache - DefaultHighlighter string - DefaultKVStore string - DefaultMemKVStore string - DefaultIndexType string - SlowSearchLogThreshold time.Duration - analysisQueue *index.AnalysisQueue -} - -func (c *configuration) SetAnalysisQueueSize(n int) { - if c.analysisQueue != nil { - c.analysisQueue.Close() - } - c.analysisQueue = index.NewAnalysisQueue(n) -} - -func (c *configuration) Shutdown() { - c.SetAnalysisQueueSize(0) -} - -func newConfiguration() *configuration { - return &configuration{ - Cache: registry.NewCache(), - analysisQueue: index.NewAnalysisQueue(4), - } -} - -// Config contains library level configuration -var Config *configuration - -func init() { - bootStart := time.Now() - - // build the default configuration - Config = newConfiguration() - - // set the default highlighter - Config.DefaultHighlighter = html.Name - - // default kv store - Config.DefaultKVStore = "" - - // default mem only kv store - Config.DefaultMemKVStore = gtreap.Name - - // default index - Config.DefaultIndexType = upsidedown.Name - - bootDuration := time.Since(bootStart) - bleveExpVar.Add("bootDuration", int64(bootDuration)) - indexStats = NewIndexStats() - bleveExpVar.Set("indexes", indexStats) - - initDisk() -} - -var logger = log.New(ioutil.Discard, "bleve", log.LstdFlags) - -// SetLog sets the logger used for logging -// by default log messages are sent to ioutil.Discard -func SetLog(l *log.Logger) { - logger = l -} diff --git a/vendor/github.com/blevesearch/bleve/doc.go b/vendor/github.com/blevesearch/bleve/doc.go deleted file mode 100644 index d54af7c9..00000000 --- a/vendor/github.com/blevesearch/bleve/doc.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* -Package bleve is a library for indexing and searching text. - -Example Opening New Index, Indexing Data - - message := struct{ - Id: "example" - From: "marty.schoch@gmail.com", - Body: "bleve indexing is easy", - } - - mapping := bleve.NewIndexMapping() - index, _ := bleve.New("example.bleve", mapping) - index.Index(message.Id, message) - -Example Opening Existing Index, Searching Data - - index, _ := bleve.Open("example.bleve") - query := bleve.NewQueryStringQuery("bleve") - searchRequest := bleve.NewSearchRequest(query) - searchResult, _ := index.Search(searchRequest) - -*/ -package bleve diff --git a/vendor/github.com/blevesearch/bleve/document/document.go b/vendor/github.com/blevesearch/bleve/document/document.go deleted file mode 100644 index 6ac17b9a..00000000 --- a/vendor/github.com/blevesearch/bleve/document/document.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package document - -import ( - "fmt" - "reflect" - - "github.com/blevesearch/bleve/size" -) - -var reflectStaticSizeDocument int - -func init() { - var d Document - reflectStaticSizeDocument = int(reflect.TypeOf(d).Size()) -} - -type Document struct { - ID string `json:"id"` - Fields []Field `json:"fields"` - CompositeFields []*CompositeField -} - -func NewDocument(id string) *Document { - return &Document{ - ID: id, - Fields: make([]Field, 0), - CompositeFields: make([]*CompositeField, 0), - } -} - -func (d *Document) Size() int { - sizeInBytes := reflectStaticSizeDocument + size.SizeOfPtr + - len(d.ID) - - for _, entry := range d.Fields { - sizeInBytes += entry.Size() - } - - for _, entry := range d.CompositeFields { - sizeInBytes += entry.Size() - } - - return sizeInBytes -} - -func (d *Document) AddField(f Field) *Document { - switch f := f.(type) { - case *CompositeField: - d.CompositeFields = append(d.CompositeFields, f) - default: - d.Fields = append(d.Fields, f) - } - return d -} - -func (d *Document) GoString() string { - fields := "" - for i, field := range d.Fields { - if i != 0 { - fields += ", " - } - fields += fmt.Sprintf("%#v", field) - } - compositeFields := "" - for i, field := range d.CompositeFields { - if i != 0 { - compositeFields += ", " - } - compositeFields += fmt.Sprintf("%#v", field) - } - return fmt.Sprintf("&document.Document{ID:%s, Fields: %s, CompositeFields: %s}", d.ID, fields, compositeFields) -} - -func (d *Document) NumPlainTextBytes() uint64 { - rv := uint64(0) - for _, field := range d.Fields { - rv += field.NumPlainTextBytes() - } - for _, compositeField := range d.CompositeFields { - for _, field := range d.Fields { - if compositeField.includesField(field.Name()) { - rv += field.NumPlainTextBytes() - } - } - } - return rv -} diff --git a/vendor/github.com/blevesearch/bleve/document/field.go b/vendor/github.com/blevesearch/bleve/document/field.go deleted file mode 100644 index 2fe91669..00000000 --- a/vendor/github.com/blevesearch/bleve/document/field.go +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package document - -import ( - "github.com/blevesearch/bleve/analysis" -) - -type Field interface { - // Name returns the path of the field from the root DocumentMapping. - // A root field path is "field", a subdocument field is "parent.field". - Name() string - // ArrayPositions returns the intermediate document and field indices - // required to resolve the field value in the document. For example, if the - // field path is "doc1.doc2.field" where doc1 and doc2 are slices or - // arrays, ArrayPositions returns 2 indices used to resolve "doc2" value in - // "doc1", then "field" in "doc2". - ArrayPositions() []uint64 - Options() IndexingOptions - Analyze() (int, analysis.TokenFrequencies) - Value() []byte - - // NumPlainTextBytes should return the number of plain text bytes - // that this field represents - this is a common metric for tracking - // the rate of indexing - NumPlainTextBytes() uint64 - - Size() int -} diff --git a/vendor/github.com/blevesearch/bleve/document/field_datetime.go b/vendor/github.com/blevesearch/bleve/document/field_datetime.go deleted file mode 100644 index 583b44cd..00000000 --- a/vendor/github.com/blevesearch/bleve/document/field_datetime.go +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package document - -import ( - "fmt" - "math" - "reflect" - "time" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/numeric" - "github.com/blevesearch/bleve/size" -) - -var reflectStaticSizeDateTimeField int - -func init() { - var f DateTimeField - reflectStaticSizeDateTimeField = int(reflect.TypeOf(f).Size()) -} - -const DefaultDateTimeIndexingOptions = StoreField | IndexField | DocValues -const DefaultDateTimePrecisionStep uint = 4 - -var MinTimeRepresentable = time.Unix(0, math.MinInt64) -var MaxTimeRepresentable = time.Unix(0, math.MaxInt64) - -type DateTimeField struct { - name string - arrayPositions []uint64 - options IndexingOptions - value numeric.PrefixCoded - numPlainTextBytes uint64 -} - -func (n *DateTimeField) Size() int { - return reflectStaticSizeDateTimeField + size.SizeOfPtr + - len(n.name) + - len(n.arrayPositions)*size.SizeOfUint64 -} - -func (n *DateTimeField) Name() string { - return n.name -} - -func (n *DateTimeField) ArrayPositions() []uint64 { - return n.arrayPositions -} - -func (n *DateTimeField) Options() IndexingOptions { - return n.options -} - -func (n *DateTimeField) Analyze() (int, analysis.TokenFrequencies) { - tokens := make(analysis.TokenStream, 0) - tokens = append(tokens, &analysis.Token{ - Start: 0, - End: len(n.value), - Term: n.value, - Position: 1, - Type: analysis.DateTime, - }) - - original, err := n.value.Int64() - if err == nil { - - shift := DefaultDateTimePrecisionStep - for shift < 64 { - shiftEncoded, err := numeric.NewPrefixCodedInt64(original, shift) - if err != nil { - break - } - token := analysis.Token{ - Start: 0, - End: len(shiftEncoded), - Term: shiftEncoded, - Position: 1, - Type: analysis.DateTime, - } - tokens = append(tokens, &token) - shift += DefaultDateTimePrecisionStep - } - } - - fieldLength := len(tokens) - tokenFreqs := analysis.TokenFrequency(tokens, n.arrayPositions, n.options.IncludeTermVectors()) - return fieldLength, tokenFreqs -} - -func (n *DateTimeField) Value() []byte { - return n.value -} - -func (n *DateTimeField) DateTime() (time.Time, error) { - i64, err := n.value.Int64() - if err != nil { - return time.Time{}, err - } - return time.Unix(0, i64).UTC(), nil -} - -func (n *DateTimeField) GoString() string { - return fmt.Sprintf("&document.DateField{Name:%s, Options: %s, Value: %s}", n.name, n.options, n.value) -} - -func (n *DateTimeField) NumPlainTextBytes() uint64 { - return n.numPlainTextBytes -} - -func NewDateTimeFieldFromBytes(name string, arrayPositions []uint64, value []byte) *DateTimeField { - return &DateTimeField{ - name: name, - arrayPositions: arrayPositions, - value: value, - options: DefaultDateTimeIndexingOptions, - numPlainTextBytes: uint64(len(value)), - } -} - -func NewDateTimeField(name string, arrayPositions []uint64, dt time.Time) (*DateTimeField, error) { - return NewDateTimeFieldWithIndexingOptions(name, arrayPositions, dt, DefaultDateTimeIndexingOptions) -} - -func NewDateTimeFieldWithIndexingOptions(name string, arrayPositions []uint64, dt time.Time, options IndexingOptions) (*DateTimeField, error) { - if canRepresent(dt) { - dtInt64 := dt.UnixNano() - prefixCoded := numeric.MustNewPrefixCodedInt64(dtInt64, 0) - return &DateTimeField{ - name: name, - arrayPositions: arrayPositions, - value: prefixCoded, - options: options, - // not correct, just a place holder until we revisit how fields are - // represented and can fix this better - numPlainTextBytes: uint64(8), - }, nil - } - return nil, fmt.Errorf("cannot represent %s in this type", dt) -} - -func canRepresent(dt time.Time) bool { - if dt.Before(MinTimeRepresentable) || dt.After(MaxTimeRepresentable) { - return false - } - return true -} diff --git a/vendor/github.com/blevesearch/bleve/document/field_geopoint.go b/vendor/github.com/blevesearch/bleve/document/field_geopoint.go deleted file mode 100644 index 91fe23f9..00000000 --- a/vendor/github.com/blevesearch/bleve/document/field_geopoint.go +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package document - -import ( - "fmt" - "reflect" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/geo" - "github.com/blevesearch/bleve/numeric" - "github.com/blevesearch/bleve/size" -) - -var reflectStaticSizeGeoPointField int - -func init() { - var f GeoPointField - reflectStaticSizeGeoPointField = int(reflect.TypeOf(f).Size()) -} - -var GeoPrecisionStep uint = 9 - -type GeoPointField struct { - name string - arrayPositions []uint64 - options IndexingOptions - value numeric.PrefixCoded - numPlainTextBytes uint64 -} - -func (n *GeoPointField) Size() int { - return reflectStaticSizeGeoPointField + size.SizeOfPtr + - len(n.name) + - len(n.arrayPositions)*size.SizeOfUint64 -} - -func (n *GeoPointField) Name() string { - return n.name -} - -func (n *GeoPointField) ArrayPositions() []uint64 { - return n.arrayPositions -} - -func (n *GeoPointField) Options() IndexingOptions { - return n.options -} - -func (n *GeoPointField) Analyze() (int, analysis.TokenFrequencies) { - tokens := make(analysis.TokenStream, 0) - tokens = append(tokens, &analysis.Token{ - Start: 0, - End: len(n.value), - Term: n.value, - Position: 1, - Type: analysis.Numeric, - }) - - original, err := n.value.Int64() - if err == nil { - - shift := GeoPrecisionStep - for shift < 64 { - shiftEncoded, err := numeric.NewPrefixCodedInt64(original, shift) - if err != nil { - break - } - token := analysis.Token{ - Start: 0, - End: len(shiftEncoded), - Term: shiftEncoded, - Position: 1, - Type: analysis.Numeric, - } - tokens = append(tokens, &token) - shift += GeoPrecisionStep - } - } - - fieldLength := len(tokens) - tokenFreqs := analysis.TokenFrequency(tokens, n.arrayPositions, n.options.IncludeTermVectors()) - return fieldLength, tokenFreqs -} - -func (n *GeoPointField) Value() []byte { - return n.value -} - -func (n *GeoPointField) Lon() (float64, error) { - i64, err := n.value.Int64() - if err != nil { - return 0.0, err - } - return geo.MortonUnhashLon(uint64(i64)), nil -} - -func (n *GeoPointField) Lat() (float64, error) { - i64, err := n.value.Int64() - if err != nil { - return 0.0, err - } - return geo.MortonUnhashLat(uint64(i64)), nil -} - -func (n *GeoPointField) GoString() string { - return fmt.Sprintf("&document.GeoPointField{Name:%s, Options: %s, Value: %s}", n.name, n.options, n.value) -} - -func (n *GeoPointField) NumPlainTextBytes() uint64 { - return n.numPlainTextBytes -} - -func NewGeoPointFieldFromBytes(name string, arrayPositions []uint64, value []byte) *GeoPointField { - return &GeoPointField{ - name: name, - arrayPositions: arrayPositions, - value: value, - options: DefaultNumericIndexingOptions, - numPlainTextBytes: uint64(len(value)), - } -} - -func NewGeoPointField(name string, arrayPositions []uint64, lon, lat float64) *GeoPointField { - return NewGeoPointFieldWithIndexingOptions(name, arrayPositions, lon, lat, DefaultNumericIndexingOptions) -} - -func NewGeoPointFieldWithIndexingOptions(name string, arrayPositions []uint64, lon, lat float64, options IndexingOptions) *GeoPointField { - mhash := geo.MortonHash(lon, lat) - prefixCoded := numeric.MustNewPrefixCodedInt64(int64(mhash), 0) - return &GeoPointField{ - name: name, - arrayPositions: arrayPositions, - value: prefixCoded, - options: options, - // not correct, just a place holder until we revisit how fields are - // represented and can fix this better - numPlainTextBytes: uint64(8), - } -} diff --git a/vendor/github.com/blevesearch/bleve/document/field_text.go b/vendor/github.com/blevesearch/bleve/document/field_text.go deleted file mode 100644 index 6bd74c71..00000000 --- a/vendor/github.com/blevesearch/bleve/document/field_text.go +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package document - -import ( - "fmt" - "reflect" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/size" -) - -var reflectStaticSizeTextField int - -func init() { - var f TextField - reflectStaticSizeTextField = int(reflect.TypeOf(f).Size()) -} - -const DefaultTextIndexingOptions = IndexField | DocValues - -type TextField struct { - name string - arrayPositions []uint64 - options IndexingOptions - analyzer *analysis.Analyzer - value []byte - numPlainTextBytes uint64 -} - -func (t *TextField) Size() int { - return reflectStaticSizeTextField + size.SizeOfPtr + - len(t.name) + - len(t.arrayPositions)*size.SizeOfUint64 + - len(t.value) -} - -func (t *TextField) Name() string { - return t.name -} - -func (t *TextField) ArrayPositions() []uint64 { - return t.arrayPositions -} - -func (t *TextField) Options() IndexingOptions { - return t.options -} - -func (t *TextField) Analyze() (int, analysis.TokenFrequencies) { - var tokens analysis.TokenStream - if t.analyzer != nil { - bytesToAnalyze := t.Value() - if t.options.IsStored() { - // need to copy - bytesCopied := make([]byte, len(bytesToAnalyze)) - copy(bytesCopied, bytesToAnalyze) - bytesToAnalyze = bytesCopied - } - tokens = t.analyzer.Analyze(bytesToAnalyze) - } else { - tokens = analysis.TokenStream{ - &analysis.Token{ - Start: 0, - End: len(t.value), - Term: t.value, - Position: 1, - Type: analysis.AlphaNumeric, - }, - } - } - fieldLength := len(tokens) // number of tokens in this doc field - tokenFreqs := analysis.TokenFrequency(tokens, t.arrayPositions, t.options.IncludeTermVectors()) - return fieldLength, tokenFreqs -} - -func (t *TextField) Analyzer() *analysis.Analyzer { - return t.analyzer -} - -func (t *TextField) Value() []byte { - return t.value -} - -func (t *TextField) GoString() string { - return fmt.Sprintf("&document.TextField{Name:%s, Options: %s, Analyzer: %v, Value: %s, ArrayPositions: %v}", t.name, t.options, t.analyzer, t.value, t.arrayPositions) -} - -func (t *TextField) NumPlainTextBytes() uint64 { - return t.numPlainTextBytes -} - -func NewTextField(name string, arrayPositions []uint64, value []byte) *TextField { - return NewTextFieldWithIndexingOptions(name, arrayPositions, value, DefaultTextIndexingOptions) -} - -func NewTextFieldWithIndexingOptions(name string, arrayPositions []uint64, value []byte, options IndexingOptions) *TextField { - return &TextField{ - name: name, - arrayPositions: arrayPositions, - options: options, - value: value, - numPlainTextBytes: uint64(len(value)), - } -} - -func NewTextFieldWithAnalyzer(name string, arrayPositions []uint64, value []byte, analyzer *analysis.Analyzer) *TextField { - return &TextField{ - name: name, - arrayPositions: arrayPositions, - options: DefaultTextIndexingOptions, - analyzer: analyzer, - value: value, - numPlainTextBytes: uint64(len(value)), - } -} - -func NewTextFieldCustom(name string, arrayPositions []uint64, value []byte, options IndexingOptions, analyzer *analysis.Analyzer) *TextField { - return &TextField{ - name: name, - arrayPositions: arrayPositions, - options: options, - analyzer: analyzer, - value: value, - numPlainTextBytes: uint64(len(value)), - } -} diff --git a/vendor/github.com/blevesearch/bleve/document/indexing_options.go b/vendor/github.com/blevesearch/bleve/document/indexing_options.go deleted file mode 100644 index 44498a8e..00000000 --- a/vendor/github.com/blevesearch/bleve/document/indexing_options.go +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package document - -type IndexingOptions int - -const ( - IndexField IndexingOptions = 1 << iota - StoreField - IncludeTermVectors - DocValues -) - -func (o IndexingOptions) IsIndexed() bool { - return o&IndexField != 0 -} - -func (o IndexingOptions) IsStored() bool { - return o&StoreField != 0 -} - -func (o IndexingOptions) IncludeTermVectors() bool { - return o&IncludeTermVectors != 0 -} - -func (o IndexingOptions) IncludeDocValues() bool { - return o&DocValues != 0 -} - -func (o IndexingOptions) String() string { - rv := "" - if o.IsIndexed() { - rv += "INDEXED" - } - if o.IsStored() { - if rv != "" { - rv += ", " - } - rv += "STORE" - } - if o.IncludeTermVectors() { - if rv != "" { - rv += ", " - } - rv += "TV" - } - if o.IncludeDocValues() { - if rv != "" { - rv += ", " - } - rv += "DV" - } - return rv -} diff --git a/vendor/github.com/blevesearch/bleve/error.go b/vendor/github.com/blevesearch/bleve/error.go deleted file mode 100644 index 7402dfcf..00000000 --- a/vendor/github.com/blevesearch/bleve/error.go +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bleve - -// Constant Error values which can be compared to determine the type of error -const ( - ErrorIndexPathExists Error = iota - ErrorIndexPathDoesNotExist - ErrorIndexMetaMissing - ErrorIndexMetaCorrupt - ErrorUnknownStorageType - ErrorIndexClosed - ErrorAliasMulti - ErrorAliasEmpty - ErrorUnknownIndexType - ErrorEmptyID - ErrorIndexReadInconsistency -) - -// Error represents a more strongly typed bleve error for detecting -// and handling specific types of errors. -type Error int - -func (e Error) Error() string { - return errorMessages[e] -} - -var errorMessages = map[Error]string{ - ErrorIndexPathExists: "cannot create new index, path already exists", - ErrorIndexPathDoesNotExist: "cannot open index, path does not exist", - ErrorIndexMetaMissing: "cannot open index, metadata missing", - ErrorIndexMetaCorrupt: "cannot open index, metadata corrupt", - ErrorUnknownStorageType: "unknown storage type", - ErrorIndexClosed: "index is closed", - ErrorAliasMulti: "cannot perform single index operation on multiple index alias", - ErrorAliasEmpty: "cannot perform operation on empty alias", - ErrorUnknownIndexType: "unknown index type", - ErrorEmptyID: "document ID cannot be empty", - ErrorIndexReadInconsistency: "index read inconsistency detected", -} diff --git a/vendor/github.com/blevesearch/bleve/geo/README.md b/vendor/github.com/blevesearch/bleve/geo/README.md deleted file mode 100644 index 43bcd98f..00000000 --- a/vendor/github.com/blevesearch/bleve/geo/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# geo support in bleve - -First, all of this geo code is a Go adaptation of the [Lucene 5.3.2 sandbox geo support](https://lucene.apache.org/core/5_3_2/sandbox/org/apache/lucene/util/package-summary.html). - -## Notes - -- All of the APIs will use float64 for lon/lat values. -- When describing a point in function arguments or return values, we always use the order lon, lat. -- High level APIs will use TopLeft and BottomRight to describe bounding boxes. This may not map cleanly to min/max lon/lat when crossing the dateline. The lower level APIs will use min/max lon/lat and require the higher-level code to split boxes accordingly. diff --git a/vendor/github.com/blevesearch/bleve/geo/parse.go b/vendor/github.com/blevesearch/bleve/geo/parse.go deleted file mode 100644 index 8286805f..00000000 --- a/vendor/github.com/blevesearch/bleve/geo/parse.go +++ /dev/null @@ -1,181 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package geo - -import ( - "reflect" - "strconv" - "strings" -) - -// ExtractGeoPoint takes an arbitrary interface{} and tries it's best to -// interpret it is as geo point. Supported formats: -// Container: -// slice length 2 (GeoJSON) -// first element lon, second element lat -// string (coordinates separated by comma, or a geohash) -// first element lat, second element lon -// map[string]interface{} -// exact keys lat and lon or lng -// struct -// w/exported fields case-insensitive match on lat and lon or lng -// struct -// satisfying Later and Loner or Lnger interfaces -// -// in all cases values must be some sort of numeric-like thing: int/uint/float -func ExtractGeoPoint(thing interface{}) (lon, lat float64, success bool) { - var foundLon, foundLat bool - - thingVal := reflect.ValueOf(thing) - if !thingVal.IsValid() { - return lon, lat, false - } - - thingTyp := thingVal.Type() - - // is it a slice - if thingVal.Kind() == reflect.Slice { - // must be length 2 - if thingVal.Len() == 2 { - first := thingVal.Index(0) - if first.CanInterface() { - firstVal := first.Interface() - lon, foundLon = extractNumericVal(firstVal) - } - second := thingVal.Index(1) - if second.CanInterface() { - secondVal := second.Interface() - lat, foundLat = extractNumericVal(secondVal) - } - } - } - - // is it a string - if thingVal.Kind() == reflect.String { - geoStr := thingVal.Interface().(string) - if strings.Contains(geoStr, ",") { - // geo point with coordinates split by comma - points := strings.Split(geoStr, ",") - for i, point := range points { - // trim any leading or trailing white spaces - points[i] = strings.TrimSpace(point) - } - if len(points) == 2 { - var err error - lat, err = strconv.ParseFloat(points[0], 64) - if err == nil { - foundLat = true - } - lon, err = strconv.ParseFloat(points[1], 64) - if err == nil { - foundLon = true - } - } - } else { - // geohash - if len(geoStr) <= geoHashMaxLength { - lat, lon = DecodeGeoHash(geoStr) - foundLat = true - foundLon = true - } - } - } - - // is it a map - if l, ok := thing.(map[string]interface{}); ok { - if lval, ok := l["lon"]; ok { - lon, foundLon = extractNumericVal(lval) - } else if lval, ok := l["lng"]; ok { - lon, foundLon = extractNumericVal(lval) - } - if lval, ok := l["lat"]; ok { - lat, foundLat = extractNumericVal(lval) - } - } - - // now try reflection on struct fields - if thingVal.Kind() == reflect.Struct { - for i := 0; i < thingVal.NumField(); i++ { - fieldName := thingTyp.Field(i).Name - if strings.HasPrefix(strings.ToLower(fieldName), "lon") { - if thingVal.Field(i).CanInterface() { - fieldVal := thingVal.Field(i).Interface() - lon, foundLon = extractNumericVal(fieldVal) - } - } - if strings.HasPrefix(strings.ToLower(fieldName), "lng") { - if thingVal.Field(i).CanInterface() { - fieldVal := thingVal.Field(i).Interface() - lon, foundLon = extractNumericVal(fieldVal) - } - } - if strings.HasPrefix(strings.ToLower(fieldName), "lat") { - if thingVal.Field(i).CanInterface() { - fieldVal := thingVal.Field(i).Interface() - lat, foundLat = extractNumericVal(fieldVal) - } - } - } - } - - // last hope, some interfaces - // lon - if l, ok := thing.(loner); ok { - lon = l.Lon() - foundLon = true - } else if l, ok := thing.(lnger); ok { - lon = l.Lng() - foundLon = true - } - // lat - if l, ok := thing.(later); ok { - lat = l.Lat() - foundLat = true - } - - return lon, lat, foundLon && foundLat -} - -// extract numeric value (if possible) and returns a float64 -func extractNumericVal(v interface{}) (float64, bool) { - val := reflect.ValueOf(v) - if !val.IsValid() { - return 0, false - } - typ := val.Type() - switch typ.Kind() { - case reflect.Float32, reflect.Float64: - return val.Float(), true - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - return float64(val.Int()), true - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: - return float64(val.Uint()), true - } - - return 0, false -} - -// various support interfaces which can be used to find lat/lon -type loner interface { - Lon() float64 -} - -type later interface { - Lat() float64 -} - -type lnger interface { - Lng() float64 -} diff --git a/vendor/github.com/blevesearch/bleve/index.go b/vendor/github.com/blevesearch/bleve/index.go deleted file mode 100644 index 974358b8..00000000 --- a/vendor/github.com/blevesearch/bleve/index.go +++ /dev/null @@ -1,309 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bleve - -import ( - "context" - - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/store" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/size" -) - -// A Batch groups together multiple Index and Delete -// operations you would like performed at the same -// time. The Batch structure is NOT thread-safe. -// You should only perform operations on a batch -// from a single thread at a time. Once batch -// execution has started, you may not modify it. -type Batch struct { - index Index - internal *index.Batch - - lastDocSize uint64 - totalSize uint64 -} - -// Index adds the specified index operation to the -// batch. NOTE: the bleve Index is not updated -// until the batch is executed. -func (b *Batch) Index(id string, data interface{}) error { - if id == "" { - return ErrorEmptyID - } - doc := document.NewDocument(id) - err := b.index.Mapping().MapDocument(doc, data) - if err != nil { - return err - } - b.internal.Update(doc) - - b.lastDocSize = uint64(doc.Size() + - len(id) + size.SizeOfString) // overhead from internal - b.totalSize += b.lastDocSize - - return nil -} - -func (b *Batch) LastDocSize() uint64 { - return b.lastDocSize -} - -func (b *Batch) TotalDocsSize() uint64 { - return b.totalSize -} - -// IndexAdvanced adds the specified index operation to the -// batch which skips the mapping. NOTE: the bleve Index is not updated -// until the batch is executed. -func (b *Batch) IndexAdvanced(doc *document.Document) (err error) { - if doc.ID == "" { - return ErrorEmptyID - } - b.internal.Update(doc) - return nil -} - -// Delete adds the specified delete operation to the -// batch. NOTE: the bleve Index is not updated until -// the batch is executed. -func (b *Batch) Delete(id string) { - if id != "" { - b.internal.Delete(id) - } -} - -// SetInternal adds the specified set internal -// operation to the batch. NOTE: the bleve Index is -// not updated until the batch is executed. -func (b *Batch) SetInternal(key, val []byte) { - b.internal.SetInternal(key, val) -} - -// DeleteInternal adds the specified delete internal -// operation to the batch. NOTE: the bleve Index is -// not updated until the batch is executed. -func (b *Batch) DeleteInternal(key []byte) { - b.internal.DeleteInternal(key) -} - -// Size returns the total number of operations inside the batch -// including normal index operations and internal operations. -func (b *Batch) Size() int { - return len(b.internal.IndexOps) + len(b.internal.InternalOps) -} - -// String prints a user friendly string representation of what -// is inside this batch. -func (b *Batch) String() string { - return b.internal.String() -} - -// Reset returns a Batch to the empty state so that it can -// be re-used in the future. -func (b *Batch) Reset() { - b.internal.Reset() - b.lastDocSize = 0 - b.totalSize = 0 -} - -func (b *Batch) Merge(o *Batch) { - if o != nil && o.internal != nil { - b.internal.Merge(o.internal) - if o.LastDocSize() > 0 { - b.lastDocSize = o.LastDocSize() - } - b.totalSize = uint64(b.internal.TotalDocSize()) - } -} - -func (b *Batch) SetPersistedCallback(f index.BatchCallback) { - b.internal.SetPersistedCallback(f) -} - -func (b *Batch) PersistedCallback() index.BatchCallback { - return b.internal.PersistedCallback() -} - -// An Index implements all the indexing and searching -// capabilities of bleve. An Index can be created -// using the New() and Open() methods. -// -// Index() takes an input value, deduces a DocumentMapping for its type, -// assigns string paths to its fields or values then applies field mappings on -// them. -// -// The DocumentMapping used to index a value is deduced by the following rules: -// 1) If value implements mapping.bleveClassifier interface, resolve the mapping -// from BleveType(). -// 2) If value implements mapping.Classifier interface, resolve the mapping -// from Type(). -// 3) If value has a string field or value at IndexMapping.TypeField. -// (defaulting to "_type"), use it to resolve the mapping. Fields addressing -// is described below. -// 4) If IndexMapping.DefaultType is registered, return it. -// 5) Return IndexMapping.DefaultMapping. -// -// Each field or nested field of the value is identified by a string path, then -// mapped to one or several FieldMappings which extract the result for analysis. -// -// Struct values fields are identified by their "json:" tag, or by their name. -// Nested fields are identified by prefixing with their parent identifier, -// separated by a dot. -// -// Map values entries are identified by their string key. Entries not indexed -// by strings are ignored. Entry values are identified recursively like struct -// fields. -// -// Slice and array values are identified by their field name. Their elements -// are processed sequentially with the same FieldMapping. -// -// String, float64 and time.Time values are identified by their field name. -// Other types are ignored. -// -// Each value identifier is decomposed in its parts and recursively address -// SubDocumentMappings in the tree starting at the root DocumentMapping. If a -// mapping is found, all its FieldMappings are applied to the value. If no -// mapping is found and the root DocumentMapping is dynamic, default mappings -// are used based on value type and IndexMapping default configurations. -// -// Finally, mapped values are analyzed, indexed or stored. See -// FieldMapping.Analyzer to know how an analyzer is resolved for a given field. -// -// Examples: -// -// type Date struct { -// Day string `json:"day"` -// Month string -// Year string -// } -// -// type Person struct { -// FirstName string `json:"first_name"` -// LastName string -// BirthDate Date `json:"birth_date"` -// } -// -// A Person value FirstName is mapped by the SubDocumentMapping at -// "first_name". Its LastName is mapped by the one at "LastName". The day of -// BirthDate is mapped to the SubDocumentMapping "day" of the root -// SubDocumentMapping "birth_date". It will appear as the "birth_date.day" -// field in the index. The month is mapped to "birth_date.Month". -type Index interface { - // Index analyzes, indexes or stores mapped data fields. Supplied - // identifier is bound to analyzed data and will be retrieved by search - // requests. See Index interface documentation for details about mapping - // rules. - Index(id string, data interface{}) error - Delete(id string) error - - NewBatch() *Batch - Batch(b *Batch) error - - // Document returns specified document or nil if the document is not - // indexed or stored. - Document(id string) (*document.Document, error) - // DocCount returns the number of documents in the index. - DocCount() (uint64, error) - - Search(req *SearchRequest) (*SearchResult, error) - SearchInContext(ctx context.Context, req *SearchRequest) (*SearchResult, error) - - Fields() ([]string, error) - - FieldDict(field string) (index.FieldDict, error) - FieldDictRange(field string, startTerm []byte, endTerm []byte) (index.FieldDict, error) - FieldDictPrefix(field string, termPrefix []byte) (index.FieldDict, error) - - Close() error - - Mapping() mapping.IndexMapping - - Stats() *IndexStat - StatsMap() map[string]interface{} - - GetInternal(key []byte) ([]byte, error) - SetInternal(key, val []byte) error - DeleteInternal(key []byte) error - - // Name returns the name of the index (by default this is the path) - Name() string - // SetName lets you assign your own logical name to this index - SetName(string) - - // Advanced returns the indexer and data store, exposing lower level - // methods to enumerate records and access data. - Advanced() (index.Index, store.KVStore, error) -} - -// New index at the specified path, must not exist. -// The provided mapping will be used for all -// Index/Search operations. -func New(path string, mapping mapping.IndexMapping) (Index, error) { - return newIndexUsing(path, mapping, Config.DefaultIndexType, Config.DefaultKVStore, nil) -} - -// NewMemOnly creates a memory-only index. -// The contents of the index is NOT persisted, -// and will be lost once closed. -// The provided mapping will be used for all -// Index/Search operations. -func NewMemOnly(mapping mapping.IndexMapping) (Index, error) { - return newIndexUsing("", mapping, Config.DefaultIndexType, Config.DefaultMemKVStore, nil) -} - -// NewUsing creates index at the specified path, -// which must not already exist. -// The provided mapping will be used for all -// Index/Search operations. -// The specified index type will be used. -// The specified kvstore implementation will be used -// and the provided kvconfig will be passed to its -// constructor. Note that currently the values of kvconfig must -// be able to be marshaled and unmarshaled using the encoding/json library (used -// when reading/writing the index metadata file). -func NewUsing(path string, mapping mapping.IndexMapping, indexType string, kvstore string, kvconfig map[string]interface{}) (Index, error) { - return newIndexUsing(path, mapping, indexType, kvstore, kvconfig) -} - -// Open index at the specified path, must exist. -// The mapping used when it was created will be used for all Index/Search operations. -func Open(path string) (Index, error) { - return openIndexUsing(path, nil) -} - -// OpenUsing opens index at the specified path, must exist. -// The mapping used when it was created will be used for all Index/Search operations. -// The provided runtimeConfig can override settings -// persisted when the kvstore was created. -func OpenUsing(path string, runtimeConfig map[string]interface{}) (Index, error) { - return openIndexUsing(path, runtimeConfig) -} - -// Builder is a limited interface, used to build indexes in an offline mode. -// Items cannot be updated or deleted, and the caller MUST ensure a document is -// indexed only once. -type Builder interface { - Index(id string, data interface{}) error - Close() error -} - -// NewBuilder creates a builder, which will build an index at the specified path, -// using the specified mapping and options. -func NewBuilder(path string, mapping mapping.IndexMapping, config map[string]interface{}) (Builder, error) { - return newBuilder(path, mapping, config) -} diff --git a/vendor/github.com/blevesearch/bleve/index/analysis.go b/vendor/github.com/blevesearch/bleve/index/analysis.go deleted file mode 100644 index 82883af0..00000000 --- a/vendor/github.com/blevesearch/bleve/index/analysis.go +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright (c) 2015 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package index - -import ( - "reflect" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/size" -) - -var reflectStaticSizeAnalysisResult int - -func init() { - var ar AnalysisResult - reflectStaticSizeAnalysisResult = int(reflect.TypeOf(ar).Size()) -} - -type IndexRow interface { - KeySize() int - KeyTo([]byte) (int, error) - Key() []byte - - ValueSize() int - ValueTo([]byte) (int, error) - Value() []byte -} - -type AnalysisResult struct { - DocID string - Rows []IndexRow - - // scorch - Document *document.Document - Analyzed []analysis.TokenFrequencies - Length []int -} - -func (a *AnalysisResult) Size() int { - rv := reflectStaticSizeAnalysisResult - for _, analyzedI := range a.Analyzed { - rv += analyzedI.Size() - } - rv += len(a.Length) * size.SizeOfInt - return rv -} - -type AnalysisWork struct { - i Index - d *document.Document - rc chan *AnalysisResult -} - -func NewAnalysisWork(i Index, d *document.Document, rc chan *AnalysisResult) *AnalysisWork { - return &AnalysisWork{ - i: i, - d: d, - rc: rc, - } -} - -type AnalysisQueue struct { - queue chan *AnalysisWork - done chan struct{} -} - -func (q *AnalysisQueue) Queue(work *AnalysisWork) { - q.queue <- work -} - -func (q *AnalysisQueue) Close() { - close(q.done) -} - -func NewAnalysisQueue(numWorkers int) *AnalysisQueue { - rv := AnalysisQueue{ - queue: make(chan *AnalysisWork), - done: make(chan struct{}), - } - for i := 0; i < numWorkers; i++ { - go AnalysisWorker(rv) - } - return &rv -} - -func AnalysisWorker(q AnalysisQueue) { - // read work off the queue - for { - select { - case <-q.done: - return - case w := <-q.queue: - r := w.i.Analyze(w.d) - w.rc <- r - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/index/index.go b/vendor/github.com/blevesearch/bleve/index/index.go deleted file mode 100644 index 551f8de8..00000000 --- a/vendor/github.com/blevesearch/bleve/index/index.go +++ /dev/null @@ -1,376 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package index - -import ( - "bytes" - "encoding/json" - "fmt" - "reflect" - - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/index/store" - "github.com/blevesearch/bleve/size" -) - -var reflectStaticSizeTermFieldDoc int -var reflectStaticSizeTermFieldVector int - -func init() { - var tfd TermFieldDoc - reflectStaticSizeTermFieldDoc = int(reflect.TypeOf(tfd).Size()) - var tfv TermFieldVector - reflectStaticSizeTermFieldVector = int(reflect.TypeOf(tfv).Size()) -} - -var ErrorUnknownStorageType = fmt.Errorf("unknown storage type") - -type Index interface { - Open() error - Close() error - - Update(doc *document.Document) error - Delete(id string) error - Batch(batch *Batch) error - - SetInternal(key, val []byte) error - DeleteInternal(key []byte) error - - // Reader returns a low-level accessor on the index data. Close it to - // release associated resources. - Reader() (IndexReader, error) - - Stats() json.Marshaler - StatsMap() map[string]interface{} - - Analyze(d *document.Document) *AnalysisResult - - Advanced() (store.KVStore, error) -} - -type DocumentFieldTermVisitor func(field string, term []byte) - -type IndexReader interface { - TermFieldReader(term []byte, field string, includeFreq, includeNorm, includeTermVectors bool) (TermFieldReader, error) - - // DocIDReader returns an iterator over all doc ids - // The caller must close returned instance to release associated resources. - DocIDReaderAll() (DocIDReader, error) - - DocIDReaderOnly(ids []string) (DocIDReader, error) - - FieldDict(field string) (FieldDict, error) - - // FieldDictRange is currently defined to include the start and end terms - FieldDictRange(field string, startTerm []byte, endTerm []byte) (FieldDict, error) - FieldDictPrefix(field string, termPrefix []byte) (FieldDict, error) - - Document(id string) (*document.Document, error) - DocumentVisitFieldTerms(id IndexInternalID, fields []string, visitor DocumentFieldTermVisitor) error - - DocValueReader(fields []string) (DocValueReader, error) - - Fields() ([]string, error) - - GetInternal(key []byte) ([]byte, error) - - DocCount() (uint64, error) - - ExternalID(id IndexInternalID) (string, error) - InternalID(id string) (IndexInternalID, error) - - DumpAll() chan interface{} - DumpDoc(id string) chan interface{} - DumpFields() chan interface{} - - Close() error -} - -// The Regexp interface defines the subset of the regexp.Regexp API -// methods that are used by bleve indexes, allowing callers to pass in -// alternate implementations. -type Regexp interface { - FindStringIndex(s string) (loc []int) - - LiteralPrefix() (prefix string, complete bool) - - String() string -} - -type IndexReaderRegexp interface { - FieldDictRegexp(field string, regex string) (FieldDict, error) -} - -type IndexReaderFuzzy interface { - FieldDictFuzzy(field string, term string, fuzziness int, prefix string) (FieldDict, error) -} - -type IndexReaderOnly interface { - FieldDictOnly(field string, onlyTerms [][]byte, includeCount bool) (FieldDict, error) -} - -type IndexReaderContains interface { - FieldDictContains(field string) (FieldDictContains, error) -} - -// FieldTerms contains the terms used by a document, keyed by field -type FieldTerms map[string][]string - -// FieldsNotYetCached returns a list of fields not yet cached out of a larger list of fields -func (f FieldTerms) FieldsNotYetCached(fields []string) []string { - rv := make([]string, 0, len(fields)) - for _, field := range fields { - if _, ok := f[field]; !ok { - rv = append(rv, field) - } - } - return rv -} - -// Merge will combine two FieldTerms -// it assumes that the terms lists are complete (thus do not need to be merged) -// field terms from the other list always replace the ones in the receiver -func (f FieldTerms) Merge(other FieldTerms) { - for field, terms := range other { - f[field] = terms - } -} - -type TermFieldVector struct { - Field string - ArrayPositions []uint64 - Pos uint64 - Start uint64 - End uint64 -} - -func (tfv *TermFieldVector) Size() int { - return reflectStaticSizeTermFieldVector + size.SizeOfPtr + - len(tfv.Field) + len(tfv.ArrayPositions)*size.SizeOfUint64 -} - -// IndexInternalID is an opaque document identifier interal to the index impl -type IndexInternalID []byte - -func (id IndexInternalID) Equals(other IndexInternalID) bool { - return id.Compare(other) == 0 -} - -func (id IndexInternalID) Compare(other IndexInternalID) int { - return bytes.Compare(id, other) -} - -type TermFieldDoc struct { - Term string - ID IndexInternalID - Freq uint64 - Norm float64 - Vectors []*TermFieldVector -} - -func (tfd *TermFieldDoc) Size() int { - sizeInBytes := reflectStaticSizeTermFieldDoc + size.SizeOfPtr + - len(tfd.Term) + len(tfd.ID) - - for _, entry := range tfd.Vectors { - sizeInBytes += entry.Size() - } - - return sizeInBytes -} - -// Reset allows an already allocated TermFieldDoc to be reused -func (tfd *TermFieldDoc) Reset() *TermFieldDoc { - // remember the []byte used for the ID - id := tfd.ID - vectors := tfd.Vectors - // idiom to copy over from empty TermFieldDoc (0 allocations) - *tfd = TermFieldDoc{} - // reuse the []byte already allocated (and reset len to 0) - tfd.ID = id[:0] - tfd.Vectors = vectors[:0] - return tfd -} - -// TermFieldReader is the interface exposing the enumeration of documents -// containing a given term in a given field. Documents are returned in byte -// lexicographic order over their identifiers. -type TermFieldReader interface { - // Next returns the next document containing the term in this field, or nil - // when it reaches the end of the enumeration. The preAlloced TermFieldDoc - // is optional, and when non-nil, will be used instead of allocating memory. - Next(preAlloced *TermFieldDoc) (*TermFieldDoc, error) - - // Advance resets the enumeration at specified document or its immediate - // follower. - Advance(ID IndexInternalID, preAlloced *TermFieldDoc) (*TermFieldDoc, error) - - // Count returns the number of documents contains the term in this field. - Count() uint64 - Close() error - - Size() int -} - -type DictEntry struct { - Term string - Count uint64 -} - -type FieldDict interface { - Next() (*DictEntry, error) - Close() error -} - -type FieldDictContains interface { - Contains(key []byte) (bool, error) -} - -// DocIDReader is the interface exposing enumeration of documents identifiers. -// Close the reader to release associated resources. -type DocIDReader interface { - // Next returns the next document internal identifier in the natural - // index order, nil when the end of the sequence is reached. - Next() (IndexInternalID, error) - - // Advance resets the iteration to the first internal identifier greater than - // or equal to ID. If ID is smaller than the start of the range, the iteration - // will start there instead. If ID is greater than or equal to the end of - // the range, Next() call will return io.EOF. - Advance(ID IndexInternalID) (IndexInternalID, error) - - Size() int - - Close() error -} - -type BatchCallback func(error) - -type Batch struct { - IndexOps map[string]*document.Document - InternalOps map[string][]byte - persistedCallback BatchCallback -} - -func NewBatch() *Batch { - return &Batch{ - IndexOps: make(map[string]*document.Document), - InternalOps: make(map[string][]byte), - } -} - -func (b *Batch) Update(doc *document.Document) { - b.IndexOps[doc.ID] = doc -} - -func (b *Batch) Delete(id string) { - b.IndexOps[id] = nil -} - -func (b *Batch) SetInternal(key, val []byte) { - b.InternalOps[string(key)] = val -} - -func (b *Batch) DeleteInternal(key []byte) { - b.InternalOps[string(key)] = nil -} - -func (b *Batch) SetPersistedCallback(f BatchCallback) { - b.persistedCallback = f -} - -func (b *Batch) PersistedCallback() BatchCallback { - return b.persistedCallback -} - -func (b *Batch) String() string { - rv := fmt.Sprintf("Batch (%d ops, %d internal ops)\n", len(b.IndexOps), len(b.InternalOps)) - for k, v := range b.IndexOps { - if v != nil { - rv += fmt.Sprintf("\tINDEX - '%s'\n", k) - } else { - rv += fmt.Sprintf("\tDELETE - '%s'\n", k) - } - } - for k, v := range b.InternalOps { - if v != nil { - rv += fmt.Sprintf("\tSET INTERNAL - '%s'\n", k) - } else { - rv += fmt.Sprintf("\tDELETE INTERNAL - '%s'\n", k) - } - } - return rv -} - -func (b *Batch) Reset() { - b.IndexOps = make(map[string]*document.Document) - b.InternalOps = make(map[string][]byte) - b.persistedCallback = nil -} - -func (b *Batch) Merge(o *Batch) { - for k, v := range o.IndexOps { - b.IndexOps[k] = v - } - for k, v := range o.InternalOps { - b.InternalOps[k] = v - } -} - -func (b *Batch) TotalDocSize() int { - var s int - for k, v := range b.IndexOps { - if v != nil { - s += v.Size() + size.SizeOfString - } - s += len(k) - } - return s -} - -// Optimizable represents an optional interface that implementable by -// optimizable resources (e.g., TermFieldReaders, Searchers). These -// optimizable resources are provided the same OptimizableContext -// instance, so that they can coordinate via dynamic interface -// casting. -type Optimizable interface { - Optimize(kind string, octx OptimizableContext) (OptimizableContext, error) -} - -// Represents a result of optimization -- see the Finish() method. -type Optimized interface{} - -type OptimizableContext interface { - // Once all the optimzable resources have been provided the same - // OptimizableContext instance, the optimization preparations are - // finished or completed via the Finish() method. - // - // Depending on the optimization being performed, the Finish() - // method might return a non-nil Optimized instance. For example, - // the Optimized instance might represent an optimized - // TermFieldReader instance. - Finish() (Optimized, error) -} - -type DocValueReader interface { - VisitDocValues(id IndexInternalID, visitor DocumentFieldTermVisitor) error -} - -// IndexBuilder is an interface supported by some index schemes -// to allow direct write-only index building -type IndexBuilder interface { - Index(doc *document.Document) error - Close() error -} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/builder.go b/vendor/github.com/blevesearch/bleve/index/scorch/builder.go deleted file mode 100644 index 1f4b41d6..00000000 --- a/vendor/github.com/blevesearch/bleve/index/scorch/builder.go +++ /dev/null @@ -1,334 +0,0 @@ -// Copyright (c) 2019 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package scorch - -import ( - "fmt" - "io/ioutil" - "os" - "sync" - - "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" - bolt "go.etcd.io/bbolt" -) - -const DefaultBuilderBatchSize = 1000 -const DefaultBuilderMergeMax = 10 - -type Builder struct { - m sync.Mutex - segCount uint64 - path string - buildPath string - segPaths []string - batchSize int - mergeMax int - batch *index.Batch - internal map[string][]byte - segPlugin segment.Plugin -} - -func NewBuilder(config map[string]interface{}) (*Builder, error) { - path, ok := config["path"].(string) - if !ok { - return nil, fmt.Errorf("must specify path") - } - - buildPathPrefix, _ := config["buildPathPrefix"].(string) - buildPath, err := ioutil.TempDir(buildPathPrefix, "scorch-offline-build") - if err != nil { - return nil, err - } - - rv := &Builder{ - path: path, - buildPath: buildPath, - mergeMax: DefaultBuilderMergeMax, - batchSize: DefaultBuilderBatchSize, - batch: index.NewBatch(), - segPlugin: defaultSegmentPlugin, - } - - err = rv.parseConfig(config) - if err != nil { - return nil, fmt.Errorf("error parsing builder config: %v", err) - } - - return rv, nil -} - -func (o *Builder) parseConfig(config map[string]interface{}) (err error) { - if v, ok := config["mergeMax"]; ok { - var t int - if t, err = parseToInteger(v); err != nil { - return fmt.Errorf("mergeMax parse err: %v", err) - } - if t > 0 { - o.mergeMax = t - } - } - - if v, ok := config["batchSize"]; ok { - var t int - if t, err = parseToInteger(v); err != nil { - return fmt.Errorf("batchSize parse err: %v", err) - } - if t > 0 { - o.batchSize = t - } - } - - if v, ok := config["internal"]; ok { - if vinternal, ok := v.(map[string][]byte); ok { - o.internal = vinternal - } - } - - forcedSegmentType, forcedSegmentVersion, err := configForceSegmentTypeVersion(config) - if err != nil { - return err - } - if forcedSegmentType != "" && forcedSegmentVersion != 0 { - segPlugin, err := chooseSegmentPlugin(forcedSegmentType, - uint32(forcedSegmentVersion)) - if err != nil { - return err - } - o.segPlugin = segPlugin - } - - return nil -} - -// Index will place the document into the index. -// It is invalid to index the same document multiple times. -func (o *Builder) Index(doc *document.Document) error { - o.m.Lock() - defer o.m.Unlock() - - o.batch.Update(doc) - - return o.maybeFlushBatchLOCKED(o.batchSize) -} - -func (o *Builder) maybeFlushBatchLOCKED(moreThan int) error { - if len(o.batch.IndexOps) >= moreThan { - defer o.batch.Reset() - return o.executeBatchLOCKED(o.batch) - } - return nil -} - -func (o *Builder) executeBatchLOCKED(batch *index.Batch) (err error) { - analysisResults := make([]*index.AnalysisResult, 0, len(batch.IndexOps)) - for _, doc := range batch.IndexOps { - if doc != nil { - // insert _id field - doc.AddField(document.NewTextFieldCustom("_id", nil, []byte(doc.ID), document.IndexField|document.StoreField, nil)) - // perform analysis directly - analysisResult := analyze(doc) - analysisResults = append(analysisResults, analysisResult) - } - } - - seg, _, err := o.segPlugin.New(analysisResults) - if err != nil { - return fmt.Errorf("error building segment base: %v", err) - } - - filename := zapFileName(o.segCount) - o.segCount++ - path := o.buildPath + string(os.PathSeparator) + filename - - if segUnpersisted, ok := seg.(segment.UnpersistedSegment); ok { - err = segUnpersisted.Persist(path) - if err != nil { - return fmt.Errorf("error persisting segment base to %s: %v", path, err) - } - - o.segPaths = append(o.segPaths, path) - return nil - } - - return fmt.Errorf("new segment does not implement unpersisted: %T", seg) -} - -func (o *Builder) doMerge() error { - // as long as we have more than 1 segment, keep merging - for len(o.segPaths) > 1 { - - // merge the next number of segments into one new one - // or, if there are fewer than remaining, merge them all - mergeCount := o.mergeMax - if mergeCount > len(o.segPaths) { - mergeCount = len(o.segPaths) - } - - mergePaths := o.segPaths[0:mergeCount] - o.segPaths = o.segPaths[mergeCount:] - - // open each of the segments to be merged - mergeSegs := make([]segment.Segment, 0, mergeCount) - - // closeOpenedSegs attempts to close all opened - // segments even if an error occurs, in which case - // the first error is returned - closeOpenedSegs := func() error { - var err error - for _, seg := range mergeSegs { - clErr := seg.Close() - if clErr != nil && err == nil { - err = clErr - } - } - return err - } - - for _, mergePath := range mergePaths { - seg, err := o.segPlugin.Open(mergePath) - if err != nil { - _ = closeOpenedSegs() - return fmt.Errorf("error opening segment (%s) for merge: %v", mergePath, err) - } - mergeSegs = append(mergeSegs, seg) - } - - // do the merge - mergedSegPath := o.buildPath + string(os.PathSeparator) + zapFileName(o.segCount) - drops := make([]*roaring.Bitmap, mergeCount) - _, _, err := o.segPlugin.Merge(mergeSegs, drops, mergedSegPath, nil, nil) - if err != nil { - _ = closeOpenedSegs() - return fmt.Errorf("error merging segments (%v): %v", mergePaths, err) - } - o.segCount++ - o.segPaths = append(o.segPaths, mergedSegPath) - - // close segments opened for merge - err = closeOpenedSegs() - if err != nil { - return fmt.Errorf("error closing opened segments: %v", err) - } - - // remove merged segments - for _, mergePath := range mergePaths { - err = os.RemoveAll(mergePath) - if err != nil { - return fmt.Errorf("error removing segment %s after merge: %v", mergePath, err) - } - } - } - - return nil -} - -func (o *Builder) Close() error { - o.m.Lock() - defer o.m.Unlock() - - // see if there is a partial batch - err := o.maybeFlushBatchLOCKED(1) - if err != nil { - return fmt.Errorf("error flushing batch before close: %v", err) - } - - // perform all the merging - err = o.doMerge() - if err != nil { - return fmt.Errorf("error while merging: %v", err) - } - - // ensure the store path exists - err = os.MkdirAll(o.path, 0700) - if err != nil { - return err - } - - // move final segment into place - // segment id 2 is chosen to match the behavior of a scorch - // index which indexes a single batch of data - finalSegPath := o.path + string(os.PathSeparator) + zapFileName(2) - err = os.Rename(o.segPaths[0], finalSegPath) - if err != nil { - return fmt.Errorf("error moving final segment into place: %v", err) - } - - // remove the buildPath, as it is no longer needed - err = os.RemoveAll(o.buildPath) - if err != nil { - return fmt.Errorf("error removing build path: %v", err) - } - - // prepare wrapping - seg, err := o.segPlugin.Open(finalSegPath) - if err != nil { - return fmt.Errorf("error opening final segment") - } - - // create a segment snapshot for this segment - ss := &SegmentSnapshot{ - segment: seg, - } - is := &IndexSnapshot{ - epoch: 3, // chosen to match scorch behavior when indexing a single batch - segment: []*SegmentSnapshot{ss}, - creator: "scorch-builder", - internal: o.internal, - } - - // create the root bolt - rootBoltPath := o.path + string(os.PathSeparator) + "root.bolt" - rootBolt, err := bolt.Open(rootBoltPath, 0600, nil) - if err != nil { - return err - } - - // start a write transaction - tx, err := rootBolt.Begin(true) - if err != nil { - return err - } - - // fill the root bolt with this fake index snapshot - _, _, err = prepareBoltSnapshot(is, tx, o.path, o.segPlugin) - if err != nil { - _ = tx.Rollback() - _ = rootBolt.Close() - return fmt.Errorf("error preparing bolt snapshot in root.bolt: %v", err) - } - - // commit bolt data - err = tx.Commit() - if err != nil { - _ = rootBolt.Close() - return fmt.Errorf("error committing bolt tx in root.bolt: %v", err) - } - - // close bolt - err = rootBolt.Close() - if err != nil { - return fmt.Errorf("error closing root.bolt: %v", err) - } - - // close final segment - err = seg.Close() - if err != nil { - return fmt.Errorf("error closing final segment: %v", err) - } - return nil -} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/merge.go b/vendor/github.com/blevesearch/bleve/index/scorch/merge.go deleted file mode 100644 index 56c0953f..00000000 --- a/vendor/github.com/blevesearch/bleve/index/scorch/merge.go +++ /dev/null @@ -1,504 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package scorch - -import ( - "context" - "encoding/json" - "fmt" - "os" - "strings" - "sync/atomic" - "time" - - "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/index/scorch/mergeplan" - "github.com/blevesearch/bleve/index/scorch/segment" -) - -func (s *Scorch) mergerLoop() { - var lastEpochMergePlanned uint64 - var ctrlMsg *mergerCtrl - mergePlannerOptions, err := s.parseMergePlannerOptions() - if err != nil { - s.fireAsyncError(fmt.Errorf("mergePlannerOption json parsing err: %v", err)) - s.asyncTasks.Done() - return - } - ctrlMsgDflt := &mergerCtrl{ctx: context.Background(), - options: mergePlannerOptions, - doneCh: nil} - -OUTER: - for { - atomic.AddUint64(&s.stats.TotFileMergeLoopBeg, 1) - - select { - case <-s.closeCh: - break OUTER - - default: - // check to see if there is a new snapshot to persist - s.rootLock.Lock() - ourSnapshot := s.root - ourSnapshot.AddRef() - atomic.StoreUint64(&s.iStats.mergeSnapshotSize, uint64(ourSnapshot.Size())) - atomic.StoreUint64(&s.iStats.mergeEpoch, ourSnapshot.epoch) - s.rootLock.Unlock() - - if ctrlMsg == nil && ourSnapshot.epoch != lastEpochMergePlanned { - ctrlMsg = ctrlMsgDflt - } - if ctrlMsg != nil { - startTime := time.Now() - - // lets get started - err := s.planMergeAtSnapshot(ctrlMsg.ctx, ctrlMsg.options, - ourSnapshot) - if err != nil { - atomic.StoreUint64(&s.iStats.mergeEpoch, 0) - if err == segment.ErrClosed { - // index has been closed - _ = ourSnapshot.DecRef() - - // continue the workloop on a user triggered cancel - if ctrlMsg.doneCh != nil { - close(ctrlMsg.doneCh) - ctrlMsg = nil - continue OUTER - } - - // exit the workloop on index closure - ctrlMsg = nil - break OUTER - } - s.fireAsyncError(fmt.Errorf("merging err: %v", err)) - _ = ourSnapshot.DecRef() - atomic.AddUint64(&s.stats.TotFileMergeLoopErr, 1) - continue OUTER - } - - if ctrlMsg.doneCh != nil { - close(ctrlMsg.doneCh) - } - ctrlMsg = nil - - lastEpochMergePlanned = ourSnapshot.epoch - - atomic.StoreUint64(&s.stats.LastMergedEpoch, ourSnapshot.epoch) - - s.fireEvent(EventKindMergerProgress, time.Since(startTime)) - } - _ = ourSnapshot.DecRef() - - // tell the persister we're waiting for changes - // first make a epochWatcher chan - ew := &epochWatcher{ - epoch: lastEpochMergePlanned, - notifyCh: make(notificationChan, 1), - } - - // give it to the persister - select { - case <-s.closeCh: - break OUTER - case s.persisterNotifier <- ew: - case ctrlMsg = <-s.forceMergeRequestCh: - continue OUTER - } - - // now wait for persister (but also detect close) - select { - case <-s.closeCh: - break OUTER - case <-ew.notifyCh: - case ctrlMsg = <-s.forceMergeRequestCh: - } - } - - atomic.AddUint64(&s.stats.TotFileMergeLoopEnd, 1) - } - - s.asyncTasks.Done() -} - -type mergerCtrl struct { - ctx context.Context - options *mergeplan.MergePlanOptions - doneCh chan struct{} -} - -// ForceMerge helps users trigger a merge operation on -// an online scorch index. -func (s *Scorch) ForceMerge(ctx context.Context, - mo *mergeplan.MergePlanOptions) error { - // check whether force merge is already under processing - s.rootLock.Lock() - if s.stats.TotFileMergeForceOpsStarted > - s.stats.TotFileMergeForceOpsCompleted { - s.rootLock.Unlock() - return fmt.Errorf("force merge already in progress") - } - - s.stats.TotFileMergeForceOpsStarted++ - s.rootLock.Unlock() - - if mo != nil { - err := mergeplan.ValidateMergePlannerOptions(mo) - if err != nil { - return err - } - } else { - // assume the default single segment merge policy - mo = &mergeplan.SingleSegmentMergePlanOptions - } - msg := &mergerCtrl{options: mo, - doneCh: make(chan struct{}), - ctx: ctx, - } - - // request the merger perform a force merge - select { - case s.forceMergeRequestCh <- msg: - case <-s.closeCh: - return nil - } - - // wait for the force merge operation completion - select { - case <-msg.doneCh: - atomic.AddUint64(&s.stats.TotFileMergeForceOpsCompleted, 1) - case <-s.closeCh: - } - - return nil -} - -func (s *Scorch) parseMergePlannerOptions() (*mergeplan.MergePlanOptions, - error) { - mergePlannerOptions := mergeplan.DefaultMergePlanOptions - if v, ok := s.config["scorchMergePlanOptions"]; ok { - b, err := json.Marshal(v) - if err != nil { - return &mergePlannerOptions, err - } - - err = json.Unmarshal(b, &mergePlannerOptions) - if err != nil { - return &mergePlannerOptions, err - } - - err = mergeplan.ValidateMergePlannerOptions(&mergePlannerOptions) - if err != nil { - return nil, err - } - } - return &mergePlannerOptions, nil -} - -type closeChWrapper struct { - ch1 chan struct{} - ctx context.Context - closeCh chan struct{} -} - -func newCloseChWrapper(ch1 chan struct{}, - ctx context.Context) *closeChWrapper { - return &closeChWrapper{ch1: ch1, - ctx: ctx, - closeCh: make(chan struct{})} -} - -func (w *closeChWrapper) close() { - select { - case <-w.closeCh: - default: - close(w.closeCh) - } -} - -func (w *closeChWrapper) listen() { - select { - case <-w.ch1: - w.close() - case <-w.ctx.Done(): - w.close() - case <-w.closeCh: - } -} - -func (s *Scorch) planMergeAtSnapshot(ctx context.Context, - options *mergeplan.MergePlanOptions, ourSnapshot *IndexSnapshot) error { - // build list of persisted segments in this snapshot - var onlyPersistedSnapshots []mergeplan.Segment - for _, segmentSnapshot := range ourSnapshot.segment { - if _, ok := segmentSnapshot.segment.(segment.PersistedSegment); ok { - onlyPersistedSnapshots = append(onlyPersistedSnapshots, segmentSnapshot) - } - } - - atomic.AddUint64(&s.stats.TotFileMergePlan, 1) - - // give this list to the planner - resultMergePlan, err := mergeplan.Plan(onlyPersistedSnapshots, options) - if err != nil { - atomic.AddUint64(&s.stats.TotFileMergePlanErr, 1) - return fmt.Errorf("merge planning err: %v", err) - } - if resultMergePlan == nil { - // nothing to do - atomic.AddUint64(&s.stats.TotFileMergePlanNone, 1) - return nil - } - atomic.AddUint64(&s.stats.TotFileMergePlanOk, 1) - - atomic.AddUint64(&s.stats.TotFileMergePlanTasks, uint64(len(resultMergePlan.Tasks))) - - // process tasks in serial for now - var filenames []string - - cw := newCloseChWrapper(s.closeCh, ctx) - defer cw.close() - - go cw.listen() - - for _, task := range resultMergePlan.Tasks { - if len(task.Segments) == 0 { - atomic.AddUint64(&s.stats.TotFileMergePlanTasksSegmentsEmpty, 1) - continue - } - - atomic.AddUint64(&s.stats.TotFileMergePlanTasksSegments, uint64(len(task.Segments))) - - oldMap := make(map[uint64]*SegmentSnapshot) - newSegmentID := atomic.AddUint64(&s.nextSegmentID, 1) - segmentsToMerge := make([]segment.Segment, 0, len(task.Segments)) - docsToDrop := make([]*roaring.Bitmap, 0, len(task.Segments)) - - for _, planSegment := range task.Segments { - if segSnapshot, ok := planSegment.(*SegmentSnapshot); ok { - oldMap[segSnapshot.id] = segSnapshot - if persistedSeg, ok := segSnapshot.segment.(segment.PersistedSegment); ok { - if segSnapshot.LiveSize() == 0 { - atomic.AddUint64(&s.stats.TotFileMergeSegmentsEmpty, 1) - oldMap[segSnapshot.id] = nil - } else { - segmentsToMerge = append(segmentsToMerge, segSnapshot.segment) - docsToDrop = append(docsToDrop, segSnapshot.deleted) - } - // track the files getting merged for unsetting the - // removal ineligibility. This helps to unflip files - // even with fast merger, slow persister work flows. - path := persistedSeg.Path() - filenames = append(filenames, - strings.TrimPrefix(path, s.path+string(os.PathSeparator))) - } - } - } - - var oldNewDocNums map[uint64][]uint64 - var seg segment.Segment - var filename string - if len(segmentsToMerge) > 0 { - filename = zapFileName(newSegmentID) - s.markIneligibleForRemoval(filename) - path := s.path + string(os.PathSeparator) + filename - - fileMergeZapStartTime := time.Now() - - atomic.AddUint64(&s.stats.TotFileMergeZapBeg, 1) - newDocNums, _, err := s.segPlugin.Merge(segmentsToMerge, docsToDrop, path, - cw.closeCh, s) - atomic.AddUint64(&s.stats.TotFileMergeZapEnd, 1) - - fileMergeZapTime := uint64(time.Since(fileMergeZapStartTime)) - atomic.AddUint64(&s.stats.TotFileMergeZapTime, fileMergeZapTime) - if atomic.LoadUint64(&s.stats.MaxFileMergeZapTime) < fileMergeZapTime { - atomic.StoreUint64(&s.stats.MaxFileMergeZapTime, fileMergeZapTime) - } - - if err != nil { - s.unmarkIneligibleForRemoval(filename) - atomic.AddUint64(&s.stats.TotFileMergePlanTasksErr, 1) - if err == segment.ErrClosed { - return err - } - return fmt.Errorf("merging failed: %v", err) - } - - seg, err = s.segPlugin.Open(path) - if err != nil { - s.unmarkIneligibleForRemoval(filename) - atomic.AddUint64(&s.stats.TotFileMergePlanTasksErr, 1) - return err - } - oldNewDocNums = make(map[uint64][]uint64) - for i, segNewDocNums := range newDocNums { - oldNewDocNums[task.Segments[i].Id()] = segNewDocNums - } - - atomic.AddUint64(&s.stats.TotFileMergeSegments, uint64(len(segmentsToMerge))) - } - - sm := &segmentMerge{ - id: newSegmentID, - old: oldMap, - oldNewDocNums: oldNewDocNums, - new: seg, - notifyCh: make(chan *mergeTaskIntroStatus), - } - - s.fireEvent(EventKindMergeTaskIntroductionStart, 0) - - // give it to the introducer - select { - case <-s.closeCh: - _ = seg.Close() - return segment.ErrClosed - case s.merges <- sm: - atomic.AddUint64(&s.stats.TotFileMergeIntroductions, 1) - } - - introStartTime := time.Now() - // it is safe to blockingly wait for the merge introduction - // here as the introducer is bound to handle the notify channel. - introStatus := <-sm.notifyCh - introTime := uint64(time.Since(introStartTime)) - atomic.AddUint64(&s.stats.TotFileMergeZapIntroductionTime, introTime) - if atomic.LoadUint64(&s.stats.MaxFileMergeZapIntroductionTime) < introTime { - atomic.StoreUint64(&s.stats.MaxFileMergeZapIntroductionTime, introTime) - } - atomic.AddUint64(&s.stats.TotFileMergeIntroductionsDone, 1) - if introStatus != nil && introStatus.indexSnapshot != nil { - _ = introStatus.indexSnapshot.DecRef() - if introStatus.skipped { - // close the segment on skipping introduction. - s.unmarkIneligibleForRemoval(filename) - _ = seg.Close() - } - } - - atomic.AddUint64(&s.stats.TotFileMergePlanTasksDone, 1) - - s.fireEvent(EventKindMergeTaskIntroduction, 0) - } - - // once all the newly merged segment introductions are done, - // its safe to unflip the removal ineligibility for the replaced - // older segments - for _, f := range filenames { - s.unmarkIneligibleForRemoval(f) - } - - return nil -} - -type mergeTaskIntroStatus struct { - indexSnapshot *IndexSnapshot - skipped bool -} - -type segmentMerge struct { - id uint64 - old map[uint64]*SegmentSnapshot - oldNewDocNums map[uint64][]uint64 - new segment.Segment - notifyCh chan *mergeTaskIntroStatus -} - -// perform a merging of the given SegmentBase instances into a new, -// persisted segment, and synchronously introduce that new segment -// into the root -func (s *Scorch) mergeSegmentBases(snapshot *IndexSnapshot, - sbs []segment.Segment, sbsDrops []*roaring.Bitmap, - sbsIndexes []int) (*IndexSnapshot, uint64, error) { - atomic.AddUint64(&s.stats.TotMemMergeBeg, 1) - - memMergeZapStartTime := time.Now() - - atomic.AddUint64(&s.stats.TotMemMergeZapBeg, 1) - - newSegmentID := atomic.AddUint64(&s.nextSegmentID, 1) - filename := zapFileName(newSegmentID) - path := s.path + string(os.PathSeparator) + filename - - newDocNums, _, err := - s.segPlugin.Merge(sbs, sbsDrops, path, s.closeCh, s) - - atomic.AddUint64(&s.stats.TotMemMergeZapEnd, 1) - - memMergeZapTime := uint64(time.Since(memMergeZapStartTime)) - atomic.AddUint64(&s.stats.TotMemMergeZapTime, memMergeZapTime) - if atomic.LoadUint64(&s.stats.MaxMemMergeZapTime) < memMergeZapTime { - atomic.StoreUint64(&s.stats.MaxMemMergeZapTime, memMergeZapTime) - } - - if err != nil { - atomic.AddUint64(&s.stats.TotMemMergeErr, 1) - return nil, 0, err - } - - seg, err := s.segPlugin.Open(path) - if err != nil { - atomic.AddUint64(&s.stats.TotMemMergeErr, 1) - return nil, 0, err - } - - // update persisted stats - atomic.AddUint64(&s.stats.TotPersistedItems, seg.Count()) - atomic.AddUint64(&s.stats.TotPersistedSegments, 1) - - sm := &segmentMerge{ - id: newSegmentID, - old: make(map[uint64]*SegmentSnapshot), - oldNewDocNums: make(map[uint64][]uint64), - new: seg, - notifyCh: make(chan *mergeTaskIntroStatus), - } - - for i, idx := range sbsIndexes { - ss := snapshot.segment[idx] - sm.old[ss.id] = ss - sm.oldNewDocNums[ss.id] = newDocNums[i] - } - - select { // send to introducer - case <-s.closeCh: - _ = seg.DecRef() - return nil, 0, segment.ErrClosed - case s.merges <- sm: - } - - // blockingly wait for the introduction to complete - var newSnapshot *IndexSnapshot - introStatus := <-sm.notifyCh - if introStatus != nil && introStatus.indexSnapshot != nil { - newSnapshot = introStatus.indexSnapshot - atomic.AddUint64(&s.stats.TotMemMergeSegments, uint64(len(sbs))) - atomic.AddUint64(&s.stats.TotMemMergeDone, 1) - if introStatus.skipped { - // close the segment on skipping introduction. - _ = newSnapshot.DecRef() - _ = seg.Close() - newSnapshot = nil - } - } - - return newSnapshot, newSegmentID, nil -} - -func (s *Scorch) ReportBytesWritten(bytesWritten uint64) { - atomic.AddUint64(&s.stats.TotFileMergeWrittenBytes, bytesWritten) -} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/optimize.go b/vendor/github.com/blevesearch/bleve/index/scorch/optimize.go deleted file mode 100644 index 658354cd..00000000 --- a/vendor/github.com/blevesearch/bleve/index/scorch/optimize.go +++ /dev/null @@ -1,396 +0,0 @@ -// Copyright (c) 2018 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package scorch - -import ( - "fmt" - "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" - "sync/atomic" -) - -var OptimizeConjunction = true -var OptimizeConjunctionUnadorned = true -var OptimizeDisjunctionUnadorned = true - -func (s *IndexSnapshotTermFieldReader) Optimize(kind string, - octx index.OptimizableContext) (index.OptimizableContext, error) { - if OptimizeConjunction && kind == "conjunction" { - return s.optimizeConjunction(octx) - } - - if OptimizeConjunctionUnadorned && kind == "conjunction:unadorned" { - return s.optimizeConjunctionUnadorned(octx) - } - - if OptimizeDisjunctionUnadorned && kind == "disjunction:unadorned" { - return s.optimizeDisjunctionUnadorned(octx) - } - - return nil, nil -} - -var OptimizeDisjunctionUnadornedMinChildCardinality = uint64(256) - -// ---------------------------------------------------------------- - -func (s *IndexSnapshotTermFieldReader) optimizeConjunction( - octx index.OptimizableContext) (index.OptimizableContext, error) { - if octx == nil { - octx = &OptimizeTFRConjunction{snapshot: s.snapshot} - } - - o, ok := octx.(*OptimizeTFRConjunction) - if !ok { - return octx, nil - } - - if o.snapshot != s.snapshot { - return nil, fmt.Errorf("tried to optimize conjunction across different snapshots") - } - - o.tfrs = append(o.tfrs, s) - - return o, nil -} - -type OptimizeTFRConjunction struct { - snapshot *IndexSnapshot - - tfrs []*IndexSnapshotTermFieldReader -} - -func (o *OptimizeTFRConjunction) Finish() (index.Optimized, error) { - if len(o.tfrs) <= 1 { - return nil, nil - } - - for i := range o.snapshot.segment { - itr0, ok := o.tfrs[0].iterators[i].(segment.OptimizablePostingsIterator) - if !ok || itr0.ActualBitmap() == nil { - continue - } - - itr1, ok := o.tfrs[1].iterators[i].(segment.OptimizablePostingsIterator) - if !ok || itr1.ActualBitmap() == nil { - continue - } - - bm := roaring.And(itr0.ActualBitmap(), itr1.ActualBitmap()) - - for _, tfr := range o.tfrs[2:] { - itr, ok := tfr.iterators[i].(segment.OptimizablePostingsIterator) - if !ok || itr.ActualBitmap() == nil { - continue - } - - bm.And(itr.ActualBitmap()) - } - - // in this conjunction optimization, the postings iterators - // will all share the same AND'ed together actual bitmap. The - // regular conjunction searcher machinery will still be used, - // but the underlying bitmap will be smaller. - for _, tfr := range o.tfrs { - itr, ok := tfr.iterators[i].(segment.OptimizablePostingsIterator) - if ok && itr.ActualBitmap() != nil { - itr.ReplaceActual(bm) - } - } - } - - return nil, nil -} - -// ---------------------------------------------------------------- - -// An "unadorned" conjunction optimization is appropriate when -// additional or subsidiary information like freq-norm's and -// term-vectors are not required, and instead only the internal-id's -// are needed. -func (s *IndexSnapshotTermFieldReader) optimizeConjunctionUnadorned( - octx index.OptimizableContext) (index.OptimizableContext, error) { - if octx == nil { - octx = &OptimizeTFRConjunctionUnadorned{snapshot: s.snapshot} - } - - o, ok := octx.(*OptimizeTFRConjunctionUnadorned) - if !ok { - return nil, nil - } - - if o.snapshot != s.snapshot { - return nil, fmt.Errorf("tried to optimize unadorned conjunction across different snapshots") - } - - o.tfrs = append(o.tfrs, s) - - return o, nil -} - -type OptimizeTFRConjunctionUnadorned struct { - snapshot *IndexSnapshot - - tfrs []*IndexSnapshotTermFieldReader -} - -var OptimizeTFRConjunctionUnadornedTerm = []byte("") -var OptimizeTFRConjunctionUnadornedField = "*" - -// Finish of an unadorned conjunction optimization will compute a -// termFieldReader with an "actual" bitmap that represents the -// constituent bitmaps AND'ed together. This termFieldReader cannot -// provide any freq-norm or termVector associated information. -func (o *OptimizeTFRConjunctionUnadorned) Finish() (rv index.Optimized, err error) { - if len(o.tfrs) <= 1 { - return nil, nil - } - - // We use an artificial term and field because the optimized - // termFieldReader can represent multiple terms and fields. - oTFR := o.snapshot.unadornedTermFieldReader( - OptimizeTFRConjunctionUnadornedTerm, OptimizeTFRConjunctionUnadornedField) - - var actualBMs []*roaring.Bitmap // Collected from regular posting lists. - -OUTER: - for i := range o.snapshot.segment { - actualBMs = actualBMs[:0] - - var docNum1HitLast uint64 - var docNum1HitLastOk bool - - for _, tfr := range o.tfrs { - if _, ok := tfr.iterators[i].(*segment.EmptyPostingsIterator); ok { - // An empty postings iterator means the entire AND is empty. - oTFR.iterators[i] = segment.AnEmptyPostingsIterator - continue OUTER - } - - itr, ok := tfr.iterators[i].(segment.OptimizablePostingsIterator) - if !ok { - // We only optimize postings iterators that support this operation. - return nil, nil - } - - // If the postings iterator is "1-hit" optimized, then we - // can perform several optimizations up-front here. - docNum1Hit, ok := itr.DocNum1Hit() - if ok { - if docNum1HitLastOk && docNum1HitLast != docNum1Hit { - // The docNum1Hit doesn't match the previous - // docNum1HitLast, so the entire AND is empty. - oTFR.iterators[i] = segment.AnEmptyPostingsIterator - continue OUTER - } - - docNum1HitLast = docNum1Hit - docNum1HitLastOk = true - - continue - } - - if itr.ActualBitmap() == nil { - // An empty actual bitmap means the entire AND is empty. - oTFR.iterators[i] = segment.AnEmptyPostingsIterator - continue OUTER - } - - // Collect the actual bitmap for more processing later. - actualBMs = append(actualBMs, itr.ActualBitmap()) - } - - if docNum1HitLastOk { - // We reach here if all the 1-hit optimized posting - // iterators had the same 1-hit docNum, so we can check if - // our collected actual bitmaps also have that docNum. - for _, bm := range actualBMs { - if !bm.Contains(uint32(docNum1HitLast)) { - // The docNum1Hit isn't in one of our actual - // bitmaps, so the entire AND is empty. - oTFR.iterators[i] = segment.AnEmptyPostingsIterator - continue OUTER - } - } - - // The actual bitmaps and docNum1Hits all contain or have - // the same 1-hit docNum, so that's our AND'ed result. - oTFR.iterators[i] = segment.NewUnadornedPostingsIteratorFrom1Hit(docNum1HitLast) - - continue OUTER - } - - if len(actualBMs) == 0 { - // If we've collected no actual bitmaps at this point, - // then the entire AND is empty. - oTFR.iterators[i] = segment.AnEmptyPostingsIterator - continue OUTER - } - - if len(actualBMs) == 1 { - // If we've only 1 actual bitmap, then that's our result. - oTFR.iterators[i] = segment.NewUnadornedPostingsIteratorFromBitmap(actualBMs[0]) - - continue OUTER - } - - // Else, AND together our collected bitmaps as our result. - bm := roaring.And(actualBMs[0], actualBMs[1]) - - for _, actualBM := range actualBMs[2:] { - bm.And(actualBM) - } - - oTFR.iterators[i] = segment.NewUnadornedPostingsIteratorFromBitmap(bm) - } - - atomic.AddUint64(&o.snapshot.parent.stats.TotTermSearchersStarted, uint64(1)) - return oTFR, nil -} - -// ---------------------------------------------------------------- - -// An "unadorned" disjunction optimization is appropriate when -// additional or subsidiary information like freq-norm's and -// term-vectors are not required, and instead only the internal-id's -// are needed. -func (s *IndexSnapshotTermFieldReader) optimizeDisjunctionUnadorned( - octx index.OptimizableContext) (index.OptimizableContext, error) { - if octx == nil { - octx = &OptimizeTFRDisjunctionUnadorned{ - snapshot: s.snapshot, - } - } - - o, ok := octx.(*OptimizeTFRDisjunctionUnadorned) - if !ok { - return nil, nil - } - - if o.snapshot != s.snapshot { - return nil, fmt.Errorf("tried to optimize unadorned disjunction across different snapshots") - } - - o.tfrs = append(o.tfrs, s) - - return o, nil -} - -type OptimizeTFRDisjunctionUnadorned struct { - snapshot *IndexSnapshot - - tfrs []*IndexSnapshotTermFieldReader -} - -var OptimizeTFRDisjunctionUnadornedTerm = []byte("") -var OptimizeTFRDisjunctionUnadornedField = "*" - -// Finish of an unadorned disjunction optimization will compute a -// termFieldReader with an "actual" bitmap that represents the -// constituent bitmaps OR'ed together. This termFieldReader cannot -// provide any freq-norm or termVector associated information. -func (o *OptimizeTFRDisjunctionUnadorned) Finish() (rv index.Optimized, err error) { - if len(o.tfrs) <= 1 { - return nil, nil - } - - for i := range o.snapshot.segment { - var cMax uint64 - - for _, tfr := range o.tfrs { - itr, ok := tfr.iterators[i].(segment.OptimizablePostingsIterator) - if !ok { - return nil, nil - } - - if itr.ActualBitmap() != nil { - c := itr.ActualBitmap().GetCardinality() - if cMax < c { - cMax = c - } - } - } - } - - // We use an artificial term and field because the optimized - // termFieldReader can represent multiple terms and fields. - oTFR := o.snapshot.unadornedTermFieldReader( - OptimizeTFRDisjunctionUnadornedTerm, OptimizeTFRDisjunctionUnadornedField) - - var docNums []uint32 // Collected docNum's from 1-hit posting lists. - var actualBMs []*roaring.Bitmap // Collected from regular posting lists. - - for i := range o.snapshot.segment { - docNums = docNums[:0] - actualBMs = actualBMs[:0] - - for _, tfr := range o.tfrs { - itr, ok := tfr.iterators[i].(segment.OptimizablePostingsIterator) - if !ok { - return nil, nil - } - - docNum, ok := itr.DocNum1Hit() - if ok { - docNums = append(docNums, uint32(docNum)) - continue - } - - if itr.ActualBitmap() != nil { - actualBMs = append(actualBMs, itr.ActualBitmap()) - } - } - - var bm *roaring.Bitmap - if len(actualBMs) > 2 { - bm = roaring.HeapOr(actualBMs...) - } else if len(actualBMs) == 2 { - bm = roaring.Or(actualBMs[0], actualBMs[1]) - } else if len(actualBMs) == 1 { - bm = actualBMs[0].Clone() - } - - if bm == nil { - bm = roaring.New() - } - - bm.AddMany(docNums) - - oTFR.iterators[i] = segment.NewUnadornedPostingsIteratorFromBitmap(bm) - } - - atomic.AddUint64(&o.snapshot.parent.stats.TotTermSearchersStarted, uint64(1)) - return oTFR, nil -} - -// ---------------------------------------------------------------- - -func (i *IndexSnapshot) unadornedTermFieldReader( - term []byte, field string) *IndexSnapshotTermFieldReader { - // This IndexSnapshotTermFieldReader will not be recycled, more - // conversation here: https://github.com/blevesearch/bleve/pull/1438 - return &IndexSnapshotTermFieldReader{ - term: term, - field: field, - snapshot: i, - iterators: make([]segment.PostingsIterator, len(i.segment)), - segmentOffset: 0, - includeFreq: false, - includeNorm: false, - includeTermVectors: false, - recycle: false, - } -} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/persister.go b/vendor/github.com/blevesearch/bleve/index/scorch/persister.go deleted file mode 100644 index 498378a4..00000000 --- a/vendor/github.com/blevesearch/bleve/index/scorch/persister.go +++ /dev/null @@ -1,990 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package scorch - -import ( - "bytes" - "encoding/binary" - "encoding/json" - "fmt" - "io/ioutil" - "log" - "math" - "os" - "path/filepath" - "strconv" - "strings" - "sync/atomic" - "time" - - "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" - bolt "go.etcd.io/bbolt" -) - -// DefaultPersisterNapTimeMSec is kept to zero as this helps in direct -// persistence of segments with the default safe batch option. -// If the default safe batch option results in high number of -// files on disk, then users may initialise this configuration parameter -// with higher values so that the persister will nap a bit within it's -// work loop to favour better in-memory merging of segments to result -// in fewer segment files on disk. But that may come with an indexing -// performance overhead. -// Unsafe batch users are advised to override this to higher value -// for better performance especially with high data density. -var DefaultPersisterNapTimeMSec int = 0 // ms - -// DefaultPersisterNapUnderNumFiles helps in controlling the pace of -// persister. At times of a slow merger progress with heavy file merging -// operations, its better to pace down the persister for letting the merger -// to catch up within a range defined by this parameter. -// Fewer files on disk (as per the merge plan) would result in keeping the -// file handle usage under limit, faster disk merger and a healthier index. -// Its been observed that such a loosely sync'ed introducer-persister-merger -// trio results in better overall performance. -var DefaultPersisterNapUnderNumFiles int = 1000 - -var DefaultMemoryPressurePauseThreshold uint64 = math.MaxUint64 - -type persisterOptions struct { - // PersisterNapTimeMSec controls the wait/delay injected into - // persistence workloop to improve the chances for - // a healthier and heavier in-memory merging - PersisterNapTimeMSec int - - // PersisterNapTimeMSec > 0, and the number of files is less than - // PersisterNapUnderNumFiles, then the persister will sleep - // PersisterNapTimeMSec amount of time to improve the chances for - // a healthier and heavier in-memory merging - PersisterNapUnderNumFiles int - - // MemoryPressurePauseThreshold let persister to have a better leeway - // for prudently performing the memory merge of segments on a memory - // pressure situation. Here the config value is an upper threshold - // for the number of paused application threads. The default value would - // be a very high number to always favour the merging of memory segments. - MemoryPressurePauseThreshold uint64 -} - -type notificationChan chan struct{} - -func (s *Scorch) persisterLoop() { - defer s.asyncTasks.Done() - - var persistWatchers []*epochWatcher - var lastPersistedEpoch, lastMergedEpoch uint64 - var ew *epochWatcher - - var unpersistedCallbacks []index.BatchCallback - - po, err := s.parsePersisterOptions() - if err != nil { - s.fireAsyncError(fmt.Errorf("persisterOptions json parsing err: %v", err)) - s.asyncTasks.Done() - return - } - -OUTER: - for { - atomic.AddUint64(&s.stats.TotPersistLoopBeg, 1) - - select { - case <-s.closeCh: - break OUTER - case ew = <-s.persisterNotifier: - persistWatchers = append(persistWatchers, ew) - default: - } - if ew != nil && ew.epoch > lastMergedEpoch { - lastMergedEpoch = ew.epoch - } - lastMergedEpoch, persistWatchers = s.pausePersisterForMergerCatchUp(lastPersistedEpoch, - lastMergedEpoch, persistWatchers, po) - - var ourSnapshot *IndexSnapshot - var ourPersisted []chan error - var ourPersistedCallbacks []index.BatchCallback - - // check to see if there is a new snapshot to persist - s.rootLock.Lock() - if s.root != nil && s.root.epoch > lastPersistedEpoch { - ourSnapshot = s.root - ourSnapshot.AddRef() - ourPersisted = s.rootPersisted - s.rootPersisted = nil - ourPersistedCallbacks = s.persistedCallbacks - s.persistedCallbacks = nil - atomic.StoreUint64(&s.iStats.persistSnapshotSize, uint64(ourSnapshot.Size())) - atomic.StoreUint64(&s.iStats.persistEpoch, ourSnapshot.epoch) - } - s.rootLock.Unlock() - - if ourSnapshot != nil { - startTime := time.Now() - - err := s.persistSnapshot(ourSnapshot, po) - for _, ch := range ourPersisted { - if err != nil { - ch <- err - } - close(ch) - } - if err != nil { - atomic.StoreUint64(&s.iStats.persistEpoch, 0) - if err == segment.ErrClosed { - // index has been closed - _ = ourSnapshot.DecRef() - break OUTER - } - - // save this current snapshot's persistedCallbacks, to invoke during - // the retry attempt - unpersistedCallbacks = append(unpersistedCallbacks, ourPersistedCallbacks...) - - s.fireAsyncError(fmt.Errorf("got err persisting snapshot: %v", err)) - _ = ourSnapshot.DecRef() - atomic.AddUint64(&s.stats.TotPersistLoopErr, 1) - continue OUTER - } - - if unpersistedCallbacks != nil { - // in the event of this being a retry attempt for persisting a snapshot - // that had earlier failed, prepend the persistedCallbacks associated - // with earlier segment(s) to the latest persistedCallbacks - ourPersistedCallbacks = append(unpersistedCallbacks, ourPersistedCallbacks...) - unpersistedCallbacks = nil - } - - for i := range ourPersistedCallbacks { - ourPersistedCallbacks[i](err) - } - - atomic.StoreUint64(&s.stats.LastPersistedEpoch, ourSnapshot.epoch) - - lastPersistedEpoch = ourSnapshot.epoch - for _, ew := range persistWatchers { - close(ew.notifyCh) - } - - persistWatchers = nil - _ = ourSnapshot.DecRef() - - changed := false - s.rootLock.RLock() - if s.root != nil && s.root.epoch != lastPersistedEpoch { - changed = true - } - s.rootLock.RUnlock() - - s.fireEvent(EventKindPersisterProgress, time.Since(startTime)) - - if changed { - atomic.AddUint64(&s.stats.TotPersistLoopProgress, 1) - continue OUTER - } - } - - // tell the introducer we're waiting for changes - w := &epochWatcher{ - epoch: lastPersistedEpoch, - notifyCh: make(notificationChan, 1), - } - - select { - case <-s.closeCh: - break OUTER - case s.introducerNotifier <- w: - } - - s.removeOldData() // might as well cleanup while waiting - - atomic.AddUint64(&s.stats.TotPersistLoopWait, 1) - - select { - case <-s.closeCh: - break OUTER - case <-w.notifyCh: - // woken up, next loop should pick up work - atomic.AddUint64(&s.stats.TotPersistLoopWaitNotified, 1) - case ew = <-s.persisterNotifier: - // if the watchers are already caught up then let them wait, - // else let them continue to do the catch up - persistWatchers = append(persistWatchers, ew) - } - - atomic.AddUint64(&s.stats.TotPersistLoopEnd, 1) - } -} - -func notifyMergeWatchers(lastPersistedEpoch uint64, - persistWatchers []*epochWatcher) []*epochWatcher { - var watchersNext []*epochWatcher - for _, w := range persistWatchers { - if w.epoch < lastPersistedEpoch { - close(w.notifyCh) - } else { - watchersNext = append(watchersNext, w) - } - } - return watchersNext -} - -func (s *Scorch) pausePersisterForMergerCatchUp(lastPersistedEpoch uint64, - lastMergedEpoch uint64, persistWatchers []*epochWatcher, - po *persisterOptions) (uint64, []*epochWatcher) { - - // First, let the watchers proceed if they lag behind - persistWatchers = notifyMergeWatchers(lastPersistedEpoch, persistWatchers) - - // Check the merger lag by counting the segment files on disk, - numFilesOnDisk, _, _ := s.diskFileStats(nil) - - // On finding fewer files on disk, persister takes a short pause - // for sufficient in-memory segments to pile up for the next - // memory merge cum persist loop. - if numFilesOnDisk < uint64(po.PersisterNapUnderNumFiles) && - po.PersisterNapTimeMSec > 0 && s.NumEventsBlocking() == 0 { - select { - case <-s.closeCh: - case <-time.After(time.Millisecond * time.Duration(po.PersisterNapTimeMSec)): - atomic.AddUint64(&s.stats.TotPersisterNapPauseCompleted, 1) - - case ew := <-s.persisterNotifier: - // unblock the merger in meantime - persistWatchers = append(persistWatchers, ew) - lastMergedEpoch = ew.epoch - persistWatchers = notifyMergeWatchers(lastPersistedEpoch, persistWatchers) - atomic.AddUint64(&s.stats.TotPersisterMergerNapBreak, 1) - } - return lastMergedEpoch, persistWatchers - } - - // Finding too many files on disk could be due to two reasons. - // 1. Too many older snapshots awaiting the clean up. - // 2. The merger could be lagging behind on merging the disk files. - if numFilesOnDisk > uint64(po.PersisterNapUnderNumFiles) { - s.removeOldData() - numFilesOnDisk, _, _ = s.diskFileStats(nil) - } - - // Persister pause until the merger catches up to reduce the segment - // file count under the threshold. - // But if there is memory pressure, then skip this sleep maneuvers. -OUTER: - for po.PersisterNapUnderNumFiles > 0 && - numFilesOnDisk >= uint64(po.PersisterNapUnderNumFiles) && - lastMergedEpoch < lastPersistedEpoch { - atomic.AddUint64(&s.stats.TotPersisterSlowMergerPause, 1) - - select { - case <-s.closeCh: - break OUTER - case ew := <-s.persisterNotifier: - persistWatchers = append(persistWatchers, ew) - lastMergedEpoch = ew.epoch - } - - atomic.AddUint64(&s.stats.TotPersisterSlowMergerResume, 1) - - // let the watchers proceed if they lag behind - persistWatchers = notifyMergeWatchers(lastPersistedEpoch, persistWatchers) - - numFilesOnDisk, _, _ = s.diskFileStats(nil) - } - - return lastMergedEpoch, persistWatchers -} - -func (s *Scorch) parsePersisterOptions() (*persisterOptions, error) { - po := persisterOptions{ - PersisterNapTimeMSec: DefaultPersisterNapTimeMSec, - PersisterNapUnderNumFiles: DefaultPersisterNapUnderNumFiles, - MemoryPressurePauseThreshold: DefaultMemoryPressurePauseThreshold, - } - if v, ok := s.config["scorchPersisterOptions"]; ok { - b, err := json.Marshal(v) - if err != nil { - return &po, err - } - - err = json.Unmarshal(b, &po) - if err != nil { - return &po, err - } - } - return &po, nil -} - -func (s *Scorch) persistSnapshot(snapshot *IndexSnapshot, - po *persisterOptions) error { - // Perform in-memory segment merging only when the memory pressure is - // below the configured threshold, else the persister performs the - // direct persistence of segments. - if s.NumEventsBlocking() < po.MemoryPressurePauseThreshold { - persisted, err := s.persistSnapshotMaybeMerge(snapshot) - if err != nil { - return err - } - if persisted { - return nil - } - } - - return s.persistSnapshotDirect(snapshot) -} - -// DefaultMinSegmentsForInMemoryMerge represents the default number of -// in-memory zap segments that persistSnapshotMaybeMerge() needs to -// see in an IndexSnapshot before it decides to merge and persist -// those segments -var DefaultMinSegmentsForInMemoryMerge = 2 - -// persistSnapshotMaybeMerge examines the snapshot and might merge and -// persist the in-memory zap segments if there are enough of them -func (s *Scorch) persistSnapshotMaybeMerge(snapshot *IndexSnapshot) ( - bool, error) { - // collect the in-memory zap segments (SegmentBase instances) - var sbs []segment.Segment - var sbsDrops []*roaring.Bitmap - var sbsIndexes []int - - for i, segmentSnapshot := range snapshot.segment { - if _, ok := segmentSnapshot.segment.(segment.PersistedSegment); !ok { - sbs = append(sbs, segmentSnapshot.segment) - sbsDrops = append(sbsDrops, segmentSnapshot.deleted) - sbsIndexes = append(sbsIndexes, i) - } - } - - if len(sbs) < DefaultMinSegmentsForInMemoryMerge { - return false, nil - } - - newSnapshot, newSegmentID, err := s.mergeSegmentBases( - snapshot, sbs, sbsDrops, sbsIndexes) - if err != nil { - return false, err - } - if newSnapshot == nil { - return false, nil - } - - defer func() { - _ = newSnapshot.DecRef() - }() - - mergedSegmentIDs := map[uint64]struct{}{} - for _, idx := range sbsIndexes { - mergedSegmentIDs[snapshot.segment[idx].id] = struct{}{} - } - - // construct a snapshot that's logically equivalent to the input - // snapshot, but with merged segments replaced by the new segment - equiv := &IndexSnapshot{ - parent: snapshot.parent, - segment: make([]*SegmentSnapshot, 0, len(snapshot.segment)), - internal: snapshot.internal, - epoch: snapshot.epoch, - creator: "persistSnapshotMaybeMerge", - } - - // copy to the equiv the segments that weren't replaced - for _, segment := range snapshot.segment { - if _, wasMerged := mergedSegmentIDs[segment.id]; !wasMerged { - equiv.segment = append(equiv.segment, segment) - } - } - - // append to the equiv the new segment - for _, segment := range newSnapshot.segment { - if segment.id == newSegmentID { - equiv.segment = append(equiv.segment, &SegmentSnapshot{ - id: newSegmentID, - segment: segment.segment, - deleted: nil, // nil since merging handled deletions - }) - break - } - } - - err = s.persistSnapshotDirect(equiv) - if err != nil { - return false, err - } - - return true, nil -} - -func prepareBoltSnapshot(snapshot *IndexSnapshot, tx *bolt.Tx, path string, - segPlugin segment.Plugin) ([]string, map[uint64]string, error) { - snapshotsBucket, err := tx.CreateBucketIfNotExists(boltSnapshotsBucket) - if err != nil { - return nil, nil, err - } - newSnapshotKey := segment.EncodeUvarintAscending(nil, snapshot.epoch) - snapshotBucket, err := snapshotsBucket.CreateBucketIfNotExists(newSnapshotKey) - if err != nil { - return nil, nil, err - } - - // persist meta values - metaBucket, err := snapshotBucket.CreateBucketIfNotExists(boltMetaDataKey) - if err != nil { - return nil, nil, err - } - err = metaBucket.Put(boltMetaDataSegmentTypeKey, []byte(segPlugin.Type())) - if err != nil { - return nil, nil, err - } - buf := make([]byte, binary.MaxVarintLen32) - binary.BigEndian.PutUint32(buf, segPlugin.Version()) - err = metaBucket.Put(boltMetaDataSegmentVersionKey, buf) - if err != nil { - return nil, nil, err - } - - // persist internal values - internalBucket, err := snapshotBucket.CreateBucketIfNotExists(boltInternalKey) - if err != nil { - return nil, nil, err - } - // TODO optimize writing these in order? - for k, v := range snapshot.internal { - err = internalBucket.Put([]byte(k), v) - if err != nil { - return nil, nil, err - } - } - - var filenames []string - newSegmentPaths := make(map[uint64]string) - - // first ensure that each segment in this snapshot has been persisted - for _, segmentSnapshot := range snapshot.segment { - snapshotSegmentKey := segment.EncodeUvarintAscending(nil, segmentSnapshot.id) - snapshotSegmentBucket, err := snapshotBucket.CreateBucketIfNotExists(snapshotSegmentKey) - if err != nil { - return nil, nil, err - } - switch seg := segmentSnapshot.segment.(type) { - case segment.PersistedSegment: - segPath := seg.Path() - filename := strings.TrimPrefix(segPath, path+string(os.PathSeparator)) - err = snapshotSegmentBucket.Put(boltPathKey, []byte(filename)) - if err != nil { - return nil, nil, err - } - filenames = append(filenames, filename) - case segment.UnpersistedSegment: - // need to persist this to disk - filename := zapFileName(segmentSnapshot.id) - path := path + string(os.PathSeparator) + filename - err = seg.Persist(path) - if err != nil { - return nil, nil, fmt.Errorf("error persisting segment: %v", err) - } - newSegmentPaths[segmentSnapshot.id] = path - err = snapshotSegmentBucket.Put(boltPathKey, []byte(filename)) - if err != nil { - return nil, nil, err - } - filenames = append(filenames, filename) - default: - return nil, nil, fmt.Errorf("unknown segment type: %T", seg) - } - // store current deleted bits - var roaringBuf bytes.Buffer - if segmentSnapshot.deleted != nil { - _, err = segmentSnapshot.deleted.WriteTo(&roaringBuf) - if err != nil { - return nil, nil, fmt.Errorf("error persisting roaring bytes: %v", err) - } - err = snapshotSegmentBucket.Put(boltDeletedKey, roaringBuf.Bytes()) - if err != nil { - return nil, nil, err - } - } - } - - return filenames, newSegmentPaths, nil -} - -func (s *Scorch) persistSnapshotDirect(snapshot *IndexSnapshot) (err error) { - // start a write transaction - tx, err := s.rootBolt.Begin(true) - if err != nil { - return err - } - // defer rollback on error - defer func() { - if err != nil { - _ = tx.Rollback() - } - }() - - filenames, newSegmentPaths, err := prepareBoltSnapshot(snapshot, tx, s.path, s.segPlugin) - if err != nil { - return err - } - - // we need to swap in a new root only when we've persisted 1 or - // more segments -- whereby the new root would have 1-for-1 - // replacements of in-memory segments with file-based segments - // - // other cases like updates to internal values only, and/or when - // there are only deletions, are already covered and persisted by - // the newly populated boltdb snapshotBucket above - if len(newSegmentPaths) > 0 { - // now try to open all the new snapshots - newSegments := make(map[uint64]segment.Segment) - defer func() { - for _, s := range newSegments { - if s != nil { - // cleanup segments that were opened but not - // swapped into the new root - _ = s.Close() - } - } - }() - for segmentID, path := range newSegmentPaths { - newSegments[segmentID], err = s.segPlugin.Open(path) - if err != nil { - return fmt.Errorf("error opening new segment at %s, %v", path, err) - } - } - - persist := &persistIntroduction{ - persisted: newSegments, - applied: make(notificationChan), - } - - select { - case <-s.closeCh: - return segment.ErrClosed - case s.persists <- persist: - } - - select { - case <-s.closeCh: - return segment.ErrClosed - case <-persist.applied: - } - } - - err = tx.Commit() - if err != nil { - return err - } - - err = s.rootBolt.Sync() - if err != nil { - return err - } - - // allow files to become eligible for removal after commit, such - // as file segments from snapshots that came from the merger - s.rootLock.Lock() - for _, filename := range filenames { - delete(s.ineligibleForRemoval, filename) - } - s.rootLock.Unlock() - - return nil -} - -func zapFileName(epoch uint64) string { - return fmt.Sprintf("%012x.zap", epoch) -} - -// bolt snapshot code - -var boltSnapshotsBucket = []byte{'s'} -var boltPathKey = []byte{'p'} -var boltDeletedKey = []byte{'d'} -var boltInternalKey = []byte{'i'} -var boltMetaDataKey = []byte{'m'} -var boltMetaDataSegmentTypeKey = []byte("type") -var boltMetaDataSegmentVersionKey = []byte("version") - -func (s *Scorch) loadFromBolt() error { - return s.rootBolt.View(func(tx *bolt.Tx) error { - snapshots := tx.Bucket(boltSnapshotsBucket) - if snapshots == nil { - return nil - } - foundRoot := false - c := snapshots.Cursor() - for k, _ := c.Last(); k != nil; k, _ = c.Prev() { - _, snapshotEpoch, err := segment.DecodeUvarintAscending(k) - if err != nil { - log.Printf("unable to parse segment epoch %x, continuing", k) - continue - } - if foundRoot { - s.AddEligibleForRemoval(snapshotEpoch) - continue - } - snapshot := snapshots.Bucket(k) - if snapshot == nil { - log.Printf("snapshot key, but bucket missing %x, continuing", k) - s.AddEligibleForRemoval(snapshotEpoch) - continue - } - indexSnapshot, err := s.loadSnapshot(snapshot) - if err != nil { - log.Printf("unable to load snapshot, %v, continuing", err) - s.AddEligibleForRemoval(snapshotEpoch) - continue - } - indexSnapshot.epoch = snapshotEpoch - // set the nextSegmentID - s.nextSegmentID, err = s.maxSegmentIDOnDisk() - if err != nil { - return err - } - s.nextSegmentID++ - s.rootLock.Lock() - s.nextSnapshotEpoch = snapshotEpoch + 1 - rootPrev := s.root - s.root = indexSnapshot - s.rootLock.Unlock() - - if rootPrev != nil { - _ = rootPrev.DecRef() - } - - foundRoot = true - } - return nil - }) -} - -// LoadSnapshot loads the segment with the specified epoch -// NOTE: this is currently ONLY intended to be used by the command-line tool -func (s *Scorch) LoadSnapshot(epoch uint64) (rv *IndexSnapshot, err error) { - err = s.rootBolt.View(func(tx *bolt.Tx) error { - snapshots := tx.Bucket(boltSnapshotsBucket) - if snapshots == nil { - return nil - } - snapshotKey := segment.EncodeUvarintAscending(nil, epoch) - snapshot := snapshots.Bucket(snapshotKey) - if snapshot == nil { - return fmt.Errorf("snapshot with epoch: %v - doesn't exist", epoch) - } - rv, err = s.loadSnapshot(snapshot) - return err - }) - if err != nil { - return nil, err - } - return rv, nil -} - -func (s *Scorch) loadSnapshot(snapshot *bolt.Bucket) (*IndexSnapshot, error) { - - rv := &IndexSnapshot{ - parent: s, - internal: make(map[string][]byte), - refs: 1, - creator: "loadSnapshot", - } - // first we look for the meta-data bucket, this will tell us - // which segment type/version was used for this snapshot - // all operations for this scorch will use this type/version - metaBucket := snapshot.Bucket(boltMetaDataKey) - if metaBucket == nil { - _ = rv.DecRef() - return nil, fmt.Errorf("meta-data bucket missing") - } - segmentType := string(metaBucket.Get(boltMetaDataSegmentTypeKey)) - segmentVersion := binary.BigEndian.Uint32( - metaBucket.Get(boltMetaDataSegmentVersionKey)) - err := s.loadSegmentPlugin(segmentType, segmentVersion) - if err != nil { - _ = rv.DecRef() - return nil, fmt.Errorf( - "unable to load correct segment wrapper: %v", err) - } - var running uint64 - c := snapshot.Cursor() - for k, _ := c.First(); k != nil; k, _ = c.Next() { - if k[0] == boltInternalKey[0] { - internalBucket := snapshot.Bucket(k) - err := internalBucket.ForEach(func(key []byte, val []byte) error { - copiedVal := append([]byte(nil), val...) - rv.internal[string(key)] = copiedVal - return nil - }) - if err != nil { - _ = rv.DecRef() - return nil, err - } - } else if k[0] != boltMetaDataKey[0] { - segmentBucket := snapshot.Bucket(k) - if segmentBucket == nil { - _ = rv.DecRef() - return nil, fmt.Errorf("segment key, but bucket missing % x", k) - } - segmentSnapshot, err := s.loadSegment(segmentBucket) - if err != nil { - _ = rv.DecRef() - return nil, fmt.Errorf("failed to load segment: %v", err) - } - _, segmentSnapshot.id, err = segment.DecodeUvarintAscending(k) - if err != nil { - _ = rv.DecRef() - return nil, fmt.Errorf("failed to decode segment id: %v", err) - } - rv.segment = append(rv.segment, segmentSnapshot) - rv.offsets = append(rv.offsets, running) - running += segmentSnapshot.segment.Count() - } - } - return rv, nil -} - -func (s *Scorch) loadSegment(segmentBucket *bolt.Bucket) (*SegmentSnapshot, error) { - pathBytes := segmentBucket.Get(boltPathKey) - if pathBytes == nil { - return nil, fmt.Errorf("segment path missing") - } - segmentPath := s.path + string(os.PathSeparator) + string(pathBytes) - segment, err := s.segPlugin.Open(segmentPath) - if err != nil { - return nil, fmt.Errorf("error opening bolt segment: %v", err) - } - - rv := &SegmentSnapshot{ - segment: segment, - cachedDocs: &cachedDocs{cache: nil}, - } - deletedBytes := segmentBucket.Get(boltDeletedKey) - if deletedBytes != nil { - deletedBitmap := roaring.NewBitmap() - r := bytes.NewReader(deletedBytes) - _, err := deletedBitmap.ReadFrom(r) - if err != nil { - _ = segment.Close() - return nil, fmt.Errorf("error reading deleted bytes: %v", err) - } - if !deletedBitmap.IsEmpty() { - rv.deleted = deletedBitmap - } - } - - return rv, nil -} - -func (s *Scorch) removeOldData() { - removed, err := s.removeOldBoltSnapshots() - if err != nil { - s.fireAsyncError(fmt.Errorf("got err removing old bolt snapshots: %v", err)) - } - atomic.AddUint64(&s.stats.TotSnapshotsRemovedFromMetaStore, uint64(removed)) - - err = s.removeOldZapFiles() - if err != nil { - s.fireAsyncError(fmt.Errorf("got err removing old zap files: %v", err)) - } -} - -// NumSnapshotsToKeep represents how many recent, old snapshots to -// keep around per Scorch instance. Useful for apps that require -// rollback'ability. -var NumSnapshotsToKeep = 1 - -// Removes enough snapshots from the rootBolt so that the -// s.eligibleForRemoval stays under the NumSnapshotsToKeep policy. -func (s *Scorch) removeOldBoltSnapshots() (numRemoved int, err error) { - persistedEpochs, err := s.RootBoltSnapshotEpochs() - if err != nil { - return 0, err - } - - if len(persistedEpochs) <= s.numSnapshotsToKeep { - // we need to keep everything - return 0, nil - } - - // make a map of epochs to protect from deletion - protectedEpochs := make(map[uint64]struct{}, s.numSnapshotsToKeep) - for _, epoch := range persistedEpochs[0:s.numSnapshotsToKeep] { - protectedEpochs[epoch] = struct{}{} - } - - var epochsToRemove []uint64 - var newEligible []uint64 - s.rootLock.Lock() - for _, epoch := range s.eligibleForRemoval { - if _, ok := protectedEpochs[epoch]; ok { - // protected - newEligible = append(newEligible, epoch) - } else { - epochsToRemove = append(epochsToRemove, epoch) - } - } - s.eligibleForRemoval = newEligible - s.rootLock.Unlock() - - if len(epochsToRemove) == 0 { - return 0, nil - } - - tx, err := s.rootBolt.Begin(true) - if err != nil { - return 0, err - } - defer func() { - if err == nil { - err = tx.Commit() - } else { - _ = tx.Rollback() - } - if err == nil { - err = s.rootBolt.Sync() - } - }() - - snapshots := tx.Bucket(boltSnapshotsBucket) - if snapshots == nil { - return 0, nil - } - - for _, epochToRemove := range epochsToRemove { - k := segment.EncodeUvarintAscending(nil, epochToRemove) - err = snapshots.DeleteBucket(k) - if err == bolt.ErrBucketNotFound { - err = nil - } - if err == nil { - numRemoved++ - } - } - - return numRemoved, err -} - -func (s *Scorch) maxSegmentIDOnDisk() (uint64, error) { - currFileInfos, err := ioutil.ReadDir(s.path) - if err != nil { - return 0, err - } - - var rv uint64 - for _, finfo := range currFileInfos { - fname := finfo.Name() - if filepath.Ext(fname) == ".zap" { - prefix := strings.TrimSuffix(fname, ".zap") - id, err2 := strconv.ParseUint(prefix, 16, 64) - if err2 != nil { - return 0, err2 - } - if id > rv { - rv = id - } - } - } - return rv, err -} - -// Removes any *.zap files which aren't listed in the rootBolt. -func (s *Scorch) removeOldZapFiles() error { - liveFileNames, err := s.loadZapFileNames() - if err != nil { - return err - } - - currFileInfos, err := ioutil.ReadDir(s.path) - if err != nil { - return err - } - - s.rootLock.RLock() - - for _, finfo := range currFileInfos { - fname := finfo.Name() - if filepath.Ext(fname) == ".zap" { - if _, exists := liveFileNames[fname]; !exists && !s.ineligibleForRemoval[fname] { - err := os.Remove(s.path + string(os.PathSeparator) + fname) - if err != nil { - log.Printf("got err removing file: %s, err: %v", fname, err) - } - } - } - } - - s.rootLock.RUnlock() - - return nil -} - -func (s *Scorch) RootBoltSnapshotEpochs() ([]uint64, error) { - var rv []uint64 - err := s.rootBolt.View(func(tx *bolt.Tx) error { - snapshots := tx.Bucket(boltSnapshotsBucket) - if snapshots == nil { - return nil - } - sc := snapshots.Cursor() - for sk, _ := sc.Last(); sk != nil; sk, _ = sc.Prev() { - _, snapshotEpoch, err := segment.DecodeUvarintAscending(sk) - if err != nil { - continue - } - rv = append(rv, snapshotEpoch) - } - return nil - }) - return rv, err -} - -// Returns the *.zap file names that are listed in the rootBolt. -func (s *Scorch) loadZapFileNames() (map[string]struct{}, error) { - rv := map[string]struct{}{} - err := s.rootBolt.View(func(tx *bolt.Tx) error { - snapshots := tx.Bucket(boltSnapshotsBucket) - if snapshots == nil { - return nil - } - sc := snapshots.Cursor() - for sk, _ := sc.First(); sk != nil; sk, _ = sc.Next() { - snapshot := snapshots.Bucket(sk) - if snapshot == nil { - continue - } - segc := snapshot.Cursor() - for segk, _ := segc.First(); segk != nil; segk, _ = segc.Next() { - if segk[0] == boltInternalKey[0] { - continue - } - segmentBucket := snapshot.Bucket(segk) - if segmentBucket == nil { - continue - } - pathBytes := segmentBucket.Get(boltPathKey) - if pathBytes == nil { - continue - } - pathString := string(pathBytes) - rv[string(pathString)] = struct{}{} - } - } - return nil - }) - - return rv, err -} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/segment/empty.go b/vendor/github.com/blevesearch/bleve/index/scorch/segment/empty.go deleted file mode 100644 index 340db73a..00000000 --- a/vendor/github.com/blevesearch/bleve/index/scorch/segment/empty.go +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package segment - -import ( - "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/index" - "github.com/couchbase/vellum" -) - -type EmptySegment struct{} - -func (e *EmptySegment) Dictionary(field string) (TermDictionary, error) { - return &EmptyDictionary{}, nil -} - -func (e *EmptySegment) VisitDocument(num uint64, visitor DocumentFieldValueVisitor) error { - return nil -} - -func (e *EmptySegment) DocID(num uint64) ([]byte, error) { - return nil, nil -} - -func (e *EmptySegment) Count() uint64 { - return 0 -} - -func (e *EmptySegment) DocNumbers([]string) (*roaring.Bitmap, error) { - r := roaring.NewBitmap() - return r, nil -} - -func (e *EmptySegment) Fields() []string { - return []string{} -} - -func (e *EmptySegment) Close() error { - return nil -} - -func (e *EmptySegment) Size() uint64 { - return 0 -} - -func (e *EmptySegment) AddRef() { -} - -func (e *EmptySegment) DecRef() error { - return nil -} - -type EmptyDictionary struct{} - -func (e *EmptyDictionary) PostingsList(term []byte, - except *roaring.Bitmap, prealloc PostingsList) (PostingsList, error) { - return &EmptyPostingsList{}, nil -} - -func (e *EmptyDictionary) Iterator() DictionaryIterator { - return &EmptyDictionaryIterator{} -} - -func (e *EmptyDictionary) PrefixIterator(prefix string) DictionaryIterator { - return &EmptyDictionaryIterator{} -} - -func (e *EmptyDictionary) RangeIterator(start, end string) DictionaryIterator { - return &EmptyDictionaryIterator{} -} - -func (e *EmptyDictionary) AutomatonIterator(a vellum.Automaton, - startKeyInclusive, endKeyExclusive []byte) DictionaryIterator { - return &EmptyDictionaryIterator{} -} - -func (e *EmptyDictionary) OnlyIterator(onlyTerms [][]byte, - includeCount bool) DictionaryIterator { - return &EmptyDictionaryIterator{} -} - -func (e *EmptyDictionary) Contains(key []byte) (bool, error) { - return false, nil -} - -type EmptyDictionaryIterator struct{} - -func (e *EmptyDictionaryIterator) Next() (*index.DictEntry, error) { - return nil, nil -} - -func (e *EmptyDictionaryIterator) Contains(key []byte) (bool, error) { - return false, nil -} - -type EmptyPostingsList struct{} - -func (e *EmptyPostingsList) Iterator(includeFreq, includeNorm, includeLocations bool, - prealloc PostingsIterator) PostingsIterator { - return &EmptyPostingsIterator{} -} - -func (e *EmptyPostingsList) Size() int { - return 0 -} - -func (e *EmptyPostingsList) Count() uint64 { - return 0 -} - -type EmptyPostingsIterator struct{} - -func (e *EmptyPostingsIterator) Next() (Posting, error) { - return nil, nil -} - -func (e *EmptyPostingsIterator) Advance(uint64) (Posting, error) { - return nil, nil -} - -func (e *EmptyPostingsIterator) Size() int { - return 0 -} - -var AnEmptyPostingsIterator = &EmptyPostingsIterator{} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/segment/int.go b/vendor/github.com/blevesearch/bleve/index/scorch/segment/int.go deleted file mode 100644 index 55299d8f..00000000 --- a/vendor/github.com/blevesearch/bleve/index/scorch/segment/int.go +++ /dev/null @@ -1,176 +0,0 @@ -// Copyright 2014 The Cockroach Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied. See the License for the specific language governing -// permissions and limitations under the License. - -// This code originated from: -// https://github.com/cockroachdb/cockroach/blob/2dd65dde5d90c157f4b93f92502ca1063b904e1d/pkg/util/encoding/encoding.go - -// Modified to not use pkg/errors - -package segment - -import ( - "errors" - "fmt" -) - -const ( - MaxVarintSize = 9 - - // IntMin is chosen such that the range of int tags does not overlap the - // ascii character set that is frequently used in testing. - IntMin = 0x80 // 128 - intMaxWidth = 8 - intZero = IntMin + intMaxWidth // 136 - intSmall = IntMax - intZero - intMaxWidth // 109 - // IntMax is the maximum int tag value. - IntMax = 0xfd // 253 -) - -// EncodeUvarintAscending encodes the uint64 value using a variable length -// (length-prefixed) representation. The length is encoded as a single -// byte indicating the number of encoded bytes (-8) to follow. See -// EncodeVarintAscending for rationale. The encoded bytes are appended to the -// supplied buffer and the final buffer is returned. -func EncodeUvarintAscending(b []byte, v uint64) []byte { - switch { - case v <= intSmall: - return append(b, intZero+byte(v)) - case v <= 0xff: - return append(b, IntMax-7, byte(v)) - case v <= 0xffff: - return append(b, IntMax-6, byte(v>>8), byte(v)) - case v <= 0xffffff: - return append(b, IntMax-5, byte(v>>16), byte(v>>8), byte(v)) - case v <= 0xffffffff: - return append(b, IntMax-4, byte(v>>24), byte(v>>16), byte(v>>8), byte(v)) - case v <= 0xffffffffff: - return append(b, IntMax-3, byte(v>>32), byte(v>>24), byte(v>>16), byte(v>>8), - byte(v)) - case v <= 0xffffffffffff: - return append(b, IntMax-2, byte(v>>40), byte(v>>32), byte(v>>24), byte(v>>16), - byte(v>>8), byte(v)) - case v <= 0xffffffffffffff: - return append(b, IntMax-1, byte(v>>48), byte(v>>40), byte(v>>32), byte(v>>24), - byte(v>>16), byte(v>>8), byte(v)) - default: - return append(b, IntMax, byte(v>>56), byte(v>>48), byte(v>>40), byte(v>>32), - byte(v>>24), byte(v>>16), byte(v>>8), byte(v)) - } -} - -// DecodeUvarintAscending decodes a varint encoded uint64 from the input -// buffer. The remainder of the input buffer and the decoded uint64 -// are returned. -func DecodeUvarintAscending(b []byte) ([]byte, uint64, error) { - if len(b) == 0 { - return nil, 0, fmt.Errorf("insufficient bytes to decode uvarint value") - } - length := int(b[0]) - intZero - b = b[1:] // skip length byte - if length <= intSmall { - return b, uint64(length), nil - } - length -= intSmall - if length < 0 || length > 8 { - return nil, 0, fmt.Errorf("invalid uvarint length of %d", length) - } else if len(b) < length { - return nil, 0, fmt.Errorf("insufficient bytes to decode uvarint value: %q", b) - } - var v uint64 - // It is faster to range over the elements in a slice than to index - // into the slice on each loop iteration. - for _, t := range b[:length] { - v = (v << 8) | uint64(t) - } - return b[length:], v, nil -} - -// ------------------------------------------------------------ - -type MemUvarintReader struct { - C int // index of next byte to read from S - S []byte -} - -func NewMemUvarintReader(s []byte) *MemUvarintReader { - return &MemUvarintReader{S: s} -} - -// Len returns the number of unread bytes. -func (r *MemUvarintReader) Len() int { - n := len(r.S) - r.C - if n < 0 { - return 0 - } - return n -} - -var ErrMemUvarintReaderOverflow = errors.New("MemUvarintReader overflow") - -// ReadUvarint reads an encoded uint64. The original code this was -// based on is at encoding/binary/ReadUvarint(). -func (r *MemUvarintReader) ReadUvarint() (uint64, error) { - var x uint64 - var s uint - var C = r.C - var S = r.S - - for { - b := S[C] - C++ - - if b < 0x80 { - r.C = C - - // why 63? The original code had an 'i += 1' loop var and - // checked for i > 9 || i == 9 ...; but, we no longer - // check for the i var, but instead check here for s, - // which is incremented by 7. So, 7*9 == 63. - // - // why the "extra" >= check? The normal case is that s < - // 63, so we check this single >= guard first so that we - // hit the normal, nil-error return pathway sooner. - if s >= 63 && (s > 63 || s == 63 && b > 1) { - return 0, ErrMemUvarintReaderOverflow - } - - return x | uint64(b)<= 0; i-- { - rv[i] = rv[i] + 1 - if rv[i] != 0 { - return rv // didn't overflow, so stop - } - } - return nil // overflowed -} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/segment/segment.go b/vendor/github.com/blevesearch/bleve/index/scorch/segment/segment.go deleted file mode 100644 index ddd0d091..00000000 --- a/vendor/github.com/blevesearch/bleve/index/scorch/segment/segment.go +++ /dev/null @@ -1,153 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package segment - -import ( - "fmt" - - "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/index" - "github.com/couchbase/vellum" -) - -var ErrClosed = fmt.Errorf("index closed") - -// DocumentFieldValueVisitor defines a callback to be visited for each -// stored field value. The return value determines if the visitor -// should keep going. Returning true continues visiting, false stops. -type DocumentFieldValueVisitor func(field string, typ byte, value []byte, pos []uint64) bool - -type Segment interface { - Dictionary(field string) (TermDictionary, error) - - VisitDocument(num uint64, visitor DocumentFieldValueVisitor) error - - DocID(num uint64) ([]byte, error) - - Count() uint64 - - DocNumbers([]string) (*roaring.Bitmap, error) - - Fields() []string - - Close() error - - Size() int - - AddRef() - DecRef() error -} - -type UnpersistedSegment interface { - Segment - Persist(path string) error -} - -type PersistedSegment interface { - Segment - Path() string -} - -type TermDictionary interface { - PostingsList(term []byte, except *roaring.Bitmap, prealloc PostingsList) (PostingsList, error) - - Iterator() DictionaryIterator - PrefixIterator(prefix string) DictionaryIterator - RangeIterator(start, end string) DictionaryIterator - AutomatonIterator(a vellum.Automaton, - startKeyInclusive, endKeyExclusive []byte) DictionaryIterator - OnlyIterator(onlyTerms [][]byte, includeCount bool) DictionaryIterator - - Contains(key []byte) (bool, error) -} - -type DictionaryIterator interface { - Next() (*index.DictEntry, error) -} - -type PostingsList interface { - Iterator(includeFreq, includeNorm, includeLocations bool, prealloc PostingsIterator) PostingsIterator - - Size() int - - Count() uint64 - - // NOTE deferred for future work - - // And(other PostingsList) PostingsList - // Or(other PostingsList) PostingsList -} - -type PostingsIterator interface { - // The caller is responsible for copying whatever it needs from - // the returned Posting instance before calling Next(), as some - // implementations may return a shared instance to reduce memory - // allocations. - Next() (Posting, error) - - // Advance will return the posting with the specified doc number - // or if there is no such posting, the next posting. - // Callers MUST NOT attempt to pass a docNum that is less than or - // equal to the currently visited posting doc Num. - Advance(docNum uint64) (Posting, error) - - Size() int -} - -type OptimizablePostingsIterator interface { - ActualBitmap() *roaring.Bitmap - DocNum1Hit() (uint64, bool) - ReplaceActual(*roaring.Bitmap) -} - -type Posting interface { - Number() uint64 - - Frequency() uint64 - Norm() float64 - - Locations() []Location - - Size() int -} - -type Location interface { - Field() string - Start() uint64 - End() uint64 - Pos() uint64 - ArrayPositions() []uint64 - Size() int -} - -// DocumentFieldTermVisitable is implemented by various scorch segment -// implementations with persistence for the un inverting of the -// postings or other indexed values. -type DocumentFieldTermVisitable interface { - VisitDocumentFieldTerms(localDocNum uint64, fields []string, - visitor index.DocumentFieldTermVisitor, optional DocVisitState) (DocVisitState, error) - - // VisitableDocValueFields implementation should return - // the list of fields which are document value persisted and - // therefore visitable by the above VisitDocumentFieldTerms method. - VisitableDocValueFields() ([]string, error) -} - -type DocVisitState interface { -} - -type StatsReporter interface { - ReportBytesWritten(bytesWritten uint64) -} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/segment/unadorned.go b/vendor/github.com/blevesearch/bleve/index/scorch/segment/unadorned.go deleted file mode 100644 index db06562d..00000000 --- a/vendor/github.com/blevesearch/bleve/index/scorch/segment/unadorned.go +++ /dev/null @@ -1,160 +0,0 @@ -// Copyright (c) 2020 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package segment - -import ( - "github.com/RoaringBitmap/roaring" - "math" - "reflect" -) - -var reflectStaticSizeUnadornedPostingsIteratorBitmap int -var reflectStaticSizeUnadornedPostingsIterator1Hit int -var reflectStaticSizeUnadornedPosting int - -func init() { - var pib UnadornedPostingsIteratorBitmap - reflectStaticSizeUnadornedPostingsIteratorBitmap = int(reflect.TypeOf(pib).Size()) - var pi1h UnadornedPostingsIterator1Hit - reflectStaticSizeUnadornedPostingsIterator1Hit = int(reflect.TypeOf(pi1h).Size()) - var up UnadornedPosting - reflectStaticSizeUnadornedPosting = int(reflect.TypeOf(up).Size()) -} - -type UnadornedPostingsIteratorBitmap struct { - actual roaring.IntPeekable - actualBM *roaring.Bitmap -} - -func (i *UnadornedPostingsIteratorBitmap) Next() (Posting, error) { - return i.nextAtOrAfter(0) -} - -func (i *UnadornedPostingsIteratorBitmap) Advance(docNum uint64) (Posting, error) { - return i.nextAtOrAfter(docNum) -} - -func (i *UnadornedPostingsIteratorBitmap) nextAtOrAfter(atOrAfter uint64) (Posting, error) { - docNum, exists := i.nextDocNumAtOrAfter(atOrAfter) - if !exists { - return nil, nil - } - return UnadornedPosting(docNum), nil -} - -func (i *UnadornedPostingsIteratorBitmap) nextDocNumAtOrAfter(atOrAfter uint64) (uint64, bool) { - if i.actual == nil || !i.actual.HasNext() { - return 0, false - } - i.actual.AdvanceIfNeeded(uint32(atOrAfter)) - - if !i.actual.HasNext() { - return 0, false // couldn't find anything - } - - return uint64(i.actual.Next()), true -} - -func (i *UnadornedPostingsIteratorBitmap) Size() int { - return reflectStaticSizeUnadornedPostingsIteratorBitmap -} - -func (i *UnadornedPostingsIteratorBitmap) ActualBitmap() *roaring.Bitmap { - return i.actualBM -} - -func (i *UnadornedPostingsIteratorBitmap) DocNum1Hit() (uint64, bool) { - return 0, false -} - -func (i *UnadornedPostingsIteratorBitmap) ReplaceActual(actual *roaring.Bitmap) { - i.actualBM = actual - i.actual = actual.Iterator() -} - -func NewUnadornedPostingsIteratorFromBitmap(bm *roaring.Bitmap) PostingsIterator { - return &UnadornedPostingsIteratorBitmap{ - actualBM: bm, - actual: bm.Iterator(), - } -} - -const docNum1HitFinished = math.MaxUint64 - -type UnadornedPostingsIterator1Hit struct { - docNum uint64 -} - -func (i *UnadornedPostingsIterator1Hit) Next() (Posting, error) { - return i.nextAtOrAfter(0) -} - -func (i *UnadornedPostingsIterator1Hit) Advance(docNum uint64) (Posting, error) { - return i.nextAtOrAfter(docNum) -} - -func (i *UnadornedPostingsIterator1Hit) nextAtOrAfter(atOrAfter uint64) (Posting, error) { - docNum, exists := i.nextDocNumAtOrAfter(atOrAfter) - if !exists { - return nil, nil - } - return UnadornedPosting(docNum), nil -} - -func (i *UnadornedPostingsIterator1Hit) nextDocNumAtOrAfter(atOrAfter uint64) (uint64, bool) { - if i.docNum == docNum1HitFinished { - return 0, false - } - if i.docNum < atOrAfter { - // advanced past our 1-hit - i.docNum = docNum1HitFinished // consume our 1-hit docNum - return 0, false - } - docNum := i.docNum - i.docNum = docNum1HitFinished // consume our 1-hit docNum - return docNum, true -} - -func (i *UnadornedPostingsIterator1Hit) Size() int { - return reflectStaticSizeUnadornedPostingsIterator1Hit -} - -func NewUnadornedPostingsIteratorFrom1Hit(docNum1Hit uint64) PostingsIterator { - return &UnadornedPostingsIterator1Hit{ - docNum1Hit, - } -} - -type UnadornedPosting uint64 - -func (p UnadornedPosting) Number() uint64 { - return uint64(p) -} - -func (p UnadornedPosting) Frequency() uint64 { - return 0 -} - -func (p UnadornedPosting) Norm() float64 { - return 0 -} - -func (p UnadornedPosting) Locations() []Location { - return nil -} - -func (p UnadornedPosting) Size() int { - return reflectStaticSizeUnadornedPosting -} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/segment_plugin.go b/vendor/github.com/blevesearch/bleve/index/scorch/segment_plugin.go deleted file mode 100644 index 2f7db48b..00000000 --- a/vendor/github.com/blevesearch/bleve/index/scorch/segment_plugin.go +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright (c) 2019 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package scorch - -import ( - "fmt" - - "github.com/blevesearch/bleve/index/scorch/segment" - - zapv11 "github.com/blevesearch/zap/v11" - zapv12 "github.com/blevesearch/zap/v12" - zapv13 "github.com/blevesearch/zap/v13" - zapv14 "github.com/blevesearch/zap/v14" - zapv15 "github.com/blevesearch/zap/v15" -) - -var supportedSegmentPlugins map[string]map[uint32]segment.Plugin -var defaultSegmentPlugin segment.Plugin - -func init() { - ResetPlugins() - RegisterPlugin(zapv15.Plugin(), false) - RegisterPlugin(zapv14.Plugin(), false) - RegisterPlugin(zapv13.Plugin(), false) - RegisterPlugin(zapv12.Plugin(), false) - RegisterPlugin(zapv11.Plugin(), true) -} - -func ResetPlugins() { - supportedSegmentPlugins = map[string]map[uint32]segment.Plugin{} -} - -func RegisterPlugin(plugin segment.Plugin, makeDefault bool) { - if _, ok := supportedSegmentPlugins[plugin.Type()]; !ok { - supportedSegmentPlugins[plugin.Type()] = map[uint32]segment.Plugin{} - } - supportedSegmentPlugins[plugin.Type()][plugin.Version()] = plugin - if makeDefault { - defaultSegmentPlugin = plugin - } -} - -func SupportedSegmentTypes() (rv []string) { - for k := range supportedSegmentPlugins { - rv = append(rv, k) - } - return -} - -func SupportedSegmentTypeVersions(typ string) (rv []uint32) { - for k := range supportedSegmentPlugins[typ] { - rv = append(rv, k) - } - return rv -} - -func chooseSegmentPlugin(forcedSegmentType string, - forcedSegmentVersion uint32) (segment.Plugin, error) { - if versions, ok := supportedSegmentPlugins[forcedSegmentType]; ok { - if segPlugin, ok := versions[uint32(forcedSegmentVersion)]; ok { - return segPlugin, nil - } - return nil, fmt.Errorf( - "unsupported version %d for segment type: %s, supported: %v", - forcedSegmentVersion, forcedSegmentType, - SupportedSegmentTypeVersions(forcedSegmentType)) - } - return nil, fmt.Errorf("unsupported segment type: %s, supported: %v", - forcedSegmentType, SupportedSegmentTypes()) -} - -func (s *Scorch) loadSegmentPlugin(forcedSegmentType string, - forcedSegmentVersion uint32) error { - segPlugin, err := chooseSegmentPlugin(forcedSegmentType, - forcedSegmentVersion) - if err != nil { - return err - } - s.segPlugin = segPlugin - return nil -} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/snapshot_index.go b/vendor/github.com/blevesearch/bleve/index/scorch/snapshot_index.go deleted file mode 100644 index 61204ebb..00000000 --- a/vendor/github.com/blevesearch/bleve/index/scorch/snapshot_index.go +++ /dev/null @@ -1,755 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package scorch - -import ( - "container/heap" - "encoding/binary" - "fmt" - "reflect" - "sort" - "sync" - "sync/atomic" - - "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/couchbase/vellum" - lev "github.com/couchbase/vellum/levenshtein" -) - -// re usable, threadsafe levenshtein builders -var lb1, lb2 *lev.LevenshteinAutomatonBuilder - -type asynchSegmentResult struct { - dict segment.TermDictionary - dictItr segment.DictionaryIterator - - index int - docs *roaring.Bitmap - - postings segment.PostingsList - - err error -} - -var reflectStaticSizeIndexSnapshot int - -func init() { - var is interface{} = IndexSnapshot{} - reflectStaticSizeIndexSnapshot = int(reflect.TypeOf(is).Size()) - var err error - lb1, err = lev.NewLevenshteinAutomatonBuilder(1, true) - if err != nil { - panic(fmt.Errorf("Levenshtein automaton ed1 builder err: %v", err)) - } - lb2, err = lev.NewLevenshteinAutomatonBuilder(2, true) - if err != nil { - panic(fmt.Errorf("Levenshtein automaton ed2 builder err: %v", err)) - } -} - -type IndexSnapshot struct { - parent *Scorch - segment []*SegmentSnapshot - offsets []uint64 - internal map[string][]byte - epoch uint64 - size uint64 - creator string - - m sync.Mutex // Protects the fields that follow. - refs int64 - - m2 sync.Mutex // Protects the fields that follow. - fieldTFRs map[string][]*IndexSnapshotTermFieldReader // keyed by field, recycled TFR's -} - -func (i *IndexSnapshot) Segments() []*SegmentSnapshot { - return i.segment -} - -func (i *IndexSnapshot) Internal() map[string][]byte { - return i.internal -} - -func (i *IndexSnapshot) AddRef() { - i.m.Lock() - i.refs++ - i.m.Unlock() -} - -func (i *IndexSnapshot) DecRef() (err error) { - i.m.Lock() - i.refs-- - if i.refs == 0 { - for _, s := range i.segment { - if s != nil { - err2 := s.segment.DecRef() - if err == nil { - err = err2 - } - } - } - if i.parent != nil { - go i.parent.AddEligibleForRemoval(i.epoch) - } - } - i.m.Unlock() - return err -} - -func (i *IndexSnapshot) Close() error { - return i.DecRef() -} - -func (i *IndexSnapshot) Size() int { - return int(i.size) -} - -func (i *IndexSnapshot) updateSize() { - i.size += uint64(reflectStaticSizeIndexSnapshot) - for _, s := range i.segment { - i.size += uint64(s.Size()) - } -} - -func (i *IndexSnapshot) newIndexSnapshotFieldDict(field string, - makeItr func(i segment.TermDictionary) segment.DictionaryIterator, - randomLookup bool) (*IndexSnapshotFieldDict, error) { - - results := make(chan *asynchSegmentResult) - for index, segment := range i.segment { - go func(index int, segment *SegmentSnapshot) { - dict, err := segment.segment.Dictionary(field) - if err != nil { - results <- &asynchSegmentResult{err: err} - } else { - if randomLookup { - results <- &asynchSegmentResult{dict: dict} - } else { - results <- &asynchSegmentResult{dictItr: makeItr(dict)} - } - } - }(index, segment) - } - - var err error - rv := &IndexSnapshotFieldDict{ - snapshot: i, - cursors: make([]*segmentDictCursor, 0, len(i.segment)), - } - for count := 0; count < len(i.segment); count++ { - asr := <-results - if asr.err != nil && err == nil { - err = asr.err - } else { - if !randomLookup { - next, err2 := asr.dictItr.Next() - if err2 != nil && err == nil { - err = err2 - } - if next != nil { - rv.cursors = append(rv.cursors, &segmentDictCursor{ - itr: asr.dictItr, - curr: *next, - }) - } - } else { - rv.cursors = append(rv.cursors, &segmentDictCursor{ - dict: asr.dict, - }) - } - } - } - // after ensuring we've read all items on channel - if err != nil { - return nil, err - } - - if !randomLookup { - // prepare heap - heap.Init(rv) - } - - return rv, nil -} - -func (i *IndexSnapshot) FieldDict(field string) (index.FieldDict, error) { - return i.newIndexSnapshotFieldDict(field, func(i segment.TermDictionary) segment.DictionaryIterator { - return i.Iterator() - }, false) -} - -func (i *IndexSnapshot) FieldDictRange(field string, startTerm []byte, - endTerm []byte) (index.FieldDict, error) { - return i.newIndexSnapshotFieldDict(field, func(i segment.TermDictionary) segment.DictionaryIterator { - return i.RangeIterator(string(startTerm), string(endTerm)) - }, false) -} - -func (i *IndexSnapshot) FieldDictPrefix(field string, - termPrefix []byte) (index.FieldDict, error) { - return i.newIndexSnapshotFieldDict(field, func(i segment.TermDictionary) segment.DictionaryIterator { - return i.PrefixIterator(string(termPrefix)) - }, false) -} - -func (i *IndexSnapshot) FieldDictRegexp(field string, - termRegex string) (index.FieldDict, error) { - // TODO: potential optimization where the literal prefix represents the, - // entire regexp, allowing us to use PrefixIterator(prefixTerm)? - - a, prefixBeg, prefixEnd, err := segment.ParseRegexp(termRegex) - if err != nil { - return nil, err - } - - return i.newIndexSnapshotFieldDict(field, func(i segment.TermDictionary) segment.DictionaryIterator { - return i.AutomatonIterator(a, prefixBeg, prefixEnd) - }, false) -} - -func (i *IndexSnapshot) getLevAutomaton(term string, - fuzziness uint8) (vellum.Automaton, error) { - if fuzziness == 1 { - return lb1.BuildDfa(term, fuzziness) - } else if fuzziness == 2 { - return lb2.BuildDfa(term, fuzziness) - } - return nil, fmt.Errorf("fuzziness exceeds the max limit") -} - -func (i *IndexSnapshot) FieldDictFuzzy(field string, - term string, fuzziness int, prefix string) (index.FieldDict, error) { - a, err := i.getLevAutomaton(term, uint8(fuzziness)) - if err != nil { - return nil, err - } - - var prefixBeg, prefixEnd []byte - if prefix != "" { - prefixBeg = []byte(prefix) - prefixEnd = segment.IncrementBytes(prefixBeg) - } - - return i.newIndexSnapshotFieldDict(field, func(i segment.TermDictionary) segment.DictionaryIterator { - return i.AutomatonIterator(a, prefixBeg, prefixEnd) - }, false) -} - -func (i *IndexSnapshot) FieldDictOnly(field string, - onlyTerms [][]byte, includeCount bool) (index.FieldDict, error) { - return i.newIndexSnapshotFieldDict(field, func(i segment.TermDictionary) segment.DictionaryIterator { - return i.OnlyIterator(onlyTerms, includeCount) - }, false) -} - -func (i *IndexSnapshot) FieldDictContains(field string) (index.FieldDictContains, error) { - return i.newIndexSnapshotFieldDict(field, nil, true) -} - -func (i *IndexSnapshot) DocIDReaderAll() (index.DocIDReader, error) { - results := make(chan *asynchSegmentResult) - for index, segment := range i.segment { - go func(index int, segment *SegmentSnapshot) { - results <- &asynchSegmentResult{ - index: index, - docs: segment.DocNumbersLive(), - } - }(index, segment) - } - - return i.newDocIDReader(results) -} - -func (i *IndexSnapshot) DocIDReaderOnly(ids []string) (index.DocIDReader, error) { - results := make(chan *asynchSegmentResult) - for index, segment := range i.segment { - go func(index int, segment *SegmentSnapshot) { - docs, err := segment.DocNumbers(ids) - if err != nil { - results <- &asynchSegmentResult{err: err} - } else { - results <- &asynchSegmentResult{ - index: index, - docs: docs, - } - } - }(index, segment) - } - - return i.newDocIDReader(results) -} - -func (i *IndexSnapshot) newDocIDReader(results chan *asynchSegmentResult) (index.DocIDReader, error) { - rv := &IndexSnapshotDocIDReader{ - snapshot: i, - iterators: make([]roaring.IntIterable, len(i.segment)), - } - var err error - for count := 0; count < len(i.segment); count++ { - asr := <-results - if asr.err != nil { - if err == nil { - // returns the first error encountered - err = asr.err - } - } else if err == nil { - rv.iterators[asr.index] = asr.docs.Iterator() - } - } - - if err != nil { - return nil, err - } - - return rv, nil -} - -func (i *IndexSnapshot) Fields() ([]string, error) { - // FIXME not making this concurrent for now as it's not used in hot path - // of any searches at the moment (just a debug aid) - fieldsMap := map[string]struct{}{} - for _, segment := range i.segment { - fields := segment.Fields() - for _, field := range fields { - fieldsMap[field] = struct{}{} - } - } - rv := make([]string, 0, len(fieldsMap)) - for k := range fieldsMap { - rv = append(rv, k) - } - return rv, nil -} - -func (i *IndexSnapshot) GetInternal(key []byte) ([]byte, error) { - return i.internal[string(key)], nil -} - -func (i *IndexSnapshot) DocCount() (uint64, error) { - var rv uint64 - for _, segment := range i.segment { - rv += segment.Count() - } - return rv, nil -} - -func (i *IndexSnapshot) Document(id string) (rv *document.Document, err error) { - // FIXME could be done more efficiently directly, but reusing for simplicity - tfr, err := i.TermFieldReader([]byte(id), "_id", false, false, false) - if err != nil { - return nil, err - } - defer func() { - if cerr := tfr.Close(); err == nil && cerr != nil { - err = cerr - } - }() - - next, err := tfr.Next(nil) - if err != nil { - return nil, err - } - - if next == nil { - // no such doc exists - return nil, nil - } - - docNum, err := docInternalToNumber(next.ID) - if err != nil { - return nil, err - } - segmentIndex, localDocNum := i.segmentIndexAndLocalDocNumFromGlobal(docNum) - - rv = document.NewDocument(id) - err = i.segment[segmentIndex].VisitDocument(localDocNum, func(name string, typ byte, val []byte, pos []uint64) bool { - if name == "_id" { - return true - } - - // copy value, array positions to preserve them beyond the scope of this callback - value := append([]byte(nil), val...) - arrayPos := append([]uint64(nil), pos...) - - switch typ { - case 't': - rv.AddField(document.NewTextField(name, arrayPos, value)) - case 'n': - rv.AddField(document.NewNumericFieldFromBytes(name, arrayPos, value)) - case 'd': - rv.AddField(document.NewDateTimeFieldFromBytes(name, arrayPos, value)) - case 'b': - rv.AddField(document.NewBooleanFieldFromBytes(name, arrayPos, value)) - case 'g': - rv.AddField(document.NewGeoPointFieldFromBytes(name, arrayPos, value)) - } - - return true - }) - if err != nil { - return nil, err - } - - return rv, nil -} - -func (i *IndexSnapshot) segmentIndexAndLocalDocNumFromGlobal(docNum uint64) (int, uint64) { - segmentIndex := sort.Search(len(i.offsets), - func(x int) bool { - return i.offsets[x] > docNum - }) - 1 - - localDocNum := docNum - i.offsets[segmentIndex] - return int(segmentIndex), localDocNum -} - -func (i *IndexSnapshot) ExternalID(id index.IndexInternalID) (string, error) { - docNum, err := docInternalToNumber(id) - if err != nil { - return "", err - } - segmentIndex, localDocNum := i.segmentIndexAndLocalDocNumFromGlobal(docNum) - - v, err := i.segment[segmentIndex].DocID(localDocNum) - if err != nil { - return "", err - } - if v == nil { - return "", fmt.Errorf("document number %d not found", docNum) - } - - return string(v), nil -} - -func (i *IndexSnapshot) InternalID(id string) (rv index.IndexInternalID, err error) { - // FIXME could be done more efficiently directly, but reusing for simplicity - tfr, err := i.TermFieldReader([]byte(id), "_id", false, false, false) - if err != nil { - return nil, err - } - defer func() { - if cerr := tfr.Close(); err == nil && cerr != nil { - err = cerr - } - }() - - next, err := tfr.Next(nil) - if err != nil || next == nil { - return nil, err - } - - return next.ID, nil -} - -func (i *IndexSnapshot) TermFieldReader(term []byte, field string, includeFreq, - includeNorm, includeTermVectors bool) (index.TermFieldReader, error) { - rv := i.allocTermFieldReaderDicts(field) - - rv.term = term - rv.field = field - rv.snapshot = i - if rv.postings == nil { - rv.postings = make([]segment.PostingsList, len(i.segment)) - } - if rv.iterators == nil { - rv.iterators = make([]segment.PostingsIterator, len(i.segment)) - } - rv.segmentOffset = 0 - rv.includeFreq = includeFreq - rv.includeNorm = includeNorm - rv.includeTermVectors = includeTermVectors - rv.currPosting = nil - rv.currID = rv.currID[:0] - - if rv.dicts == nil { - rv.dicts = make([]segment.TermDictionary, len(i.segment)) - for i, segment := range i.segment { - dict, err := segment.segment.Dictionary(field) - if err != nil { - return nil, err - } - rv.dicts[i] = dict - } - } - - for i, segment := range i.segment { - pl, err := rv.dicts[i].PostingsList(term, segment.deleted, rv.postings[i]) - if err != nil { - return nil, err - } - rv.postings[i] = pl - rv.iterators[i] = pl.Iterator(includeFreq, includeNorm, includeTermVectors, rv.iterators[i]) - } - atomic.AddUint64(&i.parent.stats.TotTermSearchersStarted, uint64(1)) - return rv, nil -} - -func (i *IndexSnapshot) allocTermFieldReaderDicts(field string) (tfr *IndexSnapshotTermFieldReader) { - i.m2.Lock() - if i.fieldTFRs != nil { - tfrs := i.fieldTFRs[field] - last := len(tfrs) - 1 - if last >= 0 { - tfr = tfrs[last] - tfrs[last] = nil - i.fieldTFRs[field] = tfrs[:last] - i.m2.Unlock() - return - } - } - i.m2.Unlock() - return &IndexSnapshotTermFieldReader{ - recycle: true, - } -} - -func (i *IndexSnapshot) recycleTermFieldReader(tfr *IndexSnapshotTermFieldReader) { - if !tfr.recycle { - // Do not recycle an optimized unadorned term field reader (used for - // ConjunctionUnadorned or DisjunctionUnadorned), during when a fresh - // roaring.Bitmap is built by AND-ing or OR-ing individual bitmaps, - // and we'll need to release them for GC. (See MB-40916) - return - } - - i.parent.rootLock.RLock() - obsolete := i.parent.root != i - i.parent.rootLock.RUnlock() - if obsolete { - // if we're not the current root (mutations happened), don't bother recycling - return - } - - i.m2.Lock() - if i.fieldTFRs == nil { - i.fieldTFRs = map[string][]*IndexSnapshotTermFieldReader{} - } - i.fieldTFRs[tfr.field] = append(i.fieldTFRs[tfr.field], tfr) - i.m2.Unlock() -} - -func docNumberToBytes(buf []byte, in uint64) []byte { - if len(buf) != 8 { - if cap(buf) >= 8 { - buf = buf[0:8] - } else { - buf = make([]byte, 8) - } - } - binary.BigEndian.PutUint64(buf, in) - return buf -} - -func docInternalToNumber(in index.IndexInternalID) (uint64, error) { - if len(in) != 8 { - return 0, fmt.Errorf("wrong len for IndexInternalID: %q", in) - } - return binary.BigEndian.Uint64(in), nil -} - -func (i *IndexSnapshot) DocumentVisitFieldTerms(id index.IndexInternalID, - fields []string, visitor index.DocumentFieldTermVisitor) error { - _, err := i.documentVisitFieldTerms(id, fields, visitor, nil) - return err -} - -func (i *IndexSnapshot) documentVisitFieldTerms(id index.IndexInternalID, - fields []string, visitor index.DocumentFieldTermVisitor, - dvs segment.DocVisitState) (segment.DocVisitState, error) { - docNum, err := docInternalToNumber(id) - if err != nil { - return nil, err - } - - segmentIndex, localDocNum := i.segmentIndexAndLocalDocNumFromGlobal(docNum) - if segmentIndex >= len(i.segment) { - return nil, nil - } - - _, dvs, err = i.documentVisitFieldTermsOnSegment( - segmentIndex, localDocNum, fields, nil, visitor, dvs) - - return dvs, err -} - -func (i *IndexSnapshot) documentVisitFieldTermsOnSegment( - segmentIndex int, localDocNum uint64, fields []string, cFields []string, - visitor index.DocumentFieldTermVisitor, dvs segment.DocVisitState) ( - cFieldsOut []string, dvsOut segment.DocVisitState, err error) { - ss := i.segment[segmentIndex] - - var vFields []string // fields that are visitable via the segment - - ssv, ssvOk := ss.segment.(segment.DocumentFieldTermVisitable) - if ssvOk && ssv != nil { - vFields, err = ssv.VisitableDocValueFields() - if err != nil { - return nil, nil, err - } - } - - var errCh chan error - - // cFields represents the fields that we'll need from the - // cachedDocs, and might be optionally be provided by the caller, - // if the caller happens to know we're on the same segmentIndex - // from a previous invocation - if cFields == nil { - cFields = subtractStrings(fields, vFields) - - if !ss.cachedDocs.hasFields(cFields) { - errCh = make(chan error, 1) - - go func() { - err := ss.cachedDocs.prepareFields(cFields, ss) - if err != nil { - errCh <- err - } - close(errCh) - }() - } - } - - if ssvOk && ssv != nil && len(vFields) > 0 { - dvs, err = ssv.VisitDocumentFieldTerms(localDocNum, fields, visitor, dvs) - if err != nil { - return nil, nil, err - } - } - - if errCh != nil { - err = <-errCh - if err != nil { - return nil, nil, err - } - } - - if len(cFields) > 0 { - ss.cachedDocs.visitDoc(localDocNum, cFields, visitor) - } - - return cFields, dvs, nil -} - -func (i *IndexSnapshot) DocValueReader(fields []string) ( - index.DocValueReader, error) { - return &DocValueReader{i: i, fields: fields, currSegmentIndex: -1}, nil -} - -type DocValueReader struct { - i *IndexSnapshot - fields []string - dvs segment.DocVisitState - - currSegmentIndex int - currCachedFields []string -} - -func (dvr *DocValueReader) VisitDocValues(id index.IndexInternalID, - visitor index.DocumentFieldTermVisitor) (err error) { - docNum, err := docInternalToNumber(id) - if err != nil { - return err - } - - segmentIndex, localDocNum := dvr.i.segmentIndexAndLocalDocNumFromGlobal(docNum) - if segmentIndex >= len(dvr.i.segment) { - return nil - } - - if dvr.currSegmentIndex != segmentIndex { - dvr.currSegmentIndex = segmentIndex - dvr.currCachedFields = nil - } - - dvr.currCachedFields, dvr.dvs, err = dvr.i.documentVisitFieldTermsOnSegment( - dvr.currSegmentIndex, localDocNum, dvr.fields, dvr.currCachedFields, visitor, dvr.dvs) - - return err -} - -func (i *IndexSnapshot) DumpAll() chan interface{} { - rv := make(chan interface{}) - go func() { - close(rv) - }() - return rv -} - -func (i *IndexSnapshot) DumpDoc(id string) chan interface{} { - rv := make(chan interface{}) - go func() { - close(rv) - }() - return rv -} - -func (i *IndexSnapshot) DumpFields() chan interface{} { - rv := make(chan interface{}) - go func() { - close(rv) - }() - return rv -} - -func (i *IndexSnapshot) diskSegmentsPaths() map[string]struct{} { - rv := make(map[string]struct{}, len(i.segment)) - for _, segmentSnapshot := range i.segment { - if seg, ok := segmentSnapshot.segment.(segment.PersistedSegment); ok { - rv[seg.Path()] = struct{}{} - } - } - return rv -} - -// reClaimableDocsRatio gives a ratio about the obsoleted or -// reclaimable documents present in a given index snapshot. -func (i *IndexSnapshot) reClaimableDocsRatio() float64 { - var totalCount, liveCount uint64 - for _, segmentSnapshot := range i.segment { - if _, ok := segmentSnapshot.segment.(segment.PersistedSegment); ok { - totalCount += uint64(segmentSnapshot.FullSize()) - liveCount += uint64(segmentSnapshot.Count()) - } - } - - if totalCount > 0 { - return float64(totalCount-liveCount) / float64(totalCount) - } - return 0 -} - -// subtractStrings returns set a minus elements of set b. -func subtractStrings(a, b []string) []string { - if len(b) == 0 { - return a - } - - rv := make([]string, 0, len(a)) -OUTER: - for _, as := range a { - for _, bs := range b { - if as == bs { - continue OUTER - } - } - rv = append(rv, as) - } - return rv -} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/stats.go b/vendor/github.com/blevesearch/bleve/index/scorch/stats.go deleted file mode 100644 index 626fff2e..00000000 --- a/vendor/github.com/blevesearch/bleve/index/scorch/stats.go +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package scorch - -import ( - "encoding/json" - "reflect" - "sync/atomic" -) - -// Stats tracks statistics about the index, fields that are -// prefixed like CurXxxx are gauges (can go up and down), -// and fields that are prefixed like TotXxxx are monotonically -// increasing counters. -type Stats struct { - TotUpdates uint64 - TotDeletes uint64 - - TotBatches uint64 - TotBatchesEmpty uint64 - TotBatchIntroTime uint64 - MaxBatchIntroTime uint64 - - CurRootEpoch uint64 - LastPersistedEpoch uint64 - LastMergedEpoch uint64 - - TotOnErrors uint64 - - TotAnalysisTime uint64 - TotIndexTime uint64 - - TotIndexedPlainTextBytes uint64 - - TotTermSearchersStarted uint64 - TotTermSearchersFinished uint64 - - TotEventTriggerStarted uint64 - TotEventTriggerCompleted uint64 - - TotIntroduceLoop uint64 - TotIntroduceSegmentBeg uint64 - TotIntroduceSegmentEnd uint64 - TotIntroducePersistBeg uint64 - TotIntroducePersistEnd uint64 - TotIntroduceMergeBeg uint64 - TotIntroduceMergeEnd uint64 - TotIntroduceRevertBeg uint64 - TotIntroduceRevertEnd uint64 - - TotIntroducedItems uint64 - TotIntroducedSegmentsBatch uint64 - TotIntroducedSegmentsMerge uint64 - - TotPersistLoopBeg uint64 - TotPersistLoopErr uint64 - TotPersistLoopProgress uint64 - TotPersistLoopWait uint64 - TotPersistLoopWaitNotified uint64 - TotPersistLoopEnd uint64 - - TotPersistedItems uint64 - TotItemsToPersist uint64 - TotPersistedSegments uint64 - - TotPersisterSlowMergerPause uint64 - TotPersisterSlowMergerResume uint64 - - TotPersisterNapPauseCompleted uint64 - TotPersisterMergerNapBreak uint64 - - TotFileMergeLoopBeg uint64 - TotFileMergeLoopErr uint64 - TotFileMergeLoopEnd uint64 - - TotFileMergeForceOpsStarted uint64 - TotFileMergeForceOpsCompleted uint64 - - TotFileMergePlan uint64 - TotFileMergePlanErr uint64 - TotFileMergePlanNone uint64 - TotFileMergePlanOk uint64 - - TotFileMergePlanTasks uint64 - TotFileMergePlanTasksDone uint64 - TotFileMergePlanTasksErr uint64 - TotFileMergePlanTasksSegments uint64 - TotFileMergePlanTasksSegmentsEmpty uint64 - - TotFileMergeSegmentsEmpty uint64 - TotFileMergeSegments uint64 - TotFileSegmentsAtRoot uint64 - TotFileMergeWrittenBytes uint64 - - TotFileMergeZapBeg uint64 - TotFileMergeZapEnd uint64 - TotFileMergeZapTime uint64 - MaxFileMergeZapTime uint64 - TotFileMergeZapIntroductionTime uint64 - MaxFileMergeZapIntroductionTime uint64 - - TotFileMergeIntroductions uint64 - TotFileMergeIntroductionsDone uint64 - TotFileMergeIntroductionsSkipped uint64 - TotFileMergeIntroductionsObsoleted uint64 - - CurFilesIneligibleForRemoval uint64 - TotSnapshotsRemovedFromMetaStore uint64 - - TotMemMergeBeg uint64 - TotMemMergeErr uint64 - TotMemMergeDone uint64 - TotMemMergeZapBeg uint64 - TotMemMergeZapEnd uint64 - TotMemMergeZapTime uint64 - MaxMemMergeZapTime uint64 - TotMemMergeSegments uint64 - TotMemorySegmentsAtRoot uint64 -} - -// atomically populates the returned map -func (s *Stats) ToMap() map[string]interface{} { - m := map[string]interface{}{} - sve := reflect.ValueOf(s).Elem() - svet := sve.Type() - for i := 0; i < svet.NumField(); i++ { - svef := sve.Field(i) - if svef.CanAddr() { - svefp := svef.Addr().Interface() - m[svet.Field(i).Name] = atomic.LoadUint64(svefp.(*uint64)) - } - } - return m -} - -// MarshalJSON implements json.Marshaler, and in contrast to standard -// json marshaling provides atomic safety -func (s *Stats) MarshalJSON() ([]byte, error) { - return json.Marshal(s.ToMap()) -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/boltdb/reader.go b/vendor/github.com/blevesearch/bleve/index/store/boltdb/reader.go deleted file mode 100644 index 7977ebbe..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/boltdb/reader.go +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package boltdb - -import ( - "github.com/blevesearch/bleve/index/store" - bolt "go.etcd.io/bbolt" -) - -type Reader struct { - store *Store - tx *bolt.Tx - bucket *bolt.Bucket -} - -func (r *Reader) Get(key []byte) ([]byte, error) { - var rv []byte - v := r.bucket.Get(key) - if v != nil { - rv = make([]byte, len(v)) - copy(rv, v) - } - return rv, nil -} - -func (r *Reader) MultiGet(keys [][]byte) ([][]byte, error) { - return store.MultiGet(r, keys) -} - -func (r *Reader) PrefixIterator(prefix []byte) store.KVIterator { - cursor := r.bucket.Cursor() - - rv := &Iterator{ - store: r.store, - tx: r.tx, - cursor: cursor, - prefix: prefix, - } - - rv.Seek(prefix) - return rv -} - -func (r *Reader) RangeIterator(start, end []byte) store.KVIterator { - cursor := r.bucket.Cursor() - - rv := &Iterator{ - store: r.store, - tx: r.tx, - cursor: cursor, - start: start, - end: end, - } - - rv.Seek(start) - return rv -} - -func (r *Reader) Close() error { - return r.tx.Rollback() -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/boltdb/stats.go b/vendor/github.com/blevesearch/bleve/index/store/boltdb/stats.go deleted file mode 100644 index e50e5527..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/boltdb/stats.go +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package boltdb - -import "encoding/json" - -type stats struct { - s *Store -} - -func (s *stats) MarshalJSON() ([]byte, error) { - bs := s.s.db.Stats() - return json.Marshal(bs) -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/boltdb/store.go b/vendor/github.com/blevesearch/bleve/index/store/boltdb/store.go deleted file mode 100644 index 3c749693..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/boltdb/store.go +++ /dev/null @@ -1,181 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package boltdb implements a store.KVStore on top of BoltDB. It supports the -// following options: -// -// "bucket" (string): the name of BoltDB bucket to use, defaults to "bleve". -// -// "nosync" (bool): if true, set boltdb.DB.NoSync to true. It speeds up index -// operations in exchange of losing integrity guarantees if indexation aborts -// without closing the index. Use it when rebuilding indexes from zero. -package boltdb - -import ( - "bytes" - "encoding/json" - "fmt" - "os" - - "github.com/blevesearch/bleve/index/store" - "github.com/blevesearch/bleve/registry" - bolt "go.etcd.io/bbolt" -) - -const ( - Name = "boltdb" - defaultCompactBatchSize = 100 -) - -type Store struct { - path string - bucket string - db *bolt.DB - noSync bool - fillPercent float64 - mo store.MergeOperator -} - -func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) { - path, ok := config["path"].(string) - if !ok { - return nil, fmt.Errorf("must specify path") - } - if path == "" { - return nil, os.ErrInvalid - } - - bucket, ok := config["bucket"].(string) - if !ok { - bucket = "bleve" - } - - noSync, _ := config["nosync"].(bool) - - fillPercent, ok := config["fillPercent"].(float64) - if !ok { - fillPercent = bolt.DefaultFillPercent - } - - bo := &bolt.Options{} - ro, ok := config["read_only"].(bool) - if ok { - bo.ReadOnly = ro - } - - if initialMmapSize, ok := config["initialMmapSize"].(int); ok { - bo.InitialMmapSize = initialMmapSize - } else if initialMmapSize, ok := config["initialMmapSize"].(float64); ok { - bo.InitialMmapSize = int(initialMmapSize) - } - - db, err := bolt.Open(path, 0600, bo) - if err != nil { - return nil, err - } - db.NoSync = noSync - - if !bo.ReadOnly { - err = db.Update(func(tx *bolt.Tx) error { - _, err := tx.CreateBucketIfNotExists([]byte(bucket)) - - return err - }) - if err != nil { - return nil, err - } - } - - rv := Store{ - path: path, - bucket: bucket, - db: db, - mo: mo, - noSync: noSync, - fillPercent: fillPercent, - } - return &rv, nil -} - -func (bs *Store) Close() error { - return bs.db.Close() -} - -func (bs *Store) Reader() (store.KVReader, error) { - tx, err := bs.db.Begin(false) - if err != nil { - return nil, err - } - return &Reader{ - store: bs, - tx: tx, - bucket: tx.Bucket([]byte(bs.bucket)), - }, nil -} - -func (bs *Store) Writer() (store.KVWriter, error) { - return &Writer{ - store: bs, - }, nil -} - -func (bs *Store) Stats() json.Marshaler { - return &stats{ - s: bs, - } -} - -// CompactWithBatchSize removes DictionaryTerm entries with a count of zero (in batchSize batches) -// Removing entries is a workaround for github issue #374. -func (bs *Store) CompactWithBatchSize(batchSize int) error { - for { - cnt := 0 - err := bs.db.Batch(func(tx *bolt.Tx) error { - c := tx.Bucket([]byte(bs.bucket)).Cursor() - prefix := []byte("d") - - for k, v := c.Seek(prefix); bytes.HasPrefix(k, prefix); k, v = c.Next() { - if bytes.Equal(v, []byte{0}) { - cnt++ - if err := c.Delete(); err != nil { - return err - } - if cnt == batchSize { - break - } - } - - } - return nil - }) - if err != nil { - return err - } - - if cnt == 0 { - break - } - } - return nil -} - -// Compact calls CompactWithBatchSize with a default batch size of 100. This is a workaround -// for github issue #374. -func (bs *Store) Compact() error { - return bs.CompactWithBatchSize(defaultCompactBatchSize) -} - -func init() { - registry.RegisterKVStore(Name, New) -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/boltdb/writer.go b/vendor/github.com/blevesearch/bleve/index/store/boltdb/writer.go deleted file mode 100644 index f0935745..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/boltdb/writer.go +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package boltdb - -import ( - "fmt" - - "github.com/blevesearch/bleve/index/store" -) - -type Writer struct { - store *Store -} - -func (w *Writer) NewBatch() store.KVBatch { - return store.NewEmulatedBatch(w.store.mo) -} - -func (w *Writer) NewBatchEx(options store.KVBatchOptions) ([]byte, store.KVBatch, error) { - return make([]byte, options.TotalBytes), w.NewBatch(), nil -} - -func (w *Writer) ExecuteBatch(batch store.KVBatch) (err error) { - - emulatedBatch, ok := batch.(*store.EmulatedBatch) - if !ok { - return fmt.Errorf("wrong type of batch") - } - - tx, err := w.store.db.Begin(true) - if err != nil { - return - } - // defer function to ensure that once started, - // we either Commit tx or Rollback - defer func() { - // if nothing went wrong, commit - if err == nil { - // careful to catch error here too - err = tx.Commit() - } else { - // caller should see error that caused abort, - // not success or failure of Rollback itself - _ = tx.Rollback() - } - }() - - bucket := tx.Bucket([]byte(w.store.bucket)) - bucket.FillPercent = w.store.fillPercent - - for k, mergeOps := range emulatedBatch.Merger.Merges { - kb := []byte(k) - existingVal := bucket.Get(kb) - mergedVal, fullMergeOk := w.store.mo.FullMerge(kb, existingVal, mergeOps) - if !fullMergeOk { - err = fmt.Errorf("merge operator returned failure") - return - } - err = bucket.Put(kb, mergedVal) - if err != nil { - return - } - } - - for _, op := range emulatedBatch.Ops { - if op.V != nil { - err = bucket.Put(op.K, op.V) - if err != nil { - return - } - } else { - err = bucket.Delete(op.K) - if err != nil { - return - } - } - } - return -} - -func (w *Writer) Close() error { - return nil -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/gtreap/reader.go b/vendor/github.com/blevesearch/bleve/index/store/gtreap/reader.go deleted file mode 100644 index 98254d34..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/gtreap/reader.go +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright (c) 2015 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package gtreap provides an in-memory implementation of the -// KVStore interfaces using the gtreap balanced-binary treap, -// copy-on-write data structure. -package gtreap - -import ( - "github.com/blevesearch/bleve/index/store" - - "github.com/steveyen/gtreap" -) - -type Reader struct { - t *gtreap.Treap -} - -func (w *Reader) Get(k []byte) (v []byte, err error) { - var rv []byte - itm := w.t.Get(&Item{k: k}) - if itm != nil { - rv = make([]byte, len(itm.(*Item).v)) - copy(rv, itm.(*Item).v) - return rv, nil - } - return nil, nil -} - -func (r *Reader) MultiGet(keys [][]byte) ([][]byte, error) { - return store.MultiGet(r, keys) -} - -func (w *Reader) PrefixIterator(k []byte) store.KVIterator { - rv := Iterator{ - t: w.t, - prefix: k, - } - rv.restart(&Item{k: k}) - return &rv -} - -func (w *Reader) RangeIterator(start, end []byte) store.KVIterator { - rv := Iterator{ - t: w.t, - start: start, - end: end, - } - rv.restart(&Item{k: start}) - return &rv -} - -func (w *Reader) Close() error { - return nil -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/gtreap/store.go b/vendor/github.com/blevesearch/bleve/index/store/gtreap/store.go deleted file mode 100644 index 3e6c5fec..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/gtreap/store.go +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright (c) 2015 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package gtreap provides an in-memory implementation of the -// KVStore interfaces using the gtreap balanced-binary treap, -// copy-on-write data structure. - -package gtreap - -import ( - "bytes" - "fmt" - "os" - "sync" - - "github.com/blevesearch/bleve/index/store" - "github.com/blevesearch/bleve/registry" - "github.com/steveyen/gtreap" -) - -const Name = "gtreap" - -type Store struct { - m sync.Mutex - t *gtreap.Treap - mo store.MergeOperator -} - -type Item struct { - k []byte - v []byte -} - -func itemCompare(a, b interface{}) int { - return bytes.Compare(a.(*Item).k, b.(*Item).k) -} - -func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) { - path, ok := config["path"].(string) - if !ok { - return nil, fmt.Errorf("must specify path") - } - if path != "" { - return nil, os.ErrInvalid - } - - rv := Store{ - t: gtreap.NewTreap(itemCompare), - mo: mo, - } - return &rv, nil -} - -func (s *Store) Close() error { - return nil -} - -func (s *Store) Reader() (store.KVReader, error) { - s.m.Lock() - t := s.t - s.m.Unlock() - return &Reader{t: t}, nil -} - -func (s *Store) Writer() (store.KVWriter, error) { - return &Writer{s: s}, nil -} - -func init() { - registry.RegisterKVStore(Name, New) -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/gtreap/writer.go b/vendor/github.com/blevesearch/bleve/index/store/gtreap/writer.go deleted file mode 100644 index 777aab40..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/gtreap/writer.go +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright (c) 2015 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package gtreap provides an in-memory implementation of the -// KVStore interfaces using the gtreap balanced-binary treap, -// copy-on-write data structure. -package gtreap - -import ( - "fmt" - "math/rand" - - "github.com/blevesearch/bleve/index/store" -) - -type Writer struct { - s *Store -} - -func (w *Writer) NewBatch() store.KVBatch { - return store.NewEmulatedBatch(w.s.mo) -} - -func (w *Writer) NewBatchEx(options store.KVBatchOptions) ([]byte, store.KVBatch, error) { - return make([]byte, options.TotalBytes), w.NewBatch(), nil -} - -func (w *Writer) ExecuteBatch(batch store.KVBatch) error { - - emulatedBatch, ok := batch.(*store.EmulatedBatch) - if !ok { - return fmt.Errorf("wrong type of batch") - } - - w.s.m.Lock() - for k, mergeOps := range emulatedBatch.Merger.Merges { - kb := []byte(k) - var existingVal []byte - existingItem := w.s.t.Get(&Item{k: kb}) - if existingItem != nil { - existingVal = w.s.t.Get(&Item{k: kb}).(*Item).v - } - mergedVal, fullMergeOk := w.s.mo.FullMerge(kb, existingVal, mergeOps) - if !fullMergeOk { - return fmt.Errorf("merge operator returned failure") - } - w.s.t = w.s.t.Upsert(&Item{k: kb, v: mergedVal}, rand.Int()) - } - - for _, op := range emulatedBatch.Ops { - if op.V != nil { - w.s.t = w.s.t.Upsert(&Item{k: op.K, v: op.V}, rand.Int()) - } else { - w.s.t = w.s.t.Delete(&Item{k: op.K}) - } - } - w.s.m.Unlock() - - return nil -} - -func (w *Writer) Close() error { - w.s = nil - return nil -} diff --git a/vendor/github.com/blevesearch/bleve/index/upsidedown/analysis.go b/vendor/github.com/blevesearch/bleve/index/upsidedown/analysis.go deleted file mode 100644 index d1b1fd59..00000000 --- a/vendor/github.com/blevesearch/bleve/index/upsidedown/analysis.go +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright (c) 2015 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package upsidedown - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/index" -) - -func (udc *UpsideDownCouch) Analyze(d *document.Document) *index.AnalysisResult { - rv := &index.AnalysisResult{ - DocID: d.ID, - Rows: make([]index.IndexRow, 0, 100), - } - - docIDBytes := []byte(d.ID) - - // track our back index entries - backIndexStoredEntries := make([]*BackIndexStoreEntry, 0) - - // information we collate as we merge fields with same name - fieldTermFreqs := make(map[uint16]analysis.TokenFrequencies) - fieldLengths := make(map[uint16]int) - fieldIncludeTermVectors := make(map[uint16]bool) - fieldNames := make(map[uint16]string) - - analyzeField := func(field document.Field, storable bool) { - fieldIndex, newFieldRow := udc.fieldIndexOrNewRow(field.Name()) - if newFieldRow != nil { - rv.Rows = append(rv.Rows, newFieldRow) - } - fieldNames[fieldIndex] = field.Name() - - if field.Options().IsIndexed() { - fieldLength, tokenFreqs := field.Analyze() - existingFreqs := fieldTermFreqs[fieldIndex] - if existingFreqs == nil { - fieldTermFreqs[fieldIndex] = tokenFreqs - } else { - existingFreqs.MergeAll(field.Name(), tokenFreqs) - fieldTermFreqs[fieldIndex] = existingFreqs - } - fieldLengths[fieldIndex] += fieldLength - fieldIncludeTermVectors[fieldIndex] = field.Options().IncludeTermVectors() - } - - if storable && field.Options().IsStored() { - rv.Rows, backIndexStoredEntries = udc.storeField(docIDBytes, field, fieldIndex, rv.Rows, backIndexStoredEntries) - } - } - - // walk all the fields, record stored fields now - // place information about indexed fields into map - // this collates information across fields with - // same names (arrays) - for _, field := range d.Fields { - analyzeField(field, true) - } - - if len(d.CompositeFields) > 0 { - for fieldIndex, tokenFreqs := range fieldTermFreqs { - // see if any of the composite fields need this - for _, compositeField := range d.CompositeFields { - compositeField.Compose(fieldNames[fieldIndex], fieldLengths[fieldIndex], tokenFreqs) - } - } - - for _, compositeField := range d.CompositeFields { - analyzeField(compositeField, false) - } - } - - rowsCapNeeded := len(rv.Rows) + 1 - for _, tokenFreqs := range fieldTermFreqs { - rowsCapNeeded += len(tokenFreqs) - } - - rv.Rows = append(make([]index.IndexRow, 0, rowsCapNeeded), rv.Rows...) - - backIndexTermsEntries := make([]*BackIndexTermsEntry, 0, len(fieldTermFreqs)) - - // walk through the collated information and process - // once for each indexed field (unique name) - for fieldIndex, tokenFreqs := range fieldTermFreqs { - fieldLength := fieldLengths[fieldIndex] - includeTermVectors := fieldIncludeTermVectors[fieldIndex] - - // encode this field - rv.Rows, backIndexTermsEntries = udc.indexField(docIDBytes, includeTermVectors, fieldIndex, fieldLength, tokenFreqs, rv.Rows, backIndexTermsEntries) - } - - // build the back index row - backIndexRow := NewBackIndexRow(docIDBytes, backIndexTermsEntries, backIndexStoredEntries) - rv.Rows = append(rv.Rows, backIndexRow) - - return rv -} diff --git a/vendor/github.com/blevesearch/bleve/index/upsidedown/reader.go b/vendor/github.com/blevesearch/bleve/index/upsidedown/reader.go deleted file mode 100644 index bc0fef11..00000000 --- a/vendor/github.com/blevesearch/bleve/index/upsidedown/reader.go +++ /dev/null @@ -1,376 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package upsidedown - -import ( - "bytes" - "reflect" - "sort" - "sync/atomic" - - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/store" - "github.com/blevesearch/bleve/size" -) - -var reflectStaticSizeUpsideDownCouchTermFieldReader int -var reflectStaticSizeUpsideDownCouchDocIDReader int - -func init() { - var tfr UpsideDownCouchTermFieldReader - reflectStaticSizeUpsideDownCouchTermFieldReader = - int(reflect.TypeOf(tfr).Size()) - var cdr UpsideDownCouchDocIDReader - reflectStaticSizeUpsideDownCouchDocIDReader = - int(reflect.TypeOf(cdr).Size()) -} - -type UpsideDownCouchTermFieldReader struct { - count uint64 - indexReader *IndexReader - iterator store.KVIterator - term []byte - tfrNext *TermFrequencyRow - tfrPrealloc TermFrequencyRow - keyBuf []byte - field uint16 - includeTermVectors bool -} - -func (r *UpsideDownCouchTermFieldReader) Size() int { - sizeInBytes := reflectStaticSizeUpsideDownCouchTermFieldReader + size.SizeOfPtr + - len(r.term) + - r.tfrPrealloc.Size() + - len(r.keyBuf) - - if r.tfrNext != nil { - sizeInBytes += r.tfrNext.Size() - } - - return sizeInBytes -} - -func newUpsideDownCouchTermFieldReader(indexReader *IndexReader, term []byte, field uint16, includeFreq, includeNorm, includeTermVectors bool) (*UpsideDownCouchTermFieldReader, error) { - bufNeeded := termFrequencyRowKeySize(term, nil) - if bufNeeded < dictionaryRowKeySize(term) { - bufNeeded = dictionaryRowKeySize(term) - } - buf := make([]byte, bufNeeded) - - bufUsed := dictionaryRowKeyTo(buf, field, term) - val, err := indexReader.kvreader.Get(buf[:bufUsed]) - if err != nil { - return nil, err - } - if val == nil { - atomic.AddUint64(&indexReader.index.stats.termSearchersStarted, uint64(1)) - rv := &UpsideDownCouchTermFieldReader{ - count: 0, - term: term, - field: field, - includeTermVectors: includeTermVectors, - } - rv.tfrNext = &rv.tfrPrealloc - return rv, nil - } - - count, err := dictionaryRowParseV(val) - if err != nil { - return nil, err - } - - bufUsed = termFrequencyRowKeyTo(buf, field, term, nil) - it := indexReader.kvreader.PrefixIterator(buf[:bufUsed]) - - atomic.AddUint64(&indexReader.index.stats.termSearchersStarted, uint64(1)) - return &UpsideDownCouchTermFieldReader{ - indexReader: indexReader, - iterator: it, - count: count, - term: term, - field: field, - includeTermVectors: includeTermVectors, - }, nil -} - -func (r *UpsideDownCouchTermFieldReader) Count() uint64 { - return r.count -} - -func (r *UpsideDownCouchTermFieldReader) Next(preAlloced *index.TermFieldDoc) (*index.TermFieldDoc, error) { - if r.iterator != nil { - // We treat tfrNext also like an initialization flag, which - // tells us whether we need to invoke the underlying - // iterator.Next(). The first time, don't call iterator.Next(). - if r.tfrNext != nil { - r.iterator.Next() - } else { - r.tfrNext = &r.tfrPrealloc - } - key, val, valid := r.iterator.Current() - if valid { - tfr := r.tfrNext - err := tfr.parseKDoc(key, r.term) - if err != nil { - return nil, err - } - err = tfr.parseV(val, r.includeTermVectors) - if err != nil { - return nil, err - } - rv := preAlloced - if rv == nil { - rv = &index.TermFieldDoc{} - } - rv.ID = append(rv.ID, tfr.doc...) - rv.Freq = tfr.freq - rv.Norm = float64(tfr.norm) - if tfr.vectors != nil { - rv.Vectors = r.indexReader.index.termFieldVectorsFromTermVectors(tfr.vectors) - } - return rv, nil - } - } - return nil, nil -} - -func (r *UpsideDownCouchTermFieldReader) Advance(docID index.IndexInternalID, preAlloced *index.TermFieldDoc) (rv *index.TermFieldDoc, err error) { - if r.iterator != nil { - if r.tfrNext == nil { - r.tfrNext = &TermFrequencyRow{} - } - tfr := InitTermFrequencyRow(r.tfrNext, r.term, r.field, docID, 0, 0) - r.keyBuf, err = tfr.KeyAppendTo(r.keyBuf[:0]) - if err != nil { - return nil, err - } - r.iterator.Seek(r.keyBuf) - key, val, valid := r.iterator.Current() - if valid { - err := tfr.parseKDoc(key, r.term) - if err != nil { - return nil, err - } - err = tfr.parseV(val, r.includeTermVectors) - if err != nil { - return nil, err - } - rv = preAlloced - if rv == nil { - rv = &index.TermFieldDoc{} - } - rv.ID = append(rv.ID, tfr.doc...) - rv.Freq = tfr.freq - rv.Norm = float64(tfr.norm) - if tfr.vectors != nil { - rv.Vectors = r.indexReader.index.termFieldVectorsFromTermVectors(tfr.vectors) - } - return rv, nil - } - } - return nil, nil -} - -func (r *UpsideDownCouchTermFieldReader) Close() error { - if r.indexReader != nil { - atomic.AddUint64(&r.indexReader.index.stats.termSearchersFinished, uint64(1)) - } - if r.iterator != nil { - return r.iterator.Close() - } - return nil -} - -type UpsideDownCouchDocIDReader struct { - indexReader *IndexReader - iterator store.KVIterator - only []string - onlyPos int - onlyMode bool -} - -func (r *UpsideDownCouchDocIDReader) Size() int { - sizeInBytes := reflectStaticSizeUpsideDownCouchDocIDReader + - reflectStaticSizeIndexReader + size.SizeOfPtr - - for _, entry := range r.only { - sizeInBytes += size.SizeOfString + len(entry) - } - - return sizeInBytes -} - -func newUpsideDownCouchDocIDReader(indexReader *IndexReader) (*UpsideDownCouchDocIDReader, error) { - startBytes := []byte{0x0} - endBytes := []byte{0xff} - - bisr := NewBackIndexRow(startBytes, nil, nil) - bier := NewBackIndexRow(endBytes, nil, nil) - it := indexReader.kvreader.RangeIterator(bisr.Key(), bier.Key()) - - return &UpsideDownCouchDocIDReader{ - indexReader: indexReader, - iterator: it, - }, nil -} - -func newUpsideDownCouchDocIDReaderOnly(indexReader *IndexReader, ids []string) (*UpsideDownCouchDocIDReader, error) { - // we don't actually own the list of ids, so if before we sort we must copy - idsCopy := make([]string, len(ids)) - copy(idsCopy, ids) - // ensure ids are sorted - sort.Strings(idsCopy) - startBytes := []byte{0x0} - if len(idsCopy) > 0 { - startBytes = []byte(idsCopy[0]) - } - endBytes := []byte{0xff} - if len(idsCopy) > 0 { - endBytes = incrementBytes([]byte(idsCopy[len(idsCopy)-1])) - } - bisr := NewBackIndexRow(startBytes, nil, nil) - bier := NewBackIndexRow(endBytes, nil, nil) - it := indexReader.kvreader.RangeIterator(bisr.Key(), bier.Key()) - - return &UpsideDownCouchDocIDReader{ - indexReader: indexReader, - iterator: it, - only: idsCopy, - onlyMode: true, - }, nil -} - -func (r *UpsideDownCouchDocIDReader) Next() (index.IndexInternalID, error) { - key, val, valid := r.iterator.Current() - - if r.onlyMode { - var rv index.IndexInternalID - for valid && r.onlyPos < len(r.only) { - br, err := NewBackIndexRowKV(key, val) - if err != nil { - return nil, err - } - if !bytes.Equal(br.doc, []byte(r.only[r.onlyPos])) { - ok := r.nextOnly() - if !ok { - return nil, nil - } - r.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key()) - key, val, valid = r.iterator.Current() - continue - } else { - rv = append([]byte(nil), br.doc...) - break - } - } - if valid && r.onlyPos < len(r.only) { - ok := r.nextOnly() - if ok { - r.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key()) - } - return rv, nil - } - - } else { - if valid { - br, err := NewBackIndexRowKV(key, val) - if err != nil { - return nil, err - } - rv := append([]byte(nil), br.doc...) - r.iterator.Next() - return rv, nil - } - } - return nil, nil -} - -func (r *UpsideDownCouchDocIDReader) Advance(docID index.IndexInternalID) (index.IndexInternalID, error) { - - if r.onlyMode { - r.onlyPos = sort.SearchStrings(r.only, string(docID)) - if r.onlyPos >= len(r.only) { - // advanced to key after our last only key - return nil, nil - } - r.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key()) - key, val, valid := r.iterator.Current() - - var rv index.IndexInternalID - for valid && r.onlyPos < len(r.only) { - br, err := NewBackIndexRowKV(key, val) - if err != nil { - return nil, err - } - if !bytes.Equal(br.doc, []byte(r.only[r.onlyPos])) { - // the only key we seek'd to didn't exist - // now look for the closest key that did exist in only - r.onlyPos = sort.SearchStrings(r.only, string(br.doc)) - if r.onlyPos >= len(r.only) { - // advanced to key after our last only key - return nil, nil - } - // now seek to this new only key - r.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key()) - key, val, valid = r.iterator.Current() - continue - } else { - rv = append([]byte(nil), br.doc...) - break - } - } - if valid && r.onlyPos < len(r.only) { - ok := r.nextOnly() - if ok { - r.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key()) - } - return rv, nil - } - } else { - bir := NewBackIndexRow(docID, nil, nil) - r.iterator.Seek(bir.Key()) - key, val, valid := r.iterator.Current() - if valid { - br, err := NewBackIndexRowKV(key, val) - if err != nil { - return nil, err - } - rv := append([]byte(nil), br.doc...) - r.iterator.Next() - return rv, nil - } - } - return nil, nil -} - -func (r *UpsideDownCouchDocIDReader) Close() error { - return r.iterator.Close() -} - -// move the r.only pos forward one, skipping duplicates -// return true if there is more data, or false if we got to the end of the list -func (r *UpsideDownCouchDocIDReader) nextOnly() bool { - - // advance 1 position, until we see a different key - // it's already sorted, so this skips duplicates - start := r.onlyPos - r.onlyPos++ - for r.onlyPos < len(r.only) && r.only[r.onlyPos] == r.only[start] { - start = r.onlyPos - r.onlyPos++ - } - // inidicate if we got to the end of the list - return r.onlyPos < len(r.only) -} diff --git a/vendor/github.com/blevesearch/bleve/index/upsidedown/stats.go b/vendor/github.com/blevesearch/bleve/index/upsidedown/stats.go deleted file mode 100644 index a148ab70..00000000 --- a/vendor/github.com/blevesearch/bleve/index/upsidedown/stats.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package upsidedown - -import ( - "encoding/json" - "sync/atomic" - - "github.com/blevesearch/bleve/index/store" -) - -type indexStat struct { - updates, deletes, batches, errors uint64 - analysisTime, indexTime uint64 - termSearchersStarted uint64 - termSearchersFinished uint64 - numPlainTextBytesIndexed uint64 - i *UpsideDownCouch -} - -func (i *indexStat) statsMap() map[string]interface{} { - m := map[string]interface{}{} - m["updates"] = atomic.LoadUint64(&i.updates) - m["deletes"] = atomic.LoadUint64(&i.deletes) - m["batches"] = atomic.LoadUint64(&i.batches) - m["errors"] = atomic.LoadUint64(&i.errors) - m["analysis_time"] = atomic.LoadUint64(&i.analysisTime) - m["index_time"] = atomic.LoadUint64(&i.indexTime) - m["term_searchers_started"] = atomic.LoadUint64(&i.termSearchersStarted) - m["term_searchers_finished"] = atomic.LoadUint64(&i.termSearchersFinished) - m["num_plain_text_bytes_indexed"] = atomic.LoadUint64(&i.numPlainTextBytesIndexed) - - if o, ok := i.i.store.(store.KVStoreStats); ok { - m["kv"] = o.StatsMap() - } - - return m -} - -func (i *indexStat) MarshalJSON() ([]byte, error) { - m := i.statsMap() - return json.Marshal(m) -} diff --git a/vendor/github.com/blevesearch/bleve/mapping.go b/vendor/github.com/blevesearch/bleve/mapping.go deleted file mode 100644 index 76238dc1..00000000 --- a/vendor/github.com/blevesearch/bleve/mapping.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bleve - -import "github.com/blevesearch/bleve/mapping" - -// NewIndexMapping creates a new IndexMapping that will use all the default indexing rules -func NewIndexMapping() *mapping.IndexMappingImpl { - return mapping.NewIndexMapping() -} - -// NewDocumentMapping returns a new document mapping -// with all the default values. -func NewDocumentMapping() *mapping.DocumentMapping { - return mapping.NewDocumentMapping() -} - -// NewDocumentStaticMapping returns a new document -// mapping that will not automatically index parts -// of a document without an explicit mapping. -func NewDocumentStaticMapping() *mapping.DocumentMapping { - return mapping.NewDocumentStaticMapping() -} - -// NewDocumentDisabledMapping returns a new document -// mapping that will not perform any indexing. -func NewDocumentDisabledMapping() *mapping.DocumentMapping { - return mapping.NewDocumentDisabledMapping() -} - -// NewTextFieldMapping returns a default field mapping for text -func NewTextFieldMapping() *mapping.FieldMapping { - return mapping.NewTextFieldMapping() -} - -// NewNumericFieldMapping returns a default field mapping for numbers -func NewNumericFieldMapping() *mapping.FieldMapping { - return mapping.NewNumericFieldMapping() -} - -// NewDateTimeFieldMapping returns a default field mapping for dates -func NewDateTimeFieldMapping() *mapping.FieldMapping { - return mapping.NewDateTimeFieldMapping() -} - -// NewBooleanFieldMapping returns a default field mapping for booleans -func NewBooleanFieldMapping() *mapping.FieldMapping { - return mapping.NewBooleanFieldMapping() -} - -func NewGeoPointFieldMapping() *mapping.FieldMapping { - return mapping.NewGeoPointFieldMapping() -} diff --git a/vendor/github.com/blevesearch/bleve/mapping/document.go b/vendor/github.com/blevesearch/bleve/mapping/document.go deleted file mode 100644 index dd42fab9..00000000 --- a/vendor/github.com/blevesearch/bleve/mapping/document.go +++ /dev/null @@ -1,558 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package mapping - -import ( - "encoding" - "encoding/json" - "fmt" - "reflect" - "time" - - "github.com/blevesearch/bleve/registry" -) - -// A DocumentMapping describes how a type of document -// should be indexed. -// As documents can be hierarchical, named sub-sections -// of documents are mapped using the same structure in -// the Properties field. -// Each value inside a document can be indexed 0 or more -// ways. These index entries are called fields and -// are stored in the Fields field. -// Entire sections of a document can be ignored or -// excluded by setting Enabled to false. -// If not explicitly mapped, default mapping operations -// are used. To disable this automatic handling, set -// Dynamic to false. -type DocumentMapping struct { - Enabled bool `json:"enabled"` - Dynamic bool `json:"dynamic"` - Properties map[string]*DocumentMapping `json:"properties,omitempty"` - Fields []*FieldMapping `json:"fields,omitempty"` - DefaultAnalyzer string `json:"default_analyzer,omitempty"` - - // StructTagKey overrides "json" when looking for field names in struct tags - StructTagKey string `json:"struct_tag_key,omitempty"` -} - -func (dm *DocumentMapping) Validate(cache *registry.Cache) error { - var err error - if dm.DefaultAnalyzer != "" { - _, err := cache.AnalyzerNamed(dm.DefaultAnalyzer) - if err != nil { - return err - } - } - for _, property := range dm.Properties { - err = property.Validate(cache) - if err != nil { - return err - } - } - for _, field := range dm.Fields { - if field.Analyzer != "" { - _, err = cache.AnalyzerNamed(field.Analyzer) - if err != nil { - return err - } - } - if field.DateFormat != "" { - _, err = cache.DateTimeParserNamed(field.DateFormat) - if err != nil { - return err - } - } - switch field.Type { - case "text", "datetime", "number", "boolean", "geopoint": - default: - return fmt.Errorf("unknown field type: '%s'", field.Type) - } - } - return nil -} - -// analyzerNameForPath attempts to first find the field -// described by this path, then returns the analyzer -// configured for that field -func (dm *DocumentMapping) analyzerNameForPath(path string) string { - field := dm.fieldDescribedByPath(path) - if field != nil { - return field.Analyzer - } - return "" -} - -func (dm *DocumentMapping) fieldDescribedByPath(path string) *FieldMapping { - pathElements := decodePath(path) - if len(pathElements) > 1 { - // easy case, there is more than 1 path element remaining - // the next path element must match a property name - // at this level - for propName, subDocMapping := range dm.Properties { - if propName == pathElements[0] { - return subDocMapping.fieldDescribedByPath(encodePath(pathElements[1:])) - } - } - } - - // either the path just had one element - // or it had multiple, but no match for the first element at this level - // look for match with full path - - // first look for property name with empty field - for propName, subDocMapping := range dm.Properties { - if propName == path { - // found property name match, now look at its fields - for _, field := range subDocMapping.Fields { - if field.Name == "" || field.Name == path { - // match - return field - } - } - } - } - // next, walk the properties again, looking for field overriding the name - for propName, subDocMapping := range dm.Properties { - if propName != path { - // property name isn't a match, but field name could override it - for _, field := range subDocMapping.Fields { - if field.Name == path { - return field - } - } - } - } - - return nil -} - -// documentMappingForPath only returns EXACT matches for a sub document -// or for an explicitly mapped field, if you want to find the -// closest document mapping to a field not explicitly mapped -// use closestDocMapping -func (dm *DocumentMapping) documentMappingForPath(path string) *DocumentMapping { - pathElements := decodePath(path) - current := dm -OUTER: - for i, pathElement := range pathElements { - for name, subDocMapping := range current.Properties { - if name == pathElement { - current = subDocMapping - continue OUTER - } - } - // no subDocMapping matches this pathElement - // only if this is the last element check for field name - if i == len(pathElements)-1 { - for _, field := range current.Fields { - if field.Name == pathElement { - break - } - } - } - - return nil - } - return current -} - -// closestDocMapping findest the most specific document mapping that matches -// part of the provided path -func (dm *DocumentMapping) closestDocMapping(path string) *DocumentMapping { - pathElements := decodePath(path) - current := dm -OUTER: - for _, pathElement := range pathElements { - for name, subDocMapping := range current.Properties { - if name == pathElement { - current = subDocMapping - continue OUTER - } - } - break - } - return current -} - -// NewDocumentMapping returns a new document mapping -// with all the default values. -func NewDocumentMapping() *DocumentMapping { - return &DocumentMapping{ - Enabled: true, - Dynamic: true, - } -} - -// NewDocumentStaticMapping returns a new document -// mapping that will not automatically index parts -// of a document without an explicit mapping. -func NewDocumentStaticMapping() *DocumentMapping { - return &DocumentMapping{ - Enabled: true, - } -} - -// NewDocumentDisabledMapping returns a new document -// mapping that will not perform any indexing. -func NewDocumentDisabledMapping() *DocumentMapping { - return &DocumentMapping{} -} - -// AddSubDocumentMapping adds the provided DocumentMapping as a sub-mapping -// for the specified named subsection. -func (dm *DocumentMapping) AddSubDocumentMapping(property string, sdm *DocumentMapping) { - if dm.Properties == nil { - dm.Properties = make(map[string]*DocumentMapping) - } - dm.Properties[property] = sdm -} - -// AddFieldMappingsAt adds one or more FieldMappings -// at the named sub-document. If the named sub-document -// doesn't yet exist it is created for you. -// This is a convenience function to make most common -// mappings more concise. -// Otherwise, you would: -// subMapping := NewDocumentMapping() -// subMapping.AddFieldMapping(fieldMapping) -// parentMapping.AddSubDocumentMapping(property, subMapping) -func (dm *DocumentMapping) AddFieldMappingsAt(property string, fms ...*FieldMapping) { - if dm.Properties == nil { - dm.Properties = make(map[string]*DocumentMapping) - } - sdm, ok := dm.Properties[property] - if !ok { - sdm = NewDocumentMapping() - } - for _, fm := range fms { - sdm.AddFieldMapping(fm) - } - dm.Properties[property] = sdm -} - -// AddFieldMapping adds the provided FieldMapping for this section -// of the document. -func (dm *DocumentMapping) AddFieldMapping(fm *FieldMapping) { - if dm.Fields == nil { - dm.Fields = make([]*FieldMapping, 0) - } - dm.Fields = append(dm.Fields, fm) -} - -// UnmarshalJSON offers custom unmarshaling with optional strict validation -func (dm *DocumentMapping) UnmarshalJSON(data []byte) error { - var tmp map[string]json.RawMessage - err := json.Unmarshal(data, &tmp) - if err != nil { - return err - } - - // set defaults for fields which might have been omitted - dm.Enabled = true - dm.Dynamic = true - - var invalidKeys []string - for k, v := range tmp { - switch k { - case "enabled": - err := json.Unmarshal(v, &dm.Enabled) - if err != nil { - return err - } - case "dynamic": - err := json.Unmarshal(v, &dm.Dynamic) - if err != nil { - return err - } - case "default_analyzer": - err := json.Unmarshal(v, &dm.DefaultAnalyzer) - if err != nil { - return err - } - case "properties": - err := json.Unmarshal(v, &dm.Properties) - if err != nil { - return err - } - case "fields": - err := json.Unmarshal(v, &dm.Fields) - if err != nil { - return err - } - case "struct_tag_key": - err := json.Unmarshal(v, &dm.StructTagKey) - if err != nil { - return err - } - default: - invalidKeys = append(invalidKeys, k) - } - } - - if MappingJSONStrict && len(invalidKeys) > 0 { - return fmt.Errorf("document mapping contains invalid keys: %v", invalidKeys) - } - - return nil -} - -func (dm *DocumentMapping) defaultAnalyzerName(path []string) string { - current := dm - rv := current.DefaultAnalyzer - for _, pathElement := range path { - var ok bool - current, ok = current.Properties[pathElement] - if !ok { - break - } - if current.DefaultAnalyzer != "" { - rv = current.DefaultAnalyzer - } - } - return rv -} - -func (dm *DocumentMapping) walkDocument(data interface{}, path []string, indexes []uint64, context *walkContext) { - // allow default "json" tag to be overridden - structTagKey := dm.StructTagKey - if structTagKey == "" { - structTagKey = "json" - } - - val := reflect.ValueOf(data) - if !val.IsValid() { - return - } - - typ := val.Type() - switch typ.Kind() { - case reflect.Map: - // FIXME can add support for other map keys in the future - if typ.Key().Kind() == reflect.String { - for _, key := range val.MapKeys() { - fieldName := key.String() - fieldVal := val.MapIndex(key).Interface() - dm.processProperty(fieldVal, append(path, fieldName), indexes, context) - } - } - case reflect.Struct: - for i := 0; i < val.NumField(); i++ { - field := typ.Field(i) - fieldName := field.Name - // anonymous fields of type struct can elide the type name - if field.Anonymous && field.Type.Kind() == reflect.Struct { - fieldName = "" - } - - // if the field has a name under the specified tag, prefer that - tag := field.Tag.Get(structTagKey) - tagFieldName := parseTagName(tag) - if tagFieldName == "-" { - continue - } - // allow tag to set field name to empty, only if anonymous - if field.Tag != "" && (tagFieldName != "" || field.Anonymous) { - fieldName = tagFieldName - } - - if val.Field(i).CanInterface() { - fieldVal := val.Field(i).Interface() - newpath := path - if fieldName != "" { - newpath = append(path, fieldName) - } - dm.processProperty(fieldVal, newpath, indexes, context) - } - } - case reflect.Slice, reflect.Array: - for i := 0; i < val.Len(); i++ { - if val.Index(i).CanInterface() { - fieldVal := val.Index(i).Interface() - dm.processProperty(fieldVal, path, append(indexes, uint64(i)), context) - } - } - case reflect.Ptr: - ptrElem := val.Elem() - if ptrElem.IsValid() && ptrElem.CanInterface() { - dm.processProperty(ptrElem.Interface(), path, indexes, context) - } - case reflect.String: - dm.processProperty(val.String(), path, indexes, context) - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - dm.processProperty(float64(val.Int()), path, indexes, context) - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: - dm.processProperty(float64(val.Uint()), path, indexes, context) - case reflect.Float32, reflect.Float64: - dm.processProperty(float64(val.Float()), path, indexes, context) - case reflect.Bool: - dm.processProperty(val.Bool(), path, indexes, context) - } - -} - -func (dm *DocumentMapping) processProperty(property interface{}, path []string, indexes []uint64, context *walkContext) { - pathString := encodePath(path) - // look to see if there is a mapping for this field - subDocMapping := dm.documentMappingForPath(pathString) - closestDocMapping := dm.closestDocMapping(pathString) - - // check to see if we even need to do further processing - if subDocMapping != nil && !subDocMapping.Enabled { - return - } - - propertyValue := reflect.ValueOf(property) - if !propertyValue.IsValid() { - // cannot do anything with the zero value - return - } - propertyType := propertyValue.Type() - switch propertyType.Kind() { - case reflect.String: - propertyValueString := propertyValue.String() - if subDocMapping != nil { - // index by explicit mapping - for _, fieldMapping := range subDocMapping.Fields { - if fieldMapping.Type == "geopoint" { - fieldMapping.processGeoPoint(property, pathString, path, indexes, context) - } else { - fieldMapping.processString(propertyValueString, pathString, path, indexes, context) - } - } - } else if closestDocMapping.Dynamic { - // automatic indexing behavior - - // first see if it can be parsed by the default date parser - dateTimeParser := context.im.DateTimeParserNamed(context.im.DefaultDateTimeParser) - if dateTimeParser != nil { - parsedDateTime, err := dateTimeParser.ParseDateTime(propertyValueString) - if err != nil { - // index as text - fieldMapping := newTextFieldMappingDynamic(context.im) - fieldMapping.processString(propertyValueString, pathString, path, indexes, context) - } else { - // index as datetime - fieldMapping := newDateTimeFieldMappingDynamic(context.im) - fieldMapping.processTime(parsedDateTime, pathString, path, indexes, context) - } - } - } - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - dm.processProperty(float64(propertyValue.Int()), path, indexes, context) - return - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: - dm.processProperty(float64(propertyValue.Uint()), path, indexes, context) - return - case reflect.Float64, reflect.Float32: - propertyValFloat := propertyValue.Float() - if subDocMapping != nil { - // index by explicit mapping - for _, fieldMapping := range subDocMapping.Fields { - fieldMapping.processFloat64(propertyValFloat, pathString, path, indexes, context) - } - } else if closestDocMapping.Dynamic { - // automatic indexing behavior - fieldMapping := newNumericFieldMappingDynamic(context.im) - fieldMapping.processFloat64(propertyValFloat, pathString, path, indexes, context) - } - case reflect.Bool: - propertyValBool := propertyValue.Bool() - if subDocMapping != nil { - // index by explicit mapping - for _, fieldMapping := range subDocMapping.Fields { - fieldMapping.processBoolean(propertyValBool, pathString, path, indexes, context) - } - } else if closestDocMapping.Dynamic { - // automatic indexing behavior - fieldMapping := newBooleanFieldMappingDynamic(context.im) - fieldMapping.processBoolean(propertyValBool, pathString, path, indexes, context) - } - case reflect.Struct: - switch property := property.(type) { - case time.Time: - // don't descend into the time struct - if subDocMapping != nil { - // index by explicit mapping - for _, fieldMapping := range subDocMapping.Fields { - fieldMapping.processTime(property, pathString, path, indexes, context) - } - } else if closestDocMapping.Dynamic { - fieldMapping := newDateTimeFieldMappingDynamic(context.im) - fieldMapping.processTime(property, pathString, path, indexes, context) - } - case encoding.TextMarshaler: - txt, err := property.MarshalText() - if err == nil && subDocMapping != nil { - // index by explicit mapping - for _, fieldMapping := range subDocMapping.Fields { - if fieldMapping.Type == "text" { - fieldMapping.processString(string(txt), pathString, path, indexes, context) - } - } - } - dm.walkDocument(property, path, indexes, context) - default: - if subDocMapping != nil { - for _, fieldMapping := range subDocMapping.Fields { - if fieldMapping.Type == "geopoint" { - fieldMapping.processGeoPoint(property, pathString, path, indexes, context) - } - } - } - dm.walkDocument(property, path, indexes, context) - } - case reflect.Map, reflect.Slice: - if subDocMapping != nil { - for _, fieldMapping := range subDocMapping.Fields { - if fieldMapping.Type == "geopoint" { - fieldMapping.processGeoPoint(property, pathString, path, indexes, context) - } - } - } - dm.walkDocument(property, path, indexes, context) - case reflect.Ptr: - if !propertyValue.IsNil() { - switch property := property.(type) { - case encoding.TextMarshaler: - // ONLY process TextMarshaler if there is an explicit mapping - // AND all of the fiels are of type text - // OTHERWISE process field without TextMarshaler - if subDocMapping != nil { - allFieldsText := true - for _, fieldMapping := range subDocMapping.Fields { - if fieldMapping.Type != "text" { - allFieldsText = false - break - } - } - txt, err := property.MarshalText() - if err == nil && allFieldsText { - txtStr := string(txt) - for _, fieldMapping := range subDocMapping.Fields { - fieldMapping.processString(txtStr, pathString, path, indexes, context) - } - return - } - } - dm.walkDocument(property, path, indexes, context) - default: - dm.walkDocument(property, path, indexes, context) - } - } - default: - dm.walkDocument(property, path, indexes, context) - } -} diff --git a/vendor/github.com/blevesearch/bleve/mapping/field.go b/vendor/github.com/blevesearch/bleve/mapping/field.go deleted file mode 100644 index 278faa1a..00000000 --- a/vendor/github.com/blevesearch/bleve/mapping/field.go +++ /dev/null @@ -1,343 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package mapping - -import ( - "encoding/json" - "fmt" - "time" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/geo" -) - -// control the default behavior for dynamic fields (those not explicitly mapped) -var ( - IndexDynamic = true - StoreDynamic = true - DocValuesDynamic = true // TODO revisit default? -) - -// A FieldMapping describes how a specific item -// should be put into the index. -type FieldMapping struct { - Name string `json:"name,omitempty"` - Type string `json:"type,omitempty"` - - // Analyzer specifies the name of the analyzer to use for this field. If - // Analyzer is empty, traverse the DocumentMapping tree toward the root and - // pick the first non-empty DefaultAnalyzer found. If there is none, use - // the IndexMapping.DefaultAnalyzer. - Analyzer string `json:"analyzer,omitempty"` - - // Store indicates whether to store field values in the index. Stored - // values can be retrieved from search results using SearchRequest.Fields. - Store bool `json:"store,omitempty"` - Index bool `json:"index,omitempty"` - - // IncludeTermVectors, if true, makes terms occurrences to be recorded for - // this field. It includes the term position within the terms sequence and - // the term offsets in the source document field. Term vectors are required - // to perform phrase queries or terms highlighting in source documents. - IncludeTermVectors bool `json:"include_term_vectors,omitempty"` - IncludeInAll bool `json:"include_in_all,omitempty"` - DateFormat string `json:"date_format,omitempty"` - - // DocValues, if true makes the index uninverting possible for this field - // It is useful for faceting and sorting queries. - DocValues bool `json:"docvalues,omitempty"` -} - -// NewTextFieldMapping returns a default field mapping for text -func NewTextFieldMapping() *FieldMapping { - return &FieldMapping{ - Type: "text", - Store: true, - Index: true, - IncludeTermVectors: true, - IncludeInAll: true, - DocValues: true, - } -} - -func newTextFieldMappingDynamic(im *IndexMappingImpl) *FieldMapping { - rv := NewTextFieldMapping() - rv.Store = im.StoreDynamic - rv.Index = im.IndexDynamic - rv.DocValues = im.DocValuesDynamic - return rv -} - -// NewNumericFieldMapping returns a default field mapping for numbers -func NewNumericFieldMapping() *FieldMapping { - return &FieldMapping{ - Type: "number", - Store: true, - Index: true, - IncludeInAll: true, - DocValues: true, - } -} - -func newNumericFieldMappingDynamic(im *IndexMappingImpl) *FieldMapping { - rv := NewNumericFieldMapping() - rv.Store = im.StoreDynamic - rv.Index = im.IndexDynamic - rv.DocValues = im.DocValuesDynamic - return rv -} - -// NewDateTimeFieldMapping returns a default field mapping for dates -func NewDateTimeFieldMapping() *FieldMapping { - return &FieldMapping{ - Type: "datetime", - Store: true, - Index: true, - IncludeInAll: true, - DocValues: true, - } -} - -func newDateTimeFieldMappingDynamic(im *IndexMappingImpl) *FieldMapping { - rv := NewDateTimeFieldMapping() - rv.Store = im.StoreDynamic - rv.Index = im.IndexDynamic - rv.DocValues = im.DocValuesDynamic - return rv -} - -// NewBooleanFieldMapping returns a default field mapping for booleans -func NewBooleanFieldMapping() *FieldMapping { - return &FieldMapping{ - Type: "boolean", - Store: true, - Index: true, - IncludeInAll: true, - DocValues: true, - } -} - -func newBooleanFieldMappingDynamic(im *IndexMappingImpl) *FieldMapping { - rv := NewBooleanFieldMapping() - rv.Store = im.StoreDynamic - rv.Index = im.IndexDynamic - rv.DocValues = im.DocValuesDynamic - return rv -} - -// NewGeoPointFieldMapping returns a default field mapping for geo points -func NewGeoPointFieldMapping() *FieldMapping { - return &FieldMapping{ - Type: "geopoint", - Store: true, - Index: true, - IncludeInAll: true, - DocValues: true, - } -} - -// Options returns the indexing options for this field. -func (fm *FieldMapping) Options() document.IndexingOptions { - var rv document.IndexingOptions - if fm.Store { - rv |= document.StoreField - } - if fm.Index { - rv |= document.IndexField - } - if fm.IncludeTermVectors { - rv |= document.IncludeTermVectors - } - if fm.DocValues { - rv |= document.DocValues - } - return rv -} - -func (fm *FieldMapping) processString(propertyValueString string, pathString string, path []string, indexes []uint64, context *walkContext) { - fieldName := getFieldName(pathString, path, fm) - options := fm.Options() - if fm.Type == "text" { - analyzer := fm.analyzerForField(path, context) - field := document.NewTextFieldCustom(fieldName, indexes, []byte(propertyValueString), options, analyzer) - context.doc.AddField(field) - - if !fm.IncludeInAll { - context.excludedFromAll = append(context.excludedFromAll, fieldName) - } - } else if fm.Type == "datetime" { - dateTimeFormat := context.im.DefaultDateTimeParser - if fm.DateFormat != "" { - dateTimeFormat = fm.DateFormat - } - dateTimeParser := context.im.DateTimeParserNamed(dateTimeFormat) - if dateTimeParser != nil { - parsedDateTime, err := dateTimeParser.ParseDateTime(propertyValueString) - if err == nil { - fm.processTime(parsedDateTime, pathString, path, indexes, context) - } - } - } -} - -func (fm *FieldMapping) processFloat64(propertyValFloat float64, pathString string, path []string, indexes []uint64, context *walkContext) { - fieldName := getFieldName(pathString, path, fm) - if fm.Type == "number" { - options := fm.Options() - field := document.NewNumericFieldWithIndexingOptions(fieldName, indexes, propertyValFloat, options) - context.doc.AddField(field) - - if !fm.IncludeInAll { - context.excludedFromAll = append(context.excludedFromAll, fieldName) - } - } -} - -func (fm *FieldMapping) processTime(propertyValueTime time.Time, pathString string, path []string, indexes []uint64, context *walkContext) { - fieldName := getFieldName(pathString, path, fm) - if fm.Type == "datetime" { - options := fm.Options() - field, err := document.NewDateTimeFieldWithIndexingOptions(fieldName, indexes, propertyValueTime, options) - if err == nil { - context.doc.AddField(field) - } else { - logger.Printf("could not build date %v", err) - } - - if !fm.IncludeInAll { - context.excludedFromAll = append(context.excludedFromAll, fieldName) - } - } -} - -func (fm *FieldMapping) processBoolean(propertyValueBool bool, pathString string, path []string, indexes []uint64, context *walkContext) { - fieldName := getFieldName(pathString, path, fm) - if fm.Type == "boolean" { - options := fm.Options() - field := document.NewBooleanFieldWithIndexingOptions(fieldName, indexes, propertyValueBool, options) - context.doc.AddField(field) - - if !fm.IncludeInAll { - context.excludedFromAll = append(context.excludedFromAll, fieldName) - } - } -} - -func (fm *FieldMapping) processGeoPoint(propertyMightBeGeoPoint interface{}, pathString string, path []string, indexes []uint64, context *walkContext) { - lon, lat, found := geo.ExtractGeoPoint(propertyMightBeGeoPoint) - if found { - fieldName := getFieldName(pathString, path, fm) - options := fm.Options() - field := document.NewGeoPointFieldWithIndexingOptions(fieldName, indexes, lon, lat, options) - context.doc.AddField(field) - - if !fm.IncludeInAll { - context.excludedFromAll = append(context.excludedFromAll, fieldName) - } - } -} - -func (fm *FieldMapping) analyzerForField(path []string, context *walkContext) *analysis.Analyzer { - analyzerName := fm.Analyzer - if analyzerName == "" { - analyzerName = context.dm.defaultAnalyzerName(path) - if analyzerName == "" { - analyzerName = context.im.DefaultAnalyzer - } - } - return context.im.AnalyzerNamed(analyzerName) -} - -func getFieldName(pathString string, path []string, fieldMapping *FieldMapping) string { - fieldName := pathString - if fieldMapping.Name != "" { - parentName := "" - if len(path) > 1 { - parentName = encodePath(path[:len(path)-1]) + pathSeparator - } - fieldName = parentName + fieldMapping.Name - } - return fieldName -} - -// UnmarshalJSON offers custom unmarshaling with optional strict validation -func (fm *FieldMapping) UnmarshalJSON(data []byte) error { - - var tmp map[string]json.RawMessage - err := json.Unmarshal(data, &tmp) - if err != nil { - return err - } - - var invalidKeys []string - for k, v := range tmp { - switch k { - case "name": - err := json.Unmarshal(v, &fm.Name) - if err != nil { - return err - } - case "type": - err := json.Unmarshal(v, &fm.Type) - if err != nil { - return err - } - case "analyzer": - err := json.Unmarshal(v, &fm.Analyzer) - if err != nil { - return err - } - case "store": - err := json.Unmarshal(v, &fm.Store) - if err != nil { - return err - } - case "index": - err := json.Unmarshal(v, &fm.Index) - if err != nil { - return err - } - case "include_term_vectors": - err := json.Unmarshal(v, &fm.IncludeTermVectors) - if err != nil { - return err - } - case "include_in_all": - err := json.Unmarshal(v, &fm.IncludeInAll) - if err != nil { - return err - } - case "date_format": - err := json.Unmarshal(v, &fm.DateFormat) - if err != nil { - return err - } - case "docvalues": - err := json.Unmarshal(v, &fm.DocValues) - if err != nil { - return err - } - default: - invalidKeys = append(invalidKeys, k) - } - } - - if MappingJSONStrict && len(invalidKeys) > 0 { - return fmt.Errorf("field mapping contains invalid keys: %v", invalidKeys) - } - - return nil -} diff --git a/vendor/github.com/blevesearch/bleve/mapping/index.go b/vendor/github.com/blevesearch/bleve/mapping/index.go deleted file mode 100644 index 319ba949..00000000 --- a/vendor/github.com/blevesearch/bleve/mapping/index.go +++ /dev/null @@ -1,443 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package mapping - -import ( - "encoding/json" - "fmt" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/analyzer/standard" - "github.com/blevesearch/bleve/analysis/datetime/optional" - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/registry" -) - -var MappingJSONStrict = false - -const defaultTypeField = "_type" -const defaultType = "_default" -const defaultField = "_all" -const defaultAnalyzer = standard.Name -const defaultDateTimeParser = optional.Name - -// An IndexMappingImpl controls how objects are placed -// into an index. -// First the type of the object is determined. -// Once the type is know, the appropriate -// DocumentMapping is selected by the type. -// If no mapping was determined for that type, -// a DefaultMapping will be used. -type IndexMappingImpl struct { - TypeMapping map[string]*DocumentMapping `json:"types,omitempty"` - DefaultMapping *DocumentMapping `json:"default_mapping"` - TypeField string `json:"type_field"` - DefaultType string `json:"default_type"` - DefaultAnalyzer string `json:"default_analyzer"` - DefaultDateTimeParser string `json:"default_datetime_parser"` - DefaultField string `json:"default_field"` - StoreDynamic bool `json:"store_dynamic"` - IndexDynamic bool `json:"index_dynamic"` - DocValuesDynamic bool `json:"docvalues_dynamic"` - CustomAnalysis *customAnalysis `json:"analysis,omitempty"` - cache *registry.Cache -} - -// AddCustomCharFilter defines a custom char filter for use in this mapping -func (im *IndexMappingImpl) AddCustomCharFilter(name string, config map[string]interface{}) error { - _, err := im.cache.DefineCharFilter(name, config) - if err != nil { - return err - } - im.CustomAnalysis.CharFilters[name] = config - return nil -} - -// AddCustomTokenizer defines a custom tokenizer for use in this mapping -func (im *IndexMappingImpl) AddCustomTokenizer(name string, config map[string]interface{}) error { - _, err := im.cache.DefineTokenizer(name, config) - if err != nil { - return err - } - im.CustomAnalysis.Tokenizers[name] = config - return nil -} - -// AddCustomTokenMap defines a custom token map for use in this mapping -func (im *IndexMappingImpl) AddCustomTokenMap(name string, config map[string]interface{}) error { - _, err := im.cache.DefineTokenMap(name, config) - if err != nil { - return err - } - im.CustomAnalysis.TokenMaps[name] = config - return nil -} - -// AddCustomTokenFilter defines a custom token filter for use in this mapping -func (im *IndexMappingImpl) AddCustomTokenFilter(name string, config map[string]interface{}) error { - _, err := im.cache.DefineTokenFilter(name, config) - if err != nil { - return err - } - im.CustomAnalysis.TokenFilters[name] = config - return nil -} - -// AddCustomAnalyzer defines a custom analyzer for use in this mapping. The -// config map must have a "type" string entry to resolve the analyzer -// constructor. The constructor is invoked with the remaining entries and -// returned analyzer is registered in the IndexMapping. -// -// bleve comes with predefined analyzers, like -// github.com/blevesearch/bleve/analysis/analyzer/custom. They are -// available only if their package is imported by client code. To achieve this, -// use their metadata to fill configuration entries: -// -// import ( -// "github.com/blevesearch/bleve/analysis/analyzer/custom" -// "github.com/blevesearch/bleve/analysis/char/html" -// "github.com/blevesearch/bleve/analysis/token/lowercase" -// "github.com/blevesearch/bleve/analysis/tokenizer/unicode" -// ) -// -// m := bleve.NewIndexMapping() -// err := m.AddCustomAnalyzer("html", map[string]interface{}{ -// "type": custom.Name, -// "char_filters": []string{ -// html.Name, -// }, -// "tokenizer": unicode.Name, -// "token_filters": []string{ -// lowercase.Name, -// ... -// }, -// }) -func (im *IndexMappingImpl) AddCustomAnalyzer(name string, config map[string]interface{}) error { - _, err := im.cache.DefineAnalyzer(name, config) - if err != nil { - return err - } - im.CustomAnalysis.Analyzers[name] = config - return nil -} - -// AddCustomDateTimeParser defines a custom date time parser for use in this mapping -func (im *IndexMappingImpl) AddCustomDateTimeParser(name string, config map[string]interface{}) error { - _, err := im.cache.DefineDateTimeParser(name, config) - if err != nil { - return err - } - im.CustomAnalysis.DateTimeParsers[name] = config - return nil -} - -// NewIndexMapping creates a new IndexMapping that will use all the default indexing rules -func NewIndexMapping() *IndexMappingImpl { - return &IndexMappingImpl{ - TypeMapping: make(map[string]*DocumentMapping), - DefaultMapping: NewDocumentMapping(), - TypeField: defaultTypeField, - DefaultType: defaultType, - DefaultAnalyzer: defaultAnalyzer, - DefaultDateTimeParser: defaultDateTimeParser, - DefaultField: defaultField, - IndexDynamic: IndexDynamic, - StoreDynamic: StoreDynamic, - DocValuesDynamic: DocValuesDynamic, - CustomAnalysis: newCustomAnalysis(), - cache: registry.NewCache(), - } -} - -// Validate will walk the entire structure ensuring the following -// explicitly named and default analyzers can be built -func (im *IndexMappingImpl) Validate() error { - _, err := im.cache.AnalyzerNamed(im.DefaultAnalyzer) - if err != nil { - return err - } - _, err = im.cache.DateTimeParserNamed(im.DefaultDateTimeParser) - if err != nil { - return err - } - err = im.DefaultMapping.Validate(im.cache) - if err != nil { - return err - } - for _, docMapping := range im.TypeMapping { - err = docMapping.Validate(im.cache) - if err != nil { - return err - } - } - return nil -} - -// AddDocumentMapping sets a custom document mapping for the specified type -func (im *IndexMappingImpl) AddDocumentMapping(doctype string, dm *DocumentMapping) { - im.TypeMapping[doctype] = dm -} - -func (im *IndexMappingImpl) mappingForType(docType string) *DocumentMapping { - docMapping := im.TypeMapping[docType] - if docMapping == nil { - docMapping = im.DefaultMapping - } - return docMapping -} - -// UnmarshalJSON offers custom unmarshaling with optional strict validation -func (im *IndexMappingImpl) UnmarshalJSON(data []byte) error { - - var tmp map[string]json.RawMessage - err := json.Unmarshal(data, &tmp) - if err != nil { - return err - } - - // set defaults for fields which might have been omitted - im.cache = registry.NewCache() - im.CustomAnalysis = newCustomAnalysis() - im.TypeField = defaultTypeField - im.DefaultType = defaultType - im.DefaultAnalyzer = defaultAnalyzer - im.DefaultDateTimeParser = defaultDateTimeParser - im.DefaultField = defaultField - im.DefaultMapping = NewDocumentMapping() - im.TypeMapping = make(map[string]*DocumentMapping) - im.StoreDynamic = StoreDynamic - im.IndexDynamic = IndexDynamic - im.DocValuesDynamic = DocValuesDynamic - - var invalidKeys []string - for k, v := range tmp { - switch k { - case "analysis": - err := json.Unmarshal(v, &im.CustomAnalysis) - if err != nil { - return err - } - case "type_field": - err := json.Unmarshal(v, &im.TypeField) - if err != nil { - return err - } - case "default_type": - err := json.Unmarshal(v, &im.DefaultType) - if err != nil { - return err - } - case "default_analyzer": - err := json.Unmarshal(v, &im.DefaultAnalyzer) - if err != nil { - return err - } - case "default_datetime_parser": - err := json.Unmarshal(v, &im.DefaultDateTimeParser) - if err != nil { - return err - } - case "default_field": - err := json.Unmarshal(v, &im.DefaultField) - if err != nil { - return err - } - case "default_mapping": - err := json.Unmarshal(v, &im.DefaultMapping) - if err != nil { - return err - } - case "types": - err := json.Unmarshal(v, &im.TypeMapping) - if err != nil { - return err - } - case "store_dynamic": - err := json.Unmarshal(v, &im.StoreDynamic) - if err != nil { - return err - } - case "index_dynamic": - err := json.Unmarshal(v, &im.IndexDynamic) - if err != nil { - return err - } - case "docvalues_dynamic": - err := json.Unmarshal(v, &im.DocValuesDynamic) - if err != nil { - return err - } - default: - invalidKeys = append(invalidKeys, k) - } - } - - if MappingJSONStrict && len(invalidKeys) > 0 { - return fmt.Errorf("index mapping contains invalid keys: %v", invalidKeys) - } - - err = im.CustomAnalysis.registerAll(im) - if err != nil { - return err - } - - return nil -} - -func (im *IndexMappingImpl) determineType(data interface{}) string { - // first see if the object implements bleveClassifier - bleveClassifier, ok := data.(bleveClassifier) - if ok { - return bleveClassifier.BleveType() - } - // next see if the object implements Classifier - classifier, ok := data.(Classifier) - if ok { - return classifier.Type() - } - - // now see if we can find a type using the mapping - typ, ok := mustString(lookupPropertyPath(data, im.TypeField)) - if ok { - return typ - } - - return im.DefaultType -} - -func (im *IndexMappingImpl) MapDocument(doc *document.Document, data interface{}) error { - docType := im.determineType(data) - docMapping := im.mappingForType(docType) - if docMapping.Enabled { - walkContext := im.newWalkContext(doc, docMapping) - docMapping.walkDocument(data, []string{}, []uint64{}, walkContext) - - // see if the _all field was disabled - allMapping := docMapping.documentMappingForPath("_all") - if allMapping == nil || allMapping.Enabled { - field := document.NewCompositeFieldWithIndexingOptions("_all", true, []string{}, walkContext.excludedFromAll, document.IndexField|document.IncludeTermVectors) - doc.AddField(field) - } - } - - return nil -} - -type walkContext struct { - doc *document.Document - im *IndexMappingImpl - dm *DocumentMapping - excludedFromAll []string -} - -func (im *IndexMappingImpl) newWalkContext(doc *document.Document, dm *DocumentMapping) *walkContext { - return &walkContext{ - doc: doc, - im: im, - dm: dm, - excludedFromAll: []string{"_id"}, - } -} - -// AnalyzerNameForPath attempts to find the best analyzer to use with only a -// field name will walk all the document types, look for field mappings at the -// provided path, if one exists and it has an explicit analyzer that is -// returned. -func (im *IndexMappingImpl) AnalyzerNameForPath(path string) string { - // first we look for explicit mapping on the field - for _, docMapping := range im.TypeMapping { - analyzerName := docMapping.analyzerNameForPath(path) - if analyzerName != "" { - return analyzerName - } - } - // now try the default mapping - pathMapping := im.DefaultMapping.documentMappingForPath(path) - if pathMapping != nil { - if len(pathMapping.Fields) > 0 { - if pathMapping.Fields[0].Analyzer != "" { - return pathMapping.Fields[0].Analyzer - } - } - } - - // next we will try default analyzers for the path - pathDecoded := decodePath(path) - for _, docMapping := range im.TypeMapping { - rv := docMapping.defaultAnalyzerName(pathDecoded) - if rv != "" { - return rv - } - } - - return im.DefaultAnalyzer -} - -func (im *IndexMappingImpl) AnalyzerNamed(name string) *analysis.Analyzer { - analyzer, err := im.cache.AnalyzerNamed(name) - if err != nil { - logger.Printf("error using analyzer named: %s", name) - return nil - } - return analyzer -} - -func (im *IndexMappingImpl) DateTimeParserNamed(name string) analysis.DateTimeParser { - if name == "" { - name = im.DefaultDateTimeParser - } - dateTimeParser, err := im.cache.DateTimeParserNamed(name) - if err != nil { - logger.Printf("error using datetime parser named: %s", name) - return nil - } - return dateTimeParser -} - -func (im *IndexMappingImpl) datetimeParserNameForPath(path string) string { - - // first we look for explicit mapping on the field - for _, docMapping := range im.TypeMapping { - pathMapping := docMapping.documentMappingForPath(path) - if pathMapping != nil { - if len(pathMapping.Fields) > 0 { - if pathMapping.Fields[0].Analyzer != "" { - return pathMapping.Fields[0].Analyzer - } - } - } - } - - return im.DefaultDateTimeParser -} - -func (im *IndexMappingImpl) AnalyzeText(analyzerName string, text []byte) (analysis.TokenStream, error) { - analyzer, err := im.cache.AnalyzerNamed(analyzerName) - if err != nil { - return nil, err - } - return analyzer.Analyze(text), nil -} - -// FieldAnalyzer returns the name of the analyzer used on a field. -func (im *IndexMappingImpl) FieldAnalyzer(field string) string { - return im.AnalyzerNameForPath(field) -} - -// wrapper to satisfy new interface - -func (im *IndexMappingImpl) DefaultSearchField() string { - return im.DefaultField -} diff --git a/vendor/github.com/blevesearch/bleve/mapping/mapping.go b/vendor/github.com/blevesearch/bleve/mapping/mapping.go deleted file mode 100644 index 4a472811..00000000 --- a/vendor/github.com/blevesearch/bleve/mapping/mapping.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package mapping - -import ( - "io/ioutil" - "log" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/document" -) - -// A Classifier is an interface describing any object which knows how to -// identify its own type. Alternatively, if a struct already has a Type -// field or method in conflict, one can use BleveType instead. -type Classifier interface { - Type() string -} - -// A bleveClassifier is an interface describing any object which knows how -// to identify its own type. This is introduced as an alternative to the -// Classifier interface which often has naming conflicts with existing -// structures. -type bleveClassifier interface { - BleveType() string -} - -var logger = log.New(ioutil.Discard, "bleve mapping ", log.LstdFlags) - -// SetLog sets the logger used for logging -// by default log messages are sent to ioutil.Discard -func SetLog(l *log.Logger) { - logger = l -} - -type IndexMapping interface { - MapDocument(doc *document.Document, data interface{}) error - Validate() error - - DateTimeParserNamed(name string) analysis.DateTimeParser - - DefaultSearchField() string - - AnalyzerNameForPath(path string) string - AnalyzerNamed(name string) *analysis.Analyzer -} diff --git a/vendor/github.com/blevesearch/bleve/query.go b/vendor/github.com/blevesearch/bleve/query.go deleted file mode 100644 index 523db5ec..00000000 --- a/vendor/github.com/blevesearch/bleve/query.go +++ /dev/null @@ -1,218 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bleve - -import ( - "time" - - "github.com/blevesearch/bleve/search/query" -) - -// NewBoolFieldQuery creates a new Query for boolean fields -func NewBoolFieldQuery(val bool) *query.BoolFieldQuery { - return query.NewBoolFieldQuery(val) -} - -// NewBooleanQuery creates a compound Query composed -// of several other Query objects. -// These other query objects are added using the -// AddMust() AddShould() and AddMustNot() methods. -// Result documents must satisfy ALL of the -// must Queries. -// Result documents must satisfy NONE of the must not -// Queries. -// Result documents that ALSO satisfy any of the should -// Queries will score higher. -func NewBooleanQuery() *query.BooleanQuery { - return query.NewBooleanQuery(nil, nil, nil) -} - -// NewConjunctionQuery creates a new compound Query. -// Result documents must satisfy all of the queries. -func NewConjunctionQuery(conjuncts ...query.Query) *query.ConjunctionQuery { - return query.NewConjunctionQuery(conjuncts) -} - -// NewDateRangeQuery creates a new Query for ranges -// of date values. -// Date strings are parsed using the DateTimeParser configured in the -// top-level config.QueryDateTimeParser -// Either, but not both endpoints can be nil. -func NewDateRangeQuery(start, end time.Time) *query.DateRangeQuery { - return query.NewDateRangeQuery(start, end) -} - -// NewDateRangeInclusiveQuery creates a new Query for ranges -// of date values. -// Date strings are parsed using the DateTimeParser configured in the -// top-level config.QueryDateTimeParser -// Either, but not both endpoints can be nil. -// startInclusive and endInclusive control inclusion of the endpoints. -func NewDateRangeInclusiveQuery(start, end time.Time, startInclusive, endInclusive *bool) *query.DateRangeQuery { - return query.NewDateRangeInclusiveQuery(start, end, startInclusive, endInclusive) -} - -// NewDisjunctionQuery creates a new compound Query. -// Result documents satisfy at least one Query. -func NewDisjunctionQuery(disjuncts ...query.Query) *query.DisjunctionQuery { - return query.NewDisjunctionQuery(disjuncts) -} - -// NewDocIDQuery creates a new Query object returning indexed documents among -// the specified set. Combine it with ConjunctionQuery to restrict the scope of -// other queries output. -func NewDocIDQuery(ids []string) *query.DocIDQuery { - return query.NewDocIDQuery(ids) -} - -// NewFuzzyQuery creates a new Query which finds -// documents containing terms within a specific -// fuzziness of the specified term. -// The default fuzziness is 1. -// -// The current implementation uses Levenshtein edit -// distance as the fuzziness metric. -func NewFuzzyQuery(term string) *query.FuzzyQuery { - return query.NewFuzzyQuery(term) -} - -// NewMatchAllQuery creates a Query which will -// match all documents in the index. -func NewMatchAllQuery() *query.MatchAllQuery { - return query.NewMatchAllQuery() -} - -// NewMatchNoneQuery creates a Query which will not -// match any documents in the index. -func NewMatchNoneQuery() *query.MatchNoneQuery { - return query.NewMatchNoneQuery() -} - -// NewMatchPhraseQuery creates a new Query object -// for matching phrases in the index. -// An Analyzer is chosen based on the field. -// Input text is analyzed using this analyzer. -// Token terms resulting from this analysis are -// used to build a search phrase. Result documents -// must match this phrase. Queried field must have been indexed with -// IncludeTermVectors set to true. -func NewMatchPhraseQuery(matchPhrase string) *query.MatchPhraseQuery { - return query.NewMatchPhraseQuery(matchPhrase) -} - -// NewMatchQuery creates a Query for matching text. -// An Analyzer is chosen based on the field. -// Input text is analyzed using this analyzer. -// Token terms resulting from this analysis are -// used to perform term searches. Result documents -// must satisfy at least one of these term searches. -func NewMatchQuery(match string) *query.MatchQuery { - return query.NewMatchQuery(match) -} - -// NewNumericRangeQuery creates a new Query for ranges -// of numeric values. -// Either, but not both endpoints can be nil. -// The minimum value is inclusive. -// The maximum value is exclusive. -func NewNumericRangeQuery(min, max *float64) *query.NumericRangeQuery { - return query.NewNumericRangeQuery(min, max) -} - -// NewNumericRangeInclusiveQuery creates a new Query for ranges -// of numeric values. -// Either, but not both endpoints can be nil. -// Control endpoint inclusion with inclusiveMin, inclusiveMax. -func NewNumericRangeInclusiveQuery(min, max *float64, minInclusive, maxInclusive *bool) *query.NumericRangeQuery { - return query.NewNumericRangeInclusiveQuery(min, max, minInclusive, maxInclusive) -} - -// NewTermRangeQuery creates a new Query for ranges -// of text terms. -// Either, but not both endpoints can be "". -// The minimum value is inclusive. -// The maximum value is exclusive. -func NewTermRangeQuery(min, max string) *query.TermRangeQuery { - return query.NewTermRangeQuery(min, max) -} - -// NewTermRangeInclusiveQuery creates a new Query for ranges -// of text terms. -// Either, but not both endpoints can be "". -// Control endpoint inclusion with inclusiveMin, inclusiveMax. -func NewTermRangeInclusiveQuery(min, max string, minInclusive, maxInclusive *bool) *query.TermRangeQuery { - return query.NewTermRangeInclusiveQuery(min, max, minInclusive, maxInclusive) -} - -// NewPhraseQuery creates a new Query for finding -// exact term phrases in the index. -// The provided terms must exist in the correct -// order, at the correct index offsets, in the -// specified field. Queried field must have been indexed with -// IncludeTermVectors set to true. -func NewPhraseQuery(terms []string, field string) *query.PhraseQuery { - return query.NewPhraseQuery(terms, field) -} - -// NewPrefixQuery creates a new Query which finds -// documents containing terms that start with the -// specified prefix. -func NewPrefixQuery(prefix string) *query.PrefixQuery { - return query.NewPrefixQuery(prefix) -} - -// NewRegexpQuery creates a new Query which finds -// documents containing terms that match the -// specified regular expression. -func NewRegexpQuery(regexp string) *query.RegexpQuery { - return query.NewRegexpQuery(regexp) -} - -// NewQueryStringQuery creates a new Query used for -// finding documents that satisfy a query string. The -// query string is a small query language for humans. -func NewQueryStringQuery(q string) *query.QueryStringQuery { - return query.NewQueryStringQuery(q) -} - -// NewTermQuery creates a new Query for finding an -// exact term match in the index. -func NewTermQuery(term string) *query.TermQuery { - return query.NewTermQuery(term) -} - -// NewWildcardQuery creates a new Query which finds -// documents containing terms that match the -// specified wildcard. In the wildcard pattern '*' -// will match any sequence of 0 or more characters, -// and '?' will match any single character. -func NewWildcardQuery(wildcard string) *query.WildcardQuery { - return query.NewWildcardQuery(wildcard) -} - -// NewGeoBoundingBoxQuery creates a new Query for performing geo bounding -// box searches. The arguments describe the position of the box and documents -// which have an indexed geo point inside the box will be returned. -func NewGeoBoundingBoxQuery(topLeftLon, topLeftLat, bottomRightLon, bottomRightLat float64) *query.GeoBoundingBoxQuery { - return query.NewGeoBoundingBoxQuery(topLeftLon, topLeftLat, bottomRightLon, bottomRightLat) -} - -// NewGeoDistanceQuery creates a new Query for performing geo distance -// searches. The arguments describe a position and a distance. Documents -// which have an indexed geo point which is less than or equal to the provided -// distance from the given position will be returned. -func NewGeoDistanceQuery(lon, lat float64, distance string) *query.GeoDistanceQuery { - return query.NewGeoDistanceQuery(lon, lat, distance) -} diff --git a/vendor/github.com/blevesearch/bleve/registry/highlighter.go b/vendor/github.com/blevesearch/bleve/registry/highlighter.go deleted file mode 100644 index b84219ca..00000000 --- a/vendor/github.com/blevesearch/bleve/registry/highlighter.go +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package registry - -import ( - "fmt" - - "github.com/blevesearch/bleve/search/highlight" -) - -func RegisterHighlighter(name string, constructor HighlighterConstructor) { - _, exists := highlighters[name] - if exists { - panic(fmt.Errorf("attempted to register duplicate highlighter named '%s'", name)) - } - highlighters[name] = constructor -} - -type HighlighterConstructor func(config map[string]interface{}, cache *Cache) (highlight.Highlighter, error) -type HighlighterRegistry map[string]HighlighterConstructor - -type HighlighterCache struct { - *ConcurrentCache -} - -func NewHighlighterCache() *HighlighterCache { - return &HighlighterCache{ - NewConcurrentCache(), - } -} - -func HighlighterBuild(name string, config map[string]interface{}, cache *Cache) (interface{}, error) { - cons, registered := highlighters[name] - if !registered { - return nil, fmt.Errorf("no highlighter with name or type '%s' registered", name) - } - highlighter, err := cons(config, cache) - if err != nil { - return nil, fmt.Errorf("error building highlighter: %v", err) - } - return highlighter, nil -} - -func (c *HighlighterCache) HighlighterNamed(name string, cache *Cache) (highlight.Highlighter, error) { - item, err := c.ItemNamed(name, cache, HighlighterBuild) - if err != nil { - return nil, err - } - return item.(highlight.Highlighter), nil -} - -func (c *HighlighterCache) DefineHighlighter(name string, typ string, config map[string]interface{}, cache *Cache) (highlight.Highlighter, error) { - item, err := c.DefineItem(name, typ, config, cache, HighlighterBuild) - if err != nil { - if err == ErrAlreadyDefined { - return nil, fmt.Errorf("highlighter named '%s' already defined", name) - } - return nil, err - } - return item.(highlight.Highlighter), nil -} - -func HighlighterTypesAndInstances() ([]string, []string) { - emptyConfig := map[string]interface{}{} - emptyCache := NewCache() - var types []string - var instances []string - for name, cons := range highlighters { - _, err := cons(emptyConfig, emptyCache) - if err == nil { - instances = append(instances, name) - } else { - types = append(types, name) - } - } - return types, instances -} diff --git a/vendor/github.com/blevesearch/bleve/registry/store.go b/vendor/github.com/blevesearch/bleve/registry/store.go deleted file mode 100644 index 83187763..00000000 --- a/vendor/github.com/blevesearch/bleve/registry/store.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package registry - -import ( - "fmt" - - "github.com/blevesearch/bleve/index/store" -) - -func RegisterKVStore(name string, constructor KVStoreConstructor) { - _, exists := stores[name] - if exists { - panic(fmt.Errorf("attempted to register duplicate store named '%s'", name)) - } - stores[name] = constructor -} - -// KVStoreConstructor is used to build a KVStore of a specific type when -// specificied by the index configuration. In addition to meeting the -// store.KVStore interface, KVStores must also support this constructor. -// Note that currently the values of config must -// be able to be marshaled and unmarshaled using the encoding/json library (used -// when reading/writing the index metadata file). -type KVStoreConstructor func(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) -type KVStoreRegistry map[string]KVStoreConstructor - -func KVStoreConstructorByName(name string) KVStoreConstructor { - return stores[name] -} - -func KVStoreTypesAndInstances() ([]string, []string) { - var types []string - var instances []string - for name := range stores { - types = append(types, name) - } - return types, instances -} diff --git a/vendor/github.com/blevesearch/bleve/search.go b/vendor/github.com/blevesearch/bleve/search.go deleted file mode 100644 index f6745077..00000000 --- a/vendor/github.com/blevesearch/bleve/search.go +++ /dev/null @@ -1,631 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bleve - -import ( - "encoding/json" - "fmt" - "reflect" - "sort" - "time" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/datetime/optional" - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/registry" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/collector" - "github.com/blevesearch/bleve/search/query" - "github.com/blevesearch/bleve/size" -) - -var reflectStaticSizeSearchResult int -var reflectStaticSizeSearchStatus int - -func init() { - var sr SearchResult - reflectStaticSizeSearchResult = int(reflect.TypeOf(sr).Size()) - var ss SearchStatus - reflectStaticSizeSearchStatus = int(reflect.TypeOf(ss).Size()) -} - -var cache = registry.NewCache() - -const defaultDateTimeParser = optional.Name - -type numericRange struct { - Name string `json:"name,omitempty"` - Min *float64 `json:"min,omitempty"` - Max *float64 `json:"max,omitempty"` -} - -type dateTimeRange struct { - Name string `json:"name,omitempty"` - Start time.Time `json:"start,omitempty"` - End time.Time `json:"end,omitempty"` - startString *string - endString *string -} - -func (dr *dateTimeRange) ParseDates(dateTimeParser analysis.DateTimeParser) (start, end time.Time) { - start = dr.Start - if dr.Start.IsZero() && dr.startString != nil { - s, err := dateTimeParser.ParseDateTime(*dr.startString) - if err == nil { - start = s - } - } - end = dr.End - if dr.End.IsZero() && dr.endString != nil { - e, err := dateTimeParser.ParseDateTime(*dr.endString) - if err == nil { - end = e - } - } - return start, end -} - -func (dr *dateTimeRange) UnmarshalJSON(input []byte) error { - var temp struct { - Name string `json:"name,omitempty"` - Start *string `json:"start,omitempty"` - End *string `json:"end,omitempty"` - } - - err := json.Unmarshal(input, &temp) - if err != nil { - return err - } - - dr.Name = temp.Name - if temp.Start != nil { - dr.startString = temp.Start - } - if temp.End != nil { - dr.endString = temp.End - } - - return nil -} - -func (dr *dateTimeRange) MarshalJSON() ([]byte, error) { - rv := map[string]interface{}{ - "name": dr.Name, - "start": dr.Start, - "end": dr.End, - } - if dr.Start.IsZero() && dr.startString != nil { - rv["start"] = dr.startString - } - if dr.End.IsZero() && dr.endString != nil { - rv["end"] = dr.endString - } - return json.Marshal(rv) -} - -// A FacetRequest describes a facet or aggregation -// of the result document set you would like to be -// built. -type FacetRequest struct { - Size int `json:"size"` - Field string `json:"field"` - NumericRanges []*numericRange `json:"numeric_ranges,omitempty"` - DateTimeRanges []*dateTimeRange `json:"date_ranges,omitempty"` -} - -func (fr *FacetRequest) Validate() error { - nrCount := len(fr.NumericRanges) - drCount := len(fr.DateTimeRanges) - if nrCount > 0 && drCount > 0 { - return fmt.Errorf("facet can only conain numeric ranges or date ranges, not both") - } - - if nrCount > 0 { - nrNames := map[string]interface{}{} - for _, nr := range fr.NumericRanges { - if _, ok := nrNames[nr.Name]; ok { - return fmt.Errorf("numeric ranges contains duplicate name '%s'", nr.Name) - } - nrNames[nr.Name] = struct{}{} - if nr.Min == nil && nr.Max == nil { - return fmt.Errorf("numeric range query must specify either min, max or both for range name '%s'", nr.Name) - } - } - - } else { - dateTimeParser, err := cache.DateTimeParserNamed(defaultDateTimeParser) - if err != nil { - return err - } - drNames := map[string]interface{}{} - for _, dr := range fr.DateTimeRanges { - if _, ok := drNames[dr.Name]; ok { - return fmt.Errorf("date ranges contains duplicate name '%s'", dr.Name) - } - drNames[dr.Name] = struct{}{} - start, end := dr.ParseDates(dateTimeParser) - if start.IsZero() && end.IsZero() { - return fmt.Errorf("date range query must specify either start, end or both for range name '%s'", dr.Name) - } - } - } - return nil -} - -// NewFacetRequest creates a facet on the specified -// field that limits the number of entries to the -// specified size. -func NewFacetRequest(field string, size int) *FacetRequest { - return &FacetRequest{ - Field: field, - Size: size, - } -} - -// AddDateTimeRange adds a bucket to a field -// containing date values. Documents with a -// date value falling into this range are tabulated -// as part of this bucket/range. -func (fr *FacetRequest) AddDateTimeRange(name string, start, end time.Time) { - if fr.DateTimeRanges == nil { - fr.DateTimeRanges = make([]*dateTimeRange, 0, 1) - } - fr.DateTimeRanges = append(fr.DateTimeRanges, &dateTimeRange{Name: name, Start: start, End: end}) -} - -// AddDateTimeRangeString adds a bucket to a field -// containing date values. -func (fr *FacetRequest) AddDateTimeRangeString(name string, start, end *string) { - if fr.DateTimeRanges == nil { - fr.DateTimeRanges = make([]*dateTimeRange, 0, 1) - } - fr.DateTimeRanges = append(fr.DateTimeRanges, - &dateTimeRange{Name: name, startString: start, endString: end}) -} - -// AddNumericRange adds a bucket to a field -// containing numeric values. Documents with a -// numeric value falling into this range are -// tabulated as part of this bucket/range. -func (fr *FacetRequest) AddNumericRange(name string, min, max *float64) { - if fr.NumericRanges == nil { - fr.NumericRanges = make([]*numericRange, 0, 1) - } - fr.NumericRanges = append(fr.NumericRanges, &numericRange{Name: name, Min: min, Max: max}) -} - -// FacetsRequest groups together all the -// FacetRequest objects for a single query. -type FacetsRequest map[string]*FacetRequest - -func (fr FacetsRequest) Validate() error { - for _, v := range fr { - err := v.Validate() - if err != nil { - return err - } - } - return nil -} - -// HighlightRequest describes how field matches -// should be highlighted. -type HighlightRequest struct { - Style *string `json:"style"` - Fields []string `json:"fields"` -} - -// NewHighlight creates a default -// HighlightRequest. -func NewHighlight() *HighlightRequest { - return &HighlightRequest{} -} - -// NewHighlightWithStyle creates a HighlightRequest -// with an alternate style. -func NewHighlightWithStyle(style string) *HighlightRequest { - return &HighlightRequest{ - Style: &style, - } -} - -func (h *HighlightRequest) AddField(field string) { - if h.Fields == nil { - h.Fields = make([]string, 0, 1) - } - h.Fields = append(h.Fields, field) -} - -// A SearchRequest describes all the parameters -// needed to search the index. -// Query is required. -// Size/From describe how much and which part of the -// result set to return. -// Highlight describes optional search result -// highlighting. -// Fields describes a list of field values which -// should be retrieved for result documents, provided they -// were stored while indexing. -// Facets describe the set of facets to be computed. -// Explain triggers inclusion of additional search -// result score explanations. -// Sort describes the desired order for the results to be returned. -// Score controls the kind of scoring performed -// SearchAfter supports deep paging by providing a minimum sort key -// SearchBefore supports deep paging by providing a maximum sort key -// sortFunc specifies the sort implementation to use for sorting results. -// -// A special field named "*" can be used to return all fields. -type SearchRequest struct { - Query query.Query `json:"query"` - Size int `json:"size"` - From int `json:"from"` - Highlight *HighlightRequest `json:"highlight"` - Fields []string `json:"fields"` - Facets FacetsRequest `json:"facets"` - Explain bool `json:"explain"` - Sort search.SortOrder `json:"sort"` - IncludeLocations bool `json:"includeLocations"` - Score string `json:"score,omitempty"` - SearchAfter []string `json:"search_after"` - SearchBefore []string `json:"search_before"` - - sortFunc func(sort.Interface) -} - -func (r *SearchRequest) Validate() error { - if srq, ok := r.Query.(query.ValidatableQuery); ok { - err := srq.Validate() - if err != nil { - return err - } - } - - if r.SearchAfter != nil && r.SearchBefore != nil { - return fmt.Errorf("cannot use search after and search before together") - } - - if r.SearchAfter != nil { - if r.From != 0 { - return fmt.Errorf("cannot use search after with from !=0") - } - if len(r.SearchAfter) != len(r.Sort) { - return fmt.Errorf("search after must have same size as sort order") - } - } - if r.SearchBefore != nil { - if r.From != 0 { - return fmt.Errorf("cannot use search before with from !=0") - } - if len(r.SearchBefore) != len(r.Sort) { - return fmt.Errorf("search before must have same size as sort order") - } - } - - return r.Facets.Validate() -} - -// AddFacet adds a FacetRequest to this SearchRequest -func (r *SearchRequest) AddFacet(facetName string, f *FacetRequest) { - if r.Facets == nil { - r.Facets = make(FacetsRequest, 1) - } - r.Facets[facetName] = f -} - -// SortBy changes the request to use the requested sort order -// this form uses the simplified syntax with an array of strings -// each string can either be a field name -// or the magic value _id and _score which refer to the doc id and search score -// any of these values can optionally be prefixed with - to reverse the order -func (r *SearchRequest) SortBy(order []string) { - so := search.ParseSortOrderStrings(order) - r.Sort = so -} - -// SortByCustom changes the request to use the requested sort order -func (r *SearchRequest) SortByCustom(order search.SortOrder) { - r.Sort = order -} - -// SetSearchAfter sets the request to skip over hits with a sort -// value less than the provided sort after key -func (r *SearchRequest) SetSearchAfter(after []string) { - r.SearchAfter = after -} - -// SetSearchBefore sets the request to skip over hits with a sort -// value greater than the provided sort before key -func (r *SearchRequest) SetSearchBefore(before []string) { - r.SearchBefore = before -} - -// UnmarshalJSON deserializes a JSON representation of -// a SearchRequest -func (r *SearchRequest) UnmarshalJSON(input []byte) error { - var temp struct { - Q json.RawMessage `json:"query"` - Size *int `json:"size"` - From int `json:"from"` - Highlight *HighlightRequest `json:"highlight"` - Fields []string `json:"fields"` - Facets FacetsRequest `json:"facets"` - Explain bool `json:"explain"` - Sort []json.RawMessage `json:"sort"` - IncludeLocations bool `json:"includeLocations"` - Score string `json:"score"` - SearchAfter []string `json:"search_after"` - SearchBefore []string `json:"search_before"` - } - - err := json.Unmarshal(input, &temp) - if err != nil { - return err - } - - if temp.Size == nil { - r.Size = 10 - } else { - r.Size = *temp.Size - } - if temp.Sort == nil { - r.Sort = search.SortOrder{&search.SortScore{Desc: true}} - } else { - r.Sort, err = search.ParseSortOrderJSON(temp.Sort) - if err != nil { - return err - } - } - r.From = temp.From - r.Explain = temp.Explain - r.Highlight = temp.Highlight - r.Fields = temp.Fields - r.Facets = temp.Facets - r.IncludeLocations = temp.IncludeLocations - r.Score = temp.Score - r.SearchAfter = temp.SearchAfter - r.SearchBefore = temp.SearchBefore - r.Query, err = query.ParseQuery(temp.Q) - if err != nil { - return err - } - - if r.Size < 0 { - r.Size = 10 - } - if r.From < 0 { - r.From = 0 - } - - return nil - -} - -// NewSearchRequest creates a new SearchRequest -// for the Query, using default values for all -// other search parameters. -func NewSearchRequest(q query.Query) *SearchRequest { - return NewSearchRequestOptions(q, 10, 0, false) -} - -// NewSearchRequestOptions creates a new SearchRequest -// for the Query, with the requested size, from -// and explanation search parameters. -// By default results are ordered by score, descending. -func NewSearchRequestOptions(q query.Query, size, from int, explain bool) *SearchRequest { - return &SearchRequest{ - Query: q, - Size: size, - From: from, - Explain: explain, - Sort: search.SortOrder{&search.SortScore{Desc: true}}, - } -} - -// IndexErrMap tracks errors with the name of the index where it occurred -type IndexErrMap map[string]error - -// MarshalJSON seralizes the error into a string for JSON consumption -func (iem IndexErrMap) MarshalJSON() ([]byte, error) { - tmp := make(map[string]string, len(iem)) - for k, v := range iem { - tmp[k] = v.Error() - } - return json.Marshal(tmp) -} - -func (iem IndexErrMap) UnmarshalJSON(data []byte) error { - var tmp map[string]string - err := json.Unmarshal(data, &tmp) - if err != nil { - return err - } - for k, v := range tmp { - iem[k] = fmt.Errorf("%s", v) - } - return nil -} - -// SearchStatus is a secion in the SearchResult reporting how many -// underlying indexes were queried, how many were successful/failed -// and a map of any errors that were encountered -type SearchStatus struct { - Total int `json:"total"` - Failed int `json:"failed"` - Successful int `json:"successful"` - Errors IndexErrMap `json:"errors,omitempty"` -} - -// Merge will merge together multiple SearchStatuses during a MultiSearch -func (ss *SearchStatus) Merge(other *SearchStatus) { - ss.Total += other.Total - ss.Failed += other.Failed - ss.Successful += other.Successful - if len(other.Errors) > 0 { - if ss.Errors == nil { - ss.Errors = make(map[string]error) - } - for otherIndex, otherError := range other.Errors { - ss.Errors[otherIndex] = otherError - } - } -} - -// A SearchResult describes the results of executing -// a SearchRequest. -type SearchResult struct { - Status *SearchStatus `json:"status"` - Request *SearchRequest `json:"request"` - Hits search.DocumentMatchCollection `json:"hits"` - Total uint64 `json:"total_hits"` - MaxScore float64 `json:"max_score"` - Took time.Duration `json:"took"` - Facets search.FacetResults `json:"facets"` -} - -func (sr *SearchResult) Size() int { - sizeInBytes := reflectStaticSizeSearchResult + size.SizeOfPtr + - reflectStaticSizeSearchStatus - - for _, entry := range sr.Hits { - if entry != nil { - sizeInBytes += entry.Size() - } - } - - for k, v := range sr.Facets { - sizeInBytes += size.SizeOfString + len(k) + - v.Size() - } - - return sizeInBytes -} - -func (sr *SearchResult) String() string { - rv := "" - if sr.Total > 0 { - if sr.Request.Size > 0 { - rv = fmt.Sprintf("%d matches, showing %d through %d, took %s\n", sr.Total, sr.Request.From+1, sr.Request.From+len(sr.Hits), sr.Took) - for i, hit := range sr.Hits { - rv += fmt.Sprintf("%5d. %s (%f)\n", i+sr.Request.From+1, hit.ID, hit.Score) - for fragmentField, fragments := range hit.Fragments { - rv += fmt.Sprintf("\t%s\n", fragmentField) - for _, fragment := range fragments { - rv += fmt.Sprintf("\t\t%s\n", fragment) - } - } - for otherFieldName, otherFieldValue := range hit.Fields { - if _, ok := hit.Fragments[otherFieldName]; !ok { - rv += fmt.Sprintf("\t%s\n", otherFieldName) - rv += fmt.Sprintf("\t\t%v\n", otherFieldValue) - } - } - } - } else { - rv = fmt.Sprintf("%d matches, took %s\n", sr.Total, sr.Took) - } - } else { - rv = "No matches" - } - if len(sr.Facets) > 0 { - rv += fmt.Sprintf("Facets:\n") - for fn, f := range sr.Facets { - rv += fmt.Sprintf("%s(%d)\n", fn, f.Total) - for _, t := range f.Terms { - rv += fmt.Sprintf("\t%s(%d)\n", t.Term, t.Count) - } - if f.Other != 0 { - rv += fmt.Sprintf("\tOther(%d)\n", f.Other) - } - } - } - return rv -} - -// Merge will merge together multiple SearchResults during a MultiSearch -func (sr *SearchResult) Merge(other *SearchResult) { - sr.Status.Merge(other.Status) - sr.Hits = append(sr.Hits, other.Hits...) - sr.Total += other.Total - if other.MaxScore > sr.MaxScore { - sr.MaxScore = other.MaxScore - } - if sr.Facets == nil && len(other.Facets) != 0 { - sr.Facets = other.Facets - return - } - - sr.Facets.Merge(other.Facets) -} - -// MemoryNeededForSearchResult is an exported helper function to determine the RAM -// needed to accommodate the results for a given search request. -func MemoryNeededForSearchResult(req *SearchRequest) uint64 { - if req == nil { - return 0 - } - - numDocMatches := req.Size + req.From - if req.Size+req.From > collector.PreAllocSizeSkipCap { - numDocMatches = collector.PreAllocSizeSkipCap - } - - estimate := 0 - - // overhead from the SearchResult structure - var sr SearchResult - estimate += sr.Size() - - var dm search.DocumentMatch - sizeOfDocumentMatch := dm.Size() - - // overhead from results - estimate += numDocMatches * sizeOfDocumentMatch - - // overhead from facet results - if req.Facets != nil { - var fr search.FacetResult - estimate += len(req.Facets) * fr.Size() - } - - // highlighting, store - var d document.Document - if len(req.Fields) > 0 || req.Highlight != nil { - for i := 0; i < (req.Size + req.From); i++ { - estimate += (req.Size + req.From) * d.Size() - } - } - - return uint64(estimate) -} - -// SetSortFunc sets the sort implementation to use when sorting hits. -// -// SearchRequests can specify a custom sort implementation to meet -// their needs. For instance, by specifying a parallel sort -// that uses all available cores. -func (r *SearchRequest) SetSortFunc(s func(sort.Interface)) { - r.sortFunc = s -} - -// SortFunc returns the sort implementation to use when sorting hits. -// Defaults to sort.Sort. -func (r *SearchRequest) SortFunc() func(data sort.Interface) { - if r.sortFunc != nil { - return r.sortFunc - } - - return sort.Sort -} diff --git a/vendor/github.com/blevesearch/bleve/search/facet/facet_builder_datetime.go b/vendor/github.com/blevesearch/bleve/search/facet/facet_builder_datetime.go deleted file mode 100644 index c45442e4..00000000 --- a/vendor/github.com/blevesearch/bleve/search/facet/facet_builder_datetime.go +++ /dev/null @@ -1,163 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package facet - -import ( - "reflect" - "sort" - "time" - - "github.com/blevesearch/bleve/numeric" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/size" -) - -var reflectStaticSizeDateTimeFacetBuilder int -var reflectStaticSizedateTimeRange int - -func init() { - var dtfb DateTimeFacetBuilder - reflectStaticSizeDateTimeFacetBuilder = int(reflect.TypeOf(dtfb).Size()) - var dtr dateTimeRange - reflectStaticSizedateTimeRange = int(reflect.TypeOf(dtr).Size()) -} - -type dateTimeRange struct { - start time.Time - end time.Time -} - -type DateTimeFacetBuilder struct { - size int - field string - termsCount map[string]int - total int - missing int - ranges map[string]*dateTimeRange - sawValue bool -} - -func NewDateTimeFacetBuilder(field string, size int) *DateTimeFacetBuilder { - return &DateTimeFacetBuilder{ - size: size, - field: field, - termsCount: make(map[string]int), - ranges: make(map[string]*dateTimeRange, 0), - } -} - -func (fb *DateTimeFacetBuilder) Size() int { - sizeInBytes := reflectStaticSizeDateTimeFacetBuilder + size.SizeOfPtr + - len(fb.field) - - for k, _ := range fb.termsCount { - sizeInBytes += size.SizeOfString + len(k) + - size.SizeOfInt - } - - for k, _ := range fb.ranges { - sizeInBytes += size.SizeOfString + len(k) + - size.SizeOfPtr + reflectStaticSizedateTimeRange - } - - return sizeInBytes -} - -func (fb *DateTimeFacetBuilder) AddRange(name string, start, end time.Time) { - r := dateTimeRange{ - start: start, - end: end, - } - fb.ranges[name] = &r -} - -func (fb *DateTimeFacetBuilder) Field() string { - return fb.field -} - -func (fb *DateTimeFacetBuilder) UpdateVisitor(field string, term []byte) { - if field == fb.field { - fb.sawValue = true - // only consider the values which are shifted 0 - prefixCoded := numeric.PrefixCoded(term) - shift, err := prefixCoded.Shift() - if err == nil && shift == 0 { - i64, err := prefixCoded.Int64() - if err == nil { - t := time.Unix(0, i64) - - // look at each of the ranges for a match - for rangeName, r := range fb.ranges { - if (r.start.IsZero() || t.After(r.start) || t.Equal(r.start)) && (r.end.IsZero() || t.Before(r.end)) { - fb.termsCount[rangeName] = fb.termsCount[rangeName] + 1 - fb.total++ - } - } - } - } - } -} - -func (fb *DateTimeFacetBuilder) StartDoc() { - fb.sawValue = false -} - -func (fb *DateTimeFacetBuilder) EndDoc() { - if !fb.sawValue { - fb.missing++ - } -} - -func (fb *DateTimeFacetBuilder) Result() *search.FacetResult { - rv := search.FacetResult{ - Field: fb.field, - Total: fb.total, - Missing: fb.missing, - } - - rv.DateRanges = make([]*search.DateRangeFacet, 0, len(fb.termsCount)) - - for term, count := range fb.termsCount { - dateRange := fb.ranges[term] - tf := &search.DateRangeFacet{ - Name: term, - Count: count, - } - if !dateRange.start.IsZero() { - start := dateRange.start.Format(time.RFC3339Nano) - tf.Start = &start - } - if !dateRange.end.IsZero() { - end := dateRange.end.Format(time.RFC3339Nano) - tf.End = &end - } - rv.DateRanges = append(rv.DateRanges, tf) - } - - sort.Sort(rv.DateRanges) - - // we now have the list of the top N facets - if fb.size < len(rv.DateRanges) { - rv.DateRanges = rv.DateRanges[:fb.size] - } - - notOther := 0 - for _, nr := range rv.DateRanges { - notOther += nr.Count - } - rv.Other = fb.total - notOther - - return &rv -} diff --git a/vendor/github.com/blevesearch/bleve/search/facets_builder.go b/vendor/github.com/blevesearch/bleve/search/facets_builder.go deleted file mode 100644 index 7fc0bedf..00000000 --- a/vendor/github.com/blevesearch/bleve/search/facets_builder.go +++ /dev/null @@ -1,341 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package search - -import ( - "reflect" - "sort" - - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/size" -) - -var reflectStaticSizeFacetsBuilder int -var reflectStaticSizeFacetResult int -var reflectStaticSizeTermFacet int -var reflectStaticSizeNumericRangeFacet int -var reflectStaticSizeDateRangeFacet int - -func init() { - var fb FacetsBuilder - reflectStaticSizeFacetsBuilder = int(reflect.TypeOf(fb).Size()) - var fr FacetResult - reflectStaticSizeFacetResult = int(reflect.TypeOf(fr).Size()) - var tf TermFacet - reflectStaticSizeTermFacet = int(reflect.TypeOf(tf).Size()) - var nrf NumericRangeFacet - reflectStaticSizeNumericRangeFacet = int(reflect.TypeOf(nrf).Size()) - var drf DateRangeFacet - reflectStaticSizeDateRangeFacet = int(reflect.TypeOf(drf).Size()) -} - -type FacetBuilder interface { - StartDoc() - UpdateVisitor(field string, term []byte) - EndDoc() - - Result() *FacetResult - Field() string - - Size() int -} - -type FacetsBuilder struct { - indexReader index.IndexReader - facetNames []string - facets []FacetBuilder - fields []string -} - -func NewFacetsBuilder(indexReader index.IndexReader) *FacetsBuilder { - return &FacetsBuilder{ - indexReader: indexReader, - } -} - -func (fb *FacetsBuilder) Size() int { - sizeInBytes := reflectStaticSizeFacetsBuilder + size.SizeOfPtr - - for k, v := range fb.facets { - sizeInBytes += size.SizeOfString + v.Size() + len(fb.facetNames[k]) - } - - for _, entry := range fb.fields { - sizeInBytes += size.SizeOfString + len(entry) - } - - return sizeInBytes -} - -func (fb *FacetsBuilder) Add(name string, facetBuilder FacetBuilder) { - fb.facetNames = append(fb.facetNames, name) - fb.facets = append(fb.facets, facetBuilder) - fb.fields = append(fb.fields, facetBuilder.Field()) -} - -func (fb *FacetsBuilder) RequiredFields() []string { - return fb.fields -} - -func (fb *FacetsBuilder) StartDoc() { - for _, facetBuilder := range fb.facets { - facetBuilder.StartDoc() - } -} - -func (fb *FacetsBuilder) EndDoc() { - for _, facetBuilder := range fb.facets { - facetBuilder.EndDoc() - } -} - -func (fb *FacetsBuilder) UpdateVisitor(field string, term []byte) { - for _, facetBuilder := range fb.facets { - facetBuilder.UpdateVisitor(field, term) - } -} - -type TermFacet struct { - Term string `json:"term"` - Count int `json:"count"` -} - -type TermFacets []*TermFacet - -func (tf TermFacets) Add(termFacet *TermFacet) TermFacets { - for _, existingTerm := range tf { - if termFacet.Term == existingTerm.Term { - existingTerm.Count += termFacet.Count - return tf - } - } - // if we got here it wasn't already in the existing terms - tf = append(tf, termFacet) - return tf -} - -func (tf TermFacets) Len() int { return len(tf) } -func (tf TermFacets) Swap(i, j int) { tf[i], tf[j] = tf[j], tf[i] } -func (tf TermFacets) Less(i, j int) bool { - if tf[i].Count == tf[j].Count { - return tf[i].Term < tf[j].Term - } - return tf[i].Count > tf[j].Count -} - -type NumericRangeFacet struct { - Name string `json:"name"` - Min *float64 `json:"min,omitempty"` - Max *float64 `json:"max,omitempty"` - Count int `json:"count"` -} - -func (nrf *NumericRangeFacet) Same(other *NumericRangeFacet) bool { - if nrf.Min == nil && other.Min != nil { - return false - } - if nrf.Min != nil && other.Min == nil { - return false - } - if nrf.Min != nil && other.Min != nil && *nrf.Min != *other.Min { - return false - } - if nrf.Max == nil && other.Max != nil { - return false - } - if nrf.Max != nil && other.Max == nil { - return false - } - if nrf.Max != nil && other.Max != nil && *nrf.Max != *other.Max { - return false - } - - return true -} - -type NumericRangeFacets []*NumericRangeFacet - -func (nrf NumericRangeFacets) Add(numericRangeFacet *NumericRangeFacet) NumericRangeFacets { - for _, existingNr := range nrf { - if numericRangeFacet.Same(existingNr) { - existingNr.Count += numericRangeFacet.Count - return nrf - } - } - // if we got here it wasn't already in the existing terms - nrf = append(nrf, numericRangeFacet) - return nrf -} - -func (nrf NumericRangeFacets) Len() int { return len(nrf) } -func (nrf NumericRangeFacets) Swap(i, j int) { nrf[i], nrf[j] = nrf[j], nrf[i] } -func (nrf NumericRangeFacets) Less(i, j int) bool { - if nrf[i].Count == nrf[j].Count { - return nrf[i].Name < nrf[j].Name - } - return nrf[i].Count > nrf[j].Count -} - -type DateRangeFacet struct { - Name string `json:"name"` - Start *string `json:"start,omitempty"` - End *string `json:"end,omitempty"` - Count int `json:"count"` -} - -func (drf *DateRangeFacet) Same(other *DateRangeFacet) bool { - if drf.Start == nil && other.Start != nil { - return false - } - if drf.Start != nil && other.Start == nil { - return false - } - if drf.Start != nil && other.Start != nil && *drf.Start != *other.Start { - return false - } - if drf.End == nil && other.End != nil { - return false - } - if drf.End != nil && other.End == nil { - return false - } - if drf.End != nil && other.End != nil && *drf.End != *other.End { - return false - } - - return true -} - -type DateRangeFacets []*DateRangeFacet - -func (drf DateRangeFacets) Add(dateRangeFacet *DateRangeFacet) DateRangeFacets { - for _, existingDr := range drf { - if dateRangeFacet.Same(existingDr) { - existingDr.Count += dateRangeFacet.Count - return drf - } - } - // if we got here it wasn't already in the existing terms - drf = append(drf, dateRangeFacet) - return drf -} - -func (drf DateRangeFacets) Len() int { return len(drf) } -func (drf DateRangeFacets) Swap(i, j int) { drf[i], drf[j] = drf[j], drf[i] } -func (drf DateRangeFacets) Less(i, j int) bool { - if drf[i].Count == drf[j].Count { - return drf[i].Name < drf[j].Name - } - return drf[i].Count > drf[j].Count -} - -type FacetResult struct { - Field string `json:"field"` - Total int `json:"total"` - Missing int `json:"missing"` - Other int `json:"other"` - Terms TermFacets `json:"terms,omitempty"` - NumericRanges NumericRangeFacets `json:"numeric_ranges,omitempty"` - DateRanges DateRangeFacets `json:"date_ranges,omitempty"` -} - -func (fr *FacetResult) Size() int { - return reflectStaticSizeFacetResult + size.SizeOfPtr + - len(fr.Field) + - len(fr.Terms)*(reflectStaticSizeTermFacet+size.SizeOfPtr) + - len(fr.NumericRanges)*(reflectStaticSizeNumericRangeFacet+size.SizeOfPtr) + - len(fr.DateRanges)*(reflectStaticSizeDateRangeFacet+size.SizeOfPtr) -} - -func (fr *FacetResult) Merge(other *FacetResult) { - fr.Total += other.Total - fr.Missing += other.Missing - fr.Other += other.Other - if fr.Terms != nil && other.Terms != nil { - for _, term := range other.Terms { - fr.Terms = fr.Terms.Add(term) - } - } - if fr.NumericRanges != nil && other.NumericRanges != nil { - for _, nr := range other.NumericRanges { - fr.NumericRanges = fr.NumericRanges.Add(nr) - } - } - if fr.DateRanges != nil && other.DateRanges != nil { - for _, dr := range other.DateRanges { - fr.DateRanges = fr.DateRanges.Add(dr) - } - } -} - -func (fr *FacetResult) Fixup(size int) { - if fr.Terms != nil { - sort.Sort(fr.Terms) - if len(fr.Terms) > size { - moveToOther := fr.Terms[size:] - for _, mto := range moveToOther { - fr.Other += mto.Count - } - fr.Terms = fr.Terms[0:size] - } - } else if fr.NumericRanges != nil { - sort.Sort(fr.NumericRanges) - if len(fr.NumericRanges) > size { - moveToOther := fr.NumericRanges[size:] - for _, mto := range moveToOther { - fr.Other += mto.Count - } - fr.NumericRanges = fr.NumericRanges[0:size] - } - } else if fr.DateRanges != nil { - sort.Sort(fr.DateRanges) - if len(fr.DateRanges) > size { - moveToOther := fr.DateRanges[size:] - for _, mto := range moveToOther { - fr.Other += mto.Count - } - fr.DateRanges = fr.DateRanges[0:size] - } - } -} - -type FacetResults map[string]*FacetResult - -func (fr FacetResults) Merge(other FacetResults) { - for name, oFacetResult := range other { - facetResult, ok := fr[name] - if ok { - facetResult.Merge(oFacetResult) - } else { - fr[name] = oFacetResult - } - } -} - -func (fr FacetResults) Fixup(name string, size int) { - facetResult, ok := fr[name] - if ok { - facetResult.Fixup(size) - } -} - -func (fb *FacetsBuilder) Results() FacetResults { - fr := make(FacetResults) - for i, facetBuilder := range fb.facets { - facetResult := facetBuilder.Result() - fr[fb.facetNames[i]] = facetResult - } - return fr -} diff --git a/vendor/github.com/blevesearch/bleve/search/highlight/format/html/html.go b/vendor/github.com/blevesearch/bleve/search/highlight/format/html/html.go deleted file mode 100644 index 259a0379..00000000 --- a/vendor/github.com/blevesearch/bleve/search/highlight/format/html/html.go +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package html - -import ( - "html" - - "github.com/blevesearch/bleve/registry" - "github.com/blevesearch/bleve/search/highlight" -) - -const Name = "html" - -const defaultHTMLHighlightBefore = "" -const defaultHTMLHighlightAfter = "" - -type FragmentFormatter struct { - before string - after string -} - -func NewFragmentFormatter(before, after string) *FragmentFormatter { - return &FragmentFormatter{ - before: before, - after: after, - } -} - -func (a *FragmentFormatter) Format(f *highlight.Fragment, orderedTermLocations highlight.TermLocations) string { - rv := "" - curr := f.Start - for _, termLocation := range orderedTermLocations { - if termLocation == nil { - continue - } - // make sure the array positions match - if !termLocation.ArrayPositions.Equals(f.ArrayPositions) { - continue - } - if termLocation.Start < curr { - continue - } - if termLocation.End > f.End { - break - } - // add the stuff before this location - rv += html.EscapeString(string(f.Orig[curr:termLocation.Start])) - // start the tag - rv += a.before - // add the term itself - rv += string(f.Orig[termLocation.Start:termLocation.End]) - // end the tag - rv += a.after - // update current - curr = termLocation.End - } - // add any remaining text after the last token - rv += html.EscapeString(string(f.Orig[curr:f.End])) - - return rv -} - -func Constructor(config map[string]interface{}, cache *registry.Cache) (highlight.FragmentFormatter, error) { - before := defaultHTMLHighlightBefore - beforeVal, ok := config["before"].(string) - if ok { - before = beforeVal - } - after := defaultHTMLHighlightAfter - afterVal, ok := config["after"].(string) - if ok { - after = afterVal - } - return NewFragmentFormatter(before, after), nil -} - -func init() { - registry.RegisterFragmentFormatter(Name, Constructor) -} diff --git a/vendor/github.com/blevesearch/bleve/search/highlight/highlighter.go b/vendor/github.com/blevesearch/bleve/search/highlight/highlighter.go deleted file mode 100644 index 8077985d..00000000 --- a/vendor/github.com/blevesearch/bleve/search/highlight/highlighter.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package highlight - -import ( - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/search" -) - -type Fragment struct { - Orig []byte - ArrayPositions []uint64 - Start int - End int - Score float64 - Index int // used by heap -} - -func (f *Fragment) Overlaps(other *Fragment) bool { - if other.Start >= f.Start && other.Start < f.End { - return true - } else if f.Start >= other.Start && f.Start < other.End { - return true - } - return false -} - -type Fragmenter interface { - Fragment([]byte, TermLocations) []*Fragment -} - -type FragmentFormatter interface { - Format(f *Fragment, orderedTermLocations TermLocations) string -} - -type FragmentScorer interface { - Score(f *Fragment) float64 -} - -type Highlighter interface { - Fragmenter() Fragmenter - SetFragmenter(Fragmenter) - - FragmentFormatter() FragmentFormatter - SetFragmentFormatter(FragmentFormatter) - - Separator() string - SetSeparator(string) - - BestFragmentInField(*search.DocumentMatch, *document.Document, string) string - BestFragmentsInField(*search.DocumentMatch, *document.Document, string, int) []string -} diff --git a/vendor/github.com/blevesearch/bleve/search/highlight/highlighter/html/html.go b/vendor/github.com/blevesearch/bleve/search/highlight/highlighter/html/html.go deleted file mode 100644 index 928589c4..00000000 --- a/vendor/github.com/blevesearch/bleve/search/highlight/highlighter/html/html.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (c) 2015 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package html - -import ( - "fmt" - - "github.com/blevesearch/bleve/registry" - "github.com/blevesearch/bleve/search/highlight" - htmlFormatter "github.com/blevesearch/bleve/search/highlight/format/html" - simpleFragmenter "github.com/blevesearch/bleve/search/highlight/fragmenter/simple" - simpleHighlighter "github.com/blevesearch/bleve/search/highlight/highlighter/simple" -) - -const Name = "html" - -func Constructor(config map[string]interface{}, cache *registry.Cache) (highlight.Highlighter, error) { - - fragmenter, err := cache.FragmenterNamed(simpleFragmenter.Name) - if err != nil { - return nil, fmt.Errorf("error building fragmenter: %v", err) - } - - formatter, err := cache.FragmentFormatterNamed(htmlFormatter.Name) - if err != nil { - return nil, fmt.Errorf("error building fragment formatter: %v", err) - } - - return simpleHighlighter.NewHighlighter( - fragmenter, - formatter, - simpleHighlighter.DefaultSeparator), - nil -} - -func init() { - registry.RegisterHighlighter(Name, Constructor) -} diff --git a/vendor/github.com/blevesearch/bleve/search/query/docid.go b/vendor/github.com/blevesearch/bleve/search/query/docid.go deleted file mode 100644 index 3b865f93..00000000 --- a/vendor/github.com/blevesearch/bleve/search/query/docid.go +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright (c) 2015 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package query - -import ( - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searcher" -) - -type DocIDQuery struct { - IDs []string `json:"ids"` - BoostVal *Boost `json:"boost,omitempty"` -} - -// NewDocIDQuery creates a new Query object returning indexed documents among -// the specified set. Combine it with ConjunctionQuery to restrict the scope of -// other queries output. -func NewDocIDQuery(ids []string) *DocIDQuery { - return &DocIDQuery{ - IDs: ids, - } -} - -func (q *DocIDQuery) SetBoost(b float64) { - boost := Boost(b) - q.BoostVal = &boost -} - -func (q *DocIDQuery) Boost() float64 { - return q.BoostVal.Value() -} - -func (q *DocIDQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { - return searcher.NewDocIDSearcher(i, q.IDs, q.BoostVal.Value(), options) -} diff --git a/vendor/github.com/blevesearch/bleve/search/query/geo_boundingbox.go b/vendor/github.com/blevesearch/bleve/search/query/geo_boundingbox.go deleted file mode 100644 index de6be4a5..00000000 --- a/vendor/github.com/blevesearch/bleve/search/query/geo_boundingbox.go +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package query - -import ( - "encoding/json" - "fmt" - - "github.com/blevesearch/bleve/geo" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searcher" -) - -type GeoBoundingBoxQuery struct { - TopLeft []float64 `json:"top_left,omitempty"` - BottomRight []float64 `json:"bottom_right,omitempty"` - FieldVal string `json:"field,omitempty"` - BoostVal *Boost `json:"boost,omitempty"` -} - -func NewGeoBoundingBoxQuery(topLeftLon, topLeftLat, bottomRightLon, bottomRightLat float64) *GeoBoundingBoxQuery { - return &GeoBoundingBoxQuery{ - TopLeft: []float64{topLeftLon, topLeftLat}, - BottomRight: []float64{bottomRightLon, bottomRightLat}, - } -} - -func (q *GeoBoundingBoxQuery) SetBoost(b float64) { - boost := Boost(b) - q.BoostVal = &boost -} - -func (q *GeoBoundingBoxQuery) Boost() float64 { - return q.BoostVal.Value() -} - -func (q *GeoBoundingBoxQuery) SetField(f string) { - q.FieldVal = f -} - -func (q *GeoBoundingBoxQuery) Field() string { - return q.FieldVal -} - -func (q *GeoBoundingBoxQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { - field := q.FieldVal - if q.FieldVal == "" { - field = m.DefaultSearchField() - } - - if q.BottomRight[0] < q.TopLeft[0] { - // cross date line, rewrite as two parts - - leftSearcher, err := searcher.NewGeoBoundingBoxSearcher(i, -180, q.BottomRight[1], q.BottomRight[0], q.TopLeft[1], field, q.BoostVal.Value(), options, true) - if err != nil { - return nil, err - } - rightSearcher, err := searcher.NewGeoBoundingBoxSearcher(i, q.TopLeft[0], q.BottomRight[1], 180, q.TopLeft[1], field, q.BoostVal.Value(), options, true) - if err != nil { - _ = leftSearcher.Close() - return nil, err - } - - return searcher.NewDisjunctionSearcher(i, []search.Searcher{leftSearcher, rightSearcher}, 0, options) - } - - return searcher.NewGeoBoundingBoxSearcher(i, q.TopLeft[0], q.BottomRight[1], q.BottomRight[0], q.TopLeft[1], field, q.BoostVal.Value(), options, true) -} - -func (q *GeoBoundingBoxQuery) Validate() error { - return nil -} - -func (q *GeoBoundingBoxQuery) UnmarshalJSON(data []byte) error { - tmp := struct { - TopLeft interface{} `json:"top_left,omitempty"` - BottomRight interface{} `json:"bottom_right,omitempty"` - FieldVal string `json:"field,omitempty"` - BoostVal *Boost `json:"boost,omitempty"` - }{} - err := json.Unmarshal(data, &tmp) - if err != nil { - return err - } - // now use our generic point parsing code from the geo package - lon, lat, found := geo.ExtractGeoPoint(tmp.TopLeft) - if !found { - return fmt.Errorf("geo location top_left not in a valid format") - } - q.TopLeft = []float64{lon, lat} - lon, lat, found = geo.ExtractGeoPoint(tmp.BottomRight) - if !found { - return fmt.Errorf("geo location bottom_right not in a valid format") - } - q.BottomRight = []float64{lon, lat} - q.FieldVal = tmp.FieldVal - q.BoostVal = tmp.BoostVal - return nil -} diff --git a/vendor/github.com/blevesearch/bleve/search/query/match.go b/vendor/github.com/blevesearch/bleve/search/query/match.go deleted file mode 100644 index 36c9ee4a..00000000 --- a/vendor/github.com/blevesearch/bleve/search/query/match.go +++ /dev/null @@ -1,176 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package query - -import ( - "encoding/json" - "fmt" - - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" -) - -type MatchQuery struct { - Match string `json:"match"` - FieldVal string `json:"field,omitempty"` - Analyzer string `json:"analyzer,omitempty"` - BoostVal *Boost `json:"boost,omitempty"` - Prefix int `json:"prefix_length"` - Fuzziness int `json:"fuzziness"` - Operator MatchQueryOperator `json:"operator,omitempty"` -} - -type MatchQueryOperator int - -const ( - // Document must satisfy AT LEAST ONE of term searches. - MatchQueryOperatorOr = 0 - // Document must satisfy ALL of term searches. - MatchQueryOperatorAnd = 1 -) - -func (o MatchQueryOperator) MarshalJSON() ([]byte, error) { - switch o { - case MatchQueryOperatorOr: - return json.Marshal("or") - case MatchQueryOperatorAnd: - return json.Marshal("and") - default: - return nil, fmt.Errorf("cannot marshal match operator %d to JSON", o) - } -} - -func (o *MatchQueryOperator) UnmarshalJSON(data []byte) error { - var operatorString string - err := json.Unmarshal(data, &operatorString) - if err != nil { - return err - } - - switch operatorString { - case "or": - *o = MatchQueryOperatorOr - return nil - case "and": - *o = MatchQueryOperatorAnd - return nil - default: - return fmt.Errorf("cannot unmarshal match operator '%v' from JSON", o) - } -} - -// NewMatchQuery creates a Query for matching text. -// An Analyzer is chosen based on the field. -// Input text is analyzed using this analyzer. -// Token terms resulting from this analysis are -// used to perform term searches. Result documents -// must satisfy at least one of these term searches. -func NewMatchQuery(match string) *MatchQuery { - return &MatchQuery{ - Match: match, - Operator: MatchQueryOperatorOr, - } -} - -func (q *MatchQuery) SetBoost(b float64) { - boost := Boost(b) - q.BoostVal = &boost -} - -func (q *MatchQuery) Boost() float64 { - return q.BoostVal.Value() -} - -func (q *MatchQuery) SetField(f string) { - q.FieldVal = f -} - -func (q *MatchQuery) Field() string { - return q.FieldVal -} - -func (q *MatchQuery) SetFuzziness(f int) { - q.Fuzziness = f -} - -func (q *MatchQuery) SetPrefix(p int) { - q.Prefix = p -} - -func (q *MatchQuery) SetOperator(operator MatchQueryOperator) { - q.Operator = operator -} - -func (q *MatchQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { - - field := q.FieldVal - if q.FieldVal == "" { - field = m.DefaultSearchField() - } - - analyzerName := "" - if q.Analyzer != "" { - analyzerName = q.Analyzer - } else { - analyzerName = m.AnalyzerNameForPath(field) - } - analyzer := m.AnalyzerNamed(analyzerName) - - if analyzer == nil { - return nil, fmt.Errorf("no analyzer named '%s' registered", q.Analyzer) - } - - tokens := analyzer.Analyze([]byte(q.Match)) - if len(tokens) > 0 { - - tqs := make([]Query, len(tokens)) - if q.Fuzziness != 0 { - for i, token := range tokens { - query := NewFuzzyQuery(string(token.Term)) - query.SetFuzziness(q.Fuzziness) - query.SetPrefix(q.Prefix) - query.SetField(field) - query.SetBoost(q.BoostVal.Value()) - tqs[i] = query - } - } else { - for i, token := range tokens { - tq := NewTermQuery(string(token.Term)) - tq.SetField(field) - tq.SetBoost(q.BoostVal.Value()) - tqs[i] = tq - } - } - - switch q.Operator { - case MatchQueryOperatorOr: - shouldQuery := NewDisjunctionQuery(tqs) - shouldQuery.SetMin(1) - shouldQuery.SetBoost(q.BoostVal.Value()) - return shouldQuery.Searcher(i, m, options) - - case MatchQueryOperatorAnd: - mustQuery := NewConjunctionQuery(tqs) - mustQuery.SetBoost(q.BoostVal.Value()) - return mustQuery.Searcher(i, m, options) - - default: - return nil, fmt.Errorf("unhandled operator %d", q.Operator) - } - } - noneQuery := NewMatchNoneQuery() - return noneQuery.Searcher(i, m, options) -} diff --git a/vendor/github.com/blevesearch/bleve/search/query/match_all.go b/vendor/github.com/blevesearch/bleve/search/query/match_all.go deleted file mode 100644 index 7fbe1f99..00000000 --- a/vendor/github.com/blevesearch/bleve/search/query/match_all.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package query - -import ( - "encoding/json" - - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searcher" -) - -type MatchAllQuery struct { - BoostVal *Boost `json:"boost,omitempty"` -} - -// NewMatchAllQuery creates a Query which will -// match all documents in the index. -func NewMatchAllQuery() *MatchAllQuery { - return &MatchAllQuery{} -} - -func (q *MatchAllQuery) SetBoost(b float64) { - boost := Boost(b) - q.BoostVal = &boost -} - -func (q *MatchAllQuery) Boost() float64 { - return q.BoostVal.Value() -} - -func (q *MatchAllQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { - return searcher.NewMatchAllSearcher(i, q.BoostVal.Value(), options) -} - -func (q *MatchAllQuery) MarshalJSON() ([]byte, error) { - tmp := map[string]interface{}{ - "boost": q.BoostVal, - "match_all": map[string]interface{}{}, - } - return json.Marshal(tmp) -} diff --git a/vendor/github.com/blevesearch/bleve/search/query/multi_phrase.go b/vendor/github.com/blevesearch/bleve/search/query/multi_phrase.go deleted file mode 100644 index 8a7c9b6a..00000000 --- a/vendor/github.com/blevesearch/bleve/search/query/multi_phrase.go +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package query - -import ( - "encoding/json" - "fmt" - - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searcher" -) - -type MultiPhraseQuery struct { - Terms [][]string `json:"terms"` - Field string `json:"field,omitempty"` - BoostVal *Boost `json:"boost,omitempty"` -} - -// NewMultiPhraseQuery creates a new Query for finding -// term phrases in the index. -// It is like PhraseQuery, but each position in the -// phrase may be satisfied by a list of terms -// as opposed to just one. -// At least one of the terms must exist in the correct -// order, at the correct index offsets, in the -// specified field. Queried field must have been indexed with -// IncludeTermVectors set to true. -func NewMultiPhraseQuery(terms [][]string, field string) *MultiPhraseQuery { - return &MultiPhraseQuery{ - Terms: terms, - Field: field, - } -} - -func (q *MultiPhraseQuery) SetBoost(b float64) { - boost := Boost(b) - q.BoostVal = &boost -} - -func (q *MultiPhraseQuery) Boost() float64 { - return q.BoostVal.Value() -} - -func (q *MultiPhraseQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { - return searcher.NewMultiPhraseSearcher(i, q.Terms, q.Field, options) -} - -func (q *MultiPhraseQuery) Validate() error { - if len(q.Terms) < 1 { - return fmt.Errorf("phrase query must contain at least one term") - } - return nil -} - -func (q *MultiPhraseQuery) UnmarshalJSON(data []byte) error { - type _mphraseQuery MultiPhraseQuery - tmp := _mphraseQuery{} - err := json.Unmarshal(data, &tmp) - if err != nil { - return err - } - q.Terms = tmp.Terms - q.Field = tmp.Field - q.BoostVal = tmp.BoostVal - return nil -} diff --git a/vendor/github.com/blevesearch/bleve/search/query/phrase.go b/vendor/github.com/blevesearch/bleve/search/query/phrase.go deleted file mode 100644 index dff1a02d..00000000 --- a/vendor/github.com/blevesearch/bleve/search/query/phrase.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package query - -import ( - "encoding/json" - "fmt" - - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searcher" -) - -type PhraseQuery struct { - Terms []string `json:"terms"` - Field string `json:"field,omitempty"` - BoostVal *Boost `json:"boost,omitempty"` -} - -// NewPhraseQuery creates a new Query for finding -// exact term phrases in the index. -// The provided terms must exist in the correct -// order, at the correct index offsets, in the -// specified field. Queried field must have been indexed with -// IncludeTermVectors set to true. -func NewPhraseQuery(terms []string, field string) *PhraseQuery { - return &PhraseQuery{ - Terms: terms, - Field: field, - } -} - -func (q *PhraseQuery) SetBoost(b float64) { - boost := Boost(b) - q.BoostVal = &boost -} - -func (q *PhraseQuery) Boost() float64 { - return q.BoostVal.Value() -} - -func (q *PhraseQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { - return searcher.NewPhraseSearcher(i, q.Terms, q.Field, options) -} - -func (q *PhraseQuery) Validate() error { - if len(q.Terms) < 1 { - return fmt.Errorf("phrase query must contain at least one term") - } - return nil -} - -func (q *PhraseQuery) UnmarshalJSON(data []byte) error { - type _phraseQuery PhraseQuery - tmp := _phraseQuery{} - err := json.Unmarshal(data, &tmp) - if err != nil { - return err - } - q.Terms = tmp.Terms - q.Field = tmp.Field - q.BoostVal = tmp.BoostVal - return nil -} diff --git a/vendor/github.com/blevesearch/bleve/search/query/query.go b/vendor/github.com/blevesearch/bleve/search/query/query.go deleted file mode 100644 index 18aca228..00000000 --- a/vendor/github.com/blevesearch/bleve/search/query/query.go +++ /dev/null @@ -1,361 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package query - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "log" - - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" -) - -var logger = log.New(ioutil.Discard, "bleve mapping ", log.LstdFlags) - -// SetLog sets the logger used for logging -// by default log messages are sent to ioutil.Discard -func SetLog(l *log.Logger) { - logger = l -} - -// A Query represents a description of the type -// and parameters for a query into the index. -type Query interface { - Searcher(i index.IndexReader, m mapping.IndexMapping, - options search.SearcherOptions) (search.Searcher, error) -} - -// A BoostableQuery represents a Query which can be boosted -// relative to other queries. -type BoostableQuery interface { - Query - SetBoost(b float64) - Boost() float64 -} - -// A FieldableQuery represents a Query which can be restricted -// to a single field. -type FieldableQuery interface { - Query - SetField(f string) - Field() string -} - -// A ValidatableQuery represents a Query which can be validated -// prior to execution. -type ValidatableQuery interface { - Query - Validate() error -} - -// ParseQuery deserializes a JSON representation of -// a Query object. -func ParseQuery(input []byte) (Query, error) { - var tmp map[string]interface{} - err := json.Unmarshal(input, &tmp) - if err != nil { - return nil, err - } - _, isMatchQuery := tmp["match"] - _, hasFuzziness := tmp["fuzziness"] - if hasFuzziness && !isMatchQuery { - var rv FuzzyQuery - err := json.Unmarshal(input, &rv) - if err != nil { - return nil, err - } - return &rv, nil - } - _, isTermQuery := tmp["term"] - if isTermQuery { - var rv TermQuery - err := json.Unmarshal(input, &rv) - if err != nil { - return nil, err - } - return &rv, nil - } - if isMatchQuery { - var rv MatchQuery - err := json.Unmarshal(input, &rv) - if err != nil { - return nil, err - } - return &rv, nil - } - _, isMatchPhraseQuery := tmp["match_phrase"] - if isMatchPhraseQuery { - var rv MatchPhraseQuery - err := json.Unmarshal(input, &rv) - if err != nil { - return nil, err - } - return &rv, nil - } - _, hasMust := tmp["must"] - _, hasShould := tmp["should"] - _, hasMustNot := tmp["must_not"] - if hasMust || hasShould || hasMustNot { - var rv BooleanQuery - err := json.Unmarshal(input, &rv) - if err != nil { - return nil, err - } - return &rv, nil - } - _, hasTerms := tmp["terms"] - if hasTerms { - var rv PhraseQuery - err := json.Unmarshal(input, &rv) - if err != nil { - // now try multi-phrase - var rv2 MultiPhraseQuery - err = json.Unmarshal(input, &rv2) - if err != nil { - return nil, err - } - return &rv2, nil - } - return &rv, nil - } - _, hasConjuncts := tmp["conjuncts"] - if hasConjuncts { - var rv ConjunctionQuery - err := json.Unmarshal(input, &rv) - if err != nil { - return nil, err - } - return &rv, nil - } - _, hasDisjuncts := tmp["disjuncts"] - if hasDisjuncts { - var rv DisjunctionQuery - err := json.Unmarshal(input, &rv) - if err != nil { - return nil, err - } - return &rv, nil - } - - _, hasSyntaxQuery := tmp["query"] - if hasSyntaxQuery { - var rv QueryStringQuery - err := json.Unmarshal(input, &rv) - if err != nil { - return nil, err - } - return &rv, nil - } - _, hasMin := tmp["min"].(float64) - _, hasMax := tmp["max"].(float64) - if hasMin || hasMax { - var rv NumericRangeQuery - err := json.Unmarshal(input, &rv) - if err != nil { - return nil, err - } - return &rv, nil - } - _, hasMinStr := tmp["min"].(string) - _, hasMaxStr := tmp["max"].(string) - if hasMinStr || hasMaxStr { - var rv TermRangeQuery - err := json.Unmarshal(input, &rv) - if err != nil { - return nil, err - } - return &rv, nil - } - _, hasStart := tmp["start"] - _, hasEnd := tmp["end"] - if hasStart || hasEnd { - var rv DateRangeQuery - err := json.Unmarshal(input, &rv) - if err != nil { - return nil, err - } - return &rv, nil - } - _, hasPrefix := tmp["prefix"] - if hasPrefix { - var rv PrefixQuery - err := json.Unmarshal(input, &rv) - if err != nil { - return nil, err - } - return &rv, nil - } - _, hasRegexp := tmp["regexp"] - if hasRegexp { - var rv RegexpQuery - err := json.Unmarshal(input, &rv) - if err != nil { - return nil, err - } - return &rv, nil - } - _, hasWildcard := tmp["wildcard"] - if hasWildcard { - var rv WildcardQuery - err := json.Unmarshal(input, &rv) - if err != nil { - return nil, err - } - return &rv, nil - } - _, hasMatchAll := tmp["match_all"] - if hasMatchAll { - var rv MatchAllQuery - err := json.Unmarshal(input, &rv) - if err != nil { - return nil, err - } - return &rv, nil - } - _, hasMatchNone := tmp["match_none"] - if hasMatchNone { - var rv MatchNoneQuery - err := json.Unmarshal(input, &rv) - if err != nil { - return nil, err - } - return &rv, nil - } - _, hasDocIds := tmp["ids"] - if hasDocIds { - var rv DocIDQuery - err := json.Unmarshal(input, &rv) - if err != nil { - return nil, err - } - return &rv, nil - } - _, hasBool := tmp["bool"] - if hasBool { - var rv BoolFieldQuery - err := json.Unmarshal(input, &rv) - if err != nil { - return nil, err - } - return &rv, nil - } - _, hasTopLeft := tmp["top_left"] - _, hasBottomRight := tmp["bottom_right"] - if hasTopLeft && hasBottomRight { - var rv GeoBoundingBoxQuery - err := json.Unmarshal(input, &rv) - if err != nil { - return nil, err - } - return &rv, nil - } - _, hasDistance := tmp["distance"] - if hasDistance { - var rv GeoDistanceQuery - err := json.Unmarshal(input, &rv) - if err != nil { - return nil, err - } - return &rv, nil - } - _, hasPoints := tmp["polygon_points"] - if hasPoints { - var rv GeoBoundingPolygonQuery - err := json.Unmarshal(input, &rv) - if err != nil { - return nil, err - } - return &rv, nil - } - return nil, fmt.Errorf("unknown query type") -} - -// expandQuery traverses the input query tree and returns a new tree where -// query string queries have been expanded into base queries. Returned tree may -// reference queries from the input tree or new queries. -func expandQuery(m mapping.IndexMapping, query Query) (Query, error) { - var expand func(query Query) (Query, error) - var expandSlice func(queries []Query) ([]Query, error) - - expandSlice = func(queries []Query) ([]Query, error) { - expanded := []Query{} - for _, q := range queries { - exp, err := expand(q) - if err != nil { - return nil, err - } - expanded = append(expanded, exp) - } - return expanded, nil - } - - expand = func(query Query) (Query, error) { - switch q := query.(type) { - case *QueryStringQuery: - parsed, err := parseQuerySyntax(q.Query) - if err != nil { - return nil, fmt.Errorf("could not parse '%s': %s", q.Query, err) - } - return expand(parsed) - case *ConjunctionQuery: - children, err := expandSlice(q.Conjuncts) - if err != nil { - return nil, err - } - q.Conjuncts = children - return q, nil - case *DisjunctionQuery: - children, err := expandSlice(q.Disjuncts) - if err != nil { - return nil, err - } - q.Disjuncts = children - return q, nil - case *BooleanQuery: - var err error - q.Must, err = expand(q.Must) - if err != nil { - return nil, err - } - q.Should, err = expand(q.Should) - if err != nil { - return nil, err - } - q.MustNot, err = expand(q.MustNot) - if err != nil { - return nil, err - } - return q, nil - default: - return query, nil - } - } - return expand(query) -} - -// DumpQuery returns a string representation of the query tree, where query -// string queries have been expanded into base queries. The output format is -// meant for debugging purpose and may change in the future. -func DumpQuery(m mapping.IndexMapping, query Query) (string, error) { - q, err := expandQuery(m, query) - if err != nil { - return "", err - } - data, err := json.MarshalIndent(q, "", " ") - return string(data), err -} diff --git a/vendor/github.com/blevesearch/bleve/search/query/regexp.go b/vendor/github.com/blevesearch/bleve/search/query/regexp.go deleted file mode 100644 index 0c87a6f9..00000000 --- a/vendor/github.com/blevesearch/bleve/search/query/regexp.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package query - -import ( - "strings" - - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searcher" -) - -type RegexpQuery struct { - Regexp string `json:"regexp"` - FieldVal string `json:"field,omitempty"` - BoostVal *Boost `json:"boost,omitempty"` -} - -// NewRegexpQuery creates a new Query which finds -// documents containing terms that match the -// specified regular expression. The regexp pattern -// SHOULD NOT include ^ or $ modifiers, the search -// will only match entire terms even without them. -func NewRegexpQuery(regexp string) *RegexpQuery { - return &RegexpQuery{ - Regexp: regexp, - } -} - -func (q *RegexpQuery) SetBoost(b float64) { - boost := Boost(b) - q.BoostVal = &boost -} - -func (q *RegexpQuery) Boost() float64 { - return q.BoostVal.Value() -} - -func (q *RegexpQuery) SetField(f string) { - q.FieldVal = f -} - -func (q *RegexpQuery) Field() string { - return q.FieldVal -} - -func (q *RegexpQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { - field := q.FieldVal - if q.FieldVal == "" { - field = m.DefaultSearchField() - } - - // require that pattern NOT be anchored to start and end of term. - // do not attempt to remove trailing $, its presence is not - // known to interfere with LiteralPrefix() the way ^ does - // and removing $ introduces possible ambiguities with escaped \$, \\$, etc - actualRegexp := q.Regexp - if strings.HasPrefix(actualRegexp, "^") { - actualRegexp = actualRegexp[1:] // remove leading ^ - } - - return searcher.NewRegexpStringSearcher(i, actualRegexp, field, - q.BoostVal.Value(), options) -} - -func (q *RegexpQuery) Validate() error { - return nil // real validation delayed until searcher constructor -} diff --git a/vendor/github.com/blevesearch/bleve/search/query/term.go b/vendor/github.com/blevesearch/bleve/search/query/term.go deleted file mode 100644 index 2eeb5a37..00000000 --- a/vendor/github.com/blevesearch/bleve/search/query/term.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package query - -import ( - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searcher" -) - -type TermQuery struct { - Term string `json:"term"` - FieldVal string `json:"field,omitempty"` - BoostVal *Boost `json:"boost,omitempty"` -} - -// NewTermQuery creates a new Query for finding an -// exact term match in the index. -func NewTermQuery(term string) *TermQuery { - return &TermQuery{ - Term: term, - } -} - -func (q *TermQuery) SetBoost(b float64) { - boost := Boost(b) - q.BoostVal = &boost -} - -func (q *TermQuery) Boost() float64 { - return q.BoostVal.Value() -} - -func (q *TermQuery) SetField(f string) { - q.FieldVal = f -} - -func (q *TermQuery) Field() string { - return q.FieldVal -} - -func (q *TermQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { - field := q.FieldVal - if q.FieldVal == "" { - field = m.DefaultSearchField() - } - return searcher.NewTermSearcher(i, q.Term, field, q.BoostVal.Value(), options) -} diff --git a/vendor/github.com/blevesearch/bleve/search/search.go b/vendor/github.com/blevesearch/bleve/search/search.go deleted file mode 100644 index 8ed23de4..00000000 --- a/vendor/github.com/blevesearch/bleve/search/search.go +++ /dev/null @@ -1,378 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package search - -import ( - "fmt" - "reflect" - "sort" - - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/size" -) - -var reflectStaticSizeDocumentMatch int -var reflectStaticSizeSearchContext int -var reflectStaticSizeLocation int - -func init() { - var dm DocumentMatch - reflectStaticSizeDocumentMatch = int(reflect.TypeOf(dm).Size()) - var sc SearchContext - reflectStaticSizeSearchContext = int(reflect.TypeOf(sc).Size()) - var l Location - reflectStaticSizeLocation = int(reflect.TypeOf(l).Size()) -} - -type ArrayPositions []uint64 - -func (ap ArrayPositions) Equals(other ArrayPositions) bool { - if len(ap) != len(other) { - return false - } - for i := range ap { - if ap[i] != other[i] { - return false - } - } - return true -} - -func (ap ArrayPositions) Compare(other ArrayPositions) int { - for i, p := range ap { - if i >= len(other) { - return 1 - } - if p < other[i] { - return -1 - } - if p > other[i] { - return 1 - } - } - if len(ap) < len(other) { - return -1 - } - return 0 -} - -type Location struct { - // Pos is the position of the term within the field, starting at 1 - Pos uint64 `json:"pos"` - - // Start and End are the byte offsets of the term in the field - Start uint64 `json:"start"` - End uint64 `json:"end"` - - // ArrayPositions contains the positions of the term within any elements. - ArrayPositions ArrayPositions `json:"array_positions"` -} - -func (l *Location) Size() int { - return reflectStaticSizeLocation + size.SizeOfPtr + - len(l.ArrayPositions)*size.SizeOfUint64 -} - -type Locations []*Location - -func (p Locations) Len() int { return len(p) } -func (p Locations) Swap(i, j int) { p[i], p[j] = p[j], p[i] } - -func (p Locations) Less(i, j int) bool { - c := p[i].ArrayPositions.Compare(p[j].ArrayPositions) - if c < 0 { - return true - } - if c > 0 { - return false - } - return p[i].Pos < p[j].Pos -} - -func (p Locations) Dedupe() Locations { // destructive! - if len(p) <= 1 { - return p - } - - sort.Sort(p) - - slow := 0 - - for _, pfast := range p { - pslow := p[slow] - if pslow.Pos == pfast.Pos && - pslow.Start == pfast.Start && - pslow.End == pfast.End && - pslow.ArrayPositions.Equals(pfast.ArrayPositions) { - continue // duplicate, so only move fast ahead - } - - slow++ - - p[slow] = pfast - } - - return p[:slow+1] -} - -type TermLocationMap map[string]Locations - -func (t TermLocationMap) AddLocation(term string, location *Location) { - t[term] = append(t[term], location) -} - -type FieldTermLocationMap map[string]TermLocationMap - -type FieldTermLocation struct { - Field string - Term string - Location Location -} - -type FieldFragmentMap map[string][]string - -type DocumentMatch struct { - Index string `json:"index,omitempty"` - ID string `json:"id"` - IndexInternalID index.IndexInternalID `json:"-"` - Score float64 `json:"score"` - Expl *Explanation `json:"explanation,omitempty"` - Locations FieldTermLocationMap `json:"locations,omitempty"` - Fragments FieldFragmentMap `json:"fragments,omitempty"` - Sort []string `json:"sort,omitempty"` - - // Fields contains the values for document fields listed in - // SearchRequest.Fields. Text fields are returned as strings, numeric - // fields as float64s and date fields as time.RFC3339 formatted strings. - Fields map[string]interface{} `json:"fields,omitempty"` - - // used to maintain natural index order - HitNumber uint64 `json:"-"` - - // used to temporarily hold field term location information during - // search processing in an efficient, recycle-friendly manner, to - // be later incorporated into the Locations map when search - // results are completed - FieldTermLocations []FieldTermLocation `json:"-"` -} - -func (dm *DocumentMatch) AddFieldValue(name string, value interface{}) { - if dm.Fields == nil { - dm.Fields = make(map[string]interface{}) - } - existingVal, ok := dm.Fields[name] - if !ok { - dm.Fields[name] = value - return - } - - valSlice, ok := existingVal.([]interface{}) - if ok { - // already a slice, append to it - valSlice = append(valSlice, value) - } else { - // create a slice - valSlice = []interface{}{existingVal, value} - } - dm.Fields[name] = valSlice -} - -// Reset allows an already allocated DocumentMatch to be reused -func (dm *DocumentMatch) Reset() *DocumentMatch { - // remember the []byte used for the IndexInternalID - indexInternalID := dm.IndexInternalID - // remember the []interface{} used for sort - sort := dm.Sort - // remember the FieldTermLocations backing array - ftls := dm.FieldTermLocations - for i := range ftls { // recycle the ArrayPositions of each location - ftls[i].Location.ArrayPositions = ftls[i].Location.ArrayPositions[:0] - } - // idiom to copy over from empty DocumentMatch (0 allocations) - *dm = DocumentMatch{} - // reuse the []byte already allocated (and reset len to 0) - dm.IndexInternalID = indexInternalID[:0] - // reuse the []interface{} already allocated (and reset len to 0) - dm.Sort = sort[:0] - // reuse the FieldTermLocations already allocated (and reset len to 0) - dm.FieldTermLocations = ftls[:0] - return dm -} - -func (dm *DocumentMatch) Size() int { - sizeInBytes := reflectStaticSizeDocumentMatch + size.SizeOfPtr + - len(dm.Index) + - len(dm.ID) + - len(dm.IndexInternalID) - - if dm.Expl != nil { - sizeInBytes += dm.Expl.Size() - } - - for k, v := range dm.Locations { - sizeInBytes += size.SizeOfString + len(k) - for k1, v1 := range v { - sizeInBytes += size.SizeOfString + len(k1) + - size.SizeOfSlice - for _, entry := range v1 { - sizeInBytes += entry.Size() - } - } - } - - for k, v := range dm.Fragments { - sizeInBytes += size.SizeOfString + len(k) + - size.SizeOfSlice - - for _, entry := range v { - sizeInBytes += size.SizeOfString + len(entry) - } - } - - for _, entry := range dm.Sort { - sizeInBytes += size.SizeOfString + len(entry) - } - - for k, _ := range dm.Fields { - sizeInBytes += size.SizeOfString + len(k) + - size.SizeOfPtr - } - - return sizeInBytes -} - -// Complete performs final preparation & transformation of the -// DocumentMatch at the end of search processing, also allowing the -// caller to provide an optional preallocated locations slice -func (dm *DocumentMatch) Complete(prealloc []Location) []Location { - // transform the FieldTermLocations slice into the Locations map - nlocs := len(dm.FieldTermLocations) - if nlocs > 0 { - if cap(prealloc) < nlocs { - prealloc = make([]Location, nlocs) - } - prealloc = prealloc[:nlocs] - - var lastField string - var tlm TermLocationMap - var needsDedupe bool - - for i, ftl := range dm.FieldTermLocations { - if lastField != ftl.Field { - lastField = ftl.Field - - if dm.Locations == nil { - dm.Locations = make(FieldTermLocationMap) - } - - tlm = dm.Locations[ftl.Field] - if tlm == nil { - tlm = make(TermLocationMap) - dm.Locations[ftl.Field] = tlm - } - } - - loc := &prealloc[i] - *loc = ftl.Location - - if len(loc.ArrayPositions) > 0 { // copy - loc.ArrayPositions = append(ArrayPositions(nil), loc.ArrayPositions...) - } - - locs := tlm[ftl.Term] - - // if the loc is before or at the last location, then there - // might be duplicates that need to be deduplicated - if !needsDedupe && len(locs) > 0 { - last := locs[len(locs)-1] - cmp := loc.ArrayPositions.Compare(last.ArrayPositions) - if cmp < 0 || (cmp == 0 && loc.Pos <= last.Pos) { - needsDedupe = true - } - } - - tlm[ftl.Term] = append(locs, loc) - - dm.FieldTermLocations[i] = FieldTermLocation{ // recycle - Location: Location{ - ArrayPositions: ftl.Location.ArrayPositions[:0], - }, - } - } - - if needsDedupe { - for _, tlm := range dm.Locations { - for term, locs := range tlm { - tlm[term] = locs.Dedupe() - } - } - } - } - - dm.FieldTermLocations = dm.FieldTermLocations[:0] // recycle - - return prealloc -} - -func (dm *DocumentMatch) String() string { - return fmt.Sprintf("[%s-%f]", string(dm.IndexInternalID), dm.Score) -} - -type DocumentMatchCollection []*DocumentMatch - -func (c DocumentMatchCollection) Len() int { return len(c) } -func (c DocumentMatchCollection) Swap(i, j int) { c[i], c[j] = c[j], c[i] } -func (c DocumentMatchCollection) Less(i, j int) bool { return c[i].Score > c[j].Score } - -type Searcher interface { - Next(ctx *SearchContext) (*DocumentMatch, error) - Advance(ctx *SearchContext, ID index.IndexInternalID) (*DocumentMatch, error) - Close() error - Weight() float64 - SetQueryNorm(float64) - Count() uint64 - Min() int - Size() int - - DocumentMatchPoolSize() int -} - -type SearcherOptions struct { - Explain bool - IncludeTermVectors bool - Score string -} - -// SearchContext represents the context around a single search -type SearchContext struct { - DocumentMatchPool *DocumentMatchPool - Collector Collector - IndexReader index.IndexReader -} - -func (sc *SearchContext) Size() int { - sizeInBytes := reflectStaticSizeSearchContext + size.SizeOfPtr + - reflectStaticSizeDocumentMatchPool + size.SizeOfPtr - - if sc.DocumentMatchPool != nil { - for _, entry := range sc.DocumentMatchPool.avail { - if entry != nil { - sizeInBytes += entry.Size() - } - } - } - - return sizeInBytes -} diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_fuzzy.go b/vendor/github.com/blevesearch/bleve/search/searcher/search_fuzzy.go deleted file mode 100644 index aca8a7d9..00000000 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_fuzzy.go +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package searcher - -import ( - "fmt" - - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" -) - -var MaxFuzziness = 2 - -func NewFuzzySearcher(indexReader index.IndexReader, term string, - prefix, fuzziness int, field string, boost float64, - options search.SearcherOptions) (search.Searcher, error) { - - if fuzziness > MaxFuzziness { - return nil, fmt.Errorf("fuzziness exceeds max (%d)", MaxFuzziness) - } - - if fuzziness < 0 { - return nil, fmt.Errorf("invalid fuzziness, negative") - } - - // Note: we don't byte slice the term for a prefix because of runes. - prefixTerm := "" - for i, r := range term { - if i < prefix { - prefixTerm += string(r) - } else { - break - } - } - candidateTerms, err := findFuzzyCandidateTerms(indexReader, term, fuzziness, - field, prefixTerm) - if err != nil { - return nil, err - } - - return NewMultiTermSearcher(indexReader, candidateTerms, field, - boost, options, true) -} - -func findFuzzyCandidateTerms(indexReader index.IndexReader, term string, - fuzziness int, field, prefixTerm string) (rv []string, err error) { - rv = make([]string, 0) - - // in case of advanced reader implementations directly call - // the levenshtein automaton based iterator to collect the - // candidate terms - if ir, ok := indexReader.(index.IndexReaderFuzzy); ok { - fieldDict, err := ir.FieldDictFuzzy(field, term, fuzziness, prefixTerm) - if err != nil { - return nil, err - } - defer func() { - if cerr := fieldDict.Close(); cerr != nil && err == nil { - err = cerr - } - }() - tfd, err := fieldDict.Next() - for err == nil && tfd != nil { - rv = append(rv, tfd.Term) - if tooManyClauses(len(rv)) { - return nil, tooManyClausesErr(field, len(rv)) - } - tfd, err = fieldDict.Next() - } - return rv, err - } - - var fieldDict index.FieldDict - if len(prefixTerm) > 0 { - fieldDict, err = indexReader.FieldDictPrefix(field, []byte(prefixTerm)) - } else { - fieldDict, err = indexReader.FieldDict(field) - } - if err != nil { - return nil, err - } - defer func() { - if cerr := fieldDict.Close(); cerr != nil && err == nil { - err = cerr - } - }() - - // enumerate terms and check levenshtein distance - var reuse []int - tfd, err := fieldDict.Next() - for err == nil && tfd != nil { - var ld int - var exceeded bool - ld, exceeded, reuse = search.LevenshteinDistanceMaxReuseSlice(term, tfd.Term, fuzziness, reuse) - if !exceeded && ld <= fuzziness { - rv = append(rv, tfd.Term) - if tooManyClauses(len(rv)) { - return nil, tooManyClausesErr(field, len(rv)) - } - } - tfd, err = fieldDict.Next() - } - - return rv, err -} diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_geoboundingbox.go b/vendor/github.com/blevesearch/bleve/search/searcher/search_geoboundingbox.go deleted file mode 100644 index 76157f01..00000000 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_geoboundingbox.go +++ /dev/null @@ -1,272 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package searcher - -import ( - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/geo" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/numeric" - "github.com/blevesearch/bleve/search" -) - -type filterFunc func(key []byte) bool - -var GeoBitsShift1 = geo.GeoBits << 1 -var GeoBitsShift1Minus1 = GeoBitsShift1 - 1 - -func NewGeoBoundingBoxSearcher(indexReader index.IndexReader, minLon, minLat, - maxLon, maxLat float64, field string, boost float64, - options search.SearcherOptions, checkBoundaries bool) ( - search.Searcher, error) { - - // track list of opened searchers, for cleanup on early exit - var openedSearchers []search.Searcher - cleanupOpenedSearchers := func() { - for _, s := range openedSearchers { - _ = s.Close() - } - } - - // do math to produce list of terms needed for this search - onBoundaryTerms, notOnBoundaryTerms, err := ComputeGeoRange(0, GeoBitsShift1Minus1, - minLon, minLat, maxLon, maxLat, checkBoundaries, indexReader, field) - if err != nil { - return nil, err - } - - var onBoundarySearcher search.Searcher - dvReader, err := indexReader.DocValueReader([]string{field}) - if err != nil { - return nil, err - } - - if len(onBoundaryTerms) > 0 { - rawOnBoundarySearcher, err := NewMultiTermSearcherBytes(indexReader, - onBoundaryTerms, field, boost, options, false) - if err != nil { - return nil, err - } - // add filter to check points near the boundary - onBoundarySearcher = NewFilteringSearcher(rawOnBoundarySearcher, - buildRectFilter(dvReader, field, minLon, minLat, maxLon, maxLat)) - openedSearchers = append(openedSearchers, onBoundarySearcher) - } - - var notOnBoundarySearcher search.Searcher - if len(notOnBoundaryTerms) > 0 { - var err error - notOnBoundarySearcher, err = NewMultiTermSearcherBytes(indexReader, - notOnBoundaryTerms, field, boost, options, false) - if err != nil { - cleanupOpenedSearchers() - return nil, err - } - openedSearchers = append(openedSearchers, notOnBoundarySearcher) - } - - if onBoundarySearcher != nil && notOnBoundarySearcher != nil { - rv, err := NewDisjunctionSearcher(indexReader, - []search.Searcher{ - onBoundarySearcher, - notOnBoundarySearcher, - }, - 0, options) - if err != nil { - cleanupOpenedSearchers() - return nil, err - } - return rv, nil - } else if onBoundarySearcher != nil { - return onBoundarySearcher, nil - } else if notOnBoundarySearcher != nil { - return notOnBoundarySearcher, nil - } - - return NewMatchNoneSearcher(indexReader) -} - -var geoMaxShift = document.GeoPrecisionStep * 4 -var geoDetailLevel = ((geo.GeoBits << 1) - geoMaxShift) / 2 -type closeFunc func() error - -func ComputeGeoRange(term uint64, shift uint, - sminLon, sminLat, smaxLon, smaxLat float64, checkBoundaries bool, - indexReader index.IndexReader, field string) ( - onBoundary [][]byte, notOnBoundary [][]byte, err error) { - - isIndexed, closeF, err := buildIsIndexedFunc(indexReader, field) - if closeF != nil { - defer func() { - cerr := closeF() - if cerr != nil { - err = cerr - } - }() - } - - grc := &geoRangeCompute{ - preallocBytesLen: 32, - preallocBytes: make([]byte, 32), - sminLon: sminLon, - sminLat: sminLat, - smaxLon: smaxLon, - smaxLat: smaxLat, - checkBoundaries: checkBoundaries, - isIndexed: isIndexed, - } - - grc.computeGeoRange(term, shift) - - return grc.onBoundary, grc.notOnBoundary, nil -} - -func buildIsIndexedFunc(indexReader index.IndexReader, field string) (isIndexed filterFunc, closeF closeFunc, err error) { - if irr, ok := indexReader.(index.IndexReaderContains); ok { - fieldDict, err := irr.FieldDictContains(field) - if err != nil { - return nil, nil, err - } - - isIndexed = func(term []byte) bool { - found, err := fieldDict.Contains(term) - return err == nil && found - } - - closeF = func() error { - if fd, ok := fieldDict.(index.FieldDict); ok { - err := fd.Close() - if err != nil { - return err - } - } - return nil - } - } else if indexReader != nil { - isIndexed = func(term []byte) bool { - reader, err := indexReader.TermFieldReader(term, field, false, false, false) - if err != nil || reader == nil { - return false - } - if reader.Count() == 0 { - _ = reader.Close() - return false - } - _ = reader.Close() - return true - } - - } else { - isIndexed = func([]byte) bool { - return true - } - } - return isIndexed, closeF, err -} - -func buildRectFilter(dvReader index.DocValueReader, field string, - minLon, minLat, maxLon, maxLat float64) FilterFunc { - return func(d *search.DocumentMatch) bool { - // check geo matches against all numeric type terms indexed - var lons, lats []float64 - var found bool - err := dvReader.VisitDocValues(d.IndexInternalID, func(field string, term []byte) { - // only consider the values which are shifted 0 - prefixCoded := numeric.PrefixCoded(term) - shift, err := prefixCoded.Shift() - if err == nil && shift == 0 { - var i64 int64 - i64, err = prefixCoded.Int64() - if err == nil { - lons = append(lons, geo.MortonUnhashLon(uint64(i64))) - lats = append(lats, geo.MortonUnhashLat(uint64(i64))) - found = true - } - } - }) - if err == nil && found { - for i := range lons { - if geo.BoundingBoxContains(lons[i], lats[i], - minLon, minLat, maxLon, maxLat) { - return true - } - } - } - return false - } -} - -type geoRangeCompute struct { - preallocBytesLen int - preallocBytes []byte - sminLon, sminLat, smaxLon, smaxLat float64 - checkBoundaries bool - onBoundary, notOnBoundary [][]byte - isIndexed func(term []byte) bool -} - -func (grc *geoRangeCompute) makePrefixCoded(in int64, shift uint) (rv numeric.PrefixCoded) { - if len(grc.preallocBytes) <= 0 { - grc.preallocBytesLen = grc.preallocBytesLen * 2 - grc.preallocBytes = make([]byte, grc.preallocBytesLen) - } - - rv, grc.preallocBytes, _ = - numeric.NewPrefixCodedInt64Prealloc(in, shift, grc.preallocBytes) - - return rv -} - -func (grc *geoRangeCompute) computeGeoRange(term uint64, shift uint) { - split := term | uint64(0x1)<> 1 - - within := res%document.GeoPrecisionStep == 0 && - geo.RectWithin(minLon, minLat, maxLon, maxLat, - grc.sminLon, grc.sminLat, grc.smaxLon, grc.smaxLat) - if within || (level == geoDetailLevel && - geo.RectIntersects(minLon, minLat, maxLon, maxLat, - grc.sminLon, grc.sminLat, grc.smaxLon, grc.smaxLat)) { - codedTerm := grc.makePrefixCoded(int64(start), res) - if grc.isIndexed(codedTerm) { - if !within && grc.checkBoundaries { - grc.onBoundary = append(grc.onBoundary, codedTerm) - } else { - grc.notOnBoundary = append(grc.notOnBoundary, codedTerm) - } - } - } else if level < geoDetailLevel && - geo.RectIntersects(minLon, minLat, maxLon, maxLat, - grc.sminLon, grc.sminLat, grc.smaxLon, grc.smaxLat) { - grc.computeGeoRange(start, res-1) - } -} \ No newline at end of file diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_geopointdistance.go b/vendor/github.com/blevesearch/bleve/search/searcher/search_geopointdistance.go deleted file mode 100644 index b6f29324..00000000 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_geopointdistance.go +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package searcher - -import ( - "github.com/blevesearch/bleve/geo" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/numeric" - "github.com/blevesearch/bleve/search" -) - -func NewGeoPointDistanceSearcher(indexReader index.IndexReader, centerLon, - centerLat, dist float64, field string, boost float64, - options search.SearcherOptions) (search.Searcher, error) { - // compute bounding box containing the circle - topLeftLon, topLeftLat, bottomRightLon, bottomRightLat, err := - geo.RectFromPointDistance(centerLon, centerLat, dist) - if err != nil { - return nil, err - } - - // build a searcher for the box - boxSearcher, err := boxSearcher(indexReader, - topLeftLon, topLeftLat, bottomRightLon, bottomRightLat, - field, boost, options, false) - if err != nil { - return nil, err - } - - dvReader, err := indexReader.DocValueReader([]string{field}) - if err != nil { - return nil, err - } - - // wrap it in a filtering searcher which checks the actual distance - return NewFilteringSearcher(boxSearcher, - buildDistFilter(dvReader, field, centerLon, centerLat, dist)), nil -} - -// boxSearcher builds a searcher for the described bounding box -// if the desired box crosses the dateline, it is automatically split into -// two boxes joined through a disjunction searcher -func boxSearcher(indexReader index.IndexReader, - topLeftLon, topLeftLat, bottomRightLon, bottomRightLat float64, - field string, boost float64, options search.SearcherOptions, checkBoundaries bool) ( - search.Searcher, error) { - if bottomRightLon < topLeftLon { - // cross date line, rewrite as two parts - - leftSearcher, err := NewGeoBoundingBoxSearcher(indexReader, - -180, bottomRightLat, bottomRightLon, topLeftLat, - field, boost, options, checkBoundaries) - if err != nil { - return nil, err - } - rightSearcher, err := NewGeoBoundingBoxSearcher(indexReader, - topLeftLon, bottomRightLat, 180, topLeftLat, field, boost, options, - checkBoundaries) - if err != nil { - _ = leftSearcher.Close() - return nil, err - } - - boxSearcher, err := NewDisjunctionSearcher(indexReader, - []search.Searcher{leftSearcher, rightSearcher}, 0, options) - if err != nil { - _ = leftSearcher.Close() - _ = rightSearcher.Close() - return nil, err - } - return boxSearcher, nil - } - - // build geoboundingbox searcher for that bounding box - boxSearcher, err := NewGeoBoundingBoxSearcher(indexReader, - topLeftLon, bottomRightLat, bottomRightLon, topLeftLat, field, boost, - options, checkBoundaries) - if err != nil { - return nil, err - } - return boxSearcher, nil -} - -func buildDistFilter(dvReader index.DocValueReader, field string, - centerLon, centerLat, maxDist float64) FilterFunc { - return func(d *search.DocumentMatch) bool { - // check geo matches against all numeric type terms indexed - var lons, lats []float64 - var found bool - - err := dvReader.VisitDocValues(d.IndexInternalID, func(field string, term []byte) { - // only consider the values which are shifted 0 - prefixCoded := numeric.PrefixCoded(term) - shift, err := prefixCoded.Shift() - if err == nil && shift == 0 { - i64, err := prefixCoded.Int64() - if err == nil { - lons = append(lons, geo.MortonUnhashLon(uint64(i64))) - lats = append(lats, geo.MortonUnhashLat(uint64(i64))) - found = true - } - } - }) - if err == nil && found { - for i := range lons { - dist := geo.Haversin(lons[i], lats[i], centerLon, centerLat) - if dist <= maxDist/1000 { - return true - } - } - } - return false - } -} diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_geopolygon.go b/vendor/github.com/blevesearch/bleve/search/searcher/search_geopolygon.go deleted file mode 100644 index 5f16aa8d..00000000 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_geopolygon.go +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright (c) 2019 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package searcher - -import ( - "fmt" - "github.com/blevesearch/bleve/geo" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/numeric" - "github.com/blevesearch/bleve/search" - "math" -) - -func NewGeoBoundedPolygonSearcher(indexReader index.IndexReader, - polygon []geo.Point, field string, boost float64, - options search.SearcherOptions) (search.Searcher, error) { - - if len(polygon) < 3 { - return nil, fmt.Errorf("Too few points specified for the polygon boundary") - } - - // compute the bounding box enclosing the polygon - topLeftLon, topLeftLat, bottomRightLon, bottomRightLat, err := - geo.BoundingRectangleForPolygon(polygon) - if err != nil { - return nil, err - } - - // build a searcher for the bounding box on the polygon - boxSearcher, err := boxSearcher(indexReader, - topLeftLon, topLeftLat, bottomRightLon, bottomRightLat, - field, boost, options, true) - if err != nil { - return nil, err - } - - dvReader, err := indexReader.DocValueReader([]string{field}) - if err != nil { - return nil, err - } - - // wrap it in a filtering searcher that checks for the polygon inclusivity - return NewFilteringSearcher(boxSearcher, - buildPolygonFilter(dvReader, field, polygon)), nil -} - -const float64EqualityThreshold = 1e-6 - -func almostEqual(a, b float64) bool { - return math.Abs(a-b) <= float64EqualityThreshold -} - -// buildPolygonFilter returns true if the point lies inside the -// polygon. It is based on the ray-casting technique as referred -// here: https://wrf.ecse.rpi.edu/nikola/pubdetails/pnpoly.html -func buildPolygonFilter(dvReader index.DocValueReader, field string, - polygon []geo.Point) FilterFunc { - return func(d *search.DocumentMatch) bool { - // check geo matches against all numeric type terms indexed - var lons, lats []float64 - var found bool - - err := dvReader.VisitDocValues(d.IndexInternalID, func(field string, term []byte) { - // only consider the values which are shifted 0 - prefixCoded := numeric.PrefixCoded(term) - shift, err := prefixCoded.Shift() - if err == nil && shift == 0 { - i64, err := prefixCoded.Int64() - if err == nil { - lons = append(lons, geo.MortonUnhashLon(uint64(i64))) - lats = append(lats, geo.MortonUnhashLat(uint64(i64))) - found = true - } - } - }) - - // Note: this approach works for points which are strictly inside - // the polygon. ie it might fail for certain points on the polygon boundaries. - if err == nil && found { - nVertices := len(polygon) - if len(polygon) < 3 { - return false - } - rayIntersectsSegment := func(point, a, b geo.Point) bool { - return (a.Lat > point.Lat) != (b.Lat > point.Lat) && - point.Lon < (b.Lon-a.Lon)*(point.Lat-a.Lat)/(b.Lat-a.Lat)+a.Lon - } - - for i := range lons { - pt := geo.Point{Lon: lons[i], Lat: lats[i]} - inside := rayIntersectsSegment(pt, polygon[len(polygon)-1], polygon[0]) - // check for a direct vertex match - if almostEqual(polygon[0].Lat, lats[i]) && - almostEqual(polygon[0].Lon, lons[i]) { - return true - } - - for j := 1; j < nVertices; j++ { - if almostEqual(polygon[j].Lat, lats[i]) && - almostEqual(polygon[j].Lon, lons[i]) { - return true - } - if rayIntersectsSegment(pt, polygon[j-1], polygon[j]) { - inside = !inside - } - } - if inside { - return true - } - } - } - return false - } -} diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_multi_term.go b/vendor/github.com/blevesearch/bleve/search/searcher/search_multi_term.go deleted file mode 100644 index 70a2fa38..00000000 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_multi_term.go +++ /dev/null @@ -1,215 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package searcher - -import ( - "fmt" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" -) - -func NewMultiTermSearcher(indexReader index.IndexReader, terms []string, - field string, boost float64, options search.SearcherOptions, limit bool) ( - search.Searcher, error) { - - if tooManyClauses(len(terms)) { - if optionsDisjunctionOptimizable(options) { - return optimizeMultiTermSearcher(indexReader, terms, field, boost, options) - } - if limit { - return nil, tooManyClausesErr(field, len(terms)) - } - } - - qsearchers, err := makeBatchSearchers(indexReader, terms, field, boost, options) - if err != nil { - return nil, err - } - - // build disjunction searcher of these ranges - return newMultiTermSearcherInternal(indexReader, qsearchers, field, boost, - options, limit) -} - -func NewMultiTermSearcherBytes(indexReader index.IndexReader, terms [][]byte, - field string, boost float64, options search.SearcherOptions, limit bool) ( - search.Searcher, error) { - - if tooManyClauses(len(terms)) { - if optionsDisjunctionOptimizable(options) { - return optimizeMultiTermSearcherBytes(indexReader, terms, field, boost, options) - } - - if limit { - return nil, tooManyClausesErr(field, len(terms)) - } - } - - qsearchers, err := makeBatchSearchersBytes(indexReader, terms, field, boost, options) - if err != nil { - return nil, err - } - - // build disjunction searcher of these ranges - return newMultiTermSearcherInternal(indexReader, qsearchers, field, boost, - options, limit) -} - -func newMultiTermSearcherInternal(indexReader index.IndexReader, - searchers []search.Searcher, field string, boost float64, - options search.SearcherOptions, limit bool) ( - search.Searcher, error) { - - // build disjunction searcher of these ranges - searcher, err := newDisjunctionSearcher(indexReader, searchers, 0, options, - limit) - if err != nil { - for _, s := range searchers { - _ = s.Close() - } - return nil, err - } - - return searcher, nil -} - -func optimizeMultiTermSearcher(indexReader index.IndexReader, terms []string, - field string, boost float64, options search.SearcherOptions) ( - search.Searcher, error) { - var finalSearcher search.Searcher - for len(terms) > 0 { - var batchTerms []string - if len(terms) > DisjunctionMaxClauseCount { - batchTerms = terms[:DisjunctionMaxClauseCount] - terms = terms[DisjunctionMaxClauseCount:] - } else { - batchTerms = terms - terms = nil - } - batch, err := makeBatchSearchers(indexReader, batchTerms, field, boost, options) - if err != nil { - return nil, err - } - if finalSearcher != nil { - batch = append(batch, finalSearcher) - } - cleanup := func() { - for _, searcher := range batch { - if searcher != nil { - _ = searcher.Close() - } - } - } - finalSearcher, err = optimizeCompositeSearcher("disjunction:unadorned", - indexReader, batch, options) - // all searchers in batch should be closed, regardless of error or optimization failure - // either we're returning, or continuing and only finalSearcher is needed for next loop - cleanup() - if err != nil { - return nil, err - } - if finalSearcher == nil { - return nil, fmt.Errorf("unable to optimize") - } - } - return finalSearcher, nil -} - -func makeBatchSearchers(indexReader index.IndexReader, terms []string, field string, - boost float64, options search.SearcherOptions) ([]search.Searcher, error) { - - qsearchers := make([]search.Searcher, len(terms)) - qsearchersClose := func() { - for _, searcher := range qsearchers { - if searcher != nil { - _ = searcher.Close() - } - } - } - for i, term := range terms { - var err error - qsearchers[i], err = NewTermSearcher(indexReader, term, field, boost, options) - if err != nil { - qsearchersClose() - return nil, err - } - } - return qsearchers, nil -} - -func optimizeMultiTermSearcherBytes(indexReader index.IndexReader, terms [][]byte, - field string, boost float64, options search.SearcherOptions) ( - search.Searcher, error) { - - var finalSearcher search.Searcher - for len(terms) > 0 { - var batchTerms [][]byte - if len(terms) > DisjunctionMaxClauseCount { - batchTerms = terms[:DisjunctionMaxClauseCount] - terms = terms[DisjunctionMaxClauseCount:] - } else { - batchTerms = terms - terms = nil - } - batch, err := makeBatchSearchersBytes(indexReader, batchTerms, field, boost, options) - if err != nil { - return nil, err - } - if finalSearcher != nil { - batch = append(batch, finalSearcher) - } - cleanup := func() { - for _, searcher := range batch { - if searcher != nil { - _ = searcher.Close() - } - } - } - finalSearcher, err = optimizeCompositeSearcher("disjunction:unadorned", - indexReader, batch, options) - // all searchers in batch should be closed, regardless of error or optimization failure - // either we're returning, or continuing and only finalSearcher is needed for next loop - cleanup() - if err != nil { - return nil, err - } - if finalSearcher == nil { - return nil, fmt.Errorf("unable to optimize") - } - } - return finalSearcher, nil -} - -func makeBatchSearchersBytes(indexReader index.IndexReader, terms [][]byte, field string, - boost float64, options search.SearcherOptions) ([]search.Searcher, error) { - - qsearchers := make([]search.Searcher, len(terms)) - qsearchersClose := func() { - for _, searcher := range qsearchers { - if searcher != nil { - _ = searcher.Close() - } - } - } - for i, term := range terms { - var err error - qsearchers[i], err = NewTermSearcherBytes(indexReader, term, field, boost, options) - if err != nil { - qsearchersClose() - return nil, err - } - } - return qsearchers, nil -} diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_phrase.go b/vendor/github.com/blevesearch/bleve/search/searcher/search_phrase.go deleted file mode 100644 index 51b7e5bd..00000000 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_phrase.go +++ /dev/null @@ -1,437 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package searcher - -import ( - "fmt" - "math" - "reflect" - - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/size" -) - -var reflectStaticSizePhraseSearcher int - -func init() { - var ps PhraseSearcher - reflectStaticSizePhraseSearcher = int(reflect.TypeOf(ps).Size()) -} - -type PhraseSearcher struct { - mustSearcher search.Searcher - queryNorm float64 - currMust *search.DocumentMatch - terms [][]string - path phrasePath - paths []phrasePath - locations []search.Location - initialized bool -} - -func (s *PhraseSearcher) Size() int { - sizeInBytes := reflectStaticSizePhraseSearcher + size.SizeOfPtr - - if s.mustSearcher != nil { - sizeInBytes += s.mustSearcher.Size() - } - - if s.currMust != nil { - sizeInBytes += s.currMust.Size() - } - - for _, entry := range s.terms { - sizeInBytes += size.SizeOfSlice - for _, entry1 := range entry { - sizeInBytes += size.SizeOfString + len(entry1) - } - } - - return sizeInBytes -} - -func NewPhraseSearcher(indexReader index.IndexReader, terms []string, field string, options search.SearcherOptions) (*PhraseSearcher, error) { - // turn flat terms []string into [][]string - mterms := make([][]string, len(terms)) - for i, term := range terms { - mterms[i] = []string{term} - } - return NewMultiPhraseSearcher(indexReader, mterms, field, options) -} - -func NewMultiPhraseSearcher(indexReader index.IndexReader, terms [][]string, field string, options search.SearcherOptions) (*PhraseSearcher, error) { - options.IncludeTermVectors = true - var termPositionSearchers []search.Searcher - for _, termPos := range terms { - if len(termPos) == 1 && termPos[0] != "" { - // single term - ts, err := NewTermSearcher(indexReader, termPos[0], field, 1.0, options) - if err != nil { - // close any searchers already opened - for _, ts := range termPositionSearchers { - _ = ts.Close() - } - return nil, fmt.Errorf("phrase searcher error building term searcher: %v", err) - } - termPositionSearchers = append(termPositionSearchers, ts) - } else if len(termPos) > 1 { - // multiple terms - var termSearchers []search.Searcher - for _, term := range termPos { - if term == "" { - continue - } - ts, err := NewTermSearcher(indexReader, term, field, 1.0, options) - if err != nil { - // close any searchers already opened - for _, ts := range termPositionSearchers { - _ = ts.Close() - } - return nil, fmt.Errorf("phrase searcher error building term searcher: %v", err) - } - termSearchers = append(termSearchers, ts) - } - disjunction, err := NewDisjunctionSearcher(indexReader, termSearchers, 1, options) - if err != nil { - // close any searchers already opened - for _, ts := range termPositionSearchers { - _ = ts.Close() - } - return nil, fmt.Errorf("phrase searcher error building term position disjunction searcher: %v", err) - } - termPositionSearchers = append(termPositionSearchers, disjunction) - } - } - - mustSearcher, err := NewConjunctionSearcher(indexReader, termPositionSearchers, options) - if err != nil { - // close any searchers already opened - for _, ts := range termPositionSearchers { - _ = ts.Close() - } - return nil, fmt.Errorf("phrase searcher error building conjunction searcher: %v", err) - } - - // build our searcher - rv := PhraseSearcher{ - mustSearcher: mustSearcher, - terms: terms, - } - rv.computeQueryNorm() - return &rv, nil -} - -func (s *PhraseSearcher) computeQueryNorm() { - // first calculate sum of squared weights - sumOfSquaredWeights := 0.0 - if s.mustSearcher != nil { - sumOfSquaredWeights += s.mustSearcher.Weight() - } - - // now compute query norm from this - s.queryNorm = 1.0 / math.Sqrt(sumOfSquaredWeights) - // finally tell all the downstream searchers the norm - if s.mustSearcher != nil { - s.mustSearcher.SetQueryNorm(s.queryNorm) - } -} - -func (s *PhraseSearcher) initSearchers(ctx *search.SearchContext) error { - err := s.advanceNextMust(ctx) - if err != nil { - return err - } - - s.initialized = true - return nil -} - -func (s *PhraseSearcher) advanceNextMust(ctx *search.SearchContext) error { - var err error - - if s.mustSearcher != nil { - if s.currMust != nil { - ctx.DocumentMatchPool.Put(s.currMust) - } - s.currMust, err = s.mustSearcher.Next(ctx) - if err != nil { - return err - } - } - - return nil -} - -func (s *PhraseSearcher) Weight() float64 { - return s.mustSearcher.Weight() -} - -func (s *PhraseSearcher) SetQueryNorm(qnorm float64) { - s.mustSearcher.SetQueryNorm(qnorm) -} - -func (s *PhraseSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { - if !s.initialized { - err := s.initSearchers(ctx) - if err != nil { - return nil, err - } - } - - for s.currMust != nil { - // check this match against phrase constraints - rv := s.checkCurrMustMatch(ctx) - - // prepare for next iteration (either loop or subsequent call to Next()) - err := s.advanceNextMust(ctx) - if err != nil { - return nil, err - } - - // if match satisfied phrase constraints return it as a hit - if rv != nil { - return rv, nil - } - } - - return nil, nil -} - -// checkCurrMustMatch is solely concerned with determining if the DocumentMatch -// pointed to by s.currMust (which satisifies the pre-condition searcher) -// also satisfies the phase constraints. if so, it returns a DocumentMatch -// for this document, otherwise nil -func (s *PhraseSearcher) checkCurrMustMatch(ctx *search.SearchContext) *search.DocumentMatch { - s.locations = s.currMust.Complete(s.locations) - - locations := s.currMust.Locations - s.currMust.Locations = nil - - ftls := s.currMust.FieldTermLocations - - // typically we would expect there to only actually be results in - // one field, but we allow for this to not be the case - // but, we note that phrase constraints can only be satisfied within - // a single field, so we can check them each independently - for field, tlm := range locations { - ftls = s.checkCurrMustMatchField(ctx, field, tlm, ftls) - } - - if len(ftls) > 0 { - // return match - rv := s.currMust - s.currMust = nil - rv.FieldTermLocations = ftls - return rv - } - - return nil -} - -// checkCurrMustMatchField is solely concerned with determining if one -// particular field within the currMust DocumentMatch Locations -// satisfies the phase constraints (possibly more than once). if so, -// the matching field term locations are appended to the provided -// slice -func (s *PhraseSearcher) checkCurrMustMatchField(ctx *search.SearchContext, - field string, tlm search.TermLocationMap, - ftls []search.FieldTermLocation) []search.FieldTermLocation { - if s.path == nil { - s.path = make(phrasePath, 0, len(s.terms)) - } - s.paths = findPhrasePaths(0, nil, s.terms, tlm, s.path[:0], 0, s.paths[:0]) - for _, p := range s.paths { - for _, pp := range p { - ftls = append(ftls, search.FieldTermLocation{ - Field: field, - Term: pp.term, - Location: search.Location{ - Pos: pp.loc.Pos, - Start: pp.loc.Start, - End: pp.loc.End, - ArrayPositions: pp.loc.ArrayPositions, - }, - }) - } - } - return ftls -} - -type phrasePart struct { - term string - loc *search.Location -} - -func (p *phrasePart) String() string { - return fmt.Sprintf("[%s %v]", p.term, p.loc) -} - -type phrasePath []phrasePart - -func (p phrasePath) MergeInto(in search.TermLocationMap) { - for _, pp := range p { - in[pp.term] = append(in[pp.term], pp.loc) - } -} - -func (p phrasePath) String() string { - rv := "[" - for i, pp := range p { - if i > 0 { - rv += ", " - } - rv += pp.String() - } - rv += "]" - return rv -} - -// findPhrasePaths is a function to identify phase matches from a set -// of known term locations. it recursive so care must be taken with -// arguments and return values. -// -// prevPos - the previous location, 0 on first invocation -// ap - array positions of the first candidate phrase part to -// which further recursive phrase parts must match, -// nil on initial invocation or when there are no array positions -// phraseTerms - slice containing the phrase terms, -// may contain empty string as placeholder (don't care) -// tlm - the Term Location Map containing all relevant term locations -// p - the current path being explored (appended to in recursive calls) -// this is the primary state being built during the traversal -// remainingSlop - amount of sloppiness that's allowed, which is the -// sum of the editDistances from each matching phrase part, -// where 0 means no sloppiness allowed (all editDistances must be 0), -// decremented during recursion -// rv - the final result being appended to by all the recursive calls -// -// returns slice of paths, or nil if invocation did not find any successul paths -func findPhrasePaths(prevPos uint64, ap search.ArrayPositions, phraseTerms [][]string, - tlm search.TermLocationMap, p phrasePath, remainingSlop int, rv []phrasePath) []phrasePath { - // no more terms - if len(phraseTerms) < 1 { - // snapshot or copy the recursively built phrasePath p and - // append it to the rv, also optimizing by checking if next - // phrasePath item in the rv (which we're about to overwrite) - // is available for reuse - var pcopy phrasePath - if len(rv) < cap(rv) { - pcopy = rv[:len(rv)+1][len(rv)][:0] - } - return append(rv, append(pcopy, p...)) - } - - car := phraseTerms[0] - cdr := phraseTerms[1:] - - // empty term is treated as match (continue) - if len(car) == 0 || (len(car) == 1 && car[0] == "") { - nextPos := prevPos + 1 - if prevPos == 0 { - // if prevPos was 0, don't set it to 1 (as thats not a real abs pos) - nextPos = 0 // don't advance nextPos if prevPos was 0 - } - return findPhrasePaths(nextPos, ap, cdr, tlm, p, remainingSlop, rv) - } - - // locations for this term - for _, carTerm := range car { - locations := tlm[carTerm] - LOCATIONS_LOOP: - for _, loc := range locations { - if prevPos != 0 && !loc.ArrayPositions.Equals(ap) { - // if the array positions are wrong, can't match, try next location - continue - } - - // compute distance from previous phrase term - dist := 0 - if prevPos != 0 { - dist = editDistance(prevPos+1, loc.Pos) - } - - // if enough slop remaining, continue recursively - if prevPos == 0 || (remainingSlop-dist) >= 0 { - // skip if we've already used this term+loc already - for _, ppart := range p { - if ppart.term == carTerm && ppart.loc == loc { - continue LOCATIONS_LOOP - } - } - - // this location works, add it to the path (but not for empty term) - px := append(p, phrasePart{term: carTerm, loc: loc}) - rv = findPhrasePaths(loc.Pos, loc.ArrayPositions, cdr, tlm, px, remainingSlop-dist, rv) - } - } - } - return rv -} - -func editDistance(p1, p2 uint64) int { - dist := int(p1 - p2) - if dist < 0 { - return -dist - } - return dist -} - -func (s *PhraseSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { - if !s.initialized { - err := s.initSearchers(ctx) - if err != nil { - return nil, err - } - } - if s.currMust != nil { - if s.currMust.IndexInternalID.Compare(ID) >= 0 { - return s.Next(ctx) - } - ctx.DocumentMatchPool.Put(s.currMust) - } - if s.currMust == nil { - return nil, nil - } - var err error - s.currMust, err = s.mustSearcher.Advance(ctx, ID) - if err != nil { - return nil, err - } - return s.Next(ctx) -} - -func (s *PhraseSearcher) Count() uint64 { - // for now return a worst case - return s.mustSearcher.Count() -} - -func (s *PhraseSearcher) Close() error { - if s.mustSearcher != nil { - err := s.mustSearcher.Close() - if err != nil { - return err - } - } - return nil -} - -func (s *PhraseSearcher) Min() int { - return 0 -} - -func (s *PhraseSearcher) DocumentMatchPoolSize() int { - return s.mustSearcher.DocumentMatchPoolSize() + 1 -} diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_regexp.go b/vendor/github.com/blevesearch/bleve/search/searcher/search_regexp.go deleted file mode 100644 index 11a44f15..00000000 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_regexp.go +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright (c) 2015 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package searcher - -import ( - "regexp" - - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" -) - -// NewRegexpStringSearcher is similar to NewRegexpSearcher, but -// additionally optimizes for index readers that handle regexp's. -func NewRegexpStringSearcher(indexReader index.IndexReader, pattern string, - field string, boost float64, options search.SearcherOptions) ( - search.Searcher, error) { - ir, ok := indexReader.(index.IndexReaderRegexp) - if !ok { - r, err := regexp.Compile(pattern) - if err != nil { - return nil, err - } - - return NewRegexpSearcher(indexReader, r, field, boost, options) - } - - fieldDict, err := ir.FieldDictRegexp(field, pattern) - if err != nil { - return nil, err - } - defer func() { - if cerr := fieldDict.Close(); cerr != nil && err == nil { - err = cerr - } - }() - - var candidateTerms []string - - tfd, err := fieldDict.Next() - for err == nil && tfd != nil { - candidateTerms = append(candidateTerms, tfd.Term) - tfd, err = fieldDict.Next() - } - if err != nil { - return nil, err - } - - return NewMultiTermSearcher(indexReader, candidateTerms, field, boost, - options, true) -} - -// NewRegexpSearcher creates a searcher which will match documents that -// contain terms which match the pattern regexp. The match must be EXACT -// matching the entire term. The provided regexp SHOULD NOT start with ^ -// or end with $ as this can intefere with the implementation. Separately, -// matches will be checked to ensure they match the entire term. -func NewRegexpSearcher(indexReader index.IndexReader, pattern index.Regexp, - field string, boost float64, options search.SearcherOptions) ( - search.Searcher, error) { - var candidateTerms []string - - prefixTerm, complete := pattern.LiteralPrefix() - if complete { - // there is no pattern - candidateTerms = []string{prefixTerm} - } else { - var err error - candidateTerms, err = findRegexpCandidateTerms(indexReader, pattern, field, - prefixTerm) - if err != nil { - return nil, err - } - } - - return NewMultiTermSearcher(indexReader, candidateTerms, field, boost, - options, true) -} - -func findRegexpCandidateTerms(indexReader index.IndexReader, - pattern index.Regexp, field, prefixTerm string) (rv []string, err error) { - rv = make([]string, 0) - var fieldDict index.FieldDict - if len(prefixTerm) > 0 { - fieldDict, err = indexReader.FieldDictPrefix(field, []byte(prefixTerm)) - } else { - fieldDict, err = indexReader.FieldDict(field) - } - defer func() { - if cerr := fieldDict.Close(); cerr != nil && err == nil { - err = cerr - } - }() - - // enumerate the terms and check against regexp - tfd, err := fieldDict.Next() - for err == nil && tfd != nil { - matchPos := pattern.FindStringIndex(tfd.Term) - if matchPos != nil && matchPos[0] == 0 && matchPos[1] == len(tfd.Term) { - rv = append(rv, tfd.Term) - if tooManyClauses(len(rv)) { - return rv, tooManyClausesErr(field, len(rv)) - } - } - tfd, err = fieldDict.Next() - } - - return rv, err -} diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_term.go b/vendor/github.com/blevesearch/bleve/search/searcher/search_term.go deleted file mode 100644 index e07d2533..00000000 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_term.go +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package searcher - -import ( - "reflect" - - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/scorer" - "github.com/blevesearch/bleve/size" -) - -var reflectStaticSizeTermSearcher int - -func init() { - var ts TermSearcher - reflectStaticSizeTermSearcher = int(reflect.TypeOf(ts).Size()) -} - -type TermSearcher struct { - indexReader index.IndexReader - reader index.TermFieldReader - scorer *scorer.TermQueryScorer - tfd index.TermFieldDoc -} - -func NewTermSearcher(indexReader index.IndexReader, term string, field string, boost float64, options search.SearcherOptions) (*TermSearcher, error) { - return NewTermSearcherBytes(indexReader, []byte(term), field, boost, options) -} - -func NewTermSearcherBytes(indexReader index.IndexReader, term []byte, field string, boost float64, options search.SearcherOptions) (*TermSearcher, error) { - needFreqNorm := options.Score != "none" - reader, err := indexReader.TermFieldReader(term, field, needFreqNorm, needFreqNorm, options.IncludeTermVectors) - if err != nil { - return nil, err - } - return newTermSearcherFromReader(indexReader, reader, term, field, boost, options) -} - -func newTermSearcherFromReader(indexReader index.IndexReader, reader index.TermFieldReader, - term []byte, field string, boost float64, options search.SearcherOptions) (*TermSearcher, error) { - count, err := indexReader.DocCount() - if err != nil { - _ = reader.Close() - return nil, err - } - scorer := scorer.NewTermQueryScorer(term, field, boost, count, reader.Count(), options) - return &TermSearcher{ - indexReader: indexReader, - reader: reader, - scorer: scorer, - }, nil -} - -func (s *TermSearcher) Size() int { - return reflectStaticSizeTermSearcher + size.SizeOfPtr + - s.reader.Size() + - s.tfd.Size() + - s.scorer.Size() -} - -func (s *TermSearcher) Count() uint64 { - return s.reader.Count() -} - -func (s *TermSearcher) Weight() float64 { - return s.scorer.Weight() -} - -func (s *TermSearcher) SetQueryNorm(qnorm float64) { - s.scorer.SetQueryNorm(qnorm) -} - -func (s *TermSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { - termMatch, err := s.reader.Next(s.tfd.Reset()) - if err != nil { - return nil, err - } - - if termMatch == nil { - return nil, nil - } - - // score match - docMatch := s.scorer.Score(ctx, termMatch) - // return doc match - return docMatch, nil - -} - -func (s *TermSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { - termMatch, err := s.reader.Advance(ID, s.tfd.Reset()) - if err != nil { - return nil, err - } - - if termMatch == nil { - return nil, nil - } - - // score match - docMatch := s.scorer.Score(ctx, termMatch) - - // return doc match - return docMatch, nil -} - -func (s *TermSearcher) Close() error { - return s.reader.Close() -} - -func (s *TermSearcher) Min() int { - return 0 -} - -func (s *TermSearcher) DocumentMatchPoolSize() int { - return 1 -} - -func (s *TermSearcher) Optimize(kind string, octx index.OptimizableContext) ( - index.OptimizableContext, error) { - o, ok := s.reader.(index.Optimizable) - if ok { - return o.Optimize(kind, octx) - } - - return nil, nil -} diff --git a/vendor/github.com/blevesearch/bleve/search/sort.go b/vendor/github.com/blevesearch/bleve/search/sort.go deleted file mode 100644 index dca422eb..00000000 --- a/vendor/github.com/blevesearch/bleve/search/sort.go +++ /dev/null @@ -1,746 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package search - -import ( - "bytes" - "encoding/json" - "fmt" - "math" - "sort" - "strings" - - "github.com/blevesearch/bleve/geo" - "github.com/blevesearch/bleve/numeric" -) - -var HighTerm = strings.Repeat(string([]byte{0xff}), 10) -var LowTerm = string([]byte{0x00}) - -type SearchSort interface { - UpdateVisitor(field string, term []byte) - Value(a *DocumentMatch) string - Descending() bool - - RequiresDocID() bool - RequiresScoring() bool - RequiresFields() []string - - Reverse() - - Copy() SearchSort -} - -func ParseSearchSortObj(input map[string]interface{}) (SearchSort, error) { - descending, ok := input["desc"].(bool) - by, ok := input["by"].(string) - if !ok { - return nil, fmt.Errorf("search sort must specify by") - } - switch by { - case "id": - return &SortDocID{ - Desc: descending, - }, nil - case "score": - return &SortScore{ - Desc: descending, - }, nil - case "geo_distance": - field, ok := input["field"].(string) - if !ok { - return nil, fmt.Errorf("search sort mode geo_distance must specify field") - } - lon, lat, foundLocation := geo.ExtractGeoPoint(input["location"]) - if !foundLocation { - return nil, fmt.Errorf("unable to parse geo_distance location") - } - rvd := &SortGeoDistance{ - Field: field, - Desc: descending, - Lon: lon, - Lat: lat, - unitMult: 1.0, - } - if distUnit, ok := input["unit"].(string); ok { - var err error - rvd.unitMult, err = geo.ParseDistanceUnit(distUnit) - if err != nil { - return nil, err - } - rvd.Unit = distUnit - } - return rvd, nil - case "field": - field, ok := input["field"].(string) - if !ok { - return nil, fmt.Errorf("search sort mode field must specify field") - } - rv := &SortField{ - Field: field, - Desc: descending, - } - typ, ok := input["type"].(string) - if ok { - switch typ { - case "auto": - rv.Type = SortFieldAuto - case "string": - rv.Type = SortFieldAsString - case "number": - rv.Type = SortFieldAsNumber - case "date": - rv.Type = SortFieldAsDate - default: - return nil, fmt.Errorf("unknown sort field type: %s", typ) - } - } - mode, ok := input["mode"].(string) - if ok { - switch mode { - case "default": - rv.Mode = SortFieldDefault - case "min": - rv.Mode = SortFieldMin - case "max": - rv.Mode = SortFieldMax - default: - return nil, fmt.Errorf("unknown sort field mode: %s", mode) - } - } - missing, ok := input["missing"].(string) - if ok { - switch missing { - case "first": - rv.Missing = SortFieldMissingFirst - case "last": - rv.Missing = SortFieldMissingLast - default: - return nil, fmt.Errorf("unknown sort field missing: %s", missing) - } - } - return rv, nil - } - - return nil, fmt.Errorf("unknown search sort by: %s", by) -} - -func ParseSearchSortString(input string) SearchSort { - descending := false - if strings.HasPrefix(input, "-") { - descending = true - input = input[1:] - } else if strings.HasPrefix(input, "+") { - input = input[1:] - } - if input == "_id" { - return &SortDocID{ - Desc: descending, - } - } else if input == "_score" { - return &SortScore{ - Desc: descending, - } - } - return &SortField{ - Field: input, - Desc: descending, - } -} - -func ParseSearchSortJSON(input json.RawMessage) (SearchSort, error) { - // first try to parse it as string - var sortString string - err := json.Unmarshal(input, &sortString) - if err != nil { - var sortObj map[string]interface{} - err = json.Unmarshal(input, &sortObj) - if err != nil { - return nil, err - } - return ParseSearchSortObj(sortObj) - } - return ParseSearchSortString(sortString), nil -} - -func ParseSortOrderStrings(in []string) SortOrder { - rv := make(SortOrder, 0, len(in)) - for _, i := range in { - ss := ParseSearchSortString(i) - rv = append(rv, ss) - } - return rv -} - -func ParseSortOrderJSON(in []json.RawMessage) (SortOrder, error) { - rv := make(SortOrder, 0, len(in)) - for _, i := range in { - ss, err := ParseSearchSortJSON(i) - if err != nil { - return nil, err - } - rv = append(rv, ss) - } - return rv, nil -} - -type SortOrder []SearchSort - -func (so SortOrder) Value(doc *DocumentMatch) { - for _, soi := range so { - doc.Sort = append(doc.Sort, soi.Value(doc)) - } -} - -func (so SortOrder) UpdateVisitor(field string, term []byte) { - for _, soi := range so { - soi.UpdateVisitor(field, term) - } -} - -func (so SortOrder) Copy() SortOrder { - rv := make(SortOrder, len(so)) - for i, soi := range so { - rv[i] = soi.Copy() - } - return rv -} - -// Compare will compare two document matches using the specified sort order -// if both are numbers, we avoid converting back to term -func (so SortOrder) Compare(cachedScoring, cachedDesc []bool, i, j *DocumentMatch) int { - // compare the documents on all search sorts until a differences is found - for x := range so { - c := 0 - if cachedScoring[x] { - if i.Score < j.Score { - c = -1 - } else if i.Score > j.Score { - c = 1 - } - } else { - iVal := i.Sort[x] - jVal := j.Sort[x] - if iVal < jVal { - c = -1 - } else if iVal > jVal { - c = 1 - } - } - - if c == 0 { - continue - } - if cachedDesc[x] { - c = -c - } - return c - } - // if they are the same at this point, impose order based on index natural sort order - if i.HitNumber == j.HitNumber { - return 0 - } else if i.HitNumber > j.HitNumber { - return 1 - } - return -1 -} - -func (so SortOrder) RequiresScore() bool { - for _, soi := range so { - if soi.RequiresScoring() { - return true - } - } - return false -} - -func (so SortOrder) RequiresDocID() bool { - for _, soi := range so { - if soi.RequiresDocID() { - return true - } - } - return false -} - -func (so SortOrder) RequiredFields() []string { - var rv []string - for _, soi := range so { - rv = append(rv, soi.RequiresFields()...) - } - return rv -} - -func (so SortOrder) CacheIsScore() []bool { - rv := make([]bool, 0, len(so)) - for _, soi := range so { - rv = append(rv, soi.RequiresScoring()) - } - return rv -} - -func (so SortOrder) CacheDescending() []bool { - rv := make([]bool, 0, len(so)) - for _, soi := range so { - rv = append(rv, soi.Descending()) - } - return rv -} - -func (so SortOrder) Reverse() { - for _, soi := range so { - soi.Reverse() - } -} - -// SortFieldType lets you control some internal sort behavior -// normally leaving this to the zero-value of SortFieldAuto is fine -type SortFieldType int - -const ( - // SortFieldAuto applies heuristics attempt to automatically sort correctly - SortFieldAuto SortFieldType = iota - // SortFieldAsString forces sort as string (no prefix coded terms removed) - SortFieldAsString - // SortFieldAsNumber forces sort as string (prefix coded terms with shift > 0 removed) - SortFieldAsNumber - // SortFieldAsDate forces sort as string (prefix coded terms with shift > 0 removed) - SortFieldAsDate -) - -// SortFieldMode describes the behavior if the field has multiple values -type SortFieldMode int - -const ( - // SortFieldDefault uses the first (or only) value, this is the default zero-value - SortFieldDefault SortFieldMode = iota // FIXME name is confusing - // SortFieldMin uses the minimum value - SortFieldMin - // SortFieldMax uses the maximum value - SortFieldMax -) - -// SortFieldMissing controls where documents missing a field value should be sorted -type SortFieldMissing int - -const ( - // SortFieldMissingLast sorts documents missing a field at the end - SortFieldMissingLast SortFieldMissing = iota - - // SortFieldMissingFirst sorts documents missing a field at the beginning - SortFieldMissingFirst -) - -// SortField will sort results by the value of a stored field -// Field is the name of the field -// Descending reverse the sort order (default false) -// Type allows forcing of string/number/date behavior (default auto) -// Mode controls behavior for multi-values fields (default first) -// Missing controls behavior of missing values (default last) -type SortField struct { - Field string - Desc bool - Type SortFieldType - Mode SortFieldMode - Missing SortFieldMissing - values [][]byte - tmp [][]byte -} - -// UpdateVisitor notifies this sort field that in this document -// this field has the specified term -func (s *SortField) UpdateVisitor(field string, term []byte) { - if field == s.Field { - s.values = append(s.values, term) - } -} - -// Value returns the sort value of the DocumentMatch -// it also resets the state of this SortField for -// processing the next document -func (s *SortField) Value(i *DocumentMatch) string { - iTerms := s.filterTermsByType(s.values) - iTerm := s.filterTermsByMode(iTerms) - s.values = s.values[:0] - return iTerm -} - -// Descending determines the order of the sort -func (s *SortField) Descending() bool { - return s.Desc -} - -func (s *SortField) filterTermsByMode(terms [][]byte) string { - if len(terms) == 1 || (len(terms) > 1 && s.Mode == SortFieldDefault) { - return string(terms[0]) - } else if len(terms) > 1 { - switch s.Mode { - case SortFieldMin: - sort.Sort(BytesSlice(terms)) - return string(terms[0]) - case SortFieldMax: - sort.Sort(BytesSlice(terms)) - return string(terms[len(terms)-1]) - } - } - - // handle missing terms - if s.Missing == SortFieldMissingLast { - if s.Desc { - return LowTerm - } - return HighTerm - } - if s.Desc { - return HighTerm - } - return LowTerm -} - -// filterTermsByType attempts to make one pass on the terms -// if we are in auto-mode AND all the terms look like prefix-coded numbers -// return only the terms which had shift of 0 -// if we are in explicit number or date mode, return only valid -// prefix coded numbers with shift of 0 -func (s *SortField) filterTermsByType(terms [][]byte) [][]byte { - stype := s.Type - if stype == SortFieldAuto { - allTermsPrefixCoded := true - termsWithShiftZero := s.tmp[:0] - for _, term := range terms { - valid, shift := numeric.ValidPrefixCodedTermBytes(term) - if valid && shift == 0 { - termsWithShiftZero = append(termsWithShiftZero, term) - } else if !valid { - allTermsPrefixCoded = false - } - } - // reset the terms only when valid zero shift terms are found. - if allTermsPrefixCoded && len(termsWithShiftZero) > 0 { - terms = termsWithShiftZero - s.tmp = termsWithShiftZero[:0] - } - } else if stype == SortFieldAsNumber || stype == SortFieldAsDate { - termsWithShiftZero := s.tmp[:0] - for _, term := range terms { - valid, shift := numeric.ValidPrefixCodedTermBytes(term) - if valid && shift == 0 { - termsWithShiftZero = append(termsWithShiftZero, term) - } - } - terms = termsWithShiftZero - s.tmp = termsWithShiftZero[:0] - } - return terms -} - -// RequiresDocID says this SearchSort does not require the DocID be loaded -func (s *SortField) RequiresDocID() bool { return false } - -// RequiresScoring says this SearchStore does not require scoring -func (s *SortField) RequiresScoring() bool { return false } - -// RequiresFields says this SearchStore requires the specified stored field -func (s *SortField) RequiresFields() []string { return []string{s.Field} } - -func (s *SortField) MarshalJSON() ([]byte, error) { - // see if simple format can be used - if s.Missing == SortFieldMissingLast && - s.Mode == SortFieldDefault && - s.Type == SortFieldAuto { - if s.Desc { - return json.Marshal("-" + s.Field) - } - return json.Marshal(s.Field) - } - sfm := map[string]interface{}{ - "by": "field", - "field": s.Field, - } - if s.Desc { - sfm["desc"] = true - } - if s.Missing > SortFieldMissingLast { - switch s.Missing { - case SortFieldMissingFirst: - sfm["missing"] = "first" - } - } - if s.Mode > SortFieldDefault { - switch s.Mode { - case SortFieldMin: - sfm["mode"] = "min" - case SortFieldMax: - sfm["mode"] = "max" - } - } - if s.Type > SortFieldAuto { - switch s.Type { - case SortFieldAsString: - sfm["type"] = "string" - case SortFieldAsNumber: - sfm["type"] = "number" - case SortFieldAsDate: - sfm["type"] = "date" - } - } - - return json.Marshal(sfm) -} - -func (s *SortField) Copy() SearchSort { - rv := *s - return &rv -} - -func (s *SortField) Reverse() { - s.Desc = !s.Desc - if s.Missing == SortFieldMissingFirst { - s.Missing = SortFieldMissingLast - } else { - s.Missing = SortFieldMissingFirst - } -} - -// SortDocID will sort results by the document identifier -type SortDocID struct { - Desc bool -} - -// UpdateVisitor is a no-op for SortDocID as it's value -// is not dependent on any field terms -func (s *SortDocID) UpdateVisitor(field string, term []byte) { -} - -// Value returns the sort value of the DocumentMatch -func (s *SortDocID) Value(i *DocumentMatch) string { - return i.ID -} - -// Descending determines the order of the sort -func (s *SortDocID) Descending() bool { - return s.Desc -} - -// RequiresDocID says this SearchSort does require the DocID be loaded -func (s *SortDocID) RequiresDocID() bool { return true } - -// RequiresScoring says this SearchStore does not require scoring -func (s *SortDocID) RequiresScoring() bool { return false } - -// RequiresFields says this SearchStore does not require any stored fields -func (s *SortDocID) RequiresFields() []string { return nil } - -func (s *SortDocID) MarshalJSON() ([]byte, error) { - if s.Desc { - return json.Marshal("-_id") - } - return json.Marshal("_id") -} - -func (s *SortDocID) Copy() SearchSort { - rv := *s - return &rv -} - -func (s *SortDocID) Reverse() { - s.Desc = !s.Desc -} - -// SortScore will sort results by the document match score -type SortScore struct { - Desc bool -} - -// UpdateVisitor is a no-op for SortScore as it's value -// is not dependent on any field terms -func (s *SortScore) UpdateVisitor(field string, term []byte) { -} - -// Value returns the sort value of the DocumentMatch -func (s *SortScore) Value(i *DocumentMatch) string { - return "_score" -} - -// Descending determines the order of the sort -func (s *SortScore) Descending() bool { - return s.Desc -} - -// RequiresDocID says this SearchSort does not require the DocID be loaded -func (s *SortScore) RequiresDocID() bool { return false } - -// RequiresScoring says this SearchStore does require scoring -func (s *SortScore) RequiresScoring() bool { return true } - -// RequiresFields says this SearchStore does not require any store fields -func (s *SortScore) RequiresFields() []string { return nil } - -func (s *SortScore) MarshalJSON() ([]byte, error) { - if s.Desc { - return json.Marshal("-_score") - } - return json.Marshal("_score") -} - -func (s *SortScore) Copy() SearchSort { - rv := *s - return &rv -} - -func (s *SortScore) Reverse() { - s.Desc = !s.Desc -} - -var maxDistance = string(numeric.MustNewPrefixCodedInt64(math.MaxInt64, 0)) - -// NewSortGeoDistance creates SearchSort instance for sorting documents by -// their distance from the specified point. -func NewSortGeoDistance(field, unit string, lon, lat float64, desc bool) ( - *SortGeoDistance, error) { - rv := &SortGeoDistance{ - Field: field, - Desc: desc, - Unit: unit, - Lon: lon, - Lat: lat, - } - var err error - rv.unitMult, err = geo.ParseDistanceUnit(unit) - if err != nil { - return nil, err - } - return rv, nil -} - -// SortGeoDistance will sort results by the distance of an -// indexed geo point, from the provided location. -// Field is the name of the field -// Descending reverse the sort order (default false) -type SortGeoDistance struct { - Field string - Desc bool - Unit string - values []string - Lon float64 - Lat float64 - unitMult float64 -} - -// UpdateVisitor notifies this sort field that in this document -// this field has the specified term -func (s *SortGeoDistance) UpdateVisitor(field string, term []byte) { - if field == s.Field { - s.values = append(s.values, string(term)) - } -} - -// Value returns the sort value of the DocumentMatch -// it also resets the state of this SortField for -// processing the next document -func (s *SortGeoDistance) Value(i *DocumentMatch) string { - iTerms := s.filterTermsByType(s.values) - iTerm := s.filterTermsByMode(iTerms) - s.values = s.values[:0] - - if iTerm == "" { - return maxDistance - } - - i64, err := numeric.PrefixCoded(iTerm).Int64() - if err != nil { - return maxDistance - } - docLon := geo.MortonUnhashLon(uint64(i64)) - docLat := geo.MortonUnhashLat(uint64(i64)) - - dist := geo.Haversin(s.Lon, s.Lat, docLon, docLat) - // dist is returned in km, so convert to m - dist *= 1000 - if s.unitMult != 0 { - dist /= s.unitMult - } - distInt64 := numeric.Float64ToInt64(dist) - return string(numeric.MustNewPrefixCodedInt64(distInt64, 0)) -} - -// Descending determines the order of the sort -func (s *SortGeoDistance) Descending() bool { - return s.Desc -} - -func (s *SortGeoDistance) filterTermsByMode(terms []string) string { - if len(terms) >= 1 { - return terms[0] - } - - return "" -} - -// filterTermsByType attempts to make one pass on the terms -// return only valid prefix coded numbers with shift of 0 -func (s *SortGeoDistance) filterTermsByType(terms []string) []string { - var termsWithShiftZero []string - for _, term := range terms { - valid, shift := numeric.ValidPrefixCodedTerm(term) - if valid && shift == 0 { - termsWithShiftZero = append(termsWithShiftZero, term) - } - } - return termsWithShiftZero -} - -// RequiresDocID says this SearchSort does not require the DocID be loaded -func (s *SortGeoDistance) RequiresDocID() bool { return false } - -// RequiresScoring says this SearchStore does not require scoring -func (s *SortGeoDistance) RequiresScoring() bool { return false } - -// RequiresFields says this SearchStore requires the specified stored field -func (s *SortGeoDistance) RequiresFields() []string { return []string{s.Field} } - -func (s *SortGeoDistance) MarshalJSON() ([]byte, error) { - sfm := map[string]interface{}{ - "by": "geo_distance", - "field": s.Field, - "location": map[string]interface{}{ - "lon": s.Lon, - "lat": s.Lat, - }, - } - if s.Unit != "" { - sfm["unit"] = s.Unit - } - if s.Desc { - sfm["desc"] = true - } - - return json.Marshal(sfm) -} - -func (s *SortGeoDistance) Copy() SearchSort { - rv := *s - return &rv -} - -func (s *SortGeoDistance) Reverse() { - s.Desc = !s.Desc -} - -type BytesSlice [][]byte - -func (p BytesSlice) Len() int { return len(p) } -func (p BytesSlice) Less(i, j int) bool { return bytes.Compare(p[i], p[j]) < 0 } -func (p BytesSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } diff --git a/vendor/github.com/blevesearch/bleve/search/util.go b/vendor/github.com/blevesearch/bleve/search/util.go deleted file mode 100644 index 19dd5d68..00000000 --- a/vendor/github.com/blevesearch/bleve/search/util.go +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package search - -func MergeLocations(locations []FieldTermLocationMap) FieldTermLocationMap { - rv := locations[0] - - for i := 1; i < len(locations); i++ { - nextLocations := locations[i] - for field, termLocationMap := range nextLocations { - rvTermLocationMap, rvHasField := rv[field] - if rvHasField { - rv[field] = MergeTermLocationMaps(rvTermLocationMap, termLocationMap) - } else { - rv[field] = termLocationMap - } - } - } - - return rv -} - -func MergeTermLocationMaps(rv, other TermLocationMap) TermLocationMap { - for term, locationMap := range other { - // for a given term/document there cannot be different locations - // if they came back from different clauses, overwrite is ok - rv[term] = locationMap - } - return rv -} - -func MergeFieldTermLocations(dest []FieldTermLocation, matches []*DocumentMatch) []FieldTermLocation { - n := len(dest) - for _, dm := range matches { - n += len(dm.FieldTermLocations) - } - if cap(dest) < n { - dest = append(make([]FieldTermLocation, 0, n), dest...) - } - - for _, dm := range matches { - for _, ftl := range dm.FieldTermLocations { - dest = append(dest, FieldTermLocation{ - Field: ftl.Field, - Term: ftl.Term, - Location: Location{ - Pos: ftl.Location.Pos, - Start: ftl.Location.Start, - End: ftl.Location.End, - ArrayPositions: append(ArrayPositions(nil), ftl.Location.ArrayPositions...), - }, - }) - } - } - - return dest -} diff --git a/vendor/github.com/blevesearch/bleve/v2/.gitignore b/vendor/github.com/blevesearch/bleve/v2/.gitignore new file mode 100644 index 00000000..7512de77 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/.gitignore @@ -0,0 +1,20 @@ +#* +*.sublime-* +*~ +.#* +.project +.settings +**/.idea/ +**/*.iml +.DS_Store +query_string.y.go.tmp +/analysis/token_filters/cld2/cld2-read-only +/analysis/token_filters/cld2/libcld2_full.a +/cmd/bleve/bleve +vendor/** +!vendor/manifest +/y.output +/search/query/y.output +*.test +tags +go.sum diff --git a/vendor/github.com/blevesearch/bleve/.travis.yml b/vendor/github.com/blevesearch/bleve/v2/.travis.yml similarity index 100% rename from vendor/github.com/blevesearch/bleve/.travis.yml rename to vendor/github.com/blevesearch/bleve/v2/.travis.yml diff --git a/vendor/github.com/blevesearch/bleve/CONTRIBUTING.md b/vendor/github.com/blevesearch/bleve/v2/CONTRIBUTING.md similarity index 100% rename from vendor/github.com/blevesearch/bleve/CONTRIBUTING.md rename to vendor/github.com/blevesearch/bleve/v2/CONTRIBUTING.md diff --git a/vendor/github.com/blevesearch/bleve/LICENSE b/vendor/github.com/blevesearch/bleve/v2/LICENSE similarity index 100% rename from vendor/github.com/blevesearch/bleve/LICENSE rename to vendor/github.com/blevesearch/bleve/v2/LICENSE diff --git a/vendor/github.com/blevesearch/bleve/v2/README.md b/vendor/github.com/blevesearch/bleve/v2/README.md new file mode 100644 index 00000000..a89be4dd --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/README.md @@ -0,0 +1,118 @@ +# ![bleve](docs/bleve.png) bleve + +[![Tests](https://github.com/blevesearch/bleve/workflows/Tests/badge.svg?branch=master&event=push)](https://github.com/blevesearch/bleve/actions?query=workflow%3ATests+event%3Apush+branch%3Amaster) +[![Coverage Status](https://coveralls.io/repos/github/blevesearch/bleve/badge.svg?branch=master)](https://coveralls.io/github/blevesearch/bleve?branch=master) +[![GoDoc](https://godoc.org/github.com/blevesearch/bleve?status.svg)](https://godoc.org/github.com/blevesearch/bleve) +[![Join the chat at https://gitter.im/blevesearch/bleve](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/blevesearch/bleve?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) +[![codebeat](https://codebeat.co/badges/38a7cbc9-9cf5-41c0-a315-0746178230f4)](https://codebeat.co/projects/github-com-blevesearch-bleve) +[![Go Report Card](https://goreportcard.com/badge/blevesearch/bleve)](https://goreportcard.com/report/blevesearch/bleve) +[![Sourcegraph](https://sourcegraph.com/github.com/blevesearch/bleve/-/badge.svg)](https://sourcegraph.com/github.com/blevesearch/bleve?badge) +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) + +A modern text indexing library in go + +## Features + +* Index any go data structure (including JSON) +* Intelligent defaults backed up by powerful configuration +* Supported field types: + * Text, Numeric, Datetime, Boolean +* Supported query types: + * Term, Phrase, Match, Match Phrase, Prefix, Fuzzy + * Conjunction, Disjunction, Boolean (must/should/must_not) + * Term Range, Numeric Range, Date Range + * [Geo Spatial](https://github.com/blevesearch/bleve/blob/master/geo/README.md) + * Simple [query string syntax](http://www.blevesearch.com/docs/Query-String-Query/) for human entry +* [tf-idf](https://en.wikipedia.org/wiki/Tf-idf) Scoring +* Query time boosting +* Search result match highlighting with document fragments +* Aggregations/faceting support: + * Terms Facet + * Numeric Range Facet + * Date Range Facet + +## Indexing + +```go +message := struct{ + Id string + From string + Body string +}{ + Id: "example", + From: "marty.schoch@gmail.com", + Body: "bleve indexing is easy", +} + +mapping := bleve.NewIndexMapping() +index, err := bleve.New("example.bleve", mapping) +if err != nil { + panic(err) +} +index.Index(message.Id, message) +``` + +## Querying + +```go +index, _ := bleve.Open("example.bleve") +query := bleve.NewQueryStringQuery("bleve") +searchRequest := bleve.NewSearchRequest(query) +searchResult, _ := index.Search(searchRequest) +``` + +## Command Line Interface + +To install the CLI for the latest release of bleve, run: + +```bash +$ go install github.com/blevesearch/bleve/v2/cmd/bleve@latest +``` + +``` +$ bleve --help +Bleve is a command-line tool to interact with a bleve index. + +Usage: + bleve [command] + +Available Commands: + bulk bulk loads from newline delimited JSON files + check checks the contents of the index + count counts the number documents in the index + create creates a new index + dictionary prints the term dictionary for the specified field in the index + dump dumps the contents of the index + fields lists the fields in this index + help Help about any command + index adds the files to the index + mapping prints the mapping used for this index + query queries the index + registry registry lists the bleve components compiled into this executable + scorch command-line tool to interact with a scorch index + +Flags: + -h, --help help for bleve + +Use "bleve [command] --help" for more information about a command. +``` + +## Text Analysis + +Bleve includes general-purpose analyzers (customizable) as well as pre-built text analyzers for the following languages: + +Arabic (ar), Bulgarian (bg), Catalan (ca), Chinese-Japanese-Korean (cjk), Kurdish (ckb), Danish (da), German (de), Greek (el), English (en), Spanish - Castilian (es), Basque (eu), Persian (fa), Finnish (fi), French (fr), Gaelic (ga), Spanish - Galician (gl), Hindi (hi), Croatian (hr), Hungarian (hu), Armenian (hy), Indonesian (id, in), Italian (it), Dutch (nl), Norwegian (no), Portuguese (pt), Romanian (ro), Russian (ru), Swedish (sv), Turkish (tr) + +## Text Analysis Wizard + +[bleveanalysis.couchbase.com](https://bleveanalysis.couchbase.com) + +## Discussion/Issues + +Discuss usage/development of bleve and/or report issues here: +* [Github issues](https://github.com/blevesearch/bleve/issues) +* [Google group](https://groups.google.com/forum/#!forum/bleve) + +## License + +Apache License Version 2.0 diff --git a/vendor/github.com/blevesearch/bleve/v2/SECURITY.md b/vendor/github.com/blevesearch/bleve/v2/SECURITY.md new file mode 100644 index 00000000..51c6b6bd --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/SECURITY.md @@ -0,0 +1,15 @@ +# Security Policy + +## Supported Versions + +We support the latest release (for example, bleve v2.3.x). + +## Reporting a Vulnerability + +All security issues for this project should be reported by email to security@couchbase.com and fts-team@couchbase.com. +This mail will be delivered to the owners of this project. + +- To ensure your report is NOT marked as spam, please include the word "security/vulnerability" along with the project name (blevesearch/bleve) in the subject of the email. +- Please be as descriptive as possible while explaining the issue, and a testcase highlighting the issue is always welcome. + +Your email will be acknowledged at the soonest possible. diff --git a/vendor/github.com/blevesearch/bleve/analysis/analyzer/keyword/keyword.go b/vendor/github.com/blevesearch/bleve/v2/analysis/analyzer/keyword/keyword.go similarity index 80% rename from vendor/github.com/blevesearch/bleve/analysis/analyzer/keyword/keyword.go rename to vendor/github.com/blevesearch/bleve/v2/analysis/analyzer/keyword/keyword.go index 2a6c1aff..6bb56d6f 100644 --- a/vendor/github.com/blevesearch/bleve/analysis/analyzer/keyword/keyword.go +++ b/vendor/github.com/blevesearch/bleve/v2/analysis/analyzer/keyword/keyword.go @@ -15,19 +15,19 @@ package keyword import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/tokenizer/single" - "github.com/blevesearch/bleve/registry" + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/single" + "github.com/blevesearch/bleve/v2/registry" ) const Name = "keyword" -func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { +func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) { keywordTokenizer, err := cache.TokenizerNamed(single.Name) if err != nil { return nil, err } - rv := analysis.Analyzer{ + rv := analysis.DefaultAnalyzer{ Tokenizer: keywordTokenizer, } return &rv, nil diff --git a/vendor/github.com/blevesearch/bleve/analysis/analyzer/standard/standard.go b/vendor/github.com/blevesearch/bleve/v2/analysis/analyzer/standard/standard.go similarity index 77% rename from vendor/github.com/blevesearch/bleve/analysis/analyzer/standard/standard.go rename to vendor/github.com/blevesearch/bleve/v2/analysis/analyzer/standard/standard.go index 74ea564e..96387bd7 100644 --- a/vendor/github.com/blevesearch/bleve/analysis/analyzer/standard/standard.go +++ b/vendor/github.com/blevesearch/bleve/v2/analysis/analyzer/standard/standard.go @@ -15,16 +15,16 @@ package standard import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/lang/en" - "github.com/blevesearch/bleve/analysis/token/lowercase" - "github.com/blevesearch/bleve/analysis/tokenizer/unicode" - "github.com/blevesearch/bleve/registry" + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/lang/en" + "github.com/blevesearch/bleve/v2/analysis/token/lowercase" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" + "github.com/blevesearch/bleve/v2/registry" ) const Name = "standard" -func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { +func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) { tokenizer, err := cache.TokenizerNamed(unicode.Name) if err != nil { return nil, err @@ -37,7 +37,7 @@ func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) ( if err != nil { return nil, err } - rv := analysis.Analyzer{ + rv := analysis.DefaultAnalyzer{ Tokenizer: tokenizer, TokenFilters: []analysis.TokenFilter{ toLowerFilter, diff --git a/vendor/github.com/blevesearch/bleve/analysis/datetime/flexible/flexible.go b/vendor/github.com/blevesearch/bleve/v2/analysis/datetime/flexible/flexible.go similarity index 88% rename from vendor/github.com/blevesearch/bleve/analysis/datetime/flexible/flexible.go rename to vendor/github.com/blevesearch/bleve/v2/analysis/datetime/flexible/flexible.go index cd549f55..cb5f234d 100644 --- a/vendor/github.com/blevesearch/bleve/analysis/datetime/flexible/flexible.go +++ b/vendor/github.com/blevesearch/bleve/v2/analysis/datetime/flexible/flexible.go @@ -18,8 +18,8 @@ import ( "fmt" "time" - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" ) const Name = "flexiblego" @@ -34,14 +34,14 @@ func New(layouts []string) *DateTimeParser { } } -func (p *DateTimeParser) ParseDateTime(input string) (time.Time, error) { +func (p *DateTimeParser) ParseDateTime(input string) (time.Time, string, error) { for _, layout := range p.layouts { rv, err := time.Parse(layout, input) if err == nil { - return rv, nil + return rv, layout, nil } } - return time.Time{}, analysis.ErrInvalidDateTime + return time.Time{}, "", analysis.ErrInvalidDateTime } func DateTimeParserConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.DateTimeParser, error) { diff --git a/vendor/github.com/blevesearch/bleve/analysis/datetime/optional/optional.go b/vendor/github.com/blevesearch/bleve/v2/analysis/datetime/optional/optional.go similarity index 84% rename from vendor/github.com/blevesearch/bleve/analysis/datetime/optional/optional.go rename to vendor/github.com/blevesearch/bleve/v2/analysis/datetime/optional/optional.go index 4b98de66..6dc7bfbc 100644 --- a/vendor/github.com/blevesearch/bleve/analysis/datetime/optional/optional.go +++ b/vendor/github.com/blevesearch/bleve/v2/analysis/datetime/optional/optional.go @@ -17,15 +17,16 @@ package optional import ( "time" - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/datetime/flexible" - "github.com/blevesearch/bleve/registry" + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/datetime/flexible" + "github.com/blevesearch/bleve/v2/registry" ) const Name = "dateTimeOptional" const rfc3339NoTimezone = "2006-01-02T15:04:05" const rfc3339NoTimezoneNoT = "2006-01-02 15:04:05" +const rfc3339Offset = "2006-01-02 15:04:05 -0700" const rfc3339NoTime = "2006-01-02" var layouts = []string{ @@ -33,6 +34,7 @@ var layouts = []string{ time.RFC3339, rfc3339NoTimezone, rfc3339NoTimezoneNoT, + rfc3339Offset, rfc3339NoTime, } diff --git a/vendor/github.com/blevesearch/bleve/v2/analysis/freq.go b/vendor/github.com/blevesearch/bleve/v2/analysis/freq.go new file mode 100644 index 00000000..a0fd1a41 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/analysis/freq.go @@ -0,0 +1,70 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package analysis + +import ( + index "github.com/blevesearch/bleve_index_api" +) + +func TokenFrequency(tokens TokenStream, arrayPositions []uint64, options index.FieldIndexingOptions) index.TokenFrequencies { + rv := make(map[string]*index.TokenFreq, len(tokens)) + + if options.IncludeTermVectors() { + tls := make([]index.TokenLocation, len(tokens)) + tlNext := 0 + + for _, token := range tokens { + tls[tlNext] = index.TokenLocation{ + ArrayPositions: arrayPositions, + Start: token.Start, + End: token.End, + Position: token.Position, + } + + curr, ok := rv[string(token.Term)] + if ok { + curr.Locations = append(curr.Locations, &tls[tlNext]) + } else { + curr = &index.TokenFreq{ + Term: token.Term, + Locations: []*index.TokenLocation{&tls[tlNext]}, + } + rv[string(token.Term)] = curr + } + + if !options.SkipFreqNorm() { + curr.SetFrequency(curr.Frequency() + 1) + } + + tlNext++ + } + } else { + for _, token := range tokens { + curr, exists := rv[string(token.Term)] + if !exists { + curr = &index.TokenFreq{ + Term: token.Term, + } + rv[string(token.Term)] = curr + } + + if !options.SkipFreqNorm() { + curr.SetFrequency(curr.Frequency() + 1) + } + } + } + + return rv +} diff --git a/vendor/github.com/blevesearch/bleve/analysis/lang/en/analyzer_en.go b/vendor/github.com/blevesearch/bleve/v2/analysis/lang/en/analyzer_en.go similarity index 83% rename from vendor/github.com/blevesearch/bleve/analysis/lang/en/analyzer_en.go rename to vendor/github.com/blevesearch/bleve/v2/analysis/lang/en/analyzer_en.go index 8402785f..44a8d4c2 100644 --- a/vendor/github.com/blevesearch/bleve/analysis/lang/en/analyzer_en.go +++ b/vendor/github.com/blevesearch/bleve/v2/analysis/lang/en/analyzer_en.go @@ -22,17 +22,17 @@ package en import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" - "github.com/blevesearch/bleve/analysis/token/lowercase" - "github.com/blevesearch/bleve/analysis/token/porter" - "github.com/blevesearch/bleve/analysis/tokenizer/unicode" + "github.com/blevesearch/bleve/v2/analysis/token/lowercase" + "github.com/blevesearch/bleve/v2/analysis/token/porter" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" ) const AnalyzerName = "en" -func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { +func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) { tokenizer, err := cache.TokenizerNamed(unicode.Name) if err != nil { return nil, err @@ -53,7 +53,7 @@ func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) ( if err != nil { return nil, err } - rv := analysis.Analyzer{ + rv := analysis.DefaultAnalyzer{ Tokenizer: tokenizer, TokenFilters: []analysis.TokenFilter{ possEnFilter, diff --git a/vendor/github.com/blevesearch/bleve/v2/analysis/lang/en/plural_stemmer.go b/vendor/github.com/blevesearch/bleve/v2/analysis/lang/en/plural_stemmer.go new file mode 100644 index 00000000..0de7c1bb --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/analysis/lang/en/plural_stemmer.go @@ -0,0 +1,174 @@ +/* + This code was ported from the Open Search Project + https://github.com/opensearch-project/OpenSearch/blob/main/modules/analysis-common/src/main/java/org/opensearch/analysis/common/EnglishPluralStemFilter.java + The algorithm itself was created by Mark Harwood + https://github.com/markharwood +*/ + +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package en + +import ( + "strings" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const PluralStemmerName = "stemmer_en_plural" + +type EnglishPluralStemmerFilter struct { +} + +func NewEnglishPluralStemmerFilter() *EnglishPluralStemmerFilter { + return &EnglishPluralStemmerFilter{} +} + +func (s *EnglishPluralStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + token.Term = []byte(stem(string(token.Term))) + } + + return input +} + +func EnglishPluralStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewEnglishPluralStemmerFilter(), nil +} + +func init() { + registry.RegisterTokenFilter(PluralStemmerName, EnglishPluralStemmerFilterConstructor) +} + +// ---------------------------------------------------------------------------- + +// Words ending in oes that retain the e when stemmed +var oesExceptions = []string{"shoes", "canoes", "oboes"} + +// Words ending in ches that retain the e when stemmed +var chesExceptions = []string{ + "cliches", + "avalanches", + "mustaches", + "moustaches", + "quiches", + "headaches", + "heartaches", + "porsches", + "tranches", + "caches", +} + +func stem(word string) string { + runes := []rune(strings.ToLower(word)) + + if len(runes) < 3 || runes[len(runes)-1] != 's' { + return string(runes) + } + + switch runes[len(runes)-2] { + case 'u': + fallthrough + case 's': + return string(runes) + case 'e': + // Modified ies->y logic from original s-stemmer - only work on strings > 4 + // so spies -> spy still but pies->pie. + // The original code also special-cased aies and eies for no good reason as far as I can tell. + // ( no words of consequence - eg http://www.thefreedictionary.com/words-that-end-in-aies ) + if len(runes) > 4 && runes[len(runes)-3] == 'i' { + runes[len(runes)-3] = 'y' + return string(runes[0 : len(runes)-2]) + } + + // Suffix rules to remove any dangling "e" + if len(runes) > 3 { + // xes (but >1 prefix so we can stem "boxes->box" but keep "axes->axe") + if len(runes) > 4 && runes[len(runes)-3] == 'x' { + return string(runes[0 : len(runes)-2]) + } + + // oes + if len(runes) > 3 && runes[len(runes)-3] == 'o' { + if isException(runes, oesExceptions) { + // Only remove the S + return string(runes[0 : len(runes)-1]) + } + // Remove the es + return string(runes[0 : len(runes)-2]) + } + + if len(runes) > 4 { + // shes/sses + if runes[len(runes)-4] == 's' && (runes[len(runes)-3] == 'h' || runes[len(runes)-3] == 's') { + return string(runes[0 : len(runes)-2]) + } + + // ches + if len(runes) > 4 { + if runes[len(runes)-4] == 'c' && runes[len(runes)-3] == 'h' { + if isException(runes, chesExceptions) { + // Only remove the S + return string(runes[0 : len(runes)-1]) + } + // Remove the es + return string(runes[0 : len(runes)-2]) + } + } + } + } + fallthrough + default: + return string(runes[0 : len(runes)-1]) + } +} + +func isException(word []rune, exceptions []string) bool { + for _, exception := range exceptions { + + exceptionRunes := []rune(exception) + + exceptionPos := len(exceptionRunes) - 1 + wordPos := len(word) - 1 + + matched := true + for exceptionPos >= 0 && wordPos >= 0 { + if exceptionRunes[exceptionPos] != word[wordPos] { + matched = false + break + } + exceptionPos-- + wordPos-- + } + if matched { + return true + } + } + return false +} diff --git a/vendor/github.com/blevesearch/bleve/analysis/lang/en/possessive_filter_en.go b/vendor/github.com/blevesearch/bleve/v2/analysis/lang/en/possessive_filter_en.go similarity index 96% rename from vendor/github.com/blevesearch/bleve/analysis/lang/en/possessive_filter_en.go rename to vendor/github.com/blevesearch/bleve/v2/analysis/lang/en/possessive_filter_en.go index 2c06efd6..79c2489e 100644 --- a/vendor/github.com/blevesearch/bleve/analysis/lang/en/possessive_filter_en.go +++ b/vendor/github.com/blevesearch/bleve/v2/analysis/lang/en/possessive_filter_en.go @@ -17,8 +17,8 @@ package en import ( "unicode/utf8" - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" ) // PossessiveName is the name PossessiveFilter is registered as diff --git a/vendor/github.com/blevesearch/bleve/analysis/lang/en/stemmer_en_snowball.go b/vendor/github.com/blevesearch/bleve/v2/analysis/lang/en/stemmer_en_snowball.go similarity index 94% rename from vendor/github.com/blevesearch/bleve/analysis/lang/en/stemmer_en_snowball.go rename to vendor/github.com/blevesearch/bleve/v2/analysis/lang/en/stemmer_en_snowball.go index 225bb066..ab30b8b1 100644 --- a/vendor/github.com/blevesearch/bleve/analysis/lang/en/stemmer_en_snowball.go +++ b/vendor/github.com/blevesearch/bleve/v2/analysis/lang/en/stemmer_en_snowball.go @@ -15,8 +15,8 @@ package en import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" "github.com/blevesearch/snowballstem" "github.com/blevesearch/snowballstem/english" diff --git a/vendor/github.com/blevesearch/bleve/analysis/lang/en/stop_filter_en.go b/vendor/github.com/blevesearch/bleve/v2/analysis/lang/en/stop_filter_en.go similarity index 87% rename from vendor/github.com/blevesearch/bleve/analysis/lang/en/stop_filter_en.go rename to vendor/github.com/blevesearch/bleve/v2/analysis/lang/en/stop_filter_en.go index bfdb2c97..a3f91d22 100644 --- a/vendor/github.com/blevesearch/bleve/analysis/lang/en/stop_filter_en.go +++ b/vendor/github.com/blevesearch/bleve/v2/analysis/lang/en/stop_filter_en.go @@ -15,9 +15,9 @@ package en import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token/stop" - "github.com/blevesearch/bleve/registry" + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/token/stop" + "github.com/blevesearch/bleve/v2/registry" ) func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { diff --git a/vendor/github.com/blevesearch/bleve/analysis/lang/en/stop_words_en.go b/vendor/github.com/blevesearch/bleve/v2/analysis/lang/en/stop_words_en.go similarity index 98% rename from vendor/github.com/blevesearch/bleve/analysis/lang/en/stop_words_en.go rename to vendor/github.com/blevesearch/bleve/v2/analysis/lang/en/stop_words_en.go index 6423cf2c..9b6ca86a 100644 --- a/vendor/github.com/blevesearch/bleve/analysis/lang/en/stop_words_en.go +++ b/vendor/github.com/blevesearch/bleve/v2/analysis/lang/en/stop_words_en.go @@ -1,8 +1,8 @@ package en import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" ) const StopName = "stop_en" diff --git a/vendor/github.com/blevesearch/bleve/analysis/test_words.txt b/vendor/github.com/blevesearch/bleve/v2/analysis/test_words.txt similarity index 100% rename from vendor/github.com/blevesearch/bleve/analysis/test_words.txt rename to vendor/github.com/blevesearch/bleve/v2/analysis/test_words.txt diff --git a/vendor/github.com/blevesearch/bleve/analysis/token/lowercase/lowercase.go b/vendor/github.com/blevesearch/bleve/v2/analysis/token/lowercase/lowercase.go similarity index 96% rename from vendor/github.com/blevesearch/bleve/analysis/token/lowercase/lowercase.go rename to vendor/github.com/blevesearch/bleve/v2/analysis/token/lowercase/lowercase.go index adb740c3..a1b6dbd0 100644 --- a/vendor/github.com/blevesearch/bleve/analysis/token/lowercase/lowercase.go +++ b/vendor/github.com/blevesearch/bleve/v2/analysis/token/lowercase/lowercase.go @@ -21,8 +21,8 @@ import ( "unicode" "unicode/utf8" - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" ) // Name is the name used to register LowerCaseFilter in the bleve registry diff --git a/vendor/github.com/blevesearch/bleve/analysis/token/porter/porter.go b/vendor/github.com/blevesearch/bleve/v2/analysis/token/porter/porter.go similarity index 94% rename from vendor/github.com/blevesearch/bleve/analysis/token/porter/porter.go rename to vendor/github.com/blevesearch/bleve/v2/analysis/token/porter/porter.go index 4cd08d99..95af0fa7 100644 --- a/vendor/github.com/blevesearch/bleve/analysis/token/porter/porter.go +++ b/vendor/github.com/blevesearch/bleve/v2/analysis/token/porter/porter.go @@ -17,8 +17,8 @@ package porter import ( "bytes" - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" "github.com/blevesearch/go-porterstemmer" ) diff --git a/vendor/github.com/blevesearch/bleve/analysis/token/stop/stop.go b/vendor/github.com/blevesearch/bleve/v2/analysis/token/stop/stop.go similarity index 95% rename from vendor/github.com/blevesearch/bleve/analysis/token/stop/stop.go rename to vendor/github.com/blevesearch/bleve/v2/analysis/token/stop/stop.go index cca2d8e0..bf4b98db 100644 --- a/vendor/github.com/blevesearch/bleve/analysis/token/stop/stop.go +++ b/vendor/github.com/blevesearch/bleve/v2/analysis/token/stop/stop.go @@ -24,8 +24,8 @@ package stop import ( "fmt" - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" ) const Name = "stop_tokens" diff --git a/vendor/github.com/blevesearch/bleve/analysis/tokenizer/single/single.go b/vendor/github.com/blevesearch/bleve/v2/analysis/tokenizer/single/single.go similarity index 93% rename from vendor/github.com/blevesearch/bleve/analysis/tokenizer/single/single.go rename to vendor/github.com/blevesearch/bleve/v2/analysis/tokenizer/single/single.go index 18b2b1af..a3eac789 100644 --- a/vendor/github.com/blevesearch/bleve/analysis/tokenizer/single/single.go +++ b/vendor/github.com/blevesearch/bleve/v2/analysis/tokenizer/single/single.go @@ -15,8 +15,8 @@ package single import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" ) const Name = "single" diff --git a/vendor/github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode/unicode.go b/vendor/github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode/unicode.go new file mode 100644 index 00000000..ca3cfe76 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode/unicode.go @@ -0,0 +1,131 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package unicode + +import ( + "github.com/blevesearch/segment" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "unicode" + +type UnicodeTokenizer struct { +} + +func NewUnicodeTokenizer() *UnicodeTokenizer { + return &UnicodeTokenizer{} +} + +func (rt *UnicodeTokenizer) Tokenize(input []byte) analysis.TokenStream { + rvx := make([]analysis.TokenStream, 0, 10) // When rv gets full, append to rvx. + rv := make(analysis.TokenStream, 0, 1) + + ta := []analysis.Token(nil) + taNext := 0 + + segmenter := segment.NewWordSegmenterDirect(input) + start := 0 + pos := 1 + + guessRemaining := func(end int) int { + avgSegmentLen := end / (len(rv) + 1) + if avgSegmentLen < 1 { + avgSegmentLen = 1 + } + + remainingLen := len(input) - end + + return remainingLen / avgSegmentLen + } + + for segmenter.Segment() { + segmentBytes := segmenter.Bytes() + end := start + len(segmentBytes) + if segmenter.Type() != segment.None { + if taNext >= len(ta) { + remainingSegments := guessRemaining(end) + if remainingSegments > 1000 { + remainingSegments = 1000 + } + if remainingSegments < 1 { + remainingSegments = 1 + } + + ta = make([]analysis.Token, remainingSegments) + taNext = 0 + } + + token := &ta[taNext] + taNext++ + + token.Term = segmentBytes + token.Start = start + token.End = end + token.Position = pos + token.Type = convertType(segmenter.Type()) + + if len(rv) >= cap(rv) { // When rv is full, save it into rvx. + rvx = append(rvx, rv) + + rvCap := cap(rv) * 2 + if rvCap > 256 { + rvCap = 256 + } + + rv = make(analysis.TokenStream, 0, rvCap) // Next rv cap is bigger. + } + + rv = append(rv, token) + pos++ + } + start = end + } + + if len(rvx) > 0 { + n := len(rv) + for _, r := range rvx { + n += len(r) + } + rall := make(analysis.TokenStream, 0, n) + for _, r := range rvx { + rall = append(rall, r...) + } + return append(rall, rv...) + } + + return rv +} + +func UnicodeTokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error) { + return NewUnicodeTokenizer(), nil +} + +func init() { + registry.RegisterTokenizer(Name, UnicodeTokenizerConstructor) +} + +func convertType(segmentWordType int) analysis.TokenType { + switch segmentWordType { + case segment.Ideo: + return analysis.Ideographic + case segment.Kana: + return analysis.Ideographic + case segment.Number: + return analysis.Numeric + } + return analysis.AlphaNumeric +} diff --git a/vendor/github.com/blevesearch/bleve/analysis/tokenmap.go b/vendor/github.com/blevesearch/bleve/v2/analysis/tokenmap.go similarity index 97% rename from vendor/github.com/blevesearch/bleve/analysis/tokenmap.go rename to vendor/github.com/blevesearch/bleve/v2/analysis/tokenmap.go index 7c0d0a89..aa4ea315 100644 --- a/vendor/github.com/blevesearch/bleve/analysis/tokenmap.go +++ b/vendor/github.com/blevesearch/bleve/v2/analysis/tokenmap.go @@ -18,7 +18,7 @@ import ( "bufio" "bytes" "io" - "io/ioutil" + "os" "strings" ) @@ -32,7 +32,7 @@ func NewTokenMap() TokenMap { // one per line. // Comments are supported using `#` or `|` func (t TokenMap) LoadFile(filename string) error { - data, err := ioutil.ReadFile(filename) + data, err := os.ReadFile(filename) if err != nil { return err } diff --git a/vendor/github.com/blevesearch/bleve/analysis/type.go b/vendor/github.com/blevesearch/bleve/v2/analysis/type.go similarity index 87% rename from vendor/github.com/blevesearch/bleve/analysis/type.go rename to vendor/github.com/blevesearch/bleve/v2/analysis/type.go index 589cc1ca..e3a7f201 100644 --- a/vendor/github.com/blevesearch/bleve/analysis/type.go +++ b/vendor/github.com/blevesearch/bleve/v2/analysis/type.go @@ -34,6 +34,7 @@ const ( Single Double Boolean + IP ) // Token represents one occurrence of a term at a particular location in a @@ -71,13 +72,17 @@ type TokenFilter interface { Filter(TokenStream) TokenStream } -type Analyzer struct { +type Analyzer interface { + Analyze([]byte) TokenStream +} + +type DefaultAnalyzer struct { CharFilters []CharFilter Tokenizer Tokenizer TokenFilters []TokenFilter } -func (a *Analyzer) Analyze(input []byte) TokenStream { +func (a *DefaultAnalyzer) Analyze(input []byte) TokenStream { if a.CharFilters != nil { for _, cf := range a.CharFilters { input = cf.Filter(input) @@ -94,8 +99,11 @@ func (a *Analyzer) Analyze(input []byte) TokenStream { var ErrInvalidDateTime = fmt.Errorf("unable to parse datetime with any of the layouts") +var ErrInvalidTimestampString = fmt.Errorf("unable to parse timestamp string") +var ErrInvalidTimestampRange = fmt.Errorf("timestamp out of range") + type DateTimeParser interface { - ParseDateTime(string) (time.Time, error) + ParseDateTime(string) (time.Time, string, error) } type ByteArrayConverter interface { diff --git a/vendor/github.com/blevesearch/bleve/analysis/util.go b/vendor/github.com/blevesearch/bleve/v2/analysis/util.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/analysis/util.go rename to vendor/github.com/blevesearch/bleve/v2/analysis/util.go diff --git a/vendor/github.com/blevesearch/bleve/v2/builder.go b/vendor/github.com/blevesearch/bleve/v2/builder.go new file mode 100644 index 00000000..30285a2e --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/builder.go @@ -0,0 +1,94 @@ +// Copyright (c) 2019 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bleve + +import ( + "fmt" + + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/index/scorch" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" +) + +type builderImpl struct { + b index.IndexBuilder + m mapping.IndexMapping +} + +func (b *builderImpl) Index(id string, data interface{}) error { + if id == "" { + return ErrorEmptyID + } + + doc := document.NewDocument(id) + err := b.m.MapDocument(doc, data) + if err != nil { + return err + } + err = b.b.Index(doc) + return err +} + +func (b *builderImpl) Close() error { + return b.b.Close() +} + +func newBuilder(path string, mapping mapping.IndexMapping, config map[string]interface{}) (Builder, error) { + if path == "" { + return nil, fmt.Errorf("builder requires path") + } + + err := mapping.Validate() + if err != nil { + return nil, err + } + + if config == nil { + config = map[string]interface{}{} + } + + // the builder does not have an API to interact with internal storage + // however we can pass k/v pairs through the config + mappingBytes, err := util.MarshalJSON(mapping) + if err != nil { + return nil, err + } + config["internal"] = map[string][]byte{ + string(mappingInternalKey): mappingBytes, + } + + // do not use real config, as these are options for the builder, + // not the resulting index + meta := newIndexMeta(scorch.Name, scorch.Name, map[string]interface{}{}) + err = meta.Save(path) + if err != nil { + return nil, err + } + + config["path"] = indexStorePath(path) + + b, err := scorch.NewBuilder(config) + if err != nil { + return nil, err + } + rv := &builderImpl{ + b: b, + m: mapping, + } + + return rv, nil +} diff --git a/vendor/github.com/blevesearch/bleve/v2/config.go b/vendor/github.com/blevesearch/bleve/v2/config.go new file mode 100644 index 00000000..d1490bdc --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/config.go @@ -0,0 +1,95 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bleve + +import ( + "expvar" + "io" + "log" + "time" + + "github.com/blevesearch/bleve/v2/index/scorch" + "github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap" + "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/bleve/v2/search/highlight/highlighter/html" + index "github.com/blevesearch/bleve_index_api" +) + +var bleveExpVar = expvar.NewMap("bleve") + +type configuration struct { + Cache *registry.Cache + DefaultHighlighter string + DefaultKVStore string + DefaultMemKVStore string + DefaultIndexType string + SlowSearchLogThreshold time.Duration + analysisQueue *index.AnalysisQueue +} + +func (c *configuration) SetAnalysisQueueSize(n int) { + if c.analysisQueue != nil { + c.analysisQueue.Close() + } + c.analysisQueue = index.NewAnalysisQueue(n) +} + +func (c *configuration) Shutdown() { + c.SetAnalysisQueueSize(0) +} + +func newConfiguration() *configuration { + return &configuration{ + Cache: registry.NewCache(), + analysisQueue: index.NewAnalysisQueue(4), + } +} + +// Config contains library level configuration +var Config *configuration + +func init() { + bootStart := time.Now() + + // build the default configuration + Config = newConfiguration() + + // set the default highlighter + Config.DefaultHighlighter = html.Name + + // default kv store + Config.DefaultKVStore = "" + + // default mem only kv store + Config.DefaultMemKVStore = gtreap.Name + + // default index + Config.DefaultIndexType = scorch.Name + + bootDuration := time.Since(bootStart) + bleveExpVar.Add("bootDuration", int64(bootDuration)) + indexStats = NewIndexStats() + bleveExpVar.Set("indexes", indexStats) + + initDisk() +} + +var logger = log.New(io.Discard, "bleve", log.LstdFlags) + +// SetLog sets the logger used for logging +// by default log messages are sent to io.Discard +func SetLog(l *log.Logger) { + logger = l +} diff --git a/vendor/github.com/blevesearch/bleve/config_app.go b/vendor/github.com/blevesearch/bleve/v2/config_app.go similarity index 95% rename from vendor/github.com/blevesearch/bleve/config_app.go rename to vendor/github.com/blevesearch/bleve/v2/config_app.go index 112d0b60..60b1db3e 100644 --- a/vendor/github.com/blevesearch/bleve/config_app.go +++ b/vendor/github.com/blevesearch/bleve/v2/config_app.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build appengine || appenginevm // +build appengine appenginevm package bleve diff --git a/vendor/github.com/blevesearch/bleve/config_disk.go b/vendor/github.com/blevesearch/bleve/v2/config_disk.go similarity index 87% rename from vendor/github.com/blevesearch/bleve/config_disk.go rename to vendor/github.com/blevesearch/bleve/v2/config_disk.go index d03bceb4..a9ab1e41 100644 --- a/vendor/github.com/blevesearch/bleve/config_disk.go +++ b/vendor/github.com/blevesearch/bleve/v2/config_disk.go @@ -12,11 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !appengine && !appenginevm // +build !appengine,!appenginevm package bleve -import "github.com/blevesearch/bleve/index/store/boltdb" +import "github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb" // in normal environments we configure boltdb as the default storage func initDisk() { diff --git a/vendor/github.com/blevesearch/bleve/v2/doc.go b/vendor/github.com/blevesearch/bleve/v2/doc.go new file mode 100644 index 00000000..b9580cbe --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/doc.go @@ -0,0 +1,37 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* +Package bleve is a library for indexing and searching text. + +Example Opening New Index, Indexing Data + + message := struct{ + Id: "example" + From: "marty.schoch@gmail.com", + Body: "bleve indexing is easy", + } + + mapping := bleve.NewIndexMapping() + index, _ := bleve.New("example.bleve", mapping) + index.Index(message.Id, message) + +Example Opening Existing Index, Searching Data + + index, _ := bleve.Open("example.bleve") + query := bleve.NewQueryStringQuery("bleve") + searchRequest := bleve.NewSearchRequest(query) + searchResult, _ := index.Search(searchRequest) +*/ +package bleve diff --git a/vendor/github.com/blevesearch/bleve/v2/document/document.go b/vendor/github.com/blevesearch/bleve/v2/document/document.go new file mode 100644 index 00000000..54fd6d44 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/document/document.go @@ -0,0 +1,135 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package document + +import ( + "fmt" + "reflect" + + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" +) + +var reflectStaticSizeDocument int + +func init() { + var d Document + reflectStaticSizeDocument = int(reflect.TypeOf(d).Size()) +} + +type Document struct { + id string `json:"id"` + Fields []Field `json:"fields"` + CompositeFields []*CompositeField + StoredFieldsSize uint64 +} + +func (d *Document) StoredFieldsBytes() uint64 { + return d.StoredFieldsSize +} + +func NewDocument(id string) *Document { + return &Document{ + id: id, + Fields: make([]Field, 0), + CompositeFields: make([]*CompositeField, 0), + } +} + +func (d *Document) Size() int { + sizeInBytes := reflectStaticSizeDocument + size.SizeOfPtr + + len(d.id) + + for _, entry := range d.Fields { + sizeInBytes += entry.Size() + } + + for _, entry := range d.CompositeFields { + sizeInBytes += entry.Size() + } + + return sizeInBytes +} + +func (d *Document) AddField(f Field) *Document { + switch f := f.(type) { + case *CompositeField: + d.CompositeFields = append(d.CompositeFields, f) + default: + d.Fields = append(d.Fields, f) + } + return d +} + +func (d *Document) GoString() string { + fields := "" + for i, field := range d.Fields { + if i != 0 { + fields += ", " + } + fields += fmt.Sprintf("%#v", field) + } + compositeFields := "" + for i, field := range d.CompositeFields { + if i != 0 { + compositeFields += ", " + } + compositeFields += fmt.Sprintf("%#v", field) + } + return fmt.Sprintf("&document.Document{ID:%s, Fields: %s, CompositeFields: %s}", d.ID(), fields, compositeFields) +} + +func (d *Document) NumPlainTextBytes() uint64 { + rv := uint64(0) + for _, field := range d.Fields { + rv += field.NumPlainTextBytes() + } + for _, compositeField := range d.CompositeFields { + for _, field := range d.Fields { + if compositeField.includesField(field.Name()) { + rv += field.NumPlainTextBytes() + } + } + } + return rv +} + +func (d *Document) ID() string { + return d.id +} + +func (d *Document) SetID(id string) { + d.id = id +} + +func (d *Document) AddIDField() { + d.AddField(NewTextFieldCustom("_id", nil, []byte(d.ID()), index.IndexField|index.StoreField, nil)) +} + +func (d *Document) VisitFields(visitor index.FieldVisitor) { + for _, f := range d.Fields { + visitor(f) + } +} + +func (d *Document) VisitComposite(visitor index.CompositeFieldVisitor) { + for _, f := range d.CompositeFields { + visitor(f) + } +} + +func (d *Document) HasComposite() bool { + return len(d.CompositeFields) > 0 +} diff --git a/vendor/github.com/blevesearch/bleve/v2/document/field.go b/vendor/github.com/blevesearch/bleve/v2/document/field.go new file mode 100644 index 00000000..eb104e2d --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/document/field.go @@ -0,0 +1,45 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package document + +import ( + index "github.com/blevesearch/bleve_index_api" +) + +type Field interface { + // Name returns the path of the field from the root DocumentMapping. + // A root field path is "field", a subdocument field is "parent.field". + Name() string + // ArrayPositions returns the intermediate document and field indices + // required to resolve the field value in the document. For example, if the + // field path is "doc1.doc2.field" where doc1 and doc2 are slices or + // arrays, ArrayPositions returns 2 indices used to resolve "doc2" value in + // "doc1", then "field" in "doc2". + ArrayPositions() []uint64 + Options() index.FieldIndexingOptions + Analyze() + Value() []byte + + // NumPlainTextBytes should return the number of plain text bytes + // that this field represents - this is a common metric for tracking + // the rate of indexing + NumPlainTextBytes() uint64 + + Size() int + + EncodedFieldType() byte + AnalyzedLength() int + AnalyzedTokenFrequencies() index.TokenFrequencies +} diff --git a/vendor/github.com/blevesearch/bleve/document/field_boolean.go b/vendor/github.com/blevesearch/bleve/v2/document/field_boolean.go similarity index 76% rename from vendor/github.com/blevesearch/bleve/document/field_boolean.go rename to vendor/github.com/blevesearch/bleve/v2/document/field_boolean.go index 6864b16f..fdf3cc0e 100644 --- a/vendor/github.com/blevesearch/bleve/document/field_boolean.go +++ b/vendor/github.com/blevesearch/bleve/v2/document/field_boolean.go @@ -18,8 +18,9 @@ import ( "fmt" "reflect" - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" ) var reflectStaticSizeBooleanField int @@ -29,14 +30,16 @@ func init() { reflectStaticSizeBooleanField = int(reflect.TypeOf(f).Size()) } -const DefaultBooleanIndexingOptions = StoreField | IndexField | DocValues +const DefaultBooleanIndexingOptions = index.StoreField | index.IndexField | index.DocValues type BooleanField struct { name string arrayPositions []uint64 - options IndexingOptions + options index.FieldIndexingOptions value []byte numPlainTextBytes uint64 + length int + frequencies index.TokenFrequencies } func (b *BooleanField) Size() int { @@ -54,11 +57,11 @@ func (b *BooleanField) ArrayPositions() []uint64 { return b.arrayPositions } -func (b *BooleanField) Options() IndexingOptions { +func (b *BooleanField) Options() index.FieldIndexingOptions { return b.options } -func (b *BooleanField) Analyze() (int, analysis.TokenFrequencies) { +func (b *BooleanField) Analyze() { tokens := make(analysis.TokenStream, 0) tokens = append(tokens, &analysis.Token{ Start: 0, @@ -68,9 +71,8 @@ func (b *BooleanField) Analyze() (int, analysis.TokenFrequencies) { Type: analysis.Boolean, }) - fieldLength := len(tokens) - tokenFreqs := analysis.TokenFrequency(tokens, b.arrayPositions, b.options.IncludeTermVectors()) - return fieldLength, tokenFreqs + b.length = len(tokens) + b.frequencies = analysis.TokenFrequency(tokens, b.arrayPositions, b.options) } func (b *BooleanField) Value() []byte { @@ -92,6 +94,18 @@ func (b *BooleanField) NumPlainTextBytes() uint64 { return b.numPlainTextBytes } +func (b *BooleanField) EncodedFieldType() byte { + return 'b' +} + +func (b *BooleanField) AnalyzedLength() int { + return b.length +} + +func (b *BooleanField) AnalyzedTokenFrequencies() index.TokenFrequencies { + return b.frequencies +} + func NewBooleanFieldFromBytes(name string, arrayPositions []uint64, value []byte) *BooleanField { return &BooleanField{ name: name, @@ -106,7 +120,7 @@ func NewBooleanField(name string, arrayPositions []uint64, b bool) *BooleanField return NewBooleanFieldWithIndexingOptions(name, arrayPositions, b, DefaultNumericIndexingOptions) } -func NewBooleanFieldWithIndexingOptions(name string, arrayPositions []uint64, b bool, options IndexingOptions) *BooleanField { +func NewBooleanFieldWithIndexingOptions(name string, arrayPositions []uint64, b bool, options index.FieldIndexingOptions) *BooleanField { numPlainTextBytes := 5 v := []byte("F") if b { diff --git a/vendor/github.com/blevesearch/bleve/document/field_composite.go b/vendor/github.com/blevesearch/bleve/v2/document/field_composite.go similarity index 78% rename from vendor/github.com/blevesearch/bleve/document/field_composite.go rename to vendor/github.com/blevesearch/bleve/v2/document/field_composite.go index a8285880..8c47643f 100644 --- a/vendor/github.com/blevesearch/bleve/document/field_composite.go +++ b/vendor/github.com/blevesearch/bleve/v2/document/field_composite.go @@ -17,8 +17,8 @@ package document import ( "reflect" - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" ) var reflectStaticSizeCompositeField int @@ -28,30 +28,30 @@ func init() { reflectStaticSizeCompositeField = int(reflect.TypeOf(cf).Size()) } -const DefaultCompositeIndexingOptions = IndexField +const DefaultCompositeIndexingOptions = index.IndexField type CompositeField struct { name string includedFields map[string]bool excludedFields map[string]bool defaultInclude bool - options IndexingOptions + options index.FieldIndexingOptions totalLength int - compositeFrequencies analysis.TokenFrequencies + compositeFrequencies index.TokenFrequencies } func NewCompositeField(name string, defaultInclude bool, include []string, exclude []string) *CompositeField { return NewCompositeFieldWithIndexingOptions(name, defaultInclude, include, exclude, DefaultCompositeIndexingOptions) } -func NewCompositeFieldWithIndexingOptions(name string, defaultInclude bool, include []string, exclude []string, options IndexingOptions) *CompositeField { +func NewCompositeFieldWithIndexingOptions(name string, defaultInclude bool, include []string, exclude []string, options index.FieldIndexingOptions) *CompositeField { rv := &CompositeField{ name: name, options: options, defaultInclude: defaultInclude, includedFields: make(map[string]bool, len(include)), excludedFields: make(map[string]bool, len(exclude)), - compositeFrequencies: make(analysis.TokenFrequencies), + compositeFrequencies: make(index.TokenFrequencies), } for _, i := range include { @@ -87,12 +87,11 @@ func (c *CompositeField) ArrayPositions() []uint64 { return []uint64{} } -func (c *CompositeField) Options() IndexingOptions { +func (c *CompositeField) Options() index.FieldIndexingOptions { return c.options } -func (c *CompositeField) Analyze() (int, analysis.TokenFrequencies) { - return c.totalLength, c.compositeFrequencies +func (c *CompositeField) Analyze() { } func (c *CompositeField) Value() []byte { @@ -116,9 +115,21 @@ func (c *CompositeField) includesField(field string) bool { return shouldInclude } -func (c *CompositeField) Compose(field string, length int, freq analysis.TokenFrequencies) { +func (c *CompositeField) Compose(field string, length int, freq index.TokenFrequencies) { if c.includesField(field) { c.totalLength += length c.compositeFrequencies.MergeAll(field, freq) } } + +func (c *CompositeField) EncodedFieldType() byte { + return 'c' +} + +func (c *CompositeField) AnalyzedLength() int { + return c.totalLength +} + +func (c *CompositeField) AnalyzedTokenFrequencies() index.TokenFrequencies { + return c.compositeFrequencies +} diff --git a/vendor/github.com/blevesearch/bleve/v2/document/field_datetime.go b/vendor/github.com/blevesearch/bleve/v2/document/field_datetime.go new file mode 100644 index 00000000..efdd26b6 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/document/field_datetime.go @@ -0,0 +1,196 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package document + +import ( + "bytes" + "fmt" + "math" + "reflect" + "time" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" +) + +var dateTimeValueSeperator = []byte{'\xff'} + +var reflectStaticSizeDateTimeField int + +func init() { + var f DateTimeField + reflectStaticSizeDateTimeField = int(reflect.TypeOf(f).Size()) +} + +const DefaultDateTimeIndexingOptions = index.StoreField | index.IndexField | index.DocValues +const DefaultDateTimePrecisionStep uint = 4 + +var MinTimeRepresentable = time.Unix(0, math.MinInt64) +var MaxTimeRepresentable = time.Unix(0, math.MaxInt64) + +type DateTimeField struct { + name string + arrayPositions []uint64 + options index.FieldIndexingOptions + value numeric.PrefixCoded + numPlainTextBytes uint64 + length int + frequencies index.TokenFrequencies +} + +func (n *DateTimeField) Size() int { + return reflectStaticSizeDateTimeField + size.SizeOfPtr + + len(n.name) + + len(n.arrayPositions)*size.SizeOfUint64 +} + +func (n *DateTimeField) Name() string { + return n.name +} + +func (n *DateTimeField) ArrayPositions() []uint64 { + return n.arrayPositions +} + +func (n *DateTimeField) Options() index.FieldIndexingOptions { + return n.options +} + +func (n *DateTimeField) EncodedFieldType() byte { + return 'd' +} + +func (n *DateTimeField) AnalyzedLength() int { + return n.length +} + +func (n *DateTimeField) AnalyzedTokenFrequencies() index.TokenFrequencies { + return n.frequencies +} + +// split the value into the prefix coded date and the layout +// using the dateTimeValueSeperator as the split point +func (n *DateTimeField) splitValue() (numeric.PrefixCoded, string) { + parts := bytes.SplitN(n.value, dateTimeValueSeperator, 2) + if len(parts) == 1 { + return numeric.PrefixCoded(parts[0]), "" + } + return numeric.PrefixCoded(parts[0]), string(parts[1]) +} + +func (n *DateTimeField) Analyze() { + valueWithoutLayout, _ := n.splitValue() + tokens := make(analysis.TokenStream, 0) + tokens = append(tokens, &analysis.Token{ + Start: 0, + End: len(valueWithoutLayout), + Term: valueWithoutLayout, + Position: 1, + Type: analysis.DateTime, + }) + + original, err := valueWithoutLayout.Int64() + if err == nil { + + shift := DefaultDateTimePrecisionStep + for shift < 64 { + shiftEncoded, err := numeric.NewPrefixCodedInt64(original, shift) + if err != nil { + break + } + token := analysis.Token{ + Start: 0, + End: len(shiftEncoded), + Term: shiftEncoded, + Position: 1, + Type: analysis.DateTime, + } + tokens = append(tokens, &token) + shift += DefaultDateTimePrecisionStep + } + } + + n.length = len(tokens) + n.frequencies = analysis.TokenFrequency(tokens, n.arrayPositions, n.options) +} + +func (n *DateTimeField) Value() []byte { + return n.value +} + +func (n *DateTimeField) DateTime() (time.Time, string, error) { + date, layout := n.splitValue() + i64, err := date.Int64() + if err != nil { + return time.Time{}, "", err + } + return time.Unix(0, i64).UTC(), layout, nil +} + +func (n *DateTimeField) GoString() string { + return fmt.Sprintf("&document.DateField{Name:%s, Options: %s, Value: %s}", n.name, n.options, n.value) +} + +func (n *DateTimeField) NumPlainTextBytes() uint64 { + return n.numPlainTextBytes +} + +func NewDateTimeFieldFromBytes(name string, arrayPositions []uint64, value []byte) *DateTimeField { + return &DateTimeField{ + name: name, + arrayPositions: arrayPositions, + value: value, + options: DefaultDateTimeIndexingOptions, + numPlainTextBytes: uint64(len(value)), + } +} + +func NewDateTimeField(name string, arrayPositions []uint64, dt time.Time, layout string) (*DateTimeField, error) { + return NewDateTimeFieldWithIndexingOptions(name, arrayPositions, dt, layout, DefaultDateTimeIndexingOptions) +} + +func NewDateTimeFieldWithIndexingOptions(name string, arrayPositions []uint64, dt time.Time, layout string, options index.FieldIndexingOptions) (*DateTimeField, error) { + if canRepresent(dt) { + dtInt64 := dt.UnixNano() + prefixCoded := numeric.MustNewPrefixCodedInt64(dtInt64, 0) + // The prefixCoded value is combined with the layout. + // This is necessary because the storage layer stores a fields value as a byte slice + // without storing extra information like layout. So by making value = prefixCoded + layout, + // both pieces of information are stored in the byte slice. + // During a query, the layout is extracted from the byte slice stored to correctly + // format the prefixCoded value. + valueWithLayout := append(prefixCoded, dateTimeValueSeperator...) + valueWithLayout = append(valueWithLayout, []byte(layout)...) + return &DateTimeField{ + name: name, + arrayPositions: arrayPositions, + value: valueWithLayout, + options: options, + // not correct, just a place holder until we revisit how fields are + // represented and can fix this better + numPlainTextBytes: uint64(8), + }, nil + } + return nil, fmt.Errorf("cannot represent %s in this type", dt) +} + +func canRepresent(dt time.Time) bool { + if dt.Before(MinTimeRepresentable) || dt.After(MaxTimeRepresentable) { + return false + } + return true +} diff --git a/vendor/github.com/blevesearch/bleve/v2/document/field_geopoint.go b/vendor/github.com/blevesearch/bleve/v2/document/field_geopoint.go new file mode 100644 index 00000000..719d18c3 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/document/field_geopoint.go @@ -0,0 +1,193 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package document + +import ( + "fmt" + "reflect" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" +) + +var reflectStaticSizeGeoPointField int + +func init() { + var f GeoPointField + reflectStaticSizeGeoPointField = int(reflect.TypeOf(f).Size()) +} + +var GeoPrecisionStep uint = 9 + +type GeoPointField struct { + name string + arrayPositions []uint64 + options index.FieldIndexingOptions + value numeric.PrefixCoded + numPlainTextBytes uint64 + length int + frequencies index.TokenFrequencies + + spatialplugin index.SpatialAnalyzerPlugin +} + +func (n *GeoPointField) Size() int { + return reflectStaticSizeGeoPointField + size.SizeOfPtr + + len(n.name) + + len(n.arrayPositions)*size.SizeOfUint64 +} + +func (n *GeoPointField) Name() string { + return n.name +} + +func (n *GeoPointField) ArrayPositions() []uint64 { + return n.arrayPositions +} + +func (n *GeoPointField) Options() index.FieldIndexingOptions { + return n.options +} + +func (n *GeoPointField) EncodedFieldType() byte { + return 'g' +} + +func (n *GeoPointField) AnalyzedLength() int { + return n.length +} + +func (n *GeoPointField) AnalyzedTokenFrequencies() index.TokenFrequencies { + return n.frequencies +} + +func (n *GeoPointField) Analyze() { + tokens := make(analysis.TokenStream, 0, 8) + tokens = append(tokens, &analysis.Token{ + Start: 0, + End: len(n.value), + Term: n.value, + Position: 1, + Type: analysis.Numeric, + }) + + if n.spatialplugin != nil { + lat, _ := n.Lat() + lon, _ := n.Lon() + p := &geo.Point{Lat: lat, Lon: lon} + terms := n.spatialplugin.GetIndexTokens(p) + + for _, term := range terms { + token := analysis.Token{ + Start: 0, + End: len(term), + Term: []byte(term), + Position: 1, + Type: analysis.AlphaNumeric, + } + tokens = append(tokens, &token) + } + } else { + original, err := n.value.Int64() + if err == nil { + + shift := GeoPrecisionStep + for shift < 64 { + shiftEncoded, err := numeric.NewPrefixCodedInt64(original, shift) + if err != nil { + break + } + token := analysis.Token{ + Start: 0, + End: len(shiftEncoded), + Term: shiftEncoded, + Position: 1, + Type: analysis.Numeric, + } + tokens = append(tokens, &token) + shift += GeoPrecisionStep + } + } + } + + n.length = len(tokens) + n.frequencies = analysis.TokenFrequency(tokens, n.arrayPositions, n.options) +} + +func (n *GeoPointField) Value() []byte { + return n.value +} + +func (n *GeoPointField) Lon() (float64, error) { + i64, err := n.value.Int64() + if err != nil { + return 0.0, err + } + return geo.MortonUnhashLon(uint64(i64)), nil +} + +func (n *GeoPointField) Lat() (float64, error) { + i64, err := n.value.Int64() + if err != nil { + return 0.0, err + } + return geo.MortonUnhashLat(uint64(i64)), nil +} + +func (n *GeoPointField) GoString() string { + return fmt.Sprintf("&document.GeoPointField{Name:%s, Options: %s, Value: %s}", n.name, n.options, n.value) +} + +func (n *GeoPointField) NumPlainTextBytes() uint64 { + return n.numPlainTextBytes +} + +func NewGeoPointFieldFromBytes(name string, arrayPositions []uint64, value []byte) *GeoPointField { + return &GeoPointField{ + name: name, + arrayPositions: arrayPositions, + value: value, + options: DefaultNumericIndexingOptions, + numPlainTextBytes: uint64(len(value)), + } +} + +func NewGeoPointField(name string, arrayPositions []uint64, lon, lat float64) *GeoPointField { + return NewGeoPointFieldWithIndexingOptions(name, arrayPositions, lon, lat, DefaultNumericIndexingOptions) +} + +func NewGeoPointFieldWithIndexingOptions(name string, arrayPositions []uint64, lon, lat float64, options index.FieldIndexingOptions) *GeoPointField { + mhash := geo.MortonHash(lon, lat) + prefixCoded := numeric.MustNewPrefixCodedInt64(int64(mhash), 0) + return &GeoPointField{ + name: name, + arrayPositions: arrayPositions, + value: prefixCoded, + options: options, + // not correct, just a place holder until we revisit how fields are + // represented and can fix this better + numPlainTextBytes: uint64(8), + } +} + +// SetSpatialAnalyzerPlugin implements the +// index.TokenisableSpatialField interface. +func (n *GeoPointField) SetSpatialAnalyzerPlugin( + plugin index.SpatialAnalyzerPlugin) { + n.spatialplugin = plugin +} diff --git a/vendor/github.com/blevesearch/bleve/v2/document/field_geoshape.go b/vendor/github.com/blevesearch/bleve/v2/document/field_geoshape.go new file mode 100644 index 00000000..a20ff183 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/document/field_geoshape.go @@ -0,0 +1,235 @@ +// Copyright (c) 2022 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package document + +import ( + "fmt" + "reflect" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" + "github.com/blevesearch/geo/geojson" +) + +var reflectStaticSizeGeoShapeField int + +func init() { + var f GeoShapeField + reflectStaticSizeGeoShapeField = int(reflect.TypeOf(f).Size()) +} + +const DefaultGeoShapeIndexingOptions = index.IndexField | index.DocValues + +type GeoShapeField struct { + name string + shape index.GeoJSON + arrayPositions []uint64 + options index.FieldIndexingOptions + numPlainTextBytes uint64 + length int + encodedValue []byte + value []byte + + frequencies index.TokenFrequencies +} + +func (n *GeoShapeField) Size() int { + return reflectStaticSizeGeoShapeField + size.SizeOfPtr + + len(n.name) + + len(n.arrayPositions)*size.SizeOfUint64 +} + +func (n *GeoShapeField) Name() string { + return n.name +} + +func (n *GeoShapeField) ArrayPositions() []uint64 { + return n.arrayPositions +} + +func (n *GeoShapeField) Options() index.FieldIndexingOptions { + return n.options +} + +func (n *GeoShapeField) EncodedFieldType() byte { + return 's' +} + +func (n *GeoShapeField) AnalyzedLength() int { + return n.length +} + +func (n *GeoShapeField) AnalyzedTokenFrequencies() index.TokenFrequencies { + return n.frequencies +} + +func (n *GeoShapeField) Analyze() { + // compute the bytes representation for the coordinates + tokens := make(analysis.TokenStream, 0) + tokens = append(tokens, &analysis.Token{ + Start: 0, + End: len(n.encodedValue), + Term: n.encodedValue, + Position: 1, + Type: analysis.AlphaNumeric, + }) + + rti := geo.GetSpatialAnalyzerPlugin("s2") + terms := rti.GetIndexTokens(n.shape) + + for _, term := range terms { + token := analysis.Token{ + Start: 0, + End: len(term), + Term: []byte(term), + Position: 1, + Type: analysis.AlphaNumeric, + } + tokens = append(tokens, &token) + } + + n.length = len(tokens) + n.frequencies = analysis.TokenFrequency(tokens, n.arrayPositions, n.options) +} + +func (n *GeoShapeField) Value() []byte { + return n.value +} + +func (n *GeoShapeField) GoString() string { + return fmt.Sprintf("&document.GeoShapeField{Name:%s, Options: %s, Value: %s}", + n.name, n.options, n.value) +} + +func (n *GeoShapeField) NumPlainTextBytes() uint64 { + return n.numPlainTextBytes +} + +func NewGeoShapeField(name string, arrayPositions []uint64, + coordinates [][][][]float64, typ string) *GeoShapeField { + return NewGeoShapeFieldWithIndexingOptions(name, arrayPositions, + coordinates, typ, DefaultGeoShapeIndexingOptions) +} + +func NewGeoShapeFieldFromBytes(name string, arrayPositions []uint64, + value []byte) *GeoShapeField { + return &GeoShapeField{ + name: name, + arrayPositions: arrayPositions, + value: value, + options: DefaultGeoShapeIndexingOptions, + numPlainTextBytes: uint64(len(value)), + } +} + +func NewGeoShapeFieldWithIndexingOptions(name string, arrayPositions []uint64, + coordinates [][][][]float64, typ string, + options index.FieldIndexingOptions) *GeoShapeField { + shape, encodedValue, err := geo.NewGeoJsonShape(coordinates, typ) + if err != nil { + return nil + } + + // extra glue bytes to work around the term splitting logic from interfering + // the custom encoding of the geoshape coordinates inside the docvalues. + encodedValue = append(geo.GlueBytes, append(encodedValue, geo.GlueBytes...)...) + + // get the byte value for the geoshape. + value, err := shape.Value() + if err != nil { + return nil + } + + options = options | DefaultGeoShapeIndexingOptions + + return &GeoShapeField{ + shape: shape, + name: name, + arrayPositions: arrayPositions, + options: options, + encodedValue: encodedValue, + value: value, + numPlainTextBytes: uint64(len(value)), + } +} + +func NewGeometryCollectionFieldWithIndexingOptions(name string, + arrayPositions []uint64, coordinates [][][][][]float64, types []string, + options index.FieldIndexingOptions) *GeoShapeField { + shape, encodedValue, err := geo.NewGeometryCollection(coordinates, types) + if err != nil { + return nil + } + + // extra glue bytes to work around the term splitting logic from interfering + // the custom encoding of the geoshape coordinates inside the docvalues. + encodedValue = append(geo.GlueBytes, append(encodedValue, geo.GlueBytes...)...) + + // get the byte value for the geometryCollection. + value, err := shape.Value() + if err != nil { + return nil + } + + options = options | DefaultGeoShapeIndexingOptions + + return &GeoShapeField{ + shape: shape, + name: name, + arrayPositions: arrayPositions, + options: options, + encodedValue: encodedValue, + value: value, + numPlainTextBytes: uint64(len(value)), + } +} + +func NewGeoCircleFieldWithIndexingOptions(name string, arrayPositions []uint64, + centerPoint []float64, radius string, + options index.FieldIndexingOptions) *GeoShapeField { + shape, encodedValue, err := geo.NewGeoCircleShape(centerPoint, radius) + if err != nil { + return nil + } + + // extra glue bytes to work around the term splitting logic from interfering + // the custom encoding of the geoshape coordinates inside the docvalues. + encodedValue = append(geo.GlueBytes, append(encodedValue, geo.GlueBytes...)...) + + // get the byte value for the circle. + value, err := shape.Value() + if err != nil { + return nil + } + + options = options | DefaultGeoShapeIndexingOptions + + return &GeoShapeField{ + shape: shape, + name: name, + arrayPositions: arrayPositions, + options: options, + encodedValue: encodedValue, + value: value, + numPlainTextBytes: uint64(len(value)), + } +} + +// GeoShape is an implementation of the index.GeoShapeField interface. +func (n *GeoShapeField) GeoShape() (index.GeoJSON, error) { + return geojson.ParseGeoJSONShape(n.value) +} diff --git a/vendor/github.com/blevesearch/bleve/v2/document/field_ip.go b/vendor/github.com/blevesearch/bleve/v2/document/field_ip.go new file mode 100644 index 00000000..1e5be500 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/document/field_ip.go @@ -0,0 +1,132 @@ +// Copyright (c) 2021 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package document + +import ( + "fmt" + "net" + "reflect" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" +) + +var reflectStaticSizeIPField int + +func init() { + var f IPField + reflectStaticSizeIPField = int(reflect.TypeOf(f).Size()) +} + +const DefaultIPIndexingOptions = index.StoreField | index.IndexField | index.DocValues | index.IncludeTermVectors + +type IPField struct { + name string + arrayPositions []uint64 + options index.FieldIndexingOptions + value net.IP + numPlainTextBytes uint64 + length int + frequencies index.TokenFrequencies +} + +func (b *IPField) Size() int { + return reflectStaticSizeIPField + size.SizeOfPtr + + len(b.name) + + len(b.arrayPositions)*size.SizeOfUint64 + + len(b.value) +} + +func (b *IPField) Name() string { + return b.name +} + +func (b *IPField) ArrayPositions() []uint64 { + return b.arrayPositions +} + +func (b *IPField) Options() index.FieldIndexingOptions { + return b.options +} + +func (n *IPField) EncodedFieldType() byte { + return 'i' +} + +func (n *IPField) AnalyzedLength() int { + return n.length +} + +func (n *IPField) AnalyzedTokenFrequencies() index.TokenFrequencies { + return n.frequencies +} + +func (b *IPField) Analyze() { + + tokens := analysis.TokenStream{ + &analysis.Token{ + Start: 0, + End: len(b.value), + Term: b.value, + Position: 1, + Type: analysis.IP, + }, + } + b.length = 1 + b.frequencies = analysis.TokenFrequency(tokens, b.arrayPositions, b.options) +} + +func (b *IPField) Value() []byte { + return b.value +} + +func (b *IPField) IP() (net.IP, error) { + return net.IP(b.value), nil +} + +func (b *IPField) GoString() string { + return fmt.Sprintf("&document.IPField{Name:%s, Options: %s, Value: %s}", b.name, b.options, net.IP(b.value)) +} + +func (b *IPField) NumPlainTextBytes() uint64 { + return b.numPlainTextBytes +} + +func NewIPFieldFromBytes(name string, arrayPositions []uint64, value []byte) *IPField { + return &IPField{ + name: name, + arrayPositions: arrayPositions, + value: value, + options: DefaultNumericIndexingOptions, + numPlainTextBytes: uint64(len(value)), + } +} + +func NewIPField(name string, arrayPositions []uint64, v net.IP) *IPField { + return NewIPFieldWithIndexingOptions(name, arrayPositions, v, DefaultIPIndexingOptions) +} + +func NewIPFieldWithIndexingOptions(name string, arrayPositions []uint64, b net.IP, options index.FieldIndexingOptions) *IPField { + v := b.To16() + + return &IPField{ + name: name, + arrayPositions: arrayPositions, + value: v, + options: options, + numPlainTextBytes: net.IPv6len, + } +} diff --git a/vendor/github.com/blevesearch/bleve/document/field_numeric.go b/vendor/github.com/blevesearch/bleve/v2/document/field_numeric.go similarity index 79% rename from vendor/github.com/blevesearch/bleve/document/field_numeric.go rename to vendor/github.com/blevesearch/bleve/v2/document/field_numeric.go index 46c685e8..a54b082b 100644 --- a/vendor/github.com/blevesearch/bleve/document/field_numeric.go +++ b/vendor/github.com/blevesearch/bleve/v2/document/field_numeric.go @@ -18,9 +18,10 @@ import ( "fmt" "reflect" - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/numeric" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" ) var reflectStaticSizeNumericField int @@ -30,16 +31,18 @@ func init() { reflectStaticSizeNumericField = int(reflect.TypeOf(f).Size()) } -const DefaultNumericIndexingOptions = StoreField | IndexField | DocValues +const DefaultNumericIndexingOptions = index.StoreField | index.IndexField | index.DocValues const DefaultPrecisionStep uint = 4 type NumericField struct { name string arrayPositions []uint64 - options IndexingOptions + options index.FieldIndexingOptions value numeric.PrefixCoded numPlainTextBytes uint64 + length int + frequencies index.TokenFrequencies } func (n *NumericField) Size() int { @@ -56,11 +59,23 @@ func (n *NumericField) ArrayPositions() []uint64 { return n.arrayPositions } -func (n *NumericField) Options() IndexingOptions { +func (n *NumericField) Options() index.FieldIndexingOptions { return n.options } -func (n *NumericField) Analyze() (int, analysis.TokenFrequencies) { +func (n *NumericField) EncodedFieldType() byte { + return 'n' +} + +func (n *NumericField) AnalyzedLength() int { + return n.length +} + +func (n *NumericField) AnalyzedTokenFrequencies() index.TokenFrequencies { + return n.frequencies +} + +func (n *NumericField) Analyze() { tokens := make(analysis.TokenStream, 0) tokens = append(tokens, &analysis.Token{ Start: 0, @@ -91,9 +106,8 @@ func (n *NumericField) Analyze() (int, analysis.TokenFrequencies) { } } - fieldLength := len(tokens) - tokenFreqs := analysis.TokenFrequency(tokens, n.arrayPositions, n.options.IncludeTermVectors()) - return fieldLength, tokenFreqs + n.length = len(tokens) + n.frequencies = analysis.TokenFrequency(tokens, n.arrayPositions, n.options) } func (n *NumericField) Value() []byte { @@ -130,7 +144,7 @@ func NewNumericField(name string, arrayPositions []uint64, number float64) *Nume return NewNumericFieldWithIndexingOptions(name, arrayPositions, number, DefaultNumericIndexingOptions) } -func NewNumericFieldWithIndexingOptions(name string, arrayPositions []uint64, number float64, options IndexingOptions) *NumericField { +func NewNumericFieldWithIndexingOptions(name string, arrayPositions []uint64, number float64, options index.FieldIndexingOptions) *NumericField { numberInt64 := numeric.Float64ToInt64(number) prefixCoded := numeric.MustNewPrefixCodedInt64(numberInt64, 0) return &NumericField{ diff --git a/vendor/github.com/blevesearch/bleve/v2/document/field_text.go b/vendor/github.com/blevesearch/bleve/v2/document/field_text.go new file mode 100644 index 00000000..fddc59d0 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/document/field_text.go @@ -0,0 +1,157 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package document + +import ( + "fmt" + "reflect" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" +) + +var reflectStaticSizeTextField int + +func init() { + var f TextField + reflectStaticSizeTextField = int(reflect.TypeOf(f).Size()) +} + +const DefaultTextIndexingOptions = index.IndexField | index.DocValues + +type TextField struct { + name string + arrayPositions []uint64 + options index.FieldIndexingOptions + analyzer analysis.Analyzer + value []byte + numPlainTextBytes uint64 + length int + frequencies index.TokenFrequencies +} + +func (t *TextField) Size() int { + return reflectStaticSizeTextField + size.SizeOfPtr + + len(t.name) + + len(t.arrayPositions)*size.SizeOfUint64 + + len(t.value) +} + +func (t *TextField) Name() string { + return t.name +} + +func (t *TextField) ArrayPositions() []uint64 { + return t.arrayPositions +} + +func (t *TextField) Options() index.FieldIndexingOptions { + return t.options +} + +func (t *TextField) EncodedFieldType() byte { + return 't' +} + +func (t *TextField) AnalyzedLength() int { + return t.length +} + +func (t *TextField) AnalyzedTokenFrequencies() index.TokenFrequencies { + return t.frequencies +} + +func (t *TextField) Analyze() { + var tokens analysis.TokenStream + if t.analyzer != nil { + bytesToAnalyze := t.Value() + if t.options.IsStored() { + // need to copy + bytesCopied := make([]byte, len(bytesToAnalyze)) + copy(bytesCopied, bytesToAnalyze) + bytesToAnalyze = bytesCopied + } + tokens = t.analyzer.Analyze(bytesToAnalyze) + } else { + tokens = analysis.TokenStream{ + &analysis.Token{ + Start: 0, + End: len(t.value), + Term: t.value, + Position: 1, + Type: analysis.AlphaNumeric, + }, + } + } + t.length = len(tokens) // number of tokens in this doc field + t.frequencies = analysis.TokenFrequency(tokens, t.arrayPositions, t.options) +} + +func (t *TextField) Analyzer() analysis.Analyzer { + return t.analyzer +} + +func (t *TextField) Value() []byte { + return t.value +} + +func (t *TextField) Text() string { + return string(t.value) +} + +func (t *TextField) GoString() string { + return fmt.Sprintf("&document.TextField{Name:%s, Options: %s, Analyzer: %v, Value: %s, ArrayPositions: %v}", t.name, t.options, t.analyzer, t.value, t.arrayPositions) +} + +func (t *TextField) NumPlainTextBytes() uint64 { + return t.numPlainTextBytes +} + +func NewTextField(name string, arrayPositions []uint64, value []byte) *TextField { + return NewTextFieldWithIndexingOptions(name, arrayPositions, value, DefaultTextIndexingOptions) +} + +func NewTextFieldWithIndexingOptions(name string, arrayPositions []uint64, value []byte, options index.FieldIndexingOptions) *TextField { + return &TextField{ + name: name, + arrayPositions: arrayPositions, + options: options, + value: value, + numPlainTextBytes: uint64(len(value)), + } +} + +func NewTextFieldWithAnalyzer(name string, arrayPositions []uint64, value []byte, analyzer analysis.Analyzer) *TextField { + return &TextField{ + name: name, + arrayPositions: arrayPositions, + options: DefaultTextIndexingOptions, + analyzer: analyzer, + value: value, + numPlainTextBytes: uint64(len(value)), + } +} + +func NewTextFieldCustom(name string, arrayPositions []uint64, value []byte, options index.FieldIndexingOptions, analyzer analysis.Analyzer) *TextField { + return &TextField{ + name: name, + arrayPositions: arrayPositions, + options: options, + analyzer: analyzer, + value: value, + numPlainTextBytes: uint64(len(value)), + } +} diff --git a/vendor/github.com/blevesearch/bleve/v2/error.go b/vendor/github.com/blevesearch/bleve/v2/error.go new file mode 100644 index 00000000..7dd21194 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/error.go @@ -0,0 +1,50 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bleve + +// Constant Error values which can be compared to determine the type of error +const ( + ErrorIndexPathExists Error = iota + ErrorIndexPathDoesNotExist + ErrorIndexMetaMissing + ErrorIndexMetaCorrupt + ErrorIndexClosed + ErrorAliasMulti + ErrorAliasEmpty + ErrorUnknownIndexType + ErrorEmptyID + ErrorIndexReadInconsistency +) + +// Error represents a more strongly typed bleve error for detecting +// and handling specific types of errors. +type Error int + +func (e Error) Error() string { + return errorMessages[e] +} + +var errorMessages = map[Error]string{ + ErrorIndexPathExists: "cannot create new index, path already exists", + ErrorIndexPathDoesNotExist: "cannot open index, path does not exist", + ErrorIndexMetaMissing: "cannot open index, metadata missing", + ErrorIndexMetaCorrupt: "cannot open index, metadata corrupt", + ErrorIndexClosed: "index is closed", + ErrorAliasMulti: "cannot perform single index operation on multiple index alias", + ErrorAliasEmpty: "cannot perform operation on empty alias", + ErrorUnknownIndexType: "unknown index type", + ErrorEmptyID: "document ID cannot be empty", + ErrorIndexReadInconsistency: "index read inconsistency detected", +} diff --git a/vendor/github.com/blevesearch/bleve/v2/geo/README.md b/vendor/github.com/blevesearch/bleve/v2/geo/README.md new file mode 100644 index 00000000..6112ff5d --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/geo/README.md @@ -0,0 +1,277 @@ +# geo support in bleve + +Latest bleve spatial capabilities are powered by spatial hierarchical tokens generated from s2geometry. +You can find more details about the [s2geometry basics here](http://s2geometry.io/), and explore the +extended functionality of our forked golang port of [s2geometry lib here](https://github.com/blevesearch/geo). + +Users can continue to index and query `geopoint` field type and the existing queries like, + +- Point Distance +- Bounded Rectangle +- Bounded Polygon + +as before. + +## New Spatial Field Type - geoshape + +We have introduced a field type (`geoshape`) for representing the new spatial types. + +Using the new `geoshape` field type, users can unblock the spatial capabilities +for the [geojson](https://datatracker.ietf.org/doc/html/rfc7946) shapes like, + +- Point +- LineString +- Polygon +- MultiPoint +- MultiLineString +- MultiPolygon +- GeometryCollection + +In addition to these shapes, bleve will also support additional shapes like, + +- Circle +- Envelope (Bounded box) + +To specify GeoJSON data, use a nested field with: + +- a field named type that specifies the GeoJSON object type and the type value will be case-insensitive. +- a field named coordinates that specifies the object's coordinates. + +``` + "fieldName": { + "type": "GeoJSON Type", + "coordinates": + } +``` + +- If specifying latitude and longitude coordinates, list the longitude first and then latitude. +- Valid longitude values are between -180 and 180, both inclusive. +- Valid latitude values are between -90 and 90, both inclusive. +- Shapes would be internally represented as geodesics. +- The GeoJSON specification strongly suggests splitting geometries so that neither of their parts crosses the antimeridian. + + +Examples for the various geojson shapes representations are as below. + +## Point + +The following specifies a [Point](https://tools.ietf.org/html/rfc7946#section-3.1.2) field in a document: + +``` + { + "type": "point", + "coordinates": [75.05687713623047,22.53539059204079] + } +``` + +## Linestring + +The following specifies a [Linestring](https://tools.ietf.org/html/rfc7946#section-3.1.4) field in a document: + + +``` +{ + "type": "linestring", + "coordinates": [ + [ 77.01416015625, 23.0797317624497], + [ 78.134765625, 20.385825381874263] + ] +} +``` + + +## Polygon + +The following specifies a [Polygon](https://tools.ietf.org/html/rfc7946#section-3.1.6) field in a document: + +``` +{ + "type": "polygon", + "coordinates": [ [ [ 85.605, 57.207], + [ 86.396, 55.998], + [ 87.033, 56.716], + [ 85.605, 57.207] + ] ] +} +``` + + +The first and last coordinates must match in order to close the polygon. +And the exterior coordinates have to be in Counter Clockwise Order in a polygon. (CCW) + + +## MultiPoint + +The following specifies a [Multipoint](https://tools.ietf.org/html/rfc7946#section-3.1.3) field in a document: + +``` +{ + "type": "multipoint", + "coordinates": [ + [ -115.8343505859375, 38.45789034424927], + [ -115.81237792968749, 38.19502155795575], + [ -120.80017089843749, 36.54053616262899], + [ -120.67932128906249, 36.33725319397006] + ] +} +``` + +## MultiLineString + +The following specifies a [MultiLineString](https://tools.ietf.org/html/rfc7946#section-3.1.5) field in a document: + +``` +{ + "type": "multilinestring", + "coordinates": [ + [ [ -118.31726074, 35.250105158],[ -117.509765624, 35.3756141] ], + [ [ -118.6962890, 34.624167789],[ -118.317260742, 35.03899204] ], + [ [ -117.9492187, 35.146862906], [ -117.6745605, 34.41144164] ] +] +} +``` + +## MultiPolygon + +The following specifies a [MultiPolygon](https://tools.ietf.org/html/rfc7946#section-3.1.7) field in a document: + +``` +{ + "type": "multipolygon", + "coordinates": [ + [ [ [ -73.958, 40.8003 ], [ -73.9498, 40.7968 ], + [ -73.9737, 40.7648 ], [ -73.9814, 40.7681 ], + [ -73.958, 40.8003 ] ] ], + + + [ [ [ -73.958, 40.8003 ], [ -73.9498, 40.7968 ], + [ -73.9737, 40.7648 ], [ -73.958, 40.8003 ] ] ] + ] +} +``` + + +## GeometryCollection + +The following specifies a [GeometryCollection](https://tools.ietf.org/html/rfc7946#section-3.1.8) field in a document: + +``` +{ + "type": "geometrycollection", + "geometries": [ + { + "type": "multipoint", + "coordinates": [ + [ -73.9580, 40.8003 ], + [ -73.9498, 40.7968 ], + [ -73.9737, 40.7648 ], + [ -73.9814, 40.7681 ] + ] + }, + { + "type": "multilinestring", + "coordinates": [ + [ [ -73.96943, 40.78519 ], [ -73.96082, 40.78095 ] ], + [ [ -73.96415, 40.79229 ], [ -73.95544, 40.78854 ] ], + [ [ -73.97162, 40.78205 ], [ -73.96374, 40.77715 ] ], + [ [ -73.97880, 40.77247 ], [ -73.97036, 40.76811 ] ] + ] + }, + { + "type" : "polygon", + "coordinates" : [ + [ [ 0 , 0 ] , [ 3 , 6 ] , [ 6 , 1 ] , [ 0 , 0 ] ], + [ [ 2 , 2 ] , [ 3 , 3 ] , [ 4 , 2 ] , [ 2 , 2 ] ] + ] + } +] +} +``` + + +## Circle + +If the user wishes to cover a circular region over the earth’s surface, then they could use this shape. +A sample circular shape is as below. + +``` +{ + "type": "circle", + "coordinates": [75.05687713623047,22.53539059204079], + "radius": "1000m" +} +``` + + +Circle is specified over the center point coordinates along with the radius. +Example formats supported for radius are: +"5in" , "5inch" , "7yd" , "7yards", "9ft" , "9feet", "11km", "11kilometers", "3nm" +"3nauticalmiles", "13mm" , "13millimeters", "15cm", "15centimeters", "17mi", "17miles" "19m" or "19meters". + +If the unit cannot be determined, the entire string is parsed and the unit of meters is assumed. + + +## Envelope + +Envelope type, which consists of coordinates for upper left and lower right points of the shape +to represent a bounding rectangle in the format [[minLon, maxLat], [maxLon, minLat]]. + +``` +{ + "type": "envelope", + "coordinates": [ + [72.83, 18.979], + [78.508,17.4555] + ] +} +``` + + +## GeoShape Query + +Geoshape query support three types/filters of spatial querying capability across those +heterogeneous types of documents indexed. + +### Query Structure: + +``` +{ + "query": { + "geometry": { + "shape": { + "type": "", + "coordinates": [[[ ]]] + }, + "relation": "<>" + } + } +} +``` + + +*shapeType* => can be any of the aforementioned types like Point, LineString, Polygon, MultiPoint, +Geometrycollection, MultiLineString, MultiPolygon, Circle and Envelope. + +*filterName* => can be any of the 3 types like *intersects*, *contains* and *within*. + +### Relation + +| FilterName | Description | +| :-----------:| :-----------------------------------------------------------------: | +| `intersects` | Return all documents whose shape field intersects the query geometry. | +| `contains` | Return all documents whose shape field contains the query geometry | +| `within` | Return all documents whose shape field is within the query geometry. | + +------------------------------------------------------------------------------------------------------------------------ + + + +### Older Implementation + +First, all of this geo code is a Go adaptation of the [Lucene 5.3.2 sandbox geo support](https://lucene.apache.org/core/5_3_2/sandbox/org/apache/lucene/util/package-summary.html). + +## Notes + +- All of the APIs will use float64 for lon/lat values. +- When describing a point in function arguments or return values, we always use the order lon, lat. +- High level APIs will use TopLeft and BottomRight to describe bounding boxes. This may not map cleanly to min/max lon/lat when crossing the dateline. The lower level APIs will use min/max lon/lat and require the higher-level code to split boxes accordingly. diff --git a/vendor/github.com/blevesearch/bleve/geo/geo.go b/vendor/github.com/blevesearch/bleve/v2/geo/geo.go similarity index 99% rename from vendor/github.com/blevesearch/bleve/geo/geo.go rename to vendor/github.com/blevesearch/bleve/v2/geo/geo.go index b18ace43..55eace1d 100644 --- a/vendor/github.com/blevesearch/bleve/geo/geo.go +++ b/vendor/github.com/blevesearch/bleve/v2/geo/geo.go @@ -18,7 +18,7 @@ import ( "fmt" "math" - "github.com/blevesearch/bleve/numeric" + "github.com/blevesearch/bleve/v2/numeric" ) // GeoBits is the number of bits used for a single geo point diff --git a/vendor/github.com/blevesearch/bleve/geo/geo_dist.go b/vendor/github.com/blevesearch/bleve/v2/geo/geo_dist.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/geo/geo_dist.go rename to vendor/github.com/blevesearch/bleve/v2/geo/geo_dist.go diff --git a/vendor/github.com/blevesearch/bleve/v2/geo/geo_s2plugin_impl.go b/vendor/github.com/blevesearch/bleve/v2/geo/geo_s2plugin_impl.go new file mode 100644 index 00000000..8a034bd9 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/geo/geo_s2plugin_impl.go @@ -0,0 +1,450 @@ +// Copyright (c) 2022 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package geo + +import ( + "encoding/json" + "sync" + + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" + "github.com/blevesearch/geo/geojson" + "github.com/blevesearch/geo/s2" +) + +const ( + PointType = "point" + MultiPointType = "multipoint" + LineStringType = "linestring" + MultiLineStringType = "multilinestring" + PolygonType = "polygon" + MultiPolygonType = "multipolygon" + GeometryCollectionType = "geometrycollection" + CircleType = "circle" + EnvelopeType = "envelope" +) + +// spatialPluginsMap is spatial plugin cache. +var ( + spatialPluginsMap = make(map[string]index.SpatialAnalyzerPlugin) + pluginsMapLock = sync.RWMutex{} +) + +func init() { + registerS2RegionTermIndexer() +} + +func registerS2RegionTermIndexer() { + spatialPlugin := S2SpatialAnalyzerPlugin{ + s2Indexer: s2.NewRegionTermIndexerWithOptions(initS2IndexerOptions()), + s2Searcher: s2.NewRegionTermIndexerWithOptions(initS2SearcherOptions()), + s2GeoPointsRegionTermIndexer: s2.NewRegionTermIndexerWithOptions(initS2OptionsForGeoPoints()), + } + + RegisterSpatialAnalyzerPlugin(&spatialPlugin) +} + +// RegisterSpatialAnalyzerPlugin registers the given plugin implementation. +func RegisterSpatialAnalyzerPlugin(plugin index.SpatialAnalyzerPlugin) { + pluginsMapLock.Lock() + spatialPluginsMap[plugin.Type()] = plugin + pluginsMapLock.Unlock() +} + +// GetSpatialAnalyzerPlugin retrieves the given implementation type. +func GetSpatialAnalyzerPlugin(typ string) index.SpatialAnalyzerPlugin { + pluginsMapLock.RLock() + rv := spatialPluginsMap[typ] + pluginsMapLock.RUnlock() + return rv +} + +// initS2IndexerOptions returns the options for s2's region +// term indexer for the index time tokens of geojson shapes. +func initS2IndexerOptions() s2.Options { + options := s2.Options{} + // maxLevel control the maximum size of the + // S2Cells used to approximate regions. + options.SetMaxLevel(16) + + // minLevel control the minimum size of the + // S2Cells used to approximate regions. + options.SetMinLevel(2) + + // levelMod value greater than 1 increases the effective branching + // factor of the S2Cell hierarchy by skipping some levels. + options.SetLevelMod(1) + + // maxCells controls the maximum number of cells + // when approximating each s2 region. + options.SetMaxCells(20) + + return options +} + +// initS2SearcherOptions returns the options for s2's region +// term indexer for the query time tokens of geojson shapes. +func initS2SearcherOptions() s2.Options { + options := s2.Options{} + // maxLevel control the maximum size of the + // S2Cells used to approximate regions. + options.SetMaxLevel(16) + + // minLevel control the minimum size of the + // S2Cells used to approximate regions. + options.SetMinLevel(2) + + // levelMod value greater than 1 increases the effective branching + // factor of the S2Cell hierarchy by skipping some levels. + options.SetLevelMod(1) + + // maxCells controls the maximum number of cells + // when approximating each s2 region. + options.SetMaxCells(8) + + return options +} + +// initS2OptionsForGeoPoints returns the options for +// s2's region term indexer for the original geopoints. +func initS2OptionsForGeoPoints() s2.Options { + options := s2.Options{} + // maxLevel control the maximum size of the + // S2Cells used to approximate regions. + options.SetMaxLevel(16) + + // minLevel control the minimum size of the + // S2Cells used to approximate regions. + options.SetMinLevel(4) + + // levelMod value greater than 1 increases the effective branching + // factor of the S2Cell hierarchy by skipping some levels. + options.SetLevelMod(2) + + // maxCells controls the maximum number of cells + // when approximating each s2 region. + options.SetMaxCells(8) + + // explicit for geo points. + options.SetPointsOnly(true) + + return options +} + +// S2SpatialAnalyzerPlugin is an implementation of +// the index.SpatialAnalyzerPlugin interface. +type S2SpatialAnalyzerPlugin struct { + s2Indexer *s2.RegionTermIndexer + s2Searcher *s2.RegionTermIndexer + s2GeoPointsRegionTermIndexer *s2.RegionTermIndexer +} + +func (s *S2SpatialAnalyzerPlugin) Type() string { + return "s2" +} + +func (s *S2SpatialAnalyzerPlugin) GetIndexTokens(queryShape index.GeoJSON) []string { + var rv []string + shapes := []index.GeoJSON{queryShape} + if gc, ok := queryShape.(*geojson.GeometryCollection); ok { + shapes = gc.Shapes + } + + for _, shape := range shapes { + if s2t, ok := shape.(s2Tokenizable); ok { + rv = append(rv, s2t.IndexTokens(s.s2Indexer)...) + } else if s2t, ok := shape.(s2TokenizableEx); ok { + rv = append(rv, s2t.IndexTokens(s)...) + } + } + + return geojson.DeduplicateTerms(rv) +} + +func (s *S2SpatialAnalyzerPlugin) GetQueryTokens(queryShape index.GeoJSON) []string { + var rv []string + shapes := []index.GeoJSON{queryShape} + if gc, ok := queryShape.(*geojson.GeometryCollection); ok { + shapes = gc.Shapes + } + + for _, shape := range shapes { + if s2t, ok := shape.(s2Tokenizable); ok { + rv = append(rv, s2t.QueryTokens(s.s2Searcher)...) + } else if s2t, ok := shape.(s2TokenizableEx); ok { + rv = append(rv, s2t.QueryTokens(s)...) + } + } + + return geojson.DeduplicateTerms(rv) +} + +// ------------------------------------------------------------------------ +// s2Tokenizable is an optional interface for shapes that support +// the generation of s2 based tokens that can be used for both +// indexing and querying. + +type s2Tokenizable interface { + // IndexTokens returns the tokens for indexing. + IndexTokens(*s2.RegionTermIndexer) []string + + // QueryTokens returns the tokens for searching. + QueryTokens(*s2.RegionTermIndexer) []string +} + +// ------------------------------------------------------------------------ +// s2TokenizableEx is an optional interface for shapes that support +// the generation of s2 based tokens that can be used for both +// indexing and querying. This is intended for the older geopoint +// indexing and querying. +type s2TokenizableEx interface { + // IndexTokens returns the tokens for indexing. + IndexTokens(*S2SpatialAnalyzerPlugin) []string + + // QueryTokens returns the tokens for searching. + QueryTokens(*S2SpatialAnalyzerPlugin) []string +} + +//---------------------------------------------------------------------------------- + +func (p *Point) Type() string { + return PointType +} + +func (p *Point) Value() ([]byte, error) { + return util.MarshalJSON(p) +} + +func (p *Point) Intersects(s index.GeoJSON) (bool, error) { + // placeholder implementation + return false, nil +} + +func (p *Point) Contains(s index.GeoJSON) (bool, error) { + // placeholder implementation + return false, nil +} + +func (p *Point) IndexTokens(s *S2SpatialAnalyzerPlugin) []string { + return s.s2GeoPointsRegionTermIndexer.GetIndexTermsForPoint(s2.PointFromLatLng( + s2.LatLngFromDegrees(p.Lat, p.Lon)), "") +} + +func (p *Point) QueryTokens(s *S2SpatialAnalyzerPlugin) []string { + return nil +} + +//---------------------------------------------------------------------------------- + +type boundedRectangle struct { + minLat float64 + maxLat float64 + minLon float64 + maxLon float64 +} + +func NewBoundedRectangle(minLat, minLon, maxLat, + maxLon float64) *boundedRectangle { + return &boundedRectangle{minLat: minLat, + maxLat: maxLat, minLon: minLon, maxLon: maxLon} +} + +func (br *boundedRectangle) Type() string { + // placeholder implementation + return "boundedRectangle" +} + +func (br *boundedRectangle) Value() ([]byte, error) { + return util.MarshalJSON(br) +} + +func (p *boundedRectangle) Intersects(s index.GeoJSON) (bool, error) { + // placeholder implementation + return false, nil +} + +func (p *boundedRectangle) Contains(s index.GeoJSON) (bool, error) { + // placeholder implementation + return false, nil +} + +func (br *boundedRectangle) IndexTokens(s *S2SpatialAnalyzerPlugin) []string { + return nil +} + +func (br *boundedRectangle) QueryTokens(s *S2SpatialAnalyzerPlugin) []string { + rect := s2.RectFromDegrees(br.minLat, br.minLon, br.maxLat, br.maxLon) + + // obtain the terms to be searched for the given bounding box. + terms := s.s2GeoPointsRegionTermIndexer.GetQueryTermsForRegion(rect, "") + + return geojson.StripCoveringTerms(terms) +} + +//---------------------------------------------------------------------------------- + +type boundedPolygon struct { + coordinates []Point +} + +func NewBoundedPolygon(coordinates []Point) *boundedPolygon { + return &boundedPolygon{coordinates: coordinates} +} + +func (bp *boundedPolygon) Type() string { + // placeholder implementation + return "boundedPolygon" +} + +func (bp *boundedPolygon) Value() ([]byte, error) { + return util.MarshalJSON(bp) +} + +func (p *boundedPolygon) Intersects(s index.GeoJSON) (bool, error) { + // placeholder implementation + return false, nil +} + +func (p *boundedPolygon) Contains(s index.GeoJSON) (bool, error) { + // placeholder implementation + return false, nil +} + +func (bp *boundedPolygon) IndexTokens(s *S2SpatialAnalyzerPlugin) []string { + return nil +} + +func (bp *boundedPolygon) QueryTokens(s *S2SpatialAnalyzerPlugin) []string { + vertices := make([]s2.Point, len(bp.coordinates)) + for i, point := range bp.coordinates { + vertices[i] = s2.PointFromLatLng( + s2.LatLngFromDegrees(point.Lat, point.Lon)) + } + s2polygon := s2.PolygonFromOrientedLoops([]*s2.Loop{s2.LoopFromPoints(vertices)}) + + // obtain the terms to be searched for the given polygon. + terms := s.s2GeoPointsRegionTermIndexer.GetQueryTermsForRegion( + s2polygon.CapBound(), "") + + return geojson.StripCoveringTerms(terms) +} + +//---------------------------------------------------------------------------------- + +type pointDistance struct { + dist float64 + centerLat float64 + centerLon float64 +} + +func (p *pointDistance) Type() string { + // placeholder implementation + return "pointDistance" +} + +func (p *pointDistance) Value() ([]byte, error) { + return util.MarshalJSON(p) +} + +func NewPointDistance(centerLat, centerLon, + dist float64) *pointDistance { + return &pointDistance{centerLat: centerLat, + centerLon: centerLon, dist: dist} +} + +func (p *pointDistance) Intersects(s index.GeoJSON) (bool, error) { + // placeholder implementation + return false, nil +} + +func (p *pointDistance) Contains(s index.GeoJSON) (bool, error) { + // placeholder implementation + return false, nil +} + +func (pd *pointDistance) IndexTokens(s *S2SpatialAnalyzerPlugin) []string { + return nil +} + +func (pd *pointDistance) QueryTokens(s *S2SpatialAnalyzerPlugin) []string { + // obtain the covering query region from the given points. + queryRegion := s2.CapFromCenterAndRadius(pd.centerLat, + pd.centerLon, pd.dist) + + // obtain the query terms for the query region. + terms := s.s2GeoPointsRegionTermIndexer.GetQueryTermsForRegion(queryRegion, "") + + return geojson.StripCoveringTerms(terms) +} + +// ------------------------------------------------------------------------ + +// NewGeometryCollection instantiate a geometrycollection +// and prefix the byte contents with certain glue bytes that +// can be used later while filering the doc values. +func NewGeometryCollection(coordinates [][][][][]float64, + typs []string) (index.GeoJSON, []byte, error) { + + return geojson.NewGeometryCollection(coordinates, typs) +} + +// NewGeoCircleShape instantiate a circle shape and +// prefix the byte contents with certain glue bytes that +// can be used later while filering the doc values. +func NewGeoCircleShape(cp []float64, + radius string) (index.GeoJSON, []byte, error) { + return geojson.NewGeoCircleShape(cp, radius) +} + +func NewGeoJsonShape(coordinates [][][][]float64, typ string) ( + index.GeoJSON, []byte, error) { + return geojson.NewGeoJsonShape(coordinates, typ) +} + +func NewGeoJsonPoint(points []float64) index.GeoJSON { + return geojson.NewGeoJsonPoint(points) +} + +func NewGeoJsonMultiPoint(points [][]float64) index.GeoJSON { + return geojson.NewGeoJsonMultiPoint(points) +} + +func NewGeoJsonLinestring(points [][]float64) index.GeoJSON { + return geojson.NewGeoJsonLinestring(points) +} + +func NewGeoJsonMultilinestring(points [][][]float64) index.GeoJSON { + return geojson.NewGeoJsonMultilinestring(points) +} + +func NewGeoJsonPolygon(points [][][]float64) index.GeoJSON { + return geojson.NewGeoJsonPolygon(points) +} + +func NewGeoJsonMultiPolygon(points [][][][]float64) index.GeoJSON { + return geojson.NewGeoJsonMultiPolygon(points) +} + +func NewGeoCircle(points []float64, radius string) index.GeoJSON { + return geojson.NewGeoCircle(points, radius) +} + +func NewGeoEnvelope(points [][]float64) index.GeoJSON { + return geojson.NewGeoEnvelope(points) +} + +func ParseGeoJSONShape(input json.RawMessage) (index.GeoJSON, error) { + return geojson.ParseGeoJSONShape(input) +} diff --git a/vendor/github.com/blevesearch/bleve/geo/geohash.go b/vendor/github.com/blevesearch/bleve/v2/geo/geohash.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/geo/geohash.go rename to vendor/github.com/blevesearch/bleve/v2/geo/geohash.go diff --git a/vendor/github.com/blevesearch/bleve/v2/geo/parse.go b/vendor/github.com/blevesearch/bleve/v2/geo/parse.go new file mode 100644 index 00000000..01ec1dd8 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/geo/parse.go @@ -0,0 +1,467 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package geo + +import ( + "reflect" + "strconv" + "strings" +) + +// ExtractGeoPoint takes an arbitrary interface{} and tries it's best to +// interpret it is as geo point. Supported formats: +// Container: +// slice length 2 (GeoJSON) +// +// first element lon, second element lat +// +// string (coordinates separated by comma, or a geohash) +// +// first element lat, second element lon +// +// map[string]interface{} +// +// exact keys lat and lon or lng +// +// struct +// +// w/exported fields case-insensitive match on lat and lon or lng +// +// struct +// +// satisfying Later and Loner or Lnger interfaces +// +// in all cases values must be some sort of numeric-like thing: int/uint/float +func ExtractGeoPoint(thing interface{}) (lon, lat float64, success bool) { + var foundLon, foundLat bool + + thingVal := reflect.ValueOf(thing) + if !thingVal.IsValid() { + return lon, lat, false + } + + thingTyp := thingVal.Type() + + // is it a slice + if thingVal.Kind() == reflect.Slice { + // must be length 2 + if thingVal.Len() == 2 { + first := thingVal.Index(0) + if first.CanInterface() { + firstVal := first.Interface() + lon, foundLon = extractNumericVal(firstVal) + } + second := thingVal.Index(1) + if second.CanInterface() { + secondVal := second.Interface() + lat, foundLat = extractNumericVal(secondVal) + } + } + } + + // is it a string + if thingVal.Kind() == reflect.String { + geoStr := thingVal.Interface().(string) + if strings.Contains(geoStr, ",") { + // geo point with coordinates split by comma + points := strings.Split(geoStr, ",") + for i, point := range points { + // trim any leading or trailing white spaces + points[i] = strings.TrimSpace(point) + } + if len(points) == 2 { + var err error + lat, err = strconv.ParseFloat(points[0], 64) + if err == nil { + foundLat = true + } + lon, err = strconv.ParseFloat(points[1], 64) + if err == nil { + foundLon = true + } + } + } else { + // geohash + if len(geoStr) <= geoHashMaxLength { + lat, lon = DecodeGeoHash(geoStr) + foundLat = true + foundLon = true + } + } + } + + // is it a map + if l, ok := thing.(map[string]interface{}); ok { + if lval, ok := l["lon"]; ok { + lon, foundLon = extractNumericVal(lval) + } else if lval, ok := l["lng"]; ok { + lon, foundLon = extractNumericVal(lval) + } + if lval, ok := l["lat"]; ok { + lat, foundLat = extractNumericVal(lval) + } + } + + // now try reflection on struct fields + if thingVal.Kind() == reflect.Struct { + for i := 0; i < thingVal.NumField(); i++ { + fieldName := thingTyp.Field(i).Name + if strings.HasPrefix(strings.ToLower(fieldName), "lon") { + if thingVal.Field(i).CanInterface() { + fieldVal := thingVal.Field(i).Interface() + lon, foundLon = extractNumericVal(fieldVal) + } + } + if strings.HasPrefix(strings.ToLower(fieldName), "lng") { + if thingVal.Field(i).CanInterface() { + fieldVal := thingVal.Field(i).Interface() + lon, foundLon = extractNumericVal(fieldVal) + } + } + if strings.HasPrefix(strings.ToLower(fieldName), "lat") { + if thingVal.Field(i).CanInterface() { + fieldVal := thingVal.Field(i).Interface() + lat, foundLat = extractNumericVal(fieldVal) + } + } + } + } + + // last hope, some interfaces + // lon + if l, ok := thing.(loner); ok { + lon = l.Lon() + foundLon = true + } else if l, ok := thing.(lnger); ok { + lon = l.Lng() + foundLon = true + } + // lat + if l, ok := thing.(later); ok { + lat = l.Lat() + foundLat = true + } + + return lon, lat, foundLon && foundLat +} + +// extract numeric value (if possible) and returns a float64 +func extractNumericVal(v interface{}) (float64, bool) { + val := reflect.ValueOf(v) + if !val.IsValid() { + return 0, false + } + typ := val.Type() + switch typ.Kind() { + case reflect.Float32, reflect.Float64: + return val.Float(), true + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return float64(val.Int()), true + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return float64(val.Uint()), true + } + + return 0, false +} + +// various support interfaces which can be used to find lat/lon +type loner interface { + Lon() float64 +} + +type later interface { + Lat() float64 +} + +type lnger interface { + Lng() float64 +} + +// GlueBytes primarily for quicker filtering of docvalues +// during the filtering phase. +var GlueBytes = []byte("##") + +var GlueBytesOffset = len(GlueBytes) + +func extractCoordinates(thing interface{}) []float64 { + thingVal := reflect.ValueOf(thing) + if !thingVal.IsValid() { + return nil + } + + if thingVal.Kind() == reflect.Slice { + // must be length 2 + if thingVal.Len() == 2 { + var foundLon, foundLat bool + var lon, lat float64 + first := thingVal.Index(0) + if first.CanInterface() { + firstVal := first.Interface() + lon, foundLon = extractNumericVal(firstVal) + } + second := thingVal.Index(1) + if second.CanInterface() { + secondVal := second.Interface() + lat, foundLat = extractNumericVal(secondVal) + } + + if !foundLon || !foundLat { + return nil + } + + return []float64{lon, lat} + } + } + return nil +} + +func extract2DCoordinates(thing interface{}) [][]float64 { + thingVal := reflect.ValueOf(thing) + if !thingVal.IsValid() { + return nil + } + + rv := make([][]float64, 0, 8) + if thingVal.Kind() == reflect.Slice { + for j := 0; j < thingVal.Len(); j++ { + edges := thingVal.Index(j).Interface() + if es, ok := edges.([]interface{}); ok { + v := extractCoordinates(es) + if len(v) == 2 { + rv = append(rv, v) + } + } + } + + return rv + } + + return nil +} + +func extract3DCoordinates(thing interface{}) (c [][][]float64) { + coords := reflect.ValueOf(thing) + for i := 0; i < coords.Len(); i++ { + vals := coords.Index(i) + + edges := vals.Interface() + if es, ok := edges.([]interface{}); ok { + loop := extract2DCoordinates(es) + if len(loop) > 0 { + c = append(c, loop) + } + } + } + + return c +} + +func extract4DCoordinates(thing interface{}) (rv [][][][]float64) { + thingVal := reflect.ValueOf(thing) + if !thingVal.IsValid() { + return nil + } + + if thingVal.Kind() == reflect.Slice { + for j := 0; j < thingVal.Len(); j++ { + c := extract3DCoordinates(thingVal.Index(j).Interface()) + rv = append(rv, c) + } + } + + return rv +} + +func ParseGeoShapeField(thing interface{}) (interface{}, string, error) { + thingVal := reflect.ValueOf(thing) + if !thingVal.IsValid() { + return nil, "", nil + } + + var shape string + var coordValue interface{} + + if thingVal.Kind() == reflect.Map { + iter := thingVal.MapRange() + for iter.Next() { + if iter.Key().String() == "type" { + shape = iter.Value().Interface().(string) + continue + } + + if iter.Key().String() == "coordinates" { + coordValue = iter.Value().Interface() + } + } + } + + return coordValue, strings.ToLower(shape), nil +} + +func extractGeoShape(thing interface{}) ([][][][]float64, string, bool) { + coordValue, typ, err := ParseGeoShapeField(thing) + if err != nil { + return nil, "", false + } + + return ExtractGeoShapeCoordinates(coordValue, typ) +} + +// ExtractGeometryCollection takes an interface{} and tries it's best to +// interpret all the member geojson shapes within it. +func ExtractGeometryCollection(thing interface{}) ([][][][][]float64, []string, bool) { + thingVal := reflect.ValueOf(thing) + if !thingVal.IsValid() { + return nil, nil, false + } + var rv [][][][][]float64 + var types []string + var f bool + + if thingVal.Kind() == reflect.Map { + iter := thingVal.MapRange() + for iter.Next() { + + if iter.Key().String() == "type" { + continue + } + + if iter.Key().String() == "geometries" { + collection := iter.Value().Interface() + items := reflect.ValueOf(collection) + + for j := 0; j < items.Len(); j++ { + coords, shape, found := extractGeoShape(items.Index(j).Interface()) + if found { + f = found + rv = append(rv, coords) + types = append(types, shape) + } + } + } + } + } + + return rv, types, f +} + +// ExtractCircle takes an interface{} and tries it's best to +// interpret the center point coordinates and the radius for a +// given circle shape. +func ExtractCircle(thing interface{}) ([]float64, string, bool) { + thingVal := reflect.ValueOf(thing) + if !thingVal.IsValid() { + return nil, "", false + } + var rv []float64 + var radiusStr string + + if thingVal.Kind() == reflect.Map { + iter := thingVal.MapRange() + for iter.Next() { + + if iter.Key().String() == "radius" { + radiusStr = iter.Value().Interface().(string) + continue + } + + if iter.Key().String() == "coordinates" { + lng, lat, found := ExtractGeoPoint(iter.Value().Interface()) + if !found { + return nil, radiusStr, false + } + rv = append(rv, lng) + rv = append(rv, lat) + } + } + } + + return rv, radiusStr, true +} + +// ExtractGeoShapeCoordinates takes an interface{} and tries it's best to +// interpret the coordinates for any of the given geoshape typ like +// a point, multipoint, linestring, multilinestring, polygon, multipolygon, +func ExtractGeoShapeCoordinates(coordValue interface{}, + typ string) ([][][][]float64, string, bool) { + var rv [][][][]float64 + if typ == PointType { + point := extractCoordinates(coordValue) + + // ignore the contents with invalid entry. + if len(point) < 2 { + return nil, typ, false + } + + rv = [][][][]float64{{{point}}} + return rv, typ, true + } + + if typ == MultiPointType || typ == LineStringType || + typ == EnvelopeType { + coords := extract2DCoordinates(coordValue) + + // ignore the contents with invalid entry. + if len(coords) == 0 { + return nil, typ, false + } + + if typ == EnvelopeType && len(coords) != 2 { + return nil, typ, false + } + + if typ == LineStringType && len(coords) < 2 { + return nil, typ, false + } + + rv = [][][][]float64{{coords}} + return rv, typ, true + } + + if typ == PolygonType || typ == MultiLineStringType { + coords := extract3DCoordinates(coordValue) + + // ignore the contents with invalid entry. + if len(coords) == 0 { + return nil, typ, false + } + + if typ == PolygonType && len(coords[0]) < 3 || + typ == MultiLineStringType && len(coords[0]) < 2 { + return nil, typ, false + } + + rv = [][][][]float64{coords} + return rv, typ, true + } + + if typ == MultiPolygonType { + rv = extract4DCoordinates(coordValue) + + // ignore the contents with invalid entry. + if len(rv) == 0 || len(rv[0]) == 0 { + return nil, typ, false + + } + + if len(rv[0][0]) < 3 { + return nil, typ, false + } + + return rv, typ, true + } + + return rv, typ, false +} diff --git a/vendor/github.com/blevesearch/bleve/geo/sloppy.go b/vendor/github.com/blevesearch/bleve/v2/geo/sloppy.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/geo/sloppy.go rename to vendor/github.com/blevesearch/bleve/v2/geo/sloppy.go diff --git a/vendor/github.com/blevesearch/bleve/v2/index.go b/vendor/github.com/blevesearch/bleve/v2/index.go new file mode 100644 index 00000000..7d4c9be9 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/index.go @@ -0,0 +1,322 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bleve + +import ( + "context" + + "github.com/blevesearch/bleve/v2/index/upsidedown" + + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" +) + +// A Batch groups together multiple Index and Delete +// operations you would like performed at the same +// time. The Batch structure is NOT thread-safe. +// You should only perform operations on a batch +// from a single thread at a time. Once batch +// execution has started, you may not modify it. +type Batch struct { + index Index + internal *index.Batch + + lastDocSize uint64 + totalSize uint64 +} + +// Index adds the specified index operation to the +// batch. NOTE: the bleve Index is not updated +// until the batch is executed. +func (b *Batch) Index(id string, data interface{}) error { + if id == "" { + return ErrorEmptyID + } + doc := document.NewDocument(id) + err := b.index.Mapping().MapDocument(doc, data) + if err != nil { + return err + } + b.internal.Update(doc) + + b.lastDocSize = uint64(doc.Size() + + len(id) + size.SizeOfString) // overhead from internal + b.totalSize += b.lastDocSize + + return nil +} + +func (b *Batch) LastDocSize() uint64 { + return b.lastDocSize +} + +func (b *Batch) TotalDocsSize() uint64 { + return b.totalSize +} + +// IndexAdvanced adds the specified index operation to the +// batch which skips the mapping. NOTE: the bleve Index is not updated +// until the batch is executed. +func (b *Batch) IndexAdvanced(doc *document.Document) (err error) { + if doc.ID() == "" { + return ErrorEmptyID + } + b.internal.Update(doc) + return nil +} + +// Delete adds the specified delete operation to the +// batch. NOTE: the bleve Index is not updated until +// the batch is executed. +func (b *Batch) Delete(id string) { + if id != "" { + b.internal.Delete(id) + } +} + +// SetInternal adds the specified set internal +// operation to the batch. NOTE: the bleve Index is +// not updated until the batch is executed. +func (b *Batch) SetInternal(key, val []byte) { + b.internal.SetInternal(key, val) +} + +// DeleteInternal adds the specified delete internal +// operation to the batch. NOTE: the bleve Index is +// not updated until the batch is executed. +func (b *Batch) DeleteInternal(key []byte) { + b.internal.DeleteInternal(key) +} + +// Size returns the total number of operations inside the batch +// including normal index operations and internal operations. +func (b *Batch) Size() int { + return len(b.internal.IndexOps) + len(b.internal.InternalOps) +} + +// String prints a user friendly string representation of what +// is inside this batch. +func (b *Batch) String() string { + return b.internal.String() +} + +// Reset returns a Batch to the empty state so that it can +// be re-used in the future. +func (b *Batch) Reset() { + b.internal.Reset() + b.lastDocSize = 0 + b.totalSize = 0 +} + +func (b *Batch) Merge(o *Batch) { + if o != nil && o.internal != nil { + b.internal.Merge(o.internal) + if o.LastDocSize() > 0 { + b.lastDocSize = o.LastDocSize() + } + b.totalSize = uint64(b.internal.TotalDocSize()) + } +} + +func (b *Batch) SetPersistedCallback(f index.BatchCallback) { + b.internal.SetPersistedCallback(f) +} + +func (b *Batch) PersistedCallback() index.BatchCallback { + return b.internal.PersistedCallback() +} + +// An Index implements all the indexing and searching +// capabilities of bleve. An Index can be created +// using the New() and Open() methods. +// +// Index() takes an input value, deduces a DocumentMapping for its type, +// assigns string paths to its fields or values then applies field mappings on +// them. +// +// The DocumentMapping used to index a value is deduced by the following rules: +// 1. If value implements mapping.bleveClassifier interface, resolve the mapping +// from BleveType(). +// 2. If value implements mapping.Classifier interface, resolve the mapping +// from Type(). +// 3. If value has a string field or value at IndexMapping.TypeField. +// +// (defaulting to "_type"), use it to resolve the mapping. Fields addressing +// is described below. +// 4) If IndexMapping.DefaultType is registered, return it. +// 5) Return IndexMapping.DefaultMapping. +// +// Each field or nested field of the value is identified by a string path, then +// mapped to one or several FieldMappings which extract the result for analysis. +// +// Struct values fields are identified by their "json:" tag, or by their name. +// Nested fields are identified by prefixing with their parent identifier, +// separated by a dot. +// +// Map values entries are identified by their string key. Entries not indexed +// by strings are ignored. Entry values are identified recursively like struct +// fields. +// +// Slice and array values are identified by their field name. Their elements +// are processed sequentially with the same FieldMapping. +// +// String, float64 and time.Time values are identified by their field name. +// Other types are ignored. +// +// Each value identifier is decomposed in its parts and recursively address +// SubDocumentMappings in the tree starting at the root DocumentMapping. If a +// mapping is found, all its FieldMappings are applied to the value. If no +// mapping is found and the root DocumentMapping is dynamic, default mappings +// are used based on value type and IndexMapping default configurations. +// +// Finally, mapped values are analyzed, indexed or stored. See +// FieldMapping.Analyzer to know how an analyzer is resolved for a given field. +// +// Examples: +// +// type Date struct { +// Day string `json:"day"` +// Month string +// Year string +// } +// +// type Person struct { +// FirstName string `json:"first_name"` +// LastName string +// BirthDate Date `json:"birth_date"` +// } +// +// A Person value FirstName is mapped by the SubDocumentMapping at +// "first_name". Its LastName is mapped by the one at "LastName". The day of +// BirthDate is mapped to the SubDocumentMapping "day" of the root +// SubDocumentMapping "birth_date". It will appear as the "birth_date.day" +// field in the index. The month is mapped to "birth_date.Month". +type Index interface { + // Index analyzes, indexes or stores mapped data fields. Supplied + // identifier is bound to analyzed data and will be retrieved by search + // requests. See Index interface documentation for details about mapping + // rules. + Index(id string, data interface{}) error + Delete(id string) error + + NewBatch() *Batch + Batch(b *Batch) error + + // Document returns specified document or nil if the document is not + // indexed or stored. + Document(id string) (index.Document, error) + // DocCount returns the number of documents in the index. + DocCount() (uint64, error) + + Search(req *SearchRequest) (*SearchResult, error) + SearchInContext(ctx context.Context, req *SearchRequest) (*SearchResult, error) + + Fields() ([]string, error) + + FieldDict(field string) (index.FieldDict, error) + FieldDictRange(field string, startTerm []byte, endTerm []byte) (index.FieldDict, error) + FieldDictPrefix(field string, termPrefix []byte) (index.FieldDict, error) + + Close() error + + Mapping() mapping.IndexMapping + + Stats() *IndexStat + StatsMap() map[string]interface{} + + GetInternal(key []byte) ([]byte, error) + SetInternal(key, val []byte) error + DeleteInternal(key []byte) error + + // Name returns the name of the index (by default this is the path) + Name() string + // SetName lets you assign your own logical name to this index + SetName(string) + + // Advanced returns the internal index implementation + Advanced() (index.Index, error) +} + +// New index at the specified path, must not exist. +// The provided mapping will be used for all +// Index/Search operations. +func New(path string, mapping mapping.IndexMapping) (Index, error) { + return newIndexUsing(path, mapping, Config.DefaultIndexType, Config.DefaultKVStore, nil) +} + +// NewMemOnly creates a memory-only index. +// The contents of the index is NOT persisted, +// and will be lost once closed. +// The provided mapping will be used for all +// Index/Search operations. +func NewMemOnly(mapping mapping.IndexMapping) (Index, error) { + return newIndexUsing("", mapping, upsidedown.Name, Config.DefaultMemKVStore, nil) +} + +// NewUsing creates index at the specified path, +// which must not already exist. +// The provided mapping will be used for all +// Index/Search operations. +// The specified index type will be used. +// The specified kvstore implementation will be used +// and the provided kvconfig will be passed to its +// constructor. Note that currently the values of kvconfig must +// be able to be marshaled and unmarshaled using the encoding/json library (used +// when reading/writing the index metadata file). +func NewUsing(path string, mapping mapping.IndexMapping, indexType string, kvstore string, kvconfig map[string]interface{}) (Index, error) { + return newIndexUsing(path, mapping, indexType, kvstore, kvconfig) +} + +// Open index at the specified path, must exist. +// The mapping used when it was created will be used for all Index/Search operations. +func Open(path string) (Index, error) { + return openIndexUsing(path, nil) +} + +// OpenUsing opens index at the specified path, must exist. +// The mapping used when it was created will be used for all Index/Search operations. +// The provided runtimeConfig can override settings +// persisted when the kvstore was created. +func OpenUsing(path string, runtimeConfig map[string]interface{}) (Index, error) { + return openIndexUsing(path, runtimeConfig) +} + +// Builder is a limited interface, used to build indexes in an offline mode. +// Items cannot be updated or deleted, and the caller MUST ensure a document is +// indexed only once. +type Builder interface { + Index(id string, data interface{}) error + Close() error +} + +// NewBuilder creates a builder, which will build an index at the specified path, +// using the specified mapping and options. +func NewBuilder(path string, mapping mapping.IndexMapping, config map[string]interface{}) (Builder, error) { + return newBuilder(path, mapping, config) +} + +// IndexCopyable is an index which supports an online copy operation +// of the index. +type IndexCopyable interface { + // CopyTo creates a fully functional copy of the index at the + // specified destination directory implementation. + CopyTo(d index.Directory) error +} + +// FileSystemDirectory is the default implementation for the +// index.Directory interface. +type FileSystemDirectory string diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/README.md b/vendor/github.com/blevesearch/bleve/v2/index/scorch/README.md similarity index 100% rename from vendor/github.com/blevesearch/bleve/index/scorch/README.md rename to vendor/github.com/blevesearch/bleve/v2/index/scorch/README.md diff --git a/vendor/github.com/blevesearch/bleve/v2/index/scorch/builder.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/builder.go new file mode 100644 index 00000000..04e5bd1b --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/builder.go @@ -0,0 +1,332 @@ +// Copyright (c) 2019 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "fmt" + "os" + "sync" + + "github.com/RoaringBitmap/roaring" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" + bolt "go.etcd.io/bbolt" +) + +const DefaultBuilderBatchSize = 1000 +const DefaultBuilderMergeMax = 10 + +type Builder struct { + m sync.Mutex + segCount uint64 + path string + buildPath string + segPaths []string + batchSize int + mergeMax int + batch *index.Batch + internal map[string][]byte + segPlugin SegmentPlugin +} + +func NewBuilder(config map[string]interface{}) (*Builder, error) { + path, ok := config["path"].(string) + if !ok { + return nil, fmt.Errorf("must specify path") + } + + buildPathPrefix, _ := config["buildPathPrefix"].(string) + buildPath, err := os.MkdirTemp(buildPathPrefix, "scorch-offline-build") + if err != nil { + return nil, err + } + + rv := &Builder{ + path: path, + buildPath: buildPath, + mergeMax: DefaultBuilderMergeMax, + batchSize: DefaultBuilderBatchSize, + batch: index.NewBatch(), + segPlugin: defaultSegmentPlugin, + } + + err = rv.parseConfig(config) + if err != nil { + return nil, fmt.Errorf("error parsing builder config: %v", err) + } + + return rv, nil +} + +func (o *Builder) parseConfig(config map[string]interface{}) (err error) { + if v, ok := config["mergeMax"]; ok { + var t int + if t, err = parseToInteger(v); err != nil { + return fmt.Errorf("mergeMax parse err: %v", err) + } + if t > 0 { + o.mergeMax = t + } + } + + if v, ok := config["batchSize"]; ok { + var t int + if t, err = parseToInteger(v); err != nil { + return fmt.Errorf("batchSize parse err: %v", err) + } + if t > 0 { + o.batchSize = t + } + } + + if v, ok := config["internal"]; ok { + if vinternal, ok := v.(map[string][]byte); ok { + o.internal = vinternal + } + } + + forcedSegmentType, forcedSegmentVersion, err := configForceSegmentTypeVersion(config) + if err != nil { + return err + } + if forcedSegmentType != "" && forcedSegmentVersion != 0 { + segPlugin, err := chooseSegmentPlugin(forcedSegmentType, + uint32(forcedSegmentVersion)) + if err != nil { + return err + } + o.segPlugin = segPlugin + } + + return nil +} + +// Index will place the document into the index. +// It is invalid to index the same document multiple times. +func (o *Builder) Index(doc index.Document) error { + o.m.Lock() + defer o.m.Unlock() + + o.batch.Update(doc) + + return o.maybeFlushBatchLOCKED(o.batchSize) +} + +func (o *Builder) maybeFlushBatchLOCKED(moreThan int) error { + if len(o.batch.IndexOps) >= moreThan { + defer o.batch.Reset() + return o.executeBatchLOCKED(o.batch) + } + return nil +} + +func (o *Builder) executeBatchLOCKED(batch *index.Batch) (err error) { + analysisResults := make([]index.Document, 0, len(batch.IndexOps)) + for _, doc := range batch.IndexOps { + if doc != nil { + // insert _id field + doc.AddIDField() + // perform analysis directly + analyze(doc, nil) + analysisResults = append(analysisResults, doc) + } + } + + seg, _, err := o.segPlugin.New(analysisResults) + if err != nil { + return fmt.Errorf("error building segment base: %v", err) + } + + filename := zapFileName(o.segCount) + o.segCount++ + path := o.buildPath + string(os.PathSeparator) + filename + + if segUnpersisted, ok := seg.(segment.UnpersistedSegment); ok { + err = segUnpersisted.Persist(path) + if err != nil { + return fmt.Errorf("error persisting segment base to %s: %v", path, err) + } + + o.segPaths = append(o.segPaths, path) + return nil + } + + return fmt.Errorf("new segment does not implement unpersisted: %T", seg) +} + +func (o *Builder) doMerge() error { + // as long as we have more than 1 segment, keep merging + for len(o.segPaths) > 1 { + + // merge the next number of segments into one new one + // or, if there are fewer than remaining, merge them all + mergeCount := o.mergeMax + if mergeCount > len(o.segPaths) { + mergeCount = len(o.segPaths) + } + + mergePaths := o.segPaths[0:mergeCount] + o.segPaths = o.segPaths[mergeCount:] + + // open each of the segments to be merged + mergeSegs := make([]segment.Segment, 0, mergeCount) + + // closeOpenedSegs attempts to close all opened + // segments even if an error occurs, in which case + // the first error is returned + closeOpenedSegs := func() error { + var err error + for _, seg := range mergeSegs { + clErr := seg.Close() + if clErr != nil && err == nil { + err = clErr + } + } + return err + } + + for _, mergePath := range mergePaths { + seg, err := o.segPlugin.Open(mergePath) + if err != nil { + _ = closeOpenedSegs() + return fmt.Errorf("error opening segment (%s) for merge: %v", mergePath, err) + } + mergeSegs = append(mergeSegs, seg) + } + + // do the merge + mergedSegPath := o.buildPath + string(os.PathSeparator) + zapFileName(o.segCount) + drops := make([]*roaring.Bitmap, mergeCount) + _, _, err := o.segPlugin.Merge(mergeSegs, drops, mergedSegPath, nil, nil) + if err != nil { + _ = closeOpenedSegs() + return fmt.Errorf("error merging segments (%v): %v", mergePaths, err) + } + o.segCount++ + o.segPaths = append(o.segPaths, mergedSegPath) + + // close segments opened for merge + err = closeOpenedSegs() + if err != nil { + return fmt.Errorf("error closing opened segments: %v", err) + } + + // remove merged segments + for _, mergePath := range mergePaths { + err = os.RemoveAll(mergePath) + if err != nil { + return fmt.Errorf("error removing segment %s after merge: %v", mergePath, err) + } + } + } + + return nil +} + +func (o *Builder) Close() error { + o.m.Lock() + defer o.m.Unlock() + + // see if there is a partial batch + err := o.maybeFlushBatchLOCKED(1) + if err != nil { + return fmt.Errorf("error flushing batch before close: %v", err) + } + + // perform all the merging + err = o.doMerge() + if err != nil { + return fmt.Errorf("error while merging: %v", err) + } + + // ensure the store path exists + err = os.MkdirAll(o.path, 0700) + if err != nil { + return err + } + + // move final segment into place + // segment id 2 is chosen to match the behavior of a scorch + // index which indexes a single batch of data + finalSegPath := o.path + string(os.PathSeparator) + zapFileName(2) + err = os.Rename(o.segPaths[0], finalSegPath) + if err != nil { + return fmt.Errorf("error moving final segment into place: %v", err) + } + + // remove the buildPath, as it is no longer needed + err = os.RemoveAll(o.buildPath) + if err != nil { + return fmt.Errorf("error removing build path: %v", err) + } + + // prepare wrapping + seg, err := o.segPlugin.Open(finalSegPath) + if err != nil { + return fmt.Errorf("error opening final segment") + } + + // create a segment snapshot for this segment + ss := &SegmentSnapshot{ + segment: seg, + } + is := &IndexSnapshot{ + epoch: 3, // chosen to match scorch behavior when indexing a single batch + segment: []*SegmentSnapshot{ss}, + creator: "scorch-builder", + internal: o.internal, + } + + // create the root bolt + rootBoltPath := o.path + string(os.PathSeparator) + "root.bolt" + rootBolt, err := bolt.Open(rootBoltPath, 0600, nil) + if err != nil { + return err + } + + // start a write transaction + tx, err := rootBolt.Begin(true) + if err != nil { + return err + } + + // fill the root bolt with this fake index snapshot + _, _, err = prepareBoltSnapshot(is, tx, o.path, o.segPlugin, nil) + if err != nil { + _ = tx.Rollback() + _ = rootBolt.Close() + return fmt.Errorf("error preparing bolt snapshot in root.bolt: %v", err) + } + + // commit bolt data + err = tx.Commit() + if err != nil { + _ = rootBolt.Close() + return fmt.Errorf("error committing bolt tx in root.bolt: %v", err) + } + + // close bolt + err = rootBolt.Close() + if err != nil { + return fmt.Errorf("error closing root.bolt: %v", err) + } + + // close final segment + err = seg.Close() + if err != nil { + return fmt.Errorf("error closing final segment: %v", err) + } + return nil +} diff --git a/vendor/github.com/blevesearch/bleve/v2/index/scorch/empty.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/empty.go new file mode 100644 index 00000000..34619d42 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/empty.go @@ -0,0 +1,41 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import segment "github.com/blevesearch/scorch_segment_api/v2" + +type emptyPostingsIterator struct{} + +func (e *emptyPostingsIterator) Next() (segment.Posting, error) { + return nil, nil +} + +func (e *emptyPostingsIterator) Advance(uint64) (segment.Posting, error) { + return nil, nil +} + +func (e *emptyPostingsIterator) Size() int { + return 0 +} + +func (e *emptyPostingsIterator) BytesRead() uint64 { + return 0 +} + +func (e *emptyPostingsIterator) ResetBytesRead(uint64) {} + +func (e *emptyPostingsIterator) BytesWritten() uint64 { return 0 } + +var anEmptyPostingsIterator = &emptyPostingsIterator{} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/event.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/event.go similarity index 97% rename from vendor/github.com/blevesearch/bleve/index/scorch/event.go rename to vendor/github.com/blevesearch/bleve/v2/index/scorch/event.go index 8f3fc191..31c9e80c 100644 --- a/vendor/github.com/blevesearch/bleve/index/scorch/event.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/event.go @@ -18,7 +18,7 @@ import "time" // RegistryAsyncErrorCallbacks should be treated as read-only after // process init()'ialization. -var RegistryAsyncErrorCallbacks = map[string]func(error){} +var RegistryAsyncErrorCallbacks = map[string]func(error, string){} // RegistryEventCallbacks should be treated as read-only after // process init()'ialization. diff --git a/vendor/github.com/blevesearch/bleve/v2/index/scorch/int.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/int.go new file mode 100644 index 00000000..4fa6d7f7 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/int.go @@ -0,0 +1,92 @@ +// Copyright 2014 The Cockroach Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +// This code originated from: +// https://github.com/cockroachdb/cockroach/blob/2dd65dde5d90c157f4b93f92502ca1063b904e1d/pkg/util/encoding/encoding.go + +// Modified to not use pkg/errors + +package scorch + +import "fmt" + +const ( + // intMin is chosen such that the range of int tags does not overlap the + // ascii character set that is frequently used in testing. + intMin = 0x80 // 128 + intMaxWidth = 8 + intZero = intMin + intMaxWidth // 136 + intSmall = intMax - intZero - intMaxWidth // 109 + // intMax is the maximum int tag value. + intMax = 0xfd // 253 +) + +// encodeUvarintAscending encodes the uint64 value using a variable length +// (length-prefixed) representation. The length is encoded as a single +// byte indicating the number of encoded bytes (-8) to follow. See +// EncodeVarintAscending for rationale. The encoded bytes are appended to the +// supplied buffer and the final buffer is returned. +func encodeUvarintAscending(b []byte, v uint64) []byte { + switch { + case v <= intSmall: + return append(b, intZero+byte(v)) + case v <= 0xff: + return append(b, intMax-7, byte(v)) + case v <= 0xffff: + return append(b, intMax-6, byte(v>>8), byte(v)) + case v <= 0xffffff: + return append(b, intMax-5, byte(v>>16), byte(v>>8), byte(v)) + case v <= 0xffffffff: + return append(b, intMax-4, byte(v>>24), byte(v>>16), byte(v>>8), byte(v)) + case v <= 0xffffffffff: + return append(b, intMax-3, byte(v>>32), byte(v>>24), byte(v>>16), byte(v>>8), + byte(v)) + case v <= 0xffffffffffff: + return append(b, intMax-2, byte(v>>40), byte(v>>32), byte(v>>24), byte(v>>16), + byte(v>>8), byte(v)) + case v <= 0xffffffffffffff: + return append(b, intMax-1, byte(v>>48), byte(v>>40), byte(v>>32), byte(v>>24), + byte(v>>16), byte(v>>8), byte(v)) + default: + return append(b, intMax, byte(v>>56), byte(v>>48), byte(v>>40), byte(v>>32), + byte(v>>24), byte(v>>16), byte(v>>8), byte(v)) + } +} + +// decodeUvarintAscending decodes a varint encoded uint64 from the input +// buffer. The remainder of the input buffer and the decoded uint64 +// are returned. +func decodeUvarintAscending(b []byte) ([]byte, uint64, error) { + if len(b) == 0 { + return nil, 0, fmt.Errorf("insufficient bytes to decode uvarint value") + } + length := int(b[0]) - intZero + b = b[1:] // skip length byte + if length <= intSmall { + return b, uint64(length), nil + } + length -= intSmall + if length < 0 || length > 8 { + return nil, 0, fmt.Errorf("invalid uvarint length of %d", length) + } else if len(b) < length { + return nil, 0, fmt.Errorf("insufficient bytes to decode uvarint value: %q", b) + } + var v uint64 + // It is faster to range over the elements in a slice than to index + // into the slice on each loop iteration. + for _, t := range b[:length] { + v = (v << 8) | uint64(t) + } + return b[length:], v, nil +} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/introducer.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/introducer.go similarity index 91% rename from vendor/github.com/blevesearch/bleve/index/scorch/introducer.go rename to vendor/github.com/blevesearch/bleve/v2/index/scorch/introducer.go index 7770c41c..123e71d6 100644 --- a/vendor/github.com/blevesearch/bleve/index/scorch/introducer.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/introducer.go @@ -16,11 +16,12 @@ package scorch import ( "fmt" + "path/filepath" "sync/atomic" "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" ) type segmentIntroduction struct { @@ -46,6 +47,17 @@ type epochWatcher struct { } func (s *Scorch) introducerLoop() { + defer func() { + if r := recover(); r != nil { + s.fireAsyncError(&AsyncPanicError{ + Source: "introducer", + Path: s.path, + }) + } + + s.asyncTasks.Done() + }() + var epochWatchers []*epochWatcher OUTER: for { @@ -88,8 +100,6 @@ OUTER: } epochWatchers = epochWatchersNext } - - s.asyncTasks.Done() } func (s *Scorch) introduceSegment(next *segmentIntroduction) error { @@ -118,6 +128,7 @@ func (s *Scorch) introduceSegment(next *segmentIntroduction) error { // iterate through current segments var running uint64 var docsToPersistCount, memSegments, fileSegments uint64 + var droppedSegmentFiles []string for i := range root.segment { // see if optimistic work included this segment delta, ok := next.obsoletes[root.segment[i].id] @@ -155,6 +166,9 @@ func (s *Scorch) introduceSegment(next *segmentIntroduction) error { root.segment[i].segment.AddRef() newSnapshot.offsets = append(newSnapshot.offsets, running) running += newss.segment.Count() + } else if seg, ok := newss.segment.(segment.PersistedSegment); ok { + droppedSegmentFiles = append(droppedSegmentFiles, + filepath.Base(seg.Path())) } if isMemorySegment(root.segment[i]) { @@ -219,6 +233,12 @@ func (s *Scorch) introduceSegment(next *segmentIntroduction) error { _ = rootPrev.DecRef() } + // update the removal eligibility for those segment files + // that are not a part of the latest root. + for _, filename := range droppedSegmentFiles { + s.unmarkIneligibleForRemoval(filename) + } + close(next.applied) return nil @@ -257,6 +277,7 @@ func (s *Scorch) introducePersist(persist *persistIntroduction) { deleted: segmentSnapshot.deleted, cachedDocs: segmentSnapshot.cachedDocs, creator: "introducePersist", + mmaped: 1, } newIndexSnapshot.segment[i] = newSegmentSnapshot delete(persist.persisted, segmentSnapshot.id) @@ -323,6 +344,7 @@ func (s *Scorch) introduceMerge(nextMerge *segmentMerge) { // iterate through current segments newSegmentDeleted := roaring.NewBitmap() var running, docsToPersistCount, memSegments, fileSegments uint64 + var droppedSegmentFiles []string for i := range root.segment { segmentID := root.segment[i].id if segSnapAtMerge, ok := nextMerge.old[segmentID]; ok { @@ -365,8 +387,12 @@ func (s *Scorch) introduceMerge(nextMerge *segmentMerge) { } else { fileSegments++ } + } else if root.segment[i].LiveSize() == 0 { + if seg, ok := root.segment[i].segment.(segment.PersistedSegment); ok { + droppedSegmentFiles = append(droppedSegmentFiles, + filepath.Base(seg.Path())) + } } - } // before the newMerge introduction, need to clean the newly @@ -388,6 +414,7 @@ func (s *Scorch) introduceMerge(nextMerge *segmentMerge) { // deleted by the time we reach here, can skip the introduction. if nextMerge.new != nil && nextMerge.new.Count() > newSegmentDeleted.GetCardinality() { + // put new segment at end newSnapshot.segment = append(newSnapshot.segment, &SegmentSnapshot{ id: nextMerge.id, @@ -395,6 +422,7 @@ func (s *Scorch) introduceMerge(nextMerge *segmentMerge) { deleted: newSegmentDeleted, cachedDocs: &cachedDocs{cache: nil}, creator: "introduceMerge", + mmaped: nextMerge.mmaped, }) newSnapshot.offsets = append(newSnapshot.offsets, running) atomic.AddUint64(&s.stats.TotIntroducedSegmentsMerge, 1) @@ -432,6 +460,12 @@ func (s *Scorch) introduceMerge(nextMerge *segmentMerge) { _ = rootPrev.DecRef() } + // update the removal eligibility for those segment files + // that are not a part of the latest root. + for _, filename := range droppedSegmentFiles { + s.unmarkIneligibleForRemoval(filename) + } + // notify requester that we incorporated this nextMerge.notifyCh <- &mergeTaskIntroStatus{ indexSnapshot: newSnapshot, diff --git a/vendor/github.com/blevesearch/bleve/v2/index/scorch/merge.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/merge.go new file mode 100644 index 00000000..92adc3fd --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/merge.go @@ -0,0 +1,527 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "context" + "fmt" + "os" + "strings" + "sync/atomic" + "time" + + "github.com/RoaringBitmap/roaring" + "github.com/blevesearch/bleve/v2/index/scorch/mergeplan" + "github.com/blevesearch/bleve/v2/util" + segment "github.com/blevesearch/scorch_segment_api/v2" +) + +func (s *Scorch) mergerLoop() { + defer func() { + if r := recover(); r != nil { + s.fireAsyncError(&AsyncPanicError{ + Source: "merger", + Path: s.path, + }) + } + + s.asyncTasks.Done() + }() + + var lastEpochMergePlanned uint64 + var ctrlMsg *mergerCtrl + mergePlannerOptions, err := s.parseMergePlannerOptions() + if err != nil { + s.fireAsyncError(fmt.Errorf("mergePlannerOption json parsing err: %v", err)) + return + } + ctrlMsgDflt := &mergerCtrl{ctx: context.Background(), + options: mergePlannerOptions, + doneCh: nil} + +OUTER: + for { + atomic.AddUint64(&s.stats.TotFileMergeLoopBeg, 1) + + select { + case <-s.closeCh: + break OUTER + + default: + // check to see if there is a new snapshot to persist + s.rootLock.Lock() + ourSnapshot := s.root + ourSnapshot.AddRef() + atomic.StoreUint64(&s.iStats.mergeSnapshotSize, uint64(ourSnapshot.Size())) + atomic.StoreUint64(&s.iStats.mergeEpoch, ourSnapshot.epoch) + s.rootLock.Unlock() + + if ctrlMsg == nil && ourSnapshot.epoch != lastEpochMergePlanned { + ctrlMsg = ctrlMsgDflt + } + if ctrlMsg != nil { + startTime := time.Now() + + // lets get started + err := s.planMergeAtSnapshot(ctrlMsg.ctx, ctrlMsg.options, + ourSnapshot) + if err != nil { + atomic.StoreUint64(&s.iStats.mergeEpoch, 0) + if err == segment.ErrClosed { + // index has been closed + _ = ourSnapshot.DecRef() + + // continue the workloop on a user triggered cancel + if ctrlMsg.doneCh != nil { + close(ctrlMsg.doneCh) + ctrlMsg = nil + continue OUTER + } + + // exit the workloop on index closure + ctrlMsg = nil + break OUTER + } + s.fireAsyncError(fmt.Errorf("merging err: %v", err)) + _ = ourSnapshot.DecRef() + atomic.AddUint64(&s.stats.TotFileMergeLoopErr, 1) + continue OUTER + } + + if ctrlMsg.doneCh != nil { + close(ctrlMsg.doneCh) + } + ctrlMsg = nil + + lastEpochMergePlanned = ourSnapshot.epoch + + atomic.StoreUint64(&s.stats.LastMergedEpoch, ourSnapshot.epoch) + + s.fireEvent(EventKindMergerProgress, time.Since(startTime)) + } + _ = ourSnapshot.DecRef() + + // tell the persister we're waiting for changes + // first make a epochWatcher chan + ew := &epochWatcher{ + epoch: lastEpochMergePlanned, + notifyCh: make(notificationChan, 1), + } + + // give it to the persister + select { + case <-s.closeCh: + break OUTER + case s.persisterNotifier <- ew: + case ctrlMsg = <-s.forceMergeRequestCh: + continue OUTER + } + + // now wait for persister (but also detect close) + select { + case <-s.closeCh: + break OUTER + case <-ew.notifyCh: + case ctrlMsg = <-s.forceMergeRequestCh: + } + } + + atomic.AddUint64(&s.stats.TotFileMergeLoopEnd, 1) + } +} + +type mergerCtrl struct { + ctx context.Context + options *mergeplan.MergePlanOptions + doneCh chan struct{} +} + +// ForceMerge helps users trigger a merge operation on +// an online scorch index. +func (s *Scorch) ForceMerge(ctx context.Context, + mo *mergeplan.MergePlanOptions) error { + // check whether force merge is already under processing + s.rootLock.Lock() + if s.stats.TotFileMergeForceOpsStarted > + s.stats.TotFileMergeForceOpsCompleted { + s.rootLock.Unlock() + return fmt.Errorf("force merge already in progress") + } + + s.stats.TotFileMergeForceOpsStarted++ + s.rootLock.Unlock() + + if mo != nil { + err := mergeplan.ValidateMergePlannerOptions(mo) + if err != nil { + return err + } + } else { + // assume the default single segment merge policy + mo = &mergeplan.SingleSegmentMergePlanOptions + } + msg := &mergerCtrl{options: mo, + doneCh: make(chan struct{}), + ctx: ctx, + } + + // request the merger perform a force merge + select { + case s.forceMergeRequestCh <- msg: + case <-s.closeCh: + return nil + } + + // wait for the force merge operation completion + select { + case <-msg.doneCh: + atomic.AddUint64(&s.stats.TotFileMergeForceOpsCompleted, 1) + case <-s.closeCh: + } + + return nil +} + +func (s *Scorch) parseMergePlannerOptions() (*mergeplan.MergePlanOptions, + error) { + mergePlannerOptions := mergeplan.DefaultMergePlanOptions + if v, ok := s.config["scorchMergePlanOptions"]; ok { + b, err := util.MarshalJSON(v) + if err != nil { + return &mergePlannerOptions, err + } + + err = util.UnmarshalJSON(b, &mergePlannerOptions) + if err != nil { + return &mergePlannerOptions, err + } + + err = mergeplan.ValidateMergePlannerOptions(&mergePlannerOptions) + if err != nil { + return nil, err + } + } + return &mergePlannerOptions, nil +} + +type closeChWrapper struct { + ch1 chan struct{} + ctx context.Context + closeCh chan struct{} + cancelCh chan struct{} +} + +func newCloseChWrapper(ch1 chan struct{}, + ctx context.Context) *closeChWrapper { + return &closeChWrapper{ + ch1: ch1, + ctx: ctx, + closeCh: make(chan struct{}), + cancelCh: make(chan struct{}), + } +} + +func (w *closeChWrapper) close() { + close(w.closeCh) +} + +func (w *closeChWrapper) listen() { + select { + case <-w.ch1: + close(w.cancelCh) + case <-w.ctx.Done(): + close(w.cancelCh) + case <-w.closeCh: + } +} + +func (s *Scorch) planMergeAtSnapshot(ctx context.Context, + options *mergeplan.MergePlanOptions, ourSnapshot *IndexSnapshot) error { + // build list of persisted segments in this snapshot + var onlyPersistedSnapshots []mergeplan.Segment + for _, segmentSnapshot := range ourSnapshot.segment { + if _, ok := segmentSnapshot.segment.(segment.PersistedSegment); ok { + onlyPersistedSnapshots = append(onlyPersistedSnapshots, segmentSnapshot) + } + } + + atomic.AddUint64(&s.stats.TotFileMergePlan, 1) + + // give this list to the planner + resultMergePlan, err := mergeplan.Plan(onlyPersistedSnapshots, options) + if err != nil { + atomic.AddUint64(&s.stats.TotFileMergePlanErr, 1) + return fmt.Errorf("merge planning err: %v", err) + } + if resultMergePlan == nil { + // nothing to do + atomic.AddUint64(&s.stats.TotFileMergePlanNone, 1) + return nil + } + atomic.AddUint64(&s.stats.TotFileMergePlanOk, 1) + + atomic.AddUint64(&s.stats.TotFileMergePlanTasks, uint64(len(resultMergePlan.Tasks))) + + // process tasks in serial for now + var filenames []string + + cw := newCloseChWrapper(s.closeCh, ctx) + defer cw.close() + + go cw.listen() + + for _, task := range resultMergePlan.Tasks { + if len(task.Segments) == 0 { + atomic.AddUint64(&s.stats.TotFileMergePlanTasksSegmentsEmpty, 1) + continue + } + + atomic.AddUint64(&s.stats.TotFileMergePlanTasksSegments, uint64(len(task.Segments))) + + oldMap := make(map[uint64]*SegmentSnapshot) + newSegmentID := atomic.AddUint64(&s.nextSegmentID, 1) + segmentsToMerge := make([]segment.Segment, 0, len(task.Segments)) + docsToDrop := make([]*roaring.Bitmap, 0, len(task.Segments)) + + for _, planSegment := range task.Segments { + if segSnapshot, ok := planSegment.(*SegmentSnapshot); ok { + oldMap[segSnapshot.id] = segSnapshot + if persistedSeg, ok := segSnapshot.segment.(segment.PersistedSegment); ok { + if segSnapshot.LiveSize() == 0 { + atomic.AddUint64(&s.stats.TotFileMergeSegmentsEmpty, 1) + oldMap[segSnapshot.id] = nil + } else { + segmentsToMerge = append(segmentsToMerge, segSnapshot.segment) + docsToDrop = append(docsToDrop, segSnapshot.deleted) + } + // track the files getting merged for unsetting the + // removal ineligibility. This helps to unflip files + // even with fast merger, slow persister work flows. + path := persistedSeg.Path() + filenames = append(filenames, + strings.TrimPrefix(path, s.path+string(os.PathSeparator))) + } + } + } + + var oldNewDocNums map[uint64][]uint64 + var seg segment.Segment + var filename string + if len(segmentsToMerge) > 0 { + filename = zapFileName(newSegmentID) + s.markIneligibleForRemoval(filename) + path := s.path + string(os.PathSeparator) + filename + + fileMergeZapStartTime := time.Now() + + atomic.AddUint64(&s.stats.TotFileMergeZapBeg, 1) + prevBytesReadTotal := cumulateBytesRead(segmentsToMerge) + newDocNums, _, err := s.segPlugin.Merge(segmentsToMerge, docsToDrop, path, + cw.cancelCh, s) + atomic.AddUint64(&s.stats.TotFileMergeZapEnd, 1) + + fileMergeZapTime := uint64(time.Since(fileMergeZapStartTime)) + atomic.AddUint64(&s.stats.TotFileMergeZapTime, fileMergeZapTime) + if atomic.LoadUint64(&s.stats.MaxFileMergeZapTime) < fileMergeZapTime { + atomic.StoreUint64(&s.stats.MaxFileMergeZapTime, fileMergeZapTime) + } + + if err != nil { + s.unmarkIneligibleForRemoval(filename) + atomic.AddUint64(&s.stats.TotFileMergePlanTasksErr, 1) + if err == segment.ErrClosed { + return err + } + return fmt.Errorf("merging failed: %v", err) + } + + seg, err = s.segPlugin.Open(path) + if err != nil { + s.unmarkIneligibleForRemoval(filename) + atomic.AddUint64(&s.stats.TotFileMergePlanTasksErr, 1) + return err + } + + totalBytesRead := seg.BytesRead() + prevBytesReadTotal + seg.ResetBytesRead(totalBytesRead) + + oldNewDocNums = make(map[uint64][]uint64) + for i, segNewDocNums := range newDocNums { + oldNewDocNums[task.Segments[i].Id()] = segNewDocNums + } + + atomic.AddUint64(&s.stats.TotFileMergeSegments, uint64(len(segmentsToMerge))) + } + + sm := &segmentMerge{ + id: newSegmentID, + old: oldMap, + oldNewDocNums: oldNewDocNums, + new: seg, + notifyCh: make(chan *mergeTaskIntroStatus), + mmaped: 1, + } + + s.fireEvent(EventKindMergeTaskIntroductionStart, 0) + + // give it to the introducer + select { + case <-s.closeCh: + _ = seg.Close() + return segment.ErrClosed + case s.merges <- sm: + atomic.AddUint64(&s.stats.TotFileMergeIntroductions, 1) + } + + introStartTime := time.Now() + // it is safe to blockingly wait for the merge introduction + // here as the introducer is bound to handle the notify channel. + introStatus := <-sm.notifyCh + introTime := uint64(time.Since(introStartTime)) + atomic.AddUint64(&s.stats.TotFileMergeZapIntroductionTime, introTime) + if atomic.LoadUint64(&s.stats.MaxFileMergeZapIntroductionTime) < introTime { + atomic.StoreUint64(&s.stats.MaxFileMergeZapIntroductionTime, introTime) + } + atomic.AddUint64(&s.stats.TotFileMergeIntroductionsDone, 1) + if introStatus != nil && introStatus.indexSnapshot != nil { + _ = introStatus.indexSnapshot.DecRef() + if introStatus.skipped { + // close the segment on skipping introduction. + s.unmarkIneligibleForRemoval(filename) + _ = seg.Close() + } + } + + atomic.AddUint64(&s.stats.TotFileMergePlanTasksDone, 1) + + s.fireEvent(EventKindMergeTaskIntroduction, 0) + } + + // once all the newly merged segment introductions are done, + // its safe to unflip the removal ineligibility for the replaced + // older segments + for _, f := range filenames { + s.unmarkIneligibleForRemoval(f) + } + + return nil +} + +type mergeTaskIntroStatus struct { + indexSnapshot *IndexSnapshot + skipped bool +} + +type segmentMerge struct { + id uint64 + old map[uint64]*SegmentSnapshot + oldNewDocNums map[uint64][]uint64 + new segment.Segment + notifyCh chan *mergeTaskIntroStatus + mmaped uint32 +} + +func cumulateBytesRead(sbs []segment.Segment) uint64 { + var rv uint64 + for _, seg := range sbs { + rv += seg.BytesRead() + } + return rv +} + +// perform a merging of the given SegmentBase instances into a new, +// persisted segment, and synchronously introduce that new segment +// into the root +func (s *Scorch) mergeSegmentBases(snapshot *IndexSnapshot, + sbs []segment.Segment, sbsDrops []*roaring.Bitmap, + sbsIndexes []int) (*IndexSnapshot, uint64, error) { + atomic.AddUint64(&s.stats.TotMemMergeBeg, 1) + + memMergeZapStartTime := time.Now() + + atomic.AddUint64(&s.stats.TotMemMergeZapBeg, 1) + + newSegmentID := atomic.AddUint64(&s.nextSegmentID, 1) + filename := zapFileName(newSegmentID) + path := s.path + string(os.PathSeparator) + filename + + newDocNums, _, err := + s.segPlugin.Merge(sbs, sbsDrops, path, s.closeCh, s) + + atomic.AddUint64(&s.stats.TotMemMergeZapEnd, 1) + + memMergeZapTime := uint64(time.Since(memMergeZapStartTime)) + atomic.AddUint64(&s.stats.TotMemMergeZapTime, memMergeZapTime) + if atomic.LoadUint64(&s.stats.MaxMemMergeZapTime) < memMergeZapTime { + atomic.StoreUint64(&s.stats.MaxMemMergeZapTime, memMergeZapTime) + } + + if err != nil { + atomic.AddUint64(&s.stats.TotMemMergeErr, 1) + return nil, 0, err + } + + seg, err := s.segPlugin.Open(path) + if err != nil { + atomic.AddUint64(&s.stats.TotMemMergeErr, 1) + return nil, 0, err + } + + // update persisted stats + atomic.AddUint64(&s.stats.TotPersistedItems, seg.Count()) + atomic.AddUint64(&s.stats.TotPersistedSegments, 1) + + sm := &segmentMerge{ + id: newSegmentID, + old: make(map[uint64]*SegmentSnapshot), + oldNewDocNums: make(map[uint64][]uint64), + new: seg, + notifyCh: make(chan *mergeTaskIntroStatus), + } + + for i, idx := range sbsIndexes { + ss := snapshot.segment[idx] + sm.old[ss.id] = ss + sm.oldNewDocNums[ss.id] = newDocNums[i] + } + + select { // send to introducer + case <-s.closeCh: + _ = seg.DecRef() + return nil, 0, segment.ErrClosed + case s.merges <- sm: + } + + // blockingly wait for the introduction to complete + var newSnapshot *IndexSnapshot + introStatus := <-sm.notifyCh + if introStatus != nil && introStatus.indexSnapshot != nil { + newSnapshot = introStatus.indexSnapshot + atomic.AddUint64(&s.stats.TotMemMergeSegments, uint64(len(sbs))) + atomic.AddUint64(&s.stats.TotMemMergeDone, 1) + if introStatus.skipped { + // close the segment on skipping introduction. + _ = newSnapshot.DecRef() + _ = seg.Close() + newSnapshot = nil + } + } + + return newSnapshot, newSegmentID, nil +} + +func (s *Scorch) ReportBytesWritten(bytesWritten uint64) { + atomic.AddUint64(&s.stats.TotFileMergeWrittenBytes, bytesWritten) +} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/mergeplan/merge_plan.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/mergeplan/merge_plan.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/index/scorch/mergeplan/merge_plan.go rename to vendor/github.com/blevesearch/bleve/v2/index/scorch/mergeplan/merge_plan.go diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/mergeplan/sort.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/mergeplan/sort.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/index/scorch/mergeplan/sort.go rename to vendor/github.com/blevesearch/bleve/v2/index/scorch/mergeplan/sort.go diff --git a/vendor/github.com/blevesearch/bleve/v2/index/scorch/optimize.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/optimize.go new file mode 100644 index 00000000..3c7969fa --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/optimize.go @@ -0,0 +1,396 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "fmt" + "github.com/RoaringBitmap/roaring" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" + "sync/atomic" +) + +var OptimizeConjunction = true +var OptimizeConjunctionUnadorned = true +var OptimizeDisjunctionUnadorned = true + +func (s *IndexSnapshotTermFieldReader) Optimize(kind string, + octx index.OptimizableContext) (index.OptimizableContext, error) { + if OptimizeConjunction && kind == "conjunction" { + return s.optimizeConjunction(octx) + } + + if OptimizeConjunctionUnadorned && kind == "conjunction:unadorned" { + return s.optimizeConjunctionUnadorned(octx) + } + + if OptimizeDisjunctionUnadorned && kind == "disjunction:unadorned" { + return s.optimizeDisjunctionUnadorned(octx) + } + + return nil, nil +} + +var OptimizeDisjunctionUnadornedMinChildCardinality = uint64(256) + +// ---------------------------------------------------------------- + +func (s *IndexSnapshotTermFieldReader) optimizeConjunction( + octx index.OptimizableContext) (index.OptimizableContext, error) { + if octx == nil { + octx = &OptimizeTFRConjunction{snapshot: s.snapshot} + } + + o, ok := octx.(*OptimizeTFRConjunction) + if !ok { + return octx, nil + } + + if o.snapshot != s.snapshot { + return nil, fmt.Errorf("tried to optimize conjunction across different snapshots") + } + + o.tfrs = append(o.tfrs, s) + + return o, nil +} + +type OptimizeTFRConjunction struct { + snapshot *IndexSnapshot + + tfrs []*IndexSnapshotTermFieldReader +} + +func (o *OptimizeTFRConjunction) Finish() (index.Optimized, error) { + if len(o.tfrs) <= 1 { + return nil, nil + } + + for i := range o.snapshot.segment { + itr0, ok := o.tfrs[0].iterators[i].(segment.OptimizablePostingsIterator) + if !ok || itr0.ActualBitmap() == nil { + continue + } + + itr1, ok := o.tfrs[1].iterators[i].(segment.OptimizablePostingsIterator) + if !ok || itr1.ActualBitmap() == nil { + continue + } + + bm := roaring.And(itr0.ActualBitmap(), itr1.ActualBitmap()) + + for _, tfr := range o.tfrs[2:] { + itr, ok := tfr.iterators[i].(segment.OptimizablePostingsIterator) + if !ok || itr.ActualBitmap() == nil { + continue + } + + bm.And(itr.ActualBitmap()) + } + + // in this conjunction optimization, the postings iterators + // will all share the same AND'ed together actual bitmap. The + // regular conjunction searcher machinery will still be used, + // but the underlying bitmap will be smaller. + for _, tfr := range o.tfrs { + itr, ok := tfr.iterators[i].(segment.OptimizablePostingsIterator) + if ok && itr.ActualBitmap() != nil { + itr.ReplaceActual(bm) + } + } + } + + return nil, nil +} + +// ---------------------------------------------------------------- + +// An "unadorned" conjunction optimization is appropriate when +// additional or subsidiary information like freq-norm's and +// term-vectors are not required, and instead only the internal-id's +// are needed. +func (s *IndexSnapshotTermFieldReader) optimizeConjunctionUnadorned( + octx index.OptimizableContext) (index.OptimizableContext, error) { + if octx == nil { + octx = &OptimizeTFRConjunctionUnadorned{snapshot: s.snapshot} + } + + o, ok := octx.(*OptimizeTFRConjunctionUnadorned) + if !ok { + return nil, nil + } + + if o.snapshot != s.snapshot { + return nil, fmt.Errorf("tried to optimize unadorned conjunction across different snapshots") + } + + o.tfrs = append(o.tfrs, s) + + return o, nil +} + +type OptimizeTFRConjunctionUnadorned struct { + snapshot *IndexSnapshot + + tfrs []*IndexSnapshotTermFieldReader +} + +var OptimizeTFRConjunctionUnadornedTerm = []byte("") +var OptimizeTFRConjunctionUnadornedField = "*" + +// Finish of an unadorned conjunction optimization will compute a +// termFieldReader with an "actual" bitmap that represents the +// constituent bitmaps AND'ed together. This termFieldReader cannot +// provide any freq-norm or termVector associated information. +func (o *OptimizeTFRConjunctionUnadorned) Finish() (rv index.Optimized, err error) { + if len(o.tfrs) <= 1 { + return nil, nil + } + + // We use an artificial term and field because the optimized + // termFieldReader can represent multiple terms and fields. + oTFR := o.snapshot.unadornedTermFieldReader( + OptimizeTFRConjunctionUnadornedTerm, OptimizeTFRConjunctionUnadornedField) + + var actualBMs []*roaring.Bitmap // Collected from regular posting lists. + +OUTER: + for i := range o.snapshot.segment { + actualBMs = actualBMs[:0] + + var docNum1HitLast uint64 + var docNum1HitLastOk bool + + for _, tfr := range o.tfrs { + if _, ok := tfr.iterators[i].(*emptyPostingsIterator); ok { + // An empty postings iterator means the entire AND is empty. + oTFR.iterators[i] = anEmptyPostingsIterator + continue OUTER + } + + itr, ok := tfr.iterators[i].(segment.OptimizablePostingsIterator) + if !ok { + // We only optimize postings iterators that support this operation. + return nil, nil + } + + // If the postings iterator is "1-hit" optimized, then we + // can perform several optimizations up-front here. + docNum1Hit, ok := itr.DocNum1Hit() + if ok { + if docNum1HitLastOk && docNum1HitLast != docNum1Hit { + // The docNum1Hit doesn't match the previous + // docNum1HitLast, so the entire AND is empty. + oTFR.iterators[i] = anEmptyPostingsIterator + continue OUTER + } + + docNum1HitLast = docNum1Hit + docNum1HitLastOk = true + + continue + } + + if itr.ActualBitmap() == nil { + // An empty actual bitmap means the entire AND is empty. + oTFR.iterators[i] = anEmptyPostingsIterator + continue OUTER + } + + // Collect the actual bitmap for more processing later. + actualBMs = append(actualBMs, itr.ActualBitmap()) + } + + if docNum1HitLastOk { + // We reach here if all the 1-hit optimized posting + // iterators had the same 1-hit docNum, so we can check if + // our collected actual bitmaps also have that docNum. + for _, bm := range actualBMs { + if !bm.Contains(uint32(docNum1HitLast)) { + // The docNum1Hit isn't in one of our actual + // bitmaps, so the entire AND is empty. + oTFR.iterators[i] = anEmptyPostingsIterator + continue OUTER + } + } + + // The actual bitmaps and docNum1Hits all contain or have + // the same 1-hit docNum, so that's our AND'ed result. + oTFR.iterators[i] = newUnadornedPostingsIteratorFrom1Hit(docNum1HitLast) + + continue OUTER + } + + if len(actualBMs) == 0 { + // If we've collected no actual bitmaps at this point, + // then the entire AND is empty. + oTFR.iterators[i] = anEmptyPostingsIterator + continue OUTER + } + + if len(actualBMs) == 1 { + // If we've only 1 actual bitmap, then that's our result. + oTFR.iterators[i] = newUnadornedPostingsIteratorFromBitmap(actualBMs[0]) + + continue OUTER + } + + // Else, AND together our collected bitmaps as our result. + bm := roaring.And(actualBMs[0], actualBMs[1]) + + for _, actualBM := range actualBMs[2:] { + bm.And(actualBM) + } + + oTFR.iterators[i] = newUnadornedPostingsIteratorFromBitmap(bm) + } + + atomic.AddUint64(&o.snapshot.parent.stats.TotTermSearchersStarted, uint64(1)) + return oTFR, nil +} + +// ---------------------------------------------------------------- + +// An "unadorned" disjunction optimization is appropriate when +// additional or subsidiary information like freq-norm's and +// term-vectors are not required, and instead only the internal-id's +// are needed. +func (s *IndexSnapshotTermFieldReader) optimizeDisjunctionUnadorned( + octx index.OptimizableContext) (index.OptimizableContext, error) { + if octx == nil { + octx = &OptimizeTFRDisjunctionUnadorned{ + snapshot: s.snapshot, + } + } + + o, ok := octx.(*OptimizeTFRDisjunctionUnadorned) + if !ok { + return nil, nil + } + + if o.snapshot != s.snapshot { + return nil, fmt.Errorf("tried to optimize unadorned disjunction across different snapshots") + } + + o.tfrs = append(o.tfrs, s) + + return o, nil +} + +type OptimizeTFRDisjunctionUnadorned struct { + snapshot *IndexSnapshot + + tfrs []*IndexSnapshotTermFieldReader +} + +var OptimizeTFRDisjunctionUnadornedTerm = []byte("") +var OptimizeTFRDisjunctionUnadornedField = "*" + +// Finish of an unadorned disjunction optimization will compute a +// termFieldReader with an "actual" bitmap that represents the +// constituent bitmaps OR'ed together. This termFieldReader cannot +// provide any freq-norm or termVector associated information. +func (o *OptimizeTFRDisjunctionUnadorned) Finish() (rv index.Optimized, err error) { + if len(o.tfrs) <= 1 { + return nil, nil + } + + for i := range o.snapshot.segment { + var cMax uint64 + + for _, tfr := range o.tfrs { + itr, ok := tfr.iterators[i].(segment.OptimizablePostingsIterator) + if !ok { + return nil, nil + } + + if itr.ActualBitmap() != nil { + c := itr.ActualBitmap().GetCardinality() + if cMax < c { + cMax = c + } + } + } + } + + // We use an artificial term and field because the optimized + // termFieldReader can represent multiple terms and fields. + oTFR := o.snapshot.unadornedTermFieldReader( + OptimizeTFRDisjunctionUnadornedTerm, OptimizeTFRDisjunctionUnadornedField) + + var docNums []uint32 // Collected docNum's from 1-hit posting lists. + var actualBMs []*roaring.Bitmap // Collected from regular posting lists. + + for i := range o.snapshot.segment { + docNums = docNums[:0] + actualBMs = actualBMs[:0] + + for _, tfr := range o.tfrs { + itr, ok := tfr.iterators[i].(segment.OptimizablePostingsIterator) + if !ok { + return nil, nil + } + + docNum, ok := itr.DocNum1Hit() + if ok { + docNums = append(docNums, uint32(docNum)) + continue + } + + if itr.ActualBitmap() != nil { + actualBMs = append(actualBMs, itr.ActualBitmap()) + } + } + + var bm *roaring.Bitmap + if len(actualBMs) > 2 { + bm = roaring.HeapOr(actualBMs...) + } else if len(actualBMs) == 2 { + bm = roaring.Or(actualBMs[0], actualBMs[1]) + } else if len(actualBMs) == 1 { + bm = actualBMs[0].Clone() + } + + if bm == nil { + bm = roaring.New() + } + + bm.AddMany(docNums) + + oTFR.iterators[i] = newUnadornedPostingsIteratorFromBitmap(bm) + } + + atomic.AddUint64(&o.snapshot.parent.stats.TotTermSearchersStarted, uint64(1)) + return oTFR, nil +} + +// ---------------------------------------------------------------- + +func (i *IndexSnapshot) unadornedTermFieldReader( + term []byte, field string) *IndexSnapshotTermFieldReader { + // This IndexSnapshotTermFieldReader will not be recycled, more + // conversation here: https://github.com/blevesearch/bleve/pull/1438 + return &IndexSnapshotTermFieldReader{ + term: term, + field: field, + snapshot: i, + iterators: make([]segment.PostingsIterator, len(i.segment)), + segmentOffset: 0, + includeFreq: false, + includeNorm: false, + includeTermVectors: false, + recycle: false, + } +} diff --git a/vendor/github.com/blevesearch/bleve/v2/index/scorch/persister.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/persister.go new file mode 100644 index 00000000..217582fe --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/persister.go @@ -0,0 +1,1267 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "log" + "math" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync/atomic" + "time" + + "github.com/RoaringBitmap/roaring" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" + bolt "go.etcd.io/bbolt" +) + +// DefaultPersisterNapTimeMSec is kept to zero as this helps in direct +// persistence of segments with the default safe batch option. +// If the default safe batch option results in high number of +// files on disk, then users may initialise this configuration parameter +// with higher values so that the persister will nap a bit within it's +// work loop to favour better in-memory merging of segments to result +// in fewer segment files on disk. But that may come with an indexing +// performance overhead. +// Unsafe batch users are advised to override this to higher value +// for better performance especially with high data density. +var DefaultPersisterNapTimeMSec int = 0 // ms + +// DefaultPersisterNapUnderNumFiles helps in controlling the pace of +// persister. At times of a slow merger progress with heavy file merging +// operations, its better to pace down the persister for letting the merger +// to catch up within a range defined by this parameter. +// Fewer files on disk (as per the merge plan) would result in keeping the +// file handle usage under limit, faster disk merger and a healthier index. +// Its been observed that such a loosely sync'ed introducer-persister-merger +// trio results in better overall performance. +var DefaultPersisterNapUnderNumFiles int = 1000 + +var DefaultMemoryPressurePauseThreshold uint64 = math.MaxUint64 + +type persisterOptions struct { + // PersisterNapTimeMSec controls the wait/delay injected into + // persistence workloop to improve the chances for + // a healthier and heavier in-memory merging + PersisterNapTimeMSec int + + // PersisterNapTimeMSec > 0, and the number of files is less than + // PersisterNapUnderNumFiles, then the persister will sleep + // PersisterNapTimeMSec amount of time to improve the chances for + // a healthier and heavier in-memory merging + PersisterNapUnderNumFiles int + + // MemoryPressurePauseThreshold let persister to have a better leeway + // for prudently performing the memory merge of segments on a memory + // pressure situation. Here the config value is an upper threshold + // for the number of paused application threads. The default value would + // be a very high number to always favour the merging of memory segments. + MemoryPressurePauseThreshold uint64 +} + +type notificationChan chan struct{} + +func (s *Scorch) persisterLoop() { + defer func() { + if r := recover(); r != nil { + s.fireAsyncError(&AsyncPanicError{ + Source: "persister", + Path: s.path, + }) + } + + s.asyncTasks.Done() + }() + + var persistWatchers []*epochWatcher + var lastPersistedEpoch, lastMergedEpoch uint64 + var ew *epochWatcher + + var unpersistedCallbacks []index.BatchCallback + + po, err := s.parsePersisterOptions() + if err != nil { + s.fireAsyncError(fmt.Errorf("persisterOptions json parsing err: %v", err)) + return + } + +OUTER: + for { + atomic.AddUint64(&s.stats.TotPersistLoopBeg, 1) + + select { + case <-s.closeCh: + break OUTER + case ew = <-s.persisterNotifier: + persistWatchers = append(persistWatchers, ew) + default: + } + if ew != nil && ew.epoch > lastMergedEpoch { + lastMergedEpoch = ew.epoch + } + lastMergedEpoch, persistWatchers = s.pausePersisterForMergerCatchUp(lastPersistedEpoch, + lastMergedEpoch, persistWatchers, po) + + var ourSnapshot *IndexSnapshot + var ourPersisted []chan error + var ourPersistedCallbacks []index.BatchCallback + + // check to see if there is a new snapshot to persist + s.rootLock.Lock() + if s.root != nil && s.root.epoch > lastPersistedEpoch { + ourSnapshot = s.root + ourSnapshot.AddRef() + ourPersisted = s.rootPersisted + s.rootPersisted = nil + ourPersistedCallbacks = s.persistedCallbacks + s.persistedCallbacks = nil + atomic.StoreUint64(&s.iStats.persistSnapshotSize, uint64(ourSnapshot.Size())) + atomic.StoreUint64(&s.iStats.persistEpoch, ourSnapshot.epoch) + } + s.rootLock.Unlock() + + if ourSnapshot != nil { + startTime := time.Now() + + err := s.persistSnapshot(ourSnapshot, po) + for _, ch := range ourPersisted { + if err != nil { + ch <- err + } + close(ch) + } + if err != nil { + atomic.StoreUint64(&s.iStats.persistEpoch, 0) + if err == segment.ErrClosed { + // index has been closed + _ = ourSnapshot.DecRef() + break OUTER + } + + // save this current snapshot's persistedCallbacks, to invoke during + // the retry attempt + unpersistedCallbacks = append(unpersistedCallbacks, ourPersistedCallbacks...) + + s.fireAsyncError(fmt.Errorf("got err persisting snapshot: %v", err)) + _ = ourSnapshot.DecRef() + atomic.AddUint64(&s.stats.TotPersistLoopErr, 1) + continue OUTER + } + + if unpersistedCallbacks != nil { + // in the event of this being a retry attempt for persisting a snapshot + // that had earlier failed, prepend the persistedCallbacks associated + // with earlier segment(s) to the latest persistedCallbacks + ourPersistedCallbacks = append(unpersistedCallbacks, ourPersistedCallbacks...) + unpersistedCallbacks = nil + } + + for i := range ourPersistedCallbacks { + ourPersistedCallbacks[i](err) + } + + atomic.StoreUint64(&s.stats.LastPersistedEpoch, ourSnapshot.epoch) + + lastPersistedEpoch = ourSnapshot.epoch + for _, ew := range persistWatchers { + close(ew.notifyCh) + } + + persistWatchers = nil + _ = ourSnapshot.DecRef() + + changed := false + s.rootLock.RLock() + if s.root != nil && s.root.epoch != lastPersistedEpoch { + changed = true + } + s.rootLock.RUnlock() + + s.fireEvent(EventKindPersisterProgress, time.Since(startTime)) + + if changed { + atomic.AddUint64(&s.stats.TotPersistLoopProgress, 1) + continue OUTER + } + } + + // tell the introducer we're waiting for changes + w := &epochWatcher{ + epoch: lastPersistedEpoch, + notifyCh: make(notificationChan, 1), + } + + select { + case <-s.closeCh: + break OUTER + case s.introducerNotifier <- w: + } + + s.removeOldData() // might as well cleanup while waiting + + atomic.AddUint64(&s.stats.TotPersistLoopWait, 1) + + select { + case <-s.closeCh: + break OUTER + case <-w.notifyCh: + // woken up, next loop should pick up work + atomic.AddUint64(&s.stats.TotPersistLoopWaitNotified, 1) + case ew = <-s.persisterNotifier: + // if the watchers are already caught up then let them wait, + // else let them continue to do the catch up + persistWatchers = append(persistWatchers, ew) + } + + atomic.AddUint64(&s.stats.TotPersistLoopEnd, 1) + } +} + +func notifyMergeWatchers(lastPersistedEpoch uint64, + persistWatchers []*epochWatcher) []*epochWatcher { + var watchersNext []*epochWatcher + for _, w := range persistWatchers { + if w.epoch < lastPersistedEpoch { + close(w.notifyCh) + } else { + watchersNext = append(watchersNext, w) + } + } + return watchersNext +} + +func (s *Scorch) pausePersisterForMergerCatchUp(lastPersistedEpoch uint64, + lastMergedEpoch uint64, persistWatchers []*epochWatcher, + po *persisterOptions) (uint64, []*epochWatcher) { + + // First, let the watchers proceed if they lag behind + persistWatchers = notifyMergeWatchers(lastPersistedEpoch, persistWatchers) + + // Check the merger lag by counting the segment files on disk, + numFilesOnDisk, _, _ := s.diskFileStats(nil) + + // On finding fewer files on disk, persister takes a short pause + // for sufficient in-memory segments to pile up for the next + // memory merge cum persist loop. + if numFilesOnDisk < uint64(po.PersisterNapUnderNumFiles) && + po.PersisterNapTimeMSec > 0 && s.NumEventsBlocking() == 0 { + select { + case <-s.closeCh: + case <-time.After(time.Millisecond * time.Duration(po.PersisterNapTimeMSec)): + atomic.AddUint64(&s.stats.TotPersisterNapPauseCompleted, 1) + + case ew := <-s.persisterNotifier: + // unblock the merger in meantime + persistWatchers = append(persistWatchers, ew) + lastMergedEpoch = ew.epoch + persistWatchers = notifyMergeWatchers(lastPersistedEpoch, persistWatchers) + atomic.AddUint64(&s.stats.TotPersisterMergerNapBreak, 1) + } + return lastMergedEpoch, persistWatchers + } + + // Finding too many files on disk could be due to two reasons. + // 1. Too many older snapshots awaiting the clean up. + // 2. The merger could be lagging behind on merging the disk files. + if numFilesOnDisk > uint64(po.PersisterNapUnderNumFiles) { + s.removeOldData() + numFilesOnDisk, _, _ = s.diskFileStats(nil) + } + + // Persister pause until the merger catches up to reduce the segment + // file count under the threshold. + // But if there is memory pressure, then skip this sleep maneuvers. +OUTER: + for po.PersisterNapUnderNumFiles > 0 && + numFilesOnDisk >= uint64(po.PersisterNapUnderNumFiles) && + lastMergedEpoch < lastPersistedEpoch { + atomic.AddUint64(&s.stats.TotPersisterSlowMergerPause, 1) + + select { + case <-s.closeCh: + break OUTER + case ew := <-s.persisterNotifier: + persistWatchers = append(persistWatchers, ew) + lastMergedEpoch = ew.epoch + } + + atomic.AddUint64(&s.stats.TotPersisterSlowMergerResume, 1) + + // let the watchers proceed if they lag behind + persistWatchers = notifyMergeWatchers(lastPersistedEpoch, persistWatchers) + + numFilesOnDisk, _, _ = s.diskFileStats(nil) + } + + return lastMergedEpoch, persistWatchers +} + +func (s *Scorch) parsePersisterOptions() (*persisterOptions, error) { + po := persisterOptions{ + PersisterNapTimeMSec: DefaultPersisterNapTimeMSec, + PersisterNapUnderNumFiles: DefaultPersisterNapUnderNumFiles, + MemoryPressurePauseThreshold: DefaultMemoryPressurePauseThreshold, + } + if v, ok := s.config["scorchPersisterOptions"]; ok { + b, err := util.MarshalJSON(v) + if err != nil { + return &po, err + } + + err = util.UnmarshalJSON(b, &po) + if err != nil { + return &po, err + } + } + return &po, nil +} + +func (s *Scorch) persistSnapshot(snapshot *IndexSnapshot, + po *persisterOptions) error { + // Perform in-memory segment merging only when the memory pressure is + // below the configured threshold, else the persister performs the + // direct persistence of segments. + if s.NumEventsBlocking() < po.MemoryPressurePauseThreshold { + persisted, err := s.persistSnapshotMaybeMerge(snapshot) + if err != nil { + return err + } + if persisted { + return nil + } + } + + return s.persistSnapshotDirect(snapshot) +} + +// DefaultMinSegmentsForInMemoryMerge represents the default number of +// in-memory zap segments that persistSnapshotMaybeMerge() needs to +// see in an IndexSnapshot before it decides to merge and persist +// those segments +var DefaultMinSegmentsForInMemoryMerge = 2 + +// persistSnapshotMaybeMerge examines the snapshot and might merge and +// persist the in-memory zap segments if there are enough of them +func (s *Scorch) persistSnapshotMaybeMerge(snapshot *IndexSnapshot) ( + bool, error) { + // collect the in-memory zap segments (SegmentBase instances) + var sbs []segment.Segment + var sbsDrops []*roaring.Bitmap + var sbsIndexes []int + + for i, segmentSnapshot := range snapshot.segment { + if _, ok := segmentSnapshot.segment.(segment.PersistedSegment); !ok { + sbs = append(sbs, segmentSnapshot.segment) + sbsDrops = append(sbsDrops, segmentSnapshot.deleted) + sbsIndexes = append(sbsIndexes, i) + } + } + + if len(sbs) < DefaultMinSegmentsForInMemoryMerge { + return false, nil + } + + newSnapshot, newSegmentID, err := s.mergeSegmentBases( + snapshot, sbs, sbsDrops, sbsIndexes) + if err != nil { + return false, err + } + if newSnapshot == nil { + return false, nil + } + + defer func() { + _ = newSnapshot.DecRef() + }() + + mergedSegmentIDs := map[uint64]struct{}{} + for _, idx := range sbsIndexes { + mergedSegmentIDs[snapshot.segment[idx].id] = struct{}{} + } + + // construct a snapshot that's logically equivalent to the input + // snapshot, but with merged segments replaced by the new segment + equiv := &IndexSnapshot{ + parent: snapshot.parent, + segment: make([]*SegmentSnapshot, 0, len(snapshot.segment)), + internal: snapshot.internal, + epoch: snapshot.epoch, + creator: "persistSnapshotMaybeMerge", + } + + // copy to the equiv the segments that weren't replaced + for _, segment := range snapshot.segment { + if _, wasMerged := mergedSegmentIDs[segment.id]; !wasMerged { + equiv.segment = append(equiv.segment, segment) + } + } + + // append to the equiv the new segment + for _, segment := range newSnapshot.segment { + if segment.id == newSegmentID { + equiv.segment = append(equiv.segment, &SegmentSnapshot{ + id: newSegmentID, + segment: segment.segment, + deleted: nil, // nil since merging handled deletions + }) + break + } + } + + err = s.persistSnapshotDirect(equiv) + if err != nil { + return false, err + } + + return true, nil +} + +func copyToDirectory(srcPath string, d index.Directory) (int64, error) { + if d == nil { + return 0, nil + } + + dest, err := d.GetWriter(filepath.Join("store", filepath.Base(srcPath))) + if err != nil { + return 0, fmt.Errorf("GetWriter err: %v", err) + } + + sourceFileStat, err := os.Stat(srcPath) + if err != nil { + return 0, err + } + + if !sourceFileStat.Mode().IsRegular() { + return 0, fmt.Errorf("%s is not a regular file", srcPath) + } + + source, err := os.Open(srcPath) + if err != nil { + return 0, err + } + defer source.Close() + defer dest.Close() + return io.Copy(dest, source) +} + +func persistToDirectory(seg segment.UnpersistedSegment, d index.Directory, + path string) error { + if d == nil { + return seg.Persist(path) + } + + sg, ok := seg.(io.WriterTo) + if !ok { + return fmt.Errorf("no io.WriterTo segment implementation found") + } + + w, err := d.GetWriter(filepath.Join("store", filepath.Base(path))) + if err != nil { + return err + } + + _, err = sg.WriteTo(w) + w.Close() + + return err +} + +func prepareBoltSnapshot(snapshot *IndexSnapshot, tx *bolt.Tx, path string, + segPlugin SegmentPlugin, d index.Directory) ( + []string, map[uint64]string, error) { + snapshotsBucket, err := tx.CreateBucketIfNotExists(boltSnapshotsBucket) + if err != nil { + return nil, nil, err + } + newSnapshotKey := encodeUvarintAscending(nil, snapshot.epoch) + snapshotBucket, err := snapshotsBucket.CreateBucketIfNotExists(newSnapshotKey) + if err != nil { + return nil, nil, err + } + + // persist meta values + metaBucket, err := snapshotBucket.CreateBucketIfNotExists(boltMetaDataKey) + if err != nil { + return nil, nil, err + } + err = metaBucket.Put(boltMetaDataSegmentTypeKey, []byte(segPlugin.Type())) + if err != nil { + return nil, nil, err + } + buf := make([]byte, binary.MaxVarintLen32) + binary.BigEndian.PutUint32(buf, segPlugin.Version()) + err = metaBucket.Put(boltMetaDataSegmentVersionKey, buf) + if err != nil { + return nil, nil, err + } + + // Storing the timestamp at which the current indexSnapshot + // was persisted, useful when you want to spread the + // numSnapshotsToKeep reasonably better than consecutive + // epochs. + currTimeStamp := time.Now() + timeStampBinary, err := currTimeStamp.MarshalText() + if err != nil { + return nil, nil, err + } + err = metaBucket.Put(boltMetaDataTimeStamp, timeStampBinary) + if err != nil { + return nil, nil, err + } + + // persist internal values + internalBucket, err := snapshotBucket.CreateBucketIfNotExists(boltInternalKey) + if err != nil { + return nil, nil, err + } + // TODO optimize writing these in order? + for k, v := range snapshot.internal { + err = internalBucket.Put([]byte(k), v) + if err != nil { + return nil, nil, err + } + } + + if snapshot.parent != nil { + val := make([]byte, 8) + bytesWritten := atomic.LoadUint64(&snapshot.parent.stats.TotBytesWrittenAtIndexTime) + binary.LittleEndian.PutUint64(val, bytesWritten) + internalBucket.Put(TotBytesWrittenKey, val) + } + + var filenames []string + newSegmentPaths := make(map[uint64]string) + + // first ensure that each segment in this snapshot has been persisted + for _, segmentSnapshot := range snapshot.segment { + snapshotSegmentKey := encodeUvarintAscending(nil, segmentSnapshot.id) + snapshotSegmentBucket, err := snapshotBucket.CreateBucketIfNotExists(snapshotSegmentKey) + if err != nil { + return nil, nil, err + } + switch seg := segmentSnapshot.segment.(type) { + case segment.PersistedSegment: + segPath := seg.Path() + _, err = copyToDirectory(segPath, d) + if err != nil { + return nil, nil, fmt.Errorf("segment: %s copy err: %v", segPath, err) + } + filename := filepath.Base(segPath) + err = snapshotSegmentBucket.Put(boltPathKey, []byte(filename)) + if err != nil { + return nil, nil, err + } + filenames = append(filenames, filename) + case segment.UnpersistedSegment: + // need to persist this to disk + filename := zapFileName(segmentSnapshot.id) + path := filepath.Join(path, filename) + err := persistToDirectory(seg, d, path) + if err != nil { + return nil, nil, fmt.Errorf("segment: %s persist err: %v", path, err) + } + newSegmentPaths[segmentSnapshot.id] = path + err = snapshotSegmentBucket.Put(boltPathKey, []byte(filename)) + if err != nil { + return nil, nil, err + } + filenames = append(filenames, filename) + default: + return nil, nil, fmt.Errorf("unknown segment type: %T", seg) + } + // store current deleted bits + var roaringBuf bytes.Buffer + if segmentSnapshot.deleted != nil { + _, err = segmentSnapshot.deleted.WriteTo(&roaringBuf) + if err != nil { + return nil, nil, fmt.Errorf("error persisting roaring bytes: %v", err) + } + err = snapshotSegmentBucket.Put(boltDeletedKey, roaringBuf.Bytes()) + if err != nil { + return nil, nil, err + } + } + } + + return filenames, newSegmentPaths, nil +} + +func (s *Scorch) persistSnapshotDirect(snapshot *IndexSnapshot) (err error) { + // start a write transaction + tx, err := s.rootBolt.Begin(true) + if err != nil { + return err + } + // defer rollback on error + defer func() { + if err != nil { + _ = tx.Rollback() + } + }() + + filenames, newSegmentPaths, err := prepareBoltSnapshot(snapshot, tx, s.path, s.segPlugin, nil) + if err != nil { + return err + } + + // we need to swap in a new root only when we've persisted 1 or + // more segments -- whereby the new root would have 1-for-1 + // replacements of in-memory segments with file-based segments + // + // other cases like updates to internal values only, and/or when + // there are only deletions, are already covered and persisted by + // the newly populated boltdb snapshotBucket above + if len(newSegmentPaths) > 0 { + // now try to open all the new snapshots + newSegments := make(map[uint64]segment.Segment) + defer func() { + for _, s := range newSegments { + if s != nil { + // cleanup segments that were opened but not + // swapped into the new root + _ = s.Close() + } + } + }() + for segmentID, path := range newSegmentPaths { + newSegments[segmentID], err = s.segPlugin.Open(path) + if err != nil { + return fmt.Errorf("error opening new segment at %s, %v", path, err) + } + } + + persist := &persistIntroduction{ + persisted: newSegments, + applied: make(notificationChan), + } + + select { + case <-s.closeCh: + return segment.ErrClosed + case s.persists <- persist: + } + + select { + case <-s.closeCh: + return segment.ErrClosed + case <-persist.applied: + } + } + + err = tx.Commit() + if err != nil { + return err + } + + err = s.rootBolt.Sync() + if err != nil { + return err + } + + // allow files to become eligible for removal after commit, such + // as file segments from snapshots that came from the merger + s.rootLock.Lock() + for _, filename := range filenames { + delete(s.ineligibleForRemoval, filename) + } + s.rootLock.Unlock() + + return nil +} + +func zapFileName(epoch uint64) string { + return fmt.Sprintf("%012x.zap", epoch) +} + +// bolt snapshot code + +var boltSnapshotsBucket = []byte{'s'} +var boltPathKey = []byte{'p'} +var boltDeletedKey = []byte{'d'} +var boltInternalKey = []byte{'i'} +var boltMetaDataKey = []byte{'m'} +var boltMetaDataSegmentTypeKey = []byte("type") +var boltMetaDataSegmentVersionKey = []byte("version") +var boltMetaDataTimeStamp = []byte("timeStamp") +var TotBytesWrittenKey = []byte("TotBytesWritten") + +func (s *Scorch) loadFromBolt() error { + return s.rootBolt.View(func(tx *bolt.Tx) error { + snapshots := tx.Bucket(boltSnapshotsBucket) + if snapshots == nil { + return nil + } + foundRoot := false + c := snapshots.Cursor() + for k, _ := c.Last(); k != nil; k, _ = c.Prev() { + _, snapshotEpoch, err := decodeUvarintAscending(k) + if err != nil { + log.Printf("unable to parse segment epoch %x, continuing", k) + continue + } + if foundRoot { + s.AddEligibleForRemoval(snapshotEpoch) + continue + } + snapshot := snapshots.Bucket(k) + if snapshot == nil { + log.Printf("snapshot key, but bucket missing %x, continuing", k) + s.AddEligibleForRemoval(snapshotEpoch) + continue + } + indexSnapshot, err := s.loadSnapshot(snapshot) + if err != nil { + log.Printf("unable to load snapshot, %v, continuing", err) + s.AddEligibleForRemoval(snapshotEpoch) + continue + } + indexSnapshot.epoch = snapshotEpoch + // set the nextSegmentID + s.nextSegmentID, err = s.maxSegmentIDOnDisk() + if err != nil { + return err + } + s.nextSegmentID++ + s.rootLock.Lock() + s.nextSnapshotEpoch = snapshotEpoch + 1 + rootPrev := s.root + s.root = indexSnapshot + s.rootLock.Unlock() + + if rootPrev != nil { + _ = rootPrev.DecRef() + } + + foundRoot = true + } + return nil + }) +} + +// LoadSnapshot loads the segment with the specified epoch +// NOTE: this is currently ONLY intended to be used by the command-line tool +func (s *Scorch) LoadSnapshot(epoch uint64) (rv *IndexSnapshot, err error) { + err = s.rootBolt.View(func(tx *bolt.Tx) error { + snapshots := tx.Bucket(boltSnapshotsBucket) + if snapshots == nil { + return nil + } + snapshotKey := encodeUvarintAscending(nil, epoch) + snapshot := snapshots.Bucket(snapshotKey) + if snapshot == nil { + return fmt.Errorf("snapshot with epoch: %v - doesn't exist", epoch) + } + rv, err = s.loadSnapshot(snapshot) + return err + }) + if err != nil { + return nil, err + } + return rv, nil +} + +func (s *Scorch) loadSnapshot(snapshot *bolt.Bucket) (*IndexSnapshot, error) { + + rv := &IndexSnapshot{ + parent: s, + internal: make(map[string][]byte), + refs: 1, + creator: "loadSnapshot", + } + // first we look for the meta-data bucket, this will tell us + // which segment type/version was used for this snapshot + // all operations for this scorch will use this type/version + metaBucket := snapshot.Bucket(boltMetaDataKey) + if metaBucket == nil { + _ = rv.DecRef() + return nil, fmt.Errorf("meta-data bucket missing") + } + segmentType := string(metaBucket.Get(boltMetaDataSegmentTypeKey)) + segmentVersion := binary.BigEndian.Uint32( + metaBucket.Get(boltMetaDataSegmentVersionKey)) + err := s.loadSegmentPlugin(segmentType, segmentVersion) + if err != nil { + _ = rv.DecRef() + return nil, fmt.Errorf( + "unable to load correct segment wrapper: %v", err) + } + var running uint64 + c := snapshot.Cursor() + for k, _ := c.First(); k != nil; k, _ = c.Next() { + if k[0] == boltInternalKey[0] { + internalBucket := snapshot.Bucket(k) + err := internalBucket.ForEach(func(key []byte, val []byte) error { + copiedVal := append([]byte(nil), val...) + rv.internal[string(key)] = copiedVal + return nil + }) + if err != nil { + _ = rv.DecRef() + return nil, err + } + } else if k[0] != boltMetaDataKey[0] { + segmentBucket := snapshot.Bucket(k) + if segmentBucket == nil { + _ = rv.DecRef() + return nil, fmt.Errorf("segment key, but bucket missing % x", k) + } + segmentSnapshot, err := s.loadSegment(segmentBucket) + if err != nil { + _ = rv.DecRef() + return nil, fmt.Errorf("failed to load segment: %v", err) + } + _, segmentSnapshot.id, err = decodeUvarintAscending(k) + if err != nil { + _ = rv.DecRef() + return nil, fmt.Errorf("failed to decode segment id: %v", err) + } + rv.segment = append(rv.segment, segmentSnapshot) + rv.offsets = append(rv.offsets, running) + running += segmentSnapshot.segment.Count() + } + } + return rv, nil +} + +func (s *Scorch) loadSegment(segmentBucket *bolt.Bucket) (*SegmentSnapshot, error) { + pathBytes := segmentBucket.Get(boltPathKey) + if pathBytes == nil { + return nil, fmt.Errorf("segment path missing") + } + segmentPath := s.path + string(os.PathSeparator) + string(pathBytes) + segment, err := s.segPlugin.Open(segmentPath) + if err != nil { + return nil, fmt.Errorf("error opening bolt segment: %v", err) + } + + rv := &SegmentSnapshot{ + segment: segment, + cachedDocs: &cachedDocs{cache: nil}, + } + deletedBytes := segmentBucket.Get(boltDeletedKey) + if deletedBytes != nil { + deletedBitmap := roaring.NewBitmap() + r := bytes.NewReader(deletedBytes) + _, err := deletedBitmap.ReadFrom(r) + if err != nil { + _ = segment.Close() + return nil, fmt.Errorf("error reading deleted bytes: %v", err) + } + if !deletedBitmap.IsEmpty() { + rv.deleted = deletedBitmap + } + } + + return rv, nil +} + +func (s *Scorch) removeOldData() { + removed, err := s.removeOldBoltSnapshots() + if err != nil { + s.fireAsyncError(fmt.Errorf("got err removing old bolt snapshots: %v", err)) + } + atomic.AddUint64(&s.stats.TotSnapshotsRemovedFromMetaStore, uint64(removed)) + + err = s.removeOldZapFiles() + if err != nil { + s.fireAsyncError(fmt.Errorf("got err removing old zap files: %v", err)) + } +} + +// NumSnapshotsToKeep represents how many recent, old snapshots to +// keep around per Scorch instance. Useful for apps that require +// rollback'ability. +var NumSnapshotsToKeep = 1 + +// RollbackSamplingInterval controls how far back we are looking +// in the history to get the rollback points. +// For example, a value of 10 minutes ensures that the +// protected snapshots (NumSnapshotsToKeep = 3) are: +// +// the very latest snapshot(ie the current one), +// the snapshot that was persisted 10 minutes before the current one, +// the snapshot that was persisted 20 minutes before the current one +// +// By default however, the timeseries way of protecting snapshots is +// disabled, and we protect the latest three contiguous snapshots +var RollbackSamplingInterval = 0 * time.Minute + +// Controls what portion of the earlier rollback points to retain during +// a infrequent/sparse mutation scenario +var RollbackRetentionFactor = float64(0.5) + +func getTimeSeriesSnapshots(maxDataPoints int, interval time.Duration, + snapshots []*snapshotMetaData) (int, map[uint64]time.Time) { + if interval == 0 { + return len(snapshots), map[uint64]time.Time{} + } + // the map containing the time series snapshots, i.e the timeseries of snapshots + // each of which is separated by rollbackSamplingInterval + rv := make(map[uint64]time.Time) + // the last point in the "time series", i.e. the timeseries of snapshots + // each of which is separated by rollbackSamplingInterval + ptr := len(snapshots) - 1 + rv[snapshots[ptr].epoch] = snapshots[ptr].timeStamp + numSnapshotsProtected := 1 + + // traverse the list in reverse order, older timestamps to newer ones. + for i := ptr - 1; i >= 0; i-- { + // If we find a timeStamp which is the next datapoint in our + // timeseries of snapshots, and newer by RollbackSamplingInterval duration + // (comparison in terms of minutes), which is the interval of our time + // series. In this case, add the epoch rv + if snapshots[i].timeStamp.Sub(snapshots[ptr].timeStamp).Minutes() > + interval.Minutes() { + if _, ok := rv[snapshots[i+1].epoch]; !ok { + rv[snapshots[i+1].epoch] = snapshots[i+1].timeStamp + ptr = i + 1 + numSnapshotsProtected++ + } + } else if snapshots[i].timeStamp.Sub(snapshots[ptr].timeStamp).Minutes() == + interval.Minutes() { + if _, ok := rv[snapshots[i].epoch]; !ok { + rv[snapshots[i].epoch] = snapshots[i].timeStamp + ptr = i + numSnapshotsProtected++ + } + } + + if numSnapshotsProtected >= maxDataPoints { + break + } + } + return ptr, rv +} + +// getProtectedEpochs aims to fetch the epochs keep based on a timestamp basis. +// It tries to get NumSnapshotsToKeep snapshots, each of which are separated +// by a time duration of RollbackSamplingInterval. +func getProtectedSnapshots(rollbackSamplingInterval time.Duration, + numSnapshotsToKeep int, + persistedSnapshots []*snapshotMetaData) map[uint64]time.Time { + + lastPoint, protectedEpochs := getTimeSeriesSnapshots(numSnapshotsToKeep, + rollbackSamplingInterval, persistedSnapshots) + if len(protectedEpochs) < numSnapshotsToKeep { + numSnapshotsNeeded := numSnapshotsToKeep - len(protectedEpochs) + // we protected the contiguous snapshots from the last point in time series + for i := 0; i < numSnapshotsNeeded && i < lastPoint; i++ { + protectedEpochs[persistedSnapshots[i].epoch] = persistedSnapshots[i].timeStamp + } + } + + return protectedEpochs +} + +func newCheckPoints(snapshots map[uint64]time.Time) []*snapshotMetaData { + rv := make([]*snapshotMetaData, 0) + + keys := make([]uint64, 0, len(snapshots)) + for k := range snapshots { + keys = append(keys, k) + } + + sort.SliceStable(keys, func(i, j int) bool { + return snapshots[keys[i]].Sub(snapshots[keys[j]]) > 0 + }) + + for _, key := range keys { + rv = append(rv, &snapshotMetaData{ + epoch: key, + timeStamp: snapshots[key], + }) + } + + return rv +} + +// Removes enough snapshots from the rootBolt so that the +// s.eligibleForRemoval stays under the NumSnapshotsToKeep policy. +func (s *Scorch) removeOldBoltSnapshots() (numRemoved int, err error) { + persistedSnapshots, err := s.rootBoltSnapshotMetaData() + if err != nil { + return 0, err + } + + if len(persistedSnapshots) <= s.numSnapshotsToKeep { + // we need to keep everything + return 0, nil + } + + protectedSnapshots := getProtectedSnapshots(s.rollbackSamplingInterval, + s.numSnapshotsToKeep, persistedSnapshots) + + var epochsToRemove []uint64 + var newEligible []uint64 + s.rootLock.Lock() + for _, epoch := range s.eligibleForRemoval { + if _, ok := protectedSnapshots[epoch]; ok { + // protected + newEligible = append(newEligible, epoch) + } else { + epochsToRemove = append(epochsToRemove, epoch) + } + } + s.eligibleForRemoval = newEligible + s.rootLock.Unlock() + s.checkPoints = newCheckPoints(protectedSnapshots) + + if len(epochsToRemove) == 0 { + return 0, nil + } + + tx, err := s.rootBolt.Begin(true) + if err != nil { + return 0, err + } + defer func() { + if err == nil { + err = tx.Commit() + } else { + _ = tx.Rollback() + } + if err == nil { + err = s.rootBolt.Sync() + } + }() + + snapshots := tx.Bucket(boltSnapshotsBucket) + if snapshots == nil { + return 0, nil + } + + for _, epochToRemove := range epochsToRemove { + k := encodeUvarintAscending(nil, epochToRemove) + err = snapshots.DeleteBucket(k) + if err == bolt.ErrBucketNotFound { + err = nil + } + if err == nil { + numRemoved++ + } + } + + return numRemoved, err +} + +func (s *Scorch) maxSegmentIDOnDisk() (uint64, error) { + files, err := os.ReadDir(s.path) + if err != nil { + return 0, err + } + + var rv uint64 + for _, f := range files { + fname := f.Name() + if filepath.Ext(fname) == ".zap" { + prefix := strings.TrimSuffix(fname, ".zap") + id, err2 := strconv.ParseUint(prefix, 16, 64) + if err2 != nil { + return 0, err2 + } + if id > rv { + rv = id + } + } + } + return rv, err +} + +// Removes any *.zap files which aren't listed in the rootBolt. +func (s *Scorch) removeOldZapFiles() error { + liveFileNames, err := s.loadZapFileNames() + if err != nil { + return err + } + + files, err := os.ReadDir(s.path) + if err != nil { + return err + } + + s.rootLock.RLock() + + for _, f := range files { + fname := f.Name() + if filepath.Ext(fname) == ".zap" { + if _, exists := liveFileNames[fname]; !exists && !s.ineligibleForRemoval[fname] { + err := os.Remove(s.path + string(os.PathSeparator) + fname) + if err != nil { + log.Printf("got err removing file: %s, err: %v", fname, err) + } + } + } + } + + s.rootLock.RUnlock() + + return nil +} + +// In sparse mutation scenario, it can so happen that all protected +// snapshots are older than the numSnapshotsToKeep * rollbackSamplingInterval +// duration. This results in all of them being purged from the boltDB +// and the next iteration of the removeOldData() would end up protecting +// latest contiguous snapshot which is a poor pattern in the rollback checkpoints. +// Hence we try to retain atleast retentionFactor portion worth of old snapshots +// in such a scenario using the following function +func getBoundaryCheckPoint(retentionFactor float64, + checkPoints []*snapshotMetaData, timeStamp time.Time) time.Time { + if checkPoints != nil { + boundary := checkPoints[int(math.Floor(float64(len(checkPoints))* + retentionFactor))] + if timeStamp.Sub(boundary.timeStamp) < 0 { + // too less checkPoints would be left. + return boundary.timeStamp + } + } + return timeStamp +} + +type snapshotMetaData struct { + epoch uint64 + timeStamp time.Time +} + +func (s *Scorch) rootBoltSnapshotMetaData() ([]*snapshotMetaData, error) { + var rv []*snapshotMetaData + currTime := time.Now() + expirationDuration := time.Duration(s.numSnapshotsToKeep) * s.rollbackSamplingInterval + + err := s.rootBolt.View(func(tx *bolt.Tx) error { + snapshots := tx.Bucket(boltSnapshotsBucket) + if snapshots == nil { + return nil + } + sc := snapshots.Cursor() + var found bool + for sk, _ := sc.Last(); sk != nil; sk, _ = sc.Prev() { + _, snapshotEpoch, err := decodeUvarintAscending(sk) + if err != nil { + continue + } + + if expirationDuration == 0 { + rv = append(rv, &snapshotMetaData{ + epoch: snapshotEpoch, + }) + continue + } + + snapshot := snapshots.Bucket(sk) + metaBucket := snapshot.Bucket(boltMetaDataKey) + if metaBucket == nil { + continue + } + timeStampBytes := metaBucket.Get(boltMetaDataTimeStamp) + var timeStamp time.Time + err = timeStamp.UnmarshalText(timeStampBytes) + if err != nil { + continue + } + // Don't keep snapshots older than + // expiration duration (numSnapshotsToKeep * + // rollbackSamplingInterval, by default) + if currTime.Sub(timeStamp) <= expirationDuration { + rv = append(rv, &snapshotMetaData{ + epoch: snapshotEpoch, + timeStamp: timeStamp, + }) + } else { + if !found { + found = true + boundary := getBoundaryCheckPoint(s.rollbackRetentionFactor, + s.checkPoints, timeStamp) + expirationDuration = currTime.Sub(boundary) + continue + } + k := encodeUvarintAscending(nil, snapshotEpoch) + err = snapshots.DeleteBucket(k) + if err == bolt.ErrBucketNotFound { + err = nil + } + } + + } + return nil + }) + return rv, err +} + +func (s *Scorch) RootBoltSnapshotEpochs() ([]uint64, error) { + var rv []uint64 + err := s.rootBolt.View(func(tx *bolt.Tx) error { + snapshots := tx.Bucket(boltSnapshotsBucket) + if snapshots == nil { + return nil + } + sc := snapshots.Cursor() + for sk, _ := sc.Last(); sk != nil; sk, _ = sc.Prev() { + _, snapshotEpoch, err := decodeUvarintAscending(sk) + if err != nil { + continue + } + rv = append(rv, snapshotEpoch) + } + return nil + }) + return rv, err +} + +// Returns the *.zap file names that are listed in the rootBolt. +func (s *Scorch) loadZapFileNames() (map[string]struct{}, error) { + rv := map[string]struct{}{} + err := s.rootBolt.View(func(tx *bolt.Tx) error { + snapshots := tx.Bucket(boltSnapshotsBucket) + if snapshots == nil { + return nil + } + sc := snapshots.Cursor() + for sk, _ := sc.First(); sk != nil; sk, _ = sc.Next() { + snapshot := snapshots.Bucket(sk) + if snapshot == nil { + continue + } + segc := snapshot.Cursor() + for segk, _ := segc.First(); segk != nil; segk, _ = segc.Next() { + if segk[0] == boltInternalKey[0] { + continue + } + segmentBucket := snapshot.Bucket(segk) + if segmentBucket == nil { + continue + } + pathBytes := segmentBucket.Get(boltPathKey) + if pathBytes == nil { + continue + } + pathString := string(pathBytes) + rv[string(pathString)] = struct{}{} + } + } + return nil + }) + + return rv, err +} diff --git a/vendor/github.com/blevesearch/bleve/v2/index/scorch/regexp.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/regexp.go new file mode 100644 index 00000000..5a3584f5 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/regexp.go @@ -0,0 +1,63 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "regexp/syntax" + + "github.com/blevesearch/vellum/regexp" +) + +func parseRegexp(pattern string) (a *regexp.Regexp, prefixBeg, prefixEnd []byte, err error) { + // TODO: potential optimization where syntax.Regexp supports a Simplify() API? + + parsed, err := syntax.Parse(pattern, syntax.Perl) + if err != nil { + return nil, nil, nil, err + } + + re, err := regexp.NewParsedWithLimit(pattern, parsed, regexp.DefaultLimit) + if err != nil { + return nil, nil, nil, err + } + + prefix := literalPrefix(parsed) + if prefix != "" { + prefixBeg := []byte(prefix) + prefixEnd := calculateExclusiveEndFromPrefix(prefixBeg) + return re, prefixBeg, prefixEnd, nil + } + + return re, nil, nil, nil +} + +// Returns the literal prefix given the parse tree for a regexp +func literalPrefix(s *syntax.Regexp) string { + // traverse the left-most branch in the parse tree as long as the + // node represents a concatenation + for s != nil && s.Op == syntax.OpConcat { + if len(s.Sub) < 1 { + return "" + } + + s = s.Sub[0] + } + + if s.Op == syntax.OpLiteral && (s.Flags&syntax.FoldCase == 0) { + return string(s.Rune) + } + + return "" // no literal prefix +} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/rollback.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/rollback.go similarity index 95% rename from vendor/github.com/blevesearch/bleve/index/scorch/rollback.go rename to vendor/github.com/blevesearch/bleve/v2/index/scorch/rollback.go index 7cc87bde..067220e6 100644 --- a/vendor/github.com/blevesearch/bleve/index/scorch/rollback.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/rollback.go @@ -19,7 +19,6 @@ import ( "log" "os" - "github.com/blevesearch/bleve/index/scorch/segment" bolt "go.etcd.io/bbolt" ) @@ -71,7 +70,7 @@ func RollbackPoints(path string) ([]*RollbackPoint, error) { c1 := snapshots.Cursor() for k, _ := c1.Last(); k != nil; k, _ = c1.Prev() { - _, snapshotEpoch, err := segment.DecodeUvarintAscending(k) + _, snapshotEpoch, err := decodeUvarintAscending(k) if err != nil { log.Printf("RollbackPoints:"+ " unable to parse segment epoch %x, continuing", k) @@ -154,7 +153,7 @@ func Rollback(path string, to *RollbackPoint) error { } sc := snapshots.Cursor() for sk, _ := sc.Last(); sk != nil && !found; sk, _ = sc.Prev() { - _, snapshotEpoch, err := segment.DecodeUvarintAscending(sk) + _, snapshotEpoch, err := decodeUvarintAscending(sk) if err != nil { continue } @@ -195,7 +194,7 @@ func Rollback(path string, to *RollbackPoint) error { return nil } for _, epoch := range eligibleEpochs { - k := segment.EncodeUvarintAscending(nil, epoch) + k := encodeUvarintAscending(nil, epoch) if err != nil { continue } diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/scorch.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/scorch.go similarity index 76% rename from vendor/github.com/blevesearch/bleve/index/scorch/scorch.go rename to vendor/github.com/blevesearch/bleve/v2/index/scorch/scorch.go index fccff67a..f30d795e 100644 --- a/vendor/github.com/blevesearch/bleve/index/scorch/scorch.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/scorch.go @@ -17,19 +17,15 @@ package scorch import ( "encoding/json" "fmt" - "io/ioutil" "os" "sync" "sync/atomic" "time" "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/blevesearch/bleve/index/store" - "github.com/blevesearch/bleve/registry" + "github.com/blevesearch/bleve/v2/registry" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" bolt "go.etcd.io/bbolt" ) @@ -60,22 +56,37 @@ type Scorch struct { eligibleForRemoval []uint64 // Index snapshot epochs that are safe to GC. ineligibleForRemoval map[string]bool // Filenames that should not be GC'ed yet. - numSnapshotsToKeep int - closeCh chan struct{} - introductions chan *segmentIntroduction - persists chan *persistIntroduction - merges chan *segmentMerge - introducerNotifier chan *epochWatcher - persisterNotifier chan *epochWatcher - rootBolt *bolt.DB - asyncTasks sync.WaitGroup + numSnapshotsToKeep int + rollbackRetentionFactor float64 + checkPoints []*snapshotMetaData + rollbackSamplingInterval time.Duration + closeCh chan struct{} + introductions chan *segmentIntroduction + persists chan *persistIntroduction + merges chan *segmentMerge + introducerNotifier chan *epochWatcher + persisterNotifier chan *epochWatcher + rootBolt *bolt.DB + asyncTasks sync.WaitGroup onEvent func(event Event) - onAsyncError func(err error) + onAsyncError func(err error, path string) forceMergeRequestCh chan *mergerCtrl - segPlugin segment.Plugin + segPlugin SegmentPlugin + + spatialPlugin index.SpatialAnalyzerPlugin +} + +// AsyncPanicError is passed to scorch asyncErrorHandler when panic occurs in scorch background process +type AsyncPanicError struct { + Source string + Path string +} + +func (e *AsyncPanicError) Error() string { + return fmt.Sprintf("%s panic when processing %s", e.Source, e.Path) } type internalStats struct { @@ -115,6 +126,11 @@ func NewScorch(storeName string, } } + typ, ok := config["spatialPlugin"].(string) + if ok { + rv.loadSpatialAnalyzerPlugin(typ) + } + rv.root = &IndexSnapshot{parent: rv, refs: 1, creator: "NewScorch"} ro, ok := config["read_only"].(bool) if ok { @@ -132,6 +148,7 @@ func NewScorch(storeName string, if ok { rv.onAsyncError = RegistryAsyncErrorCallbacks[aecbName] } + return rv, nil } @@ -168,7 +185,7 @@ func (s *Scorch) fireEvent(kind EventKind, dur time.Duration) { func (s *Scorch) fireAsyncError(err error) { if s.onAsyncError != nil { - s.onAsyncError(err) + s.onAsyncError(err, s.path) } atomic.AddUint64(&s.stats.TotOnErrors, 1) } @@ -202,10 +219,17 @@ func (s *Scorch) openBolt() error { s.unsafeBatch = true } - var rootBoltOpt *bolt.Options + var rootBoltOpt = *bolt.DefaultOptions if s.readOnly { - rootBoltOpt = &bolt.Options{ - ReadOnly: true, + rootBoltOpt.ReadOnly = true + rootBoltOpt.OpenFile = func(path string, flag int, mode os.FileMode) (*os.File, error) { + // Bolt appends an O_CREATE flag regardless. + // See - https://github.com/etcd-io/bbolt/blob/v1.3.5/db.go#L210 + // Use os.O_RDONLY only if path exists (#1623) + if _, err := os.Stat(path); os.IsNotExist(err) { + return os.OpenFile(path, flag, mode) + } + return os.OpenFile(path, os.O_RDONLY, mode) } } else { if s.path != "" { @@ -216,10 +240,19 @@ func (s *Scorch) openBolt() error { } } + if boltTimeoutStr, ok := s.config["bolt_timeout"].(string); ok { + var err error + boltTimeout, err := time.ParseDuration(boltTimeoutStr) + if err != nil { + return fmt.Errorf("invalid duration specified for bolt_timeout: %v", err) + } + rootBoltOpt.Timeout = boltTimeout + } + rootBoltPath := s.path + string(os.PathSeparator) + "root.bolt" var err error if s.path != "" { - s.rootBolt, err = bolt.Open(rootBoltPath, 0600, rootBoltOpt) + s.rootBolt, err = bolt.Open(rootBoltPath, 0600, &rootBoltOpt) if err != nil { return err } @@ -261,6 +294,29 @@ func (s *Scorch) openBolt() error { } } + s.rollbackSamplingInterval = RollbackSamplingInterval + if v, ok := s.config["rollbackSamplingInterval"]; ok { + var t time.Duration + if t, err = parseToTimeDuration(v); err != nil { + return fmt.Errorf("rollbackSamplingInterval parse err: %v", err) + } + s.rollbackSamplingInterval = t + } + + s.rollbackRetentionFactor = RollbackRetentionFactor + if v, ok := s.config["rollbackRetentionFactor"]; ok { + var r float64 + if r, ok = v.(float64); ok { + return fmt.Errorf("rollbackRetentionFactor parse err: %v", err) + } + s.rollbackRetentionFactor = r + } + + typ, ok := s.config["spatialPlugin"].(string) + if ok { + s.loadSpatialAnalyzerPlugin(typ) + } + return nil } @@ -293,7 +349,7 @@ func (s *Scorch) Close() (err error) { return } -func (s *Scorch) Update(doc *document.Document) error { +func (s *Scorch) Update(doc index.Document) error { b := index.NewBatch() b.Update(doc) return s.Batch(b) @@ -313,7 +369,7 @@ func (s *Scorch) Batch(batch *index.Batch) (err error) { s.fireEvent(EventKindBatchIntroduction, time.Since(start)) }() - resultChan := make(chan *index.AnalysisResult, len(batch.IndexOps)) + resultChan := make(chan index.Document, len(batch.IndexOps)) var numUpdates uint64 var numDeletes uint64 @@ -322,7 +378,7 @@ func (s *Scorch) Batch(batch *index.Batch) (err error) { for docID, doc := range batch.IndexOps { if doc != nil { // insert _id field - doc.AddField(document.NewTextFieldCustom("_id", nil, []byte(doc.ID), document.IndexField|document.StoreField, nil)) + doc.AddIDField() numUpdates++ numPlainTextBytes += doc.NumPlainTextBytes() } else { @@ -335,18 +391,21 @@ func (s *Scorch) Batch(batch *index.Batch) (err error) { if numUpdates > 0 { go func() { - for _, doc := range batch.IndexOps { + for k := range batch.IndexOps { + doc := batch.IndexOps[k] if doc != nil { - aw := index.NewAnalysisWork(s, doc, resultChan) // put the work on the queue - s.analysisQueue.Queue(aw) + s.analysisQueue.Queue(func() { + analyze(doc, s.setSpatialAnalyzerPlugin) + resultChan <- doc + }) } } }() } // wait for analysis result - analysisResults := make([]*index.AnalysisResult, int(numUpdates)) + analysisResults := make([]index.Document, int(numUpdates)) var itemsDeQueued uint64 var totalAnalysisSize int for itemsDeQueued < numUpdates { @@ -374,6 +433,10 @@ func (s *Scorch) Batch(batch *index.Batch) (err error) { if err != nil { return err } + if segB, ok := newSegment.(segment.DiskStatsReporter); ok { + atomic.AddUint64(&s.stats.TotBytesWrittenAtIndexTime, + segB.BytesWritten()) + } atomic.AddUint64(&s.iStats.newSegBufBytesAdded, bufBytes) } else { atomic.AddUint64(&s.stats.TotBatchesEmpty, 1) @@ -487,20 +550,26 @@ func (s *Scorch) Stats() json.Marshaler { return &s.stats } +func (s *Scorch) BytesReadQueryTime() uint64 { + return s.stats.TotBytesReadAtQueryTime +} + func (s *Scorch) diskFileStats(rootSegmentPaths map[string]struct{}) (uint64, uint64, uint64) { var numFilesOnDisk, numBytesUsedDisk, numBytesOnDiskByRoot uint64 if s.path != "" { - finfos, err := ioutil.ReadDir(s.path) + files, err := os.ReadDir(s.path) if err == nil { - for _, finfo := range finfos { - if !finfo.IsDir() { - numBytesUsedDisk += uint64(finfo.Size()) - numFilesOnDisk++ - if rootSegmentPaths != nil { - fname := s.path + string(os.PathSeparator) + finfo.Name() - if _, fileAtRoot := rootSegmentPaths[fname]; fileAtRoot { - numBytesOnDiskByRoot += uint64(finfo.Size()) + for _, f := range files { + if !f.IsDir() { + if finfo, err := f.Info(); err == nil { + numBytesUsedDisk += uint64(finfo.Size()) + numFilesOnDisk++ + if rootSegmentPaths != nil { + fname := s.path + string(os.PathSeparator) + finfo.Name() + if _, fileAtRoot := rootSegmentPaths[fname]; fileAtRoot { + numBytesOnDiskByRoot += uint64(finfo.Size()) + } } } } @@ -519,6 +588,10 @@ func (s *Scorch) StatsMap() map[string]interface{} { m := s.stats.ToMap() indexSnapshot := s.currentSnapshot() + if indexSnapshot == nil { + return nil + } + defer func() { _ = indexSnapshot.Close() }() @@ -544,7 +617,9 @@ func (s *Scorch) StatsMap() map[string]interface{} { m["index_time"] = m["TotIndexTime"] m["term_searchers_started"] = m["TotTermSearchersStarted"] m["term_searchers_finished"] = m["TotTermSearchersFinished"] + m["num_bytes_read_at_query_time"] = m["TotBytesReadAtQueryTime"] m["num_plain_text_bytes_indexed"] = m["TotIndexedPlainTextBytes"] + m["num_bytes_written_at_index_time"] = m["TotBytesWrittenAtIndexTime"] m["num_items_introduced"] = m["TotIntroducedItems"] m["num_items_persisted"] = m["TotPersistedItems"] m["num_recs_to_persist"] = m["TotItemsToPersist"] @@ -566,37 +641,40 @@ func (s *Scorch) StatsMap() map[string]interface{} { return m } -func (s *Scorch) Analyze(d *document.Document) *index.AnalysisResult { - return analyze(d) +func (s *Scorch) Analyze(d index.Document) { + analyze(d, s.setSpatialAnalyzerPlugin) } -func analyze(d *document.Document) *index.AnalysisResult { - rv := &index.AnalysisResult{ - Document: d, - Analyzed: make([]analysis.TokenFrequencies, len(d.Fields)+len(d.CompositeFields)), - Length: make([]int, len(d.Fields)+len(d.CompositeFields)), +type customAnalyzerPluginInitFunc func(field index.Field) + +func (s *Scorch) setSpatialAnalyzerPlugin(f index.Field) { + if s.segPlugin != nil { + // check whether the current field is a custom tokenizable + // spatial field then set the spatial analyser plugin for + // overriding the tokenisation during the analysis stage. + if sf, ok := f.(index.TokenizableSpatialField); ok { + sf.SetSpatialAnalyzerPlugin(s.spatialPlugin) + } } +} - for i, field := range d.Fields { +func analyze(d index.Document, fn customAnalyzerPluginInitFunc) { + d.VisitFields(func(field index.Field) { if field.Options().IsIndexed() { - fieldLength, tokenFreqs := field.Analyze() - rv.Analyzed[i] = tokenFreqs - rv.Length[i] = fieldLength + if fn != nil { + fn(field) + } + + field.Analyze() - if len(d.CompositeFields) > 0 && field.Name() != "_id" { + if d.HasComposite() && field.Name() != "_id" { // see if any of the composite fields need this - for _, compositeField := range d.CompositeFields { - compositeField.Compose(field.Name(), fieldLength, tokenFreqs) - } + d.VisitComposite(func(cf index.CompositeField) { + cf.Compose(field.Name(), field.AnalyzedLength(), field.AnalyzedTokenFrequencies()) + }) } } - } - - return rv -} - -func (s *Scorch) Advanced() (store.KVStore, error) { - return nil, nil + }) } func (s *Scorch) AddEligibleForRemoval(epoch uint64) { @@ -663,6 +741,16 @@ func init() { registry.RegisterIndexType(Name, NewScorch) } +func parseToTimeDuration(i interface{}) (time.Duration, error) { + switch v := i.(type) { + case string: + return time.ParseDuration(v) + + default: + return 0, fmt.Errorf("expects a duration string") + } +} + func parseToInteger(i interface{}) (int, error) { switch v := i.(type) { case float64: diff --git a/vendor/github.com/blevesearch/bleve/v2/index/scorch/segment_plugin.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/segment_plugin.go new file mode 100644 index 00000000..a84d2d55 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/segment_plugin.go @@ -0,0 +1,143 @@ +// Copyright (c) 2019 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "fmt" + + "github.com/RoaringBitmap/roaring" + index "github.com/blevesearch/bleve_index_api" + + "github.com/blevesearch/bleve/v2/geo" + segment "github.com/blevesearch/scorch_segment_api/v2" + + zapv11 "github.com/blevesearch/zapx/v11" + zapv12 "github.com/blevesearch/zapx/v12" + zapv13 "github.com/blevesearch/zapx/v13" + zapv14 "github.com/blevesearch/zapx/v14" + zapv15 "github.com/blevesearch/zapx/v15" +) + +// SegmentPlugin represents the essential functions required by a package to plug in +// it's segment implementation +type SegmentPlugin interface { + + // Type is the name for this segment plugin + Type() string + + // Version is a numeric value identifying a specific version of this type. + // When incompatible changes are made to a particular type of plugin, the + // version must be incremented. + Version() uint32 + + // New takes a set of Documents and turns them into a new Segment + New(results []index.Document) (segment.Segment, uint64, error) + + // Open attempts to open the file at the specified path and + // return the corresponding Segment + Open(path string) (segment.Segment, error) + + // Merge takes a set of Segments, and creates a new segment on disk at + // the specified path. + // Drops is a set of bitmaps (one for each segment) indicating which + // documents can be dropped from the segments during the merge. + // If the closeCh channel is closed, Merge will cease doing work at + // the next opportunity, and return an error (closed). + // StatsReporter can optionally be provided, in which case progress + // made during the merge is reported while operation continues. + // Returns: + // A slice of new document numbers (one for each input segment), + // this allows the caller to know a particular document's new + // document number in the newly merged segment. + // The number of bytes written to the new segment file. + // An error, if any occurred. + Merge(segments []segment.Segment, drops []*roaring.Bitmap, path string, + closeCh chan struct{}, s segment.StatsReporter) ( + [][]uint64, uint64, error) +} + +var supportedSegmentPlugins map[string]map[uint32]SegmentPlugin +var defaultSegmentPlugin SegmentPlugin + +func init() { + ResetSegmentPlugins() + RegisterSegmentPlugin(&zapv15.ZapPlugin{}, true) + RegisterSegmentPlugin(&zapv14.ZapPlugin{}, false) + RegisterSegmentPlugin(&zapv13.ZapPlugin{}, false) + RegisterSegmentPlugin(&zapv12.ZapPlugin{}, false) + RegisterSegmentPlugin(&zapv11.ZapPlugin{}, false) +} + +func ResetSegmentPlugins() { + supportedSegmentPlugins = map[string]map[uint32]SegmentPlugin{} +} + +func RegisterSegmentPlugin(plugin SegmentPlugin, makeDefault bool) { + if _, ok := supportedSegmentPlugins[plugin.Type()]; !ok { + supportedSegmentPlugins[plugin.Type()] = map[uint32]SegmentPlugin{} + } + supportedSegmentPlugins[plugin.Type()][plugin.Version()] = plugin + if makeDefault { + defaultSegmentPlugin = plugin + } +} + +func SupportedSegmentTypes() (rv []string) { + for k := range supportedSegmentPlugins { + rv = append(rv, k) + } + return +} + +func SupportedSegmentTypeVersions(typ string) (rv []uint32) { + for k := range supportedSegmentPlugins[typ] { + rv = append(rv, k) + } + return rv +} + +func chooseSegmentPlugin(forcedSegmentType string, + forcedSegmentVersion uint32) (SegmentPlugin, error) { + if versions, ok := supportedSegmentPlugins[forcedSegmentType]; ok { + if segPlugin, ok := versions[uint32(forcedSegmentVersion)]; ok { + return segPlugin, nil + } + return nil, fmt.Errorf( + "unsupported version %d for segment type: %s, supported: %v", + forcedSegmentVersion, forcedSegmentType, + SupportedSegmentTypeVersions(forcedSegmentType)) + } + return nil, fmt.Errorf("unsupported segment type: %s, supported: %v", + forcedSegmentType, SupportedSegmentTypes()) +} + +func (s *Scorch) loadSegmentPlugin(forcedSegmentType string, + forcedSegmentVersion uint32) error { + segPlugin, err := chooseSegmentPlugin(forcedSegmentType, + forcedSegmentVersion) + if err != nil { + return err + } + s.segPlugin = segPlugin + return nil +} + +func (s *Scorch) loadSpatialAnalyzerPlugin(typ string) error { + s.spatialPlugin = geo.GetSpatialAnalyzerPlugin(typ) + if s.spatialPlugin == nil { + return fmt.Errorf("unsupported spatial plugin type: %s", typ) + } + return nil +} diff --git a/vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_index.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_index.go new file mode 100644 index 00000000..59828e87 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_index.go @@ -0,0 +1,907 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "container/heap" + "context" + "encoding/binary" + "fmt" + "os" + "path/filepath" + "reflect" + "sort" + "sync" + "sync/atomic" + + "github.com/RoaringBitmap/roaring" + "github.com/blevesearch/bleve/v2/document" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" + "github.com/blevesearch/vellum" + lev "github.com/blevesearch/vellum/levenshtein" + bolt "go.etcd.io/bbolt" +) + +// re usable, threadsafe levenshtein builders +var lb1, lb2 *lev.LevenshteinAutomatonBuilder + +type asynchSegmentResult struct { + dict segment.TermDictionary + dictItr segment.DictionaryIterator + + index int + docs *roaring.Bitmap + + postings segment.PostingsList + + err error +} + +var reflectStaticSizeIndexSnapshot int + +// DefaultFieldTFRCacheThreshold limits the number of TermFieldReaders(TFR) for +// a field in an index snapshot. Without this limit, when recycling TFRs, it is +// possible that a very large number of TFRs may be added to the recycle +// cache, which could eventually lead to significant memory consumption. +// This threshold can be overwritten by users at the library level by changing the +// exported variable, or at the index level by setting the FieldTFRCacheThreshold +// in the kvConfig. +var DefaultFieldTFRCacheThreshold uint64 = 10 + +func init() { + var is interface{} = IndexSnapshot{} + reflectStaticSizeIndexSnapshot = int(reflect.TypeOf(is).Size()) + var err error + lb1, err = lev.NewLevenshteinAutomatonBuilder(1, true) + if err != nil { + panic(fmt.Errorf("Levenshtein automaton ed1 builder err: %v", err)) + } + lb2, err = lev.NewLevenshteinAutomatonBuilder(2, true) + if err != nil { + panic(fmt.Errorf("Levenshtein automaton ed2 builder err: %v", err)) + } +} + +type IndexSnapshot struct { + parent *Scorch + segment []*SegmentSnapshot + offsets []uint64 + internal map[string][]byte + epoch uint64 + size uint64 + creator string + + m sync.Mutex // Protects the fields that follow. + refs int64 + + m2 sync.Mutex // Protects the fields that follow. + fieldTFRs map[string][]*IndexSnapshotTermFieldReader // keyed by field, recycled TFR's +} + +func (i *IndexSnapshot) Segments() []*SegmentSnapshot { + return i.segment +} + +func (i *IndexSnapshot) Internal() map[string][]byte { + return i.internal +} + +func (i *IndexSnapshot) AddRef() { + i.m.Lock() + i.refs++ + i.m.Unlock() +} + +func (i *IndexSnapshot) DecRef() (err error) { + i.m.Lock() + i.refs-- + if i.refs == 0 { + for _, s := range i.segment { + if s != nil { + err2 := s.segment.DecRef() + if err == nil { + err = err2 + } + } + } + if i.parent != nil { + go i.parent.AddEligibleForRemoval(i.epoch) + } + } + i.m.Unlock() + return err +} + +func (i *IndexSnapshot) Close() error { + return i.DecRef() +} + +func (i *IndexSnapshot) Size() int { + return int(i.size) +} + +func (i *IndexSnapshot) updateSize() { + i.size += uint64(reflectStaticSizeIndexSnapshot) + for _, s := range i.segment { + i.size += uint64(s.Size()) + } +} + +func (is *IndexSnapshot) newIndexSnapshotFieldDict(field string, + makeItr func(i segment.TermDictionary) segment.DictionaryIterator, + randomLookup bool) (*IndexSnapshotFieldDict, error) { + + results := make(chan *asynchSegmentResult) + var totalBytesRead uint64 + for _, s := range is.segment { + go func(s *SegmentSnapshot) { + dict, err := s.segment.Dictionary(field) + if err != nil { + results <- &asynchSegmentResult{err: err} + } else { + if dictStats, ok := dict.(segment.DiskStatsReporter); ok { + atomic.AddUint64(&totalBytesRead, dictStats.BytesRead()) + } + if randomLookup { + results <- &asynchSegmentResult{dict: dict} + } else { + results <- &asynchSegmentResult{dictItr: makeItr(dict)} + } + } + }(s) + } + + var err error + rv := &IndexSnapshotFieldDict{ + snapshot: is, + cursors: make([]*segmentDictCursor, 0, len(is.segment)), + } + for count := 0; count < len(is.segment); count++ { + asr := <-results + if asr.err != nil && err == nil { + err = asr.err + } else { + if !randomLookup { + next, err2 := asr.dictItr.Next() + if err2 != nil && err == nil { + err = err2 + } + if next != nil { + rv.cursors = append(rv.cursors, &segmentDictCursor{ + itr: asr.dictItr, + curr: *next, + }) + } + } else { + rv.cursors = append(rv.cursors, &segmentDictCursor{ + dict: asr.dict, + }) + } + } + } + rv.bytesRead = totalBytesRead + // after ensuring we've read all items on channel + if err != nil { + return nil, err + } + + if !randomLookup { + // prepare heap + heap.Init(rv) + } + + return rv, nil +} + +func (is *IndexSnapshot) FieldDict(field string) (index.FieldDict, error) { + return is.newIndexSnapshotFieldDict(field, func(is segment.TermDictionary) segment.DictionaryIterator { + return is.AutomatonIterator(nil, nil, nil) + }, false) +} + +// calculateExclusiveEndFromInclusiveEnd produces the next key +// when sorting using memcmp style comparisons, suitable to +// use as the end key in a traditional (inclusive, exclusive] +// start/end range +func calculateExclusiveEndFromInclusiveEnd(inclusiveEnd []byte) []byte { + rv := inclusiveEnd + if len(inclusiveEnd) > 0 { + rv = make([]byte, len(inclusiveEnd)) + copy(rv, inclusiveEnd) + if rv[len(rv)-1] < 0xff { + // last byte can be incremented by one + rv[len(rv)-1]++ + } else { + // last byte is already 0xff, so append 0 + // next key is simply one byte longer + rv = append(rv, 0x0) + } + } + return rv +} + +func (is *IndexSnapshot) FieldDictRange(field string, startTerm []byte, + endTerm []byte) (index.FieldDict, error) { + return is.newIndexSnapshotFieldDict(field, func(is segment.TermDictionary) segment.DictionaryIterator { + endTermExclusive := calculateExclusiveEndFromInclusiveEnd(endTerm) + return is.AutomatonIterator(nil, startTerm, endTermExclusive) + }, false) +} + +// calculateExclusiveEndFromPrefix produces the first key that +// does not have the same prefix as the input bytes, suitable +// to use as the end key in a traditional (inclusive, exclusive] +// start/end range +func calculateExclusiveEndFromPrefix(in []byte) []byte { + rv := make([]byte, len(in)) + copy(rv, in) + for i := len(rv) - 1; i >= 0; i-- { + rv[i] = rv[i] + 1 + if rv[i] != 0 { + return rv // didn't overflow, so stop + } + } + // all bytes were 0xff, so return nil + // as there is no end key for this prefix + return nil +} + +func (is *IndexSnapshot) FieldDictPrefix(field string, + termPrefix []byte) (index.FieldDict, error) { + termPrefixEnd := calculateExclusiveEndFromPrefix(termPrefix) + return is.newIndexSnapshotFieldDict(field, func(is segment.TermDictionary) segment.DictionaryIterator { + return is.AutomatonIterator(nil, termPrefix, termPrefixEnd) + }, false) +} + +func (is *IndexSnapshot) FieldDictRegexp(field string, + termRegex string) (index.FieldDict, error) { + // TODO: potential optimization where the literal prefix represents the, + // entire regexp, allowing us to use PrefixIterator(prefixTerm)? + + a, prefixBeg, prefixEnd, err := parseRegexp(termRegex) + if err != nil { + return nil, err + } + + return is.newIndexSnapshotFieldDict(field, func(is segment.TermDictionary) segment.DictionaryIterator { + return is.AutomatonIterator(a, prefixBeg, prefixEnd) + }, false) +} + +func (is *IndexSnapshot) getLevAutomaton(term string, + fuzziness uint8) (vellum.Automaton, error) { + if fuzziness == 1 { + return lb1.BuildDfa(term, fuzziness) + } else if fuzziness == 2 { + return lb2.BuildDfa(term, fuzziness) + } + return nil, fmt.Errorf("fuzziness exceeds the max limit") +} + +func (is *IndexSnapshot) FieldDictFuzzy(field string, + term string, fuzziness int, prefix string) (index.FieldDict, error) { + a, err := is.getLevAutomaton(term, uint8(fuzziness)) + if err != nil { + return nil, err + } + + var prefixBeg, prefixEnd []byte + if prefix != "" { + prefixBeg = []byte(prefix) + prefixEnd = calculateExclusiveEndFromPrefix(prefixBeg) + } + + return is.newIndexSnapshotFieldDict(field, func(is segment.TermDictionary) segment.DictionaryIterator { + return is.AutomatonIterator(a, prefixBeg, prefixEnd) + }, false) +} + +func (is *IndexSnapshot) FieldDictContains(field string) (index.FieldDictContains, error) { + return is.newIndexSnapshotFieldDict(field, nil, true) +} + +func (is *IndexSnapshot) DocIDReaderAll() (index.DocIDReader, error) { + results := make(chan *asynchSegmentResult) + for index, segment := range is.segment { + go func(index int, segment *SegmentSnapshot) { + results <- &asynchSegmentResult{ + index: index, + docs: segment.DocNumbersLive(), + } + }(index, segment) + } + + return is.newDocIDReader(results) +} + +func (is *IndexSnapshot) DocIDReaderOnly(ids []string) (index.DocIDReader, error) { + results := make(chan *asynchSegmentResult) + for index, segment := range is.segment { + go func(index int, segment *SegmentSnapshot) { + docs, err := segment.DocNumbers(ids) + if err != nil { + results <- &asynchSegmentResult{err: err} + } else { + results <- &asynchSegmentResult{ + index: index, + docs: docs, + } + } + }(index, segment) + } + + return is.newDocIDReader(results) +} + +func (is *IndexSnapshot) newDocIDReader(results chan *asynchSegmentResult) (index.DocIDReader, error) { + rv := &IndexSnapshotDocIDReader{ + snapshot: is, + iterators: make([]roaring.IntIterable, len(is.segment)), + } + var err error + for count := 0; count < len(is.segment); count++ { + asr := <-results + if asr.err != nil { + if err == nil { + // returns the first error encountered + err = asr.err + } + } else if err == nil { + rv.iterators[asr.index] = asr.docs.Iterator() + } + } + + if err != nil { + return nil, err + } + + return rv, nil +} + +func (is *IndexSnapshot) Fields() ([]string, error) { + // FIXME not making this concurrent for now as it's not used in hot path + // of any searches at the moment (just a debug aid) + fieldsMap := map[string]struct{}{} + for _, segment := range is.segment { + fields := segment.Fields() + for _, field := range fields { + fieldsMap[field] = struct{}{} + } + } + rv := make([]string, 0, len(fieldsMap)) + for k := range fieldsMap { + rv = append(rv, k) + } + return rv, nil +} + +func (is *IndexSnapshot) GetInternal(key []byte) ([]byte, error) { + return is.internal[string(key)], nil +} + +func (is *IndexSnapshot) DocCount() (uint64, error) { + var rv uint64 + for _, segment := range is.segment { + rv += segment.Count() + } + return rv, nil +} + +func (is *IndexSnapshot) Document(id string) (rv index.Document, err error) { + // FIXME could be done more efficiently directly, but reusing for simplicity + tfr, err := is.TermFieldReader(nil, []byte(id), "_id", false, false, false) + if err != nil { + return nil, err + } + defer func() { + if cerr := tfr.Close(); err == nil && cerr != nil { + err = cerr + } + }() + + next, err := tfr.Next(nil) + if err != nil { + return nil, err + } + + if next == nil { + // no such doc exists + return nil, nil + } + + docNum, err := docInternalToNumber(next.ID) + if err != nil { + return nil, err + } + segmentIndex, localDocNum := is.segmentIndexAndLocalDocNumFromGlobal(docNum) + + rvd := document.NewDocument(id) + + err = is.segment[segmentIndex].VisitDocument(localDocNum, func(name string, typ byte, val []byte, pos []uint64) bool { + if name == "_id" { + return true + } + + // track uncompressed stored fields bytes as part of IO stats. + // However, ideally we'd need to track the compressed on-disk value + // Keeping that TODO for now until we have a cleaner way. + rvd.StoredFieldsSize += uint64(len(val)) + + // copy value, array positions to preserve them beyond the scope of this callback + value := append([]byte(nil), val...) + arrayPos := append([]uint64(nil), pos...) + + switch typ { + case 't': + rvd.AddField(document.NewTextField(name, arrayPos, value)) + case 'n': + rvd.AddField(document.NewNumericFieldFromBytes(name, arrayPos, value)) + case 'i': + rvd.AddField(document.NewIPFieldFromBytes(name, arrayPos, value)) + case 'd': + rvd.AddField(document.NewDateTimeFieldFromBytes(name, arrayPos, value)) + case 'b': + rvd.AddField(document.NewBooleanFieldFromBytes(name, arrayPos, value)) + case 'g': + rvd.AddField(document.NewGeoPointFieldFromBytes(name, arrayPos, value)) + case 's': + rvd.AddField(document.NewGeoShapeFieldFromBytes(name, arrayPos, value)) + } + + return true + }) + if err != nil { + return nil, err + } + + return rvd, nil +} + +func (is *IndexSnapshot) segmentIndexAndLocalDocNumFromGlobal(docNum uint64) (int, uint64) { + segmentIndex := sort.Search(len(is.offsets), + func(x int) bool { + return is.offsets[x] > docNum + }) - 1 + + localDocNum := docNum - is.offsets[segmentIndex] + return int(segmentIndex), localDocNum +} + +func (is *IndexSnapshot) ExternalID(id index.IndexInternalID) (string, error) { + docNum, err := docInternalToNumber(id) + if err != nil { + return "", err + } + segmentIndex, localDocNum := is.segmentIndexAndLocalDocNumFromGlobal(docNum) + + v, err := is.segment[segmentIndex].DocID(localDocNum) + if err != nil { + return "", err + } + if v == nil { + return "", fmt.Errorf("document number %d not found", docNum) + } + + return string(v), nil +} + +func (is *IndexSnapshot) InternalID(id string) (rv index.IndexInternalID, err error) { + // FIXME could be done more efficiently directly, but reusing for simplicity + tfr, err := is.TermFieldReader(nil, []byte(id), "_id", false, false, false) + if err != nil { + return nil, err + } + defer func() { + if cerr := tfr.Close(); err == nil && cerr != nil { + err = cerr + } + }() + + next, err := tfr.Next(nil) + if err != nil || next == nil { + return nil, err + } + + return next.ID, nil +} + +func (is *IndexSnapshot) TermFieldReader(ctx context.Context, term []byte, field string, includeFreq, + includeNorm, includeTermVectors bool) (index.TermFieldReader, error) { + rv := is.allocTermFieldReaderDicts(field) + + rv.ctx = ctx + rv.term = term + rv.field = field + rv.snapshot = is + if rv.postings == nil { + rv.postings = make([]segment.PostingsList, len(is.segment)) + } + if rv.iterators == nil { + rv.iterators = make([]segment.PostingsIterator, len(is.segment)) + } + rv.segmentOffset = 0 + rv.includeFreq = includeFreq + rv.includeNorm = includeNorm + rv.includeTermVectors = includeTermVectors + rv.currPosting = nil + rv.currID = rv.currID[:0] + + if rv.dicts == nil { + rv.dicts = make([]segment.TermDictionary, len(is.segment)) + for i, s := range is.segment { + // the intention behind this compare and swap operation is + // to make sure that the accounting of the metadata is happening + // only once(which corresponds to this persisted segment's most + // recent segPlugin.Open() call), and any subsequent queries won't + // incur this cost which would essentially be a double counting. + if atomic.CompareAndSwapUint32(&s.mmaped, 1, 0) { + segBytesRead := s.segment.BytesRead() + rv.incrementBytesRead(segBytesRead) + } + dict, err := s.segment.Dictionary(field) + if err != nil { + return nil, err + } + if dictStats, ok := dict.(segment.DiskStatsReporter); ok { + bytesRead := dictStats.BytesRead() + rv.incrementBytesRead(bytesRead) + } + rv.dicts[i] = dict + } + } + + for i, s := range is.segment { + var prevBytesReadPL uint64 + if rv.postings[i] != nil { + prevBytesReadPL = rv.postings[i].BytesRead() + } + pl, err := rv.dicts[i].PostingsList(term, s.deleted, rv.postings[i]) + if err != nil { + return nil, err + } + rv.postings[i] = pl + + var prevBytesReadItr uint64 + if rv.iterators[i] != nil { + prevBytesReadItr = rv.iterators[i].BytesRead() + } + rv.iterators[i] = pl.Iterator(includeFreq, includeNorm, includeTermVectors, rv.iterators[i]) + + if bytesRead := rv.postings[i].BytesRead(); prevBytesReadPL < bytesRead { + rv.incrementBytesRead(bytesRead - prevBytesReadPL) + } + + if bytesRead := rv.iterators[i].BytesRead(); prevBytesReadItr < bytesRead { + rv.incrementBytesRead(bytesRead - prevBytesReadItr) + } + } + atomic.AddUint64(&is.parent.stats.TotTermSearchersStarted, uint64(1)) + return rv, nil +} + +func (is *IndexSnapshot) allocTermFieldReaderDicts(field string) (tfr *IndexSnapshotTermFieldReader) { + is.m2.Lock() + if is.fieldTFRs != nil { + tfrs := is.fieldTFRs[field] + last := len(tfrs) - 1 + if last >= 0 { + tfr = tfrs[last] + tfrs[last] = nil + is.fieldTFRs[field] = tfrs[:last] + is.m2.Unlock() + return + } + } + is.m2.Unlock() + return &IndexSnapshotTermFieldReader{ + recycle: true, + } +} + +func (is *IndexSnapshot) getFieldTFRCacheThreshold() uint64 { + if is.parent.config != nil { + if _, ok := is.parent.config["FieldTFRCacheThreshold"]; ok { + return is.parent.config["FieldTFRCacheThreshold"].(uint64) + } + } + return DefaultFieldTFRCacheThreshold +} + +func (is *IndexSnapshot) recycleTermFieldReader(tfr *IndexSnapshotTermFieldReader) { + if !tfr.recycle { + // Do not recycle an optimized unadorned term field reader (used for + // ConjunctionUnadorned or DisjunctionUnadorned), during when a fresh + // roaring.Bitmap is built by AND-ing or OR-ing individual bitmaps, + // and we'll need to release them for GC. (See MB-40916) + return + } + + is.parent.rootLock.RLock() + obsolete := is.parent.root != is + is.parent.rootLock.RUnlock() + if obsolete { + // if we're not the current root (mutations happened), don't bother recycling + return + } + + is.m2.Lock() + if is.fieldTFRs == nil { + is.fieldTFRs = map[string][]*IndexSnapshotTermFieldReader{} + } + if uint64(len(is.fieldTFRs[tfr.field])) < is.getFieldTFRCacheThreshold() { + tfr.bytesRead = 0 + is.fieldTFRs[tfr.field] = append(is.fieldTFRs[tfr.field], tfr) + } + is.m2.Unlock() +} + +func docNumberToBytes(buf []byte, in uint64) []byte { + if len(buf) != 8 { + if cap(buf) >= 8 { + buf = buf[0:8] + } else { + buf = make([]byte, 8) + } + } + binary.BigEndian.PutUint64(buf, in) + return buf +} + +func docInternalToNumber(in index.IndexInternalID) (uint64, error) { + if len(in) != 8 { + return 0, fmt.Errorf("wrong len for IndexInternalID: %q", in) + } + return binary.BigEndian.Uint64(in), nil +} + +func (is *IndexSnapshot) documentVisitFieldTermsOnSegment( + segmentIndex int, localDocNum uint64, fields []string, cFields []string, + visitor index.DocValueVisitor, dvs segment.DocVisitState) ( + cFieldsOut []string, dvsOut segment.DocVisitState, err error) { + ss := is.segment[segmentIndex] + + var vFields []string // fields that are visitable via the segment + + ssv, ssvOk := ss.segment.(segment.DocValueVisitable) + if ssvOk && ssv != nil { + vFields, err = ssv.VisitableDocValueFields() + if err != nil { + return nil, nil, err + } + } + + var errCh chan error + + // cFields represents the fields that we'll need from the + // cachedDocs, and might be optionally be provided by the caller, + // if the caller happens to know we're on the same segmentIndex + // from a previous invocation + if cFields == nil { + cFields = subtractStrings(fields, vFields) + + if !ss.cachedDocs.hasFields(cFields) { + errCh = make(chan error, 1) + + go func() { + err := ss.cachedDocs.prepareFields(cFields, ss) + if err != nil { + errCh <- err + } + close(errCh) + }() + } + } + + if ssvOk && ssv != nil && len(vFields) > 0 { + dvs, err = ssv.VisitDocValues(localDocNum, fields, visitor, dvs) + if err != nil { + return nil, nil, err + } + } + + if errCh != nil { + err = <-errCh + if err != nil { + return nil, nil, err + } + } + + if len(cFields) > 0 { + ss.cachedDocs.visitDoc(localDocNum, cFields, visitor) + } + + return cFields, dvs, nil +} + +func (is *IndexSnapshot) DocValueReader(fields []string) ( + index.DocValueReader, error) { + return &DocValueReader{i: is, fields: fields, currSegmentIndex: -1}, nil +} + +type DocValueReader struct { + i *IndexSnapshot + fields []string + dvs segment.DocVisitState + + currSegmentIndex int + currCachedFields []string + + totalBytesRead uint64 + bytesRead uint64 +} + +func (dvr *DocValueReader) BytesRead() uint64 { + return dvr.totalBytesRead + dvr.bytesRead +} + +func (dvr *DocValueReader) VisitDocValues(id index.IndexInternalID, + visitor index.DocValueVisitor) (err error) { + docNum, err := docInternalToNumber(id) + if err != nil { + return err + } + + segmentIndex, localDocNum := dvr.i.segmentIndexAndLocalDocNumFromGlobal(docNum) + if segmentIndex >= len(dvr.i.segment) { + return nil + } + + if dvr.currSegmentIndex != segmentIndex { + dvr.currSegmentIndex = segmentIndex + dvr.currCachedFields = nil + dvr.totalBytesRead += dvr.bytesRead + dvr.bytesRead = 0 + } + + dvr.currCachedFields, dvr.dvs, err = dvr.i.documentVisitFieldTermsOnSegment( + dvr.currSegmentIndex, localDocNum, dvr.fields, dvr.currCachedFields, visitor, dvr.dvs) + + if dvr.dvs != nil { + dvr.bytesRead = dvr.dvs.BytesRead() + } + return err +} + +func (is *IndexSnapshot) DumpAll() chan interface{} { + rv := make(chan interface{}) + go func() { + close(rv) + }() + return rv +} + +func (is *IndexSnapshot) DumpDoc(id string) chan interface{} { + rv := make(chan interface{}) + go func() { + close(rv) + }() + return rv +} + +func (is *IndexSnapshot) DumpFields() chan interface{} { + rv := make(chan interface{}) + go func() { + close(rv) + }() + return rv +} + +func (is *IndexSnapshot) diskSegmentsPaths() map[string]struct{} { + rv := make(map[string]struct{}, len(is.segment)) + for _, s := range is.segment { + if seg, ok := s.segment.(segment.PersistedSegment); ok { + rv[seg.Path()] = struct{}{} + } + } + return rv +} + +// reClaimableDocsRatio gives a ratio about the obsoleted or +// reclaimable documents present in a given index snapshot. +func (is *IndexSnapshot) reClaimableDocsRatio() float64 { + var totalCount, liveCount uint64 + for _, s := range is.segment { + if _, ok := s.segment.(segment.PersistedSegment); ok { + totalCount += uint64(s.FullSize()) + liveCount += uint64(s.Count()) + } + } + + if totalCount > 0 { + return float64(totalCount-liveCount) / float64(totalCount) + } + return 0 +} + +// subtractStrings returns set a minus elements of set b. +func subtractStrings(a, b []string) []string { + if len(b) == 0 { + return a + } + + rv := make([]string, 0, len(a)) +OUTER: + for _, as := range a { + for _, bs := range b { + if as == bs { + continue OUTER + } + } + rv = append(rv, as) + } + return rv +} + +func (is *IndexSnapshot) CopyTo(d index.Directory) error { + // get the root bolt file. + w, err := d.GetWriter(filepath.Join("store", "root.bolt")) + if err != nil || w == nil { + return fmt.Errorf("failed to create the root.bolt file, err: %v", err) + } + rootFile, ok := w.(*os.File) + if !ok { + return fmt.Errorf("invalid root.bolt file found") + } + + copyBolt, err := bolt.Open(rootFile.Name(), 0600, nil) + if err != nil { + return err + } + defer func() { + w.Close() + if cerr := copyBolt.Close(); cerr != nil && err == nil { + err = cerr + } + }() + + // start a write transaction + tx, err := copyBolt.Begin(true) + if err != nil { + return err + } + + _, _, err = prepareBoltSnapshot(is, tx, "", is.parent.segPlugin, d) + if err != nil { + _ = tx.Rollback() + return fmt.Errorf("error backing up index snapshot: %v", err) + } + + // commit bolt data + err = tx.Commit() + if err != nil { + return fmt.Errorf("error commit tx to backup root bolt: %v", err) + } + + return copyBolt.Sync() +} + +func (is *IndexSnapshot) UpdateIOStats(val uint64) { + atomic.AddUint64(&is.parent.stats.TotBytesReadAtQueryTime, val) +} + +func (is *IndexSnapshot) GetSpatialAnalyzerPlugin(typ string) ( + index.SpatialAnalyzerPlugin, error) { + var rv index.SpatialAnalyzerPlugin + is.m.Lock() + rv = is.parent.spatialPlugin + is.m.Unlock() + + if rv == nil { + return nil, fmt.Errorf("no spatial plugin type: %s found", typ) + } + return rv, nil +} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/snapshot_index_dict.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_index_dict.go similarity index 89% rename from vendor/github.com/blevesearch/bleve/index/scorch/snapshot_index_dict.go rename to vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_index_dict.go index 47486c25..658aa814 100644 --- a/vendor/github.com/blevesearch/bleve/index/scorch/snapshot_index_dict.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_index_dict.go @@ -17,8 +17,8 @@ package scorch import ( "container/heap" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" ) type segmentDictCursor struct { @@ -28,9 +28,14 @@ type segmentDictCursor struct { } type IndexSnapshotFieldDict struct { - snapshot *IndexSnapshot - cursors []*segmentDictCursor - entry index.DictEntry + snapshot *IndexSnapshot + cursors []*segmentDictCursor + entry index.DictEntry + bytesRead uint64 +} + +func (i *IndexSnapshotFieldDict) BytesRead() uint64 { + return i.bytesRead } func (i *IndexSnapshotFieldDict) Len() int { return len(i.cursors) } diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/snapshot_index_doc.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_index_doc.go similarity index 95% rename from vendor/github.com/blevesearch/bleve/index/scorch/snapshot_index_doc.go rename to vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_index_doc.go index 27da2086..fe174e7e 100644 --- a/vendor/github.com/blevesearch/bleve/index/scorch/snapshot_index_doc.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_index_doc.go @@ -19,8 +19,8 @@ import ( "reflect" "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" ) var reflectStaticSizeIndexSnapshotDocIDReader int diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/snapshot_index_tfr.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_index_tfr.go similarity index 80% rename from vendor/github.com/blevesearch/bleve/index/scorch/snapshot_index_tfr.go rename to vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_index_tfr.go index 239f68fb..9f0315fa 100644 --- a/vendor/github.com/blevesearch/bleve/index/scorch/snapshot_index_tfr.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_index_tfr.go @@ -16,13 +16,15 @@ package scorch import ( "bytes" + "context" "fmt" "reflect" "sync/atomic" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" ) var reflectStaticSizeIndexSnapshotTermFieldReader int @@ -46,6 +48,12 @@ type IndexSnapshotTermFieldReader struct { currPosting segment.Posting currID index.IndexInternalID recycle bool + bytesRead uint64 + ctx context.Context +} + +func (i *IndexSnapshotTermFieldReader) incrementBytesRead(val uint64) { + i.bytesRead += val } func (i *IndexSnapshotTermFieldReader) Size() int { @@ -76,6 +84,7 @@ func (i *IndexSnapshotTermFieldReader) Next(preAlloced *index.TermFieldDoc) (*in } // find the next hit for i.segmentOffset < len(i.iterators) { + prevBytesRead := i.iterators[i.segmentOffset].BytesRead() next, err := i.iterators[i.segmentOffset].Next() if err != nil { return nil, err @@ -89,6 +98,14 @@ func (i *IndexSnapshotTermFieldReader) Next(preAlloced *index.TermFieldDoc) (*in i.currID = rv.ID i.currPosting = next + // postingsIterators is maintain the bytesRead stat in a cumulative fashion. + // this is because there are chances of having a series of loadChunk calls, + // and they have to be added together before sending the bytesRead at this point + // upstream. + bytesRead := i.iterators[i.segmentOffset].BytesRead() + if bytesRead > prevBytesRead { + i.incrementBytesRead(bytesRead - prevBytesRead) + } return rv, nil } i.segmentOffset++ @@ -129,7 +146,7 @@ func (i *IndexSnapshotTermFieldReader) Advance(ID index.IndexInternalID, preAllo // FIXME do something better // for now, if we need to seek backwards, then restart from the beginning if i.currPosting != nil && bytes.Compare(i.currID, ID) >= 0 { - i2, err := i.snapshot.TermFieldReader(i.term, i.field, + i2, err := i.snapshot.TermFieldReader(nil, i.term, i.field, i.includeFreq, i.includeNorm, i.includeTermVectors) if err != nil { return nil, err @@ -180,6 +197,17 @@ func (i *IndexSnapshotTermFieldReader) Count() uint64 { } func (i *IndexSnapshotTermFieldReader) Close() error { + if i.ctx != nil { + statsCallbackFn := i.ctx.Value(search.SearchIOStatsCallbackKey) + if statsCallbackFn != nil { + // essentially before you close the TFR, you must report this + // reader's bytesRead value + statsCallbackFn.(search.SearchIOStatsCallbackFunc)(i.bytesRead) + } + + search.RecordSearchCost(i.ctx, search.AddM, i.bytesRead) + } + if i.snapshot != nil { atomic.AddUint64(&i.snapshot.parent.stats.TotTermSearchersFinished, uint64(1)) i.snapshot.recycleTermFieldReader(i) diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/snapshot_segment.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_segment.go similarity index 91% rename from vendor/github.com/blevesearch/bleve/index/scorch/snapshot_segment.go rename to vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_segment.go index 96742b4f..0b76ec74 100644 --- a/vendor/github.com/blevesearch/bleve/index/scorch/snapshot_segment.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_segment.go @@ -20,9 +20,9 @@ import ( "sync/atomic" "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" ) var TermSeparator byte = 0xff @@ -30,6 +30,11 @@ var TermSeparator byte = 0xff var TermSeparatorSplitSlice = []byte{TermSeparator} type SegmentSnapshot struct { + // this flag is needed to identify whether this + // segment was mmaped recently, in which case + // we consider the loading cost of the metadata + // as part of IO stats. + mmaped uint32 id uint64 segment segment.Segment deleted *roaring.Bitmap @@ -54,7 +59,7 @@ func (s *SegmentSnapshot) FullSize() int64 { return int64(s.segment.Count()) } -func (s SegmentSnapshot) LiveSize() int64 { +func (s *SegmentSnapshot) LiveSize() int64 { return int64(s.Count()) } @@ -62,8 +67,8 @@ func (s *SegmentSnapshot) Close() error { return s.segment.Close() } -func (s *SegmentSnapshot) VisitDocument(num uint64, visitor segment.DocumentFieldValueVisitor) error { - return s.segment.VisitDocument(num, visitor) +func (s *SegmentSnapshot) VisitDocument(num uint64, visitor segment.StoredFieldValueVisitor) error { + return s.segment.VisitStoredFields(num, visitor) } func (s *SegmentSnapshot) DocID(num uint64) ([]byte, error) { @@ -147,7 +152,7 @@ func (cfd *cachedFieldDocs) prepareField(field string, ss *SegmentSnapshot) { var postings segment.PostingsList var postingsItr segment.PostingsIterator - dictItr := dict.Iterator() + dictItr := dict.AutomatonIterator(nil, nil, nil) next, err := dictItr.Next() for err == nil && next != nil { var err1 error @@ -253,7 +258,7 @@ func (c *cachedDocs) updateSizeLOCKED() { } func (c *cachedDocs) visitDoc(localDocNum uint64, - fields []string, visitor index.DocumentFieldTermVisitor) { + fields []string, visitor index.DocValueVisitor) { c.m.Lock() for _, field := range fields { diff --git a/vendor/github.com/blevesearch/bleve/v2/index/scorch/stats.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/stats.go new file mode 100644 index 00000000..dc74d9f2 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/stats.go @@ -0,0 +1,156 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "reflect" + "sync/atomic" + + "github.com/blevesearch/bleve/v2/util" +) + +// Stats tracks statistics about the index, fields that are +// prefixed like CurXxxx are gauges (can go up and down), +// and fields that are prefixed like TotXxxx are monotonically +// increasing counters. +type Stats struct { + TotUpdates uint64 + TotDeletes uint64 + + TotBatches uint64 + TotBatchesEmpty uint64 + TotBatchIntroTime uint64 + MaxBatchIntroTime uint64 + + CurRootEpoch uint64 + LastPersistedEpoch uint64 + LastMergedEpoch uint64 + + TotOnErrors uint64 + + TotAnalysisTime uint64 + TotIndexTime uint64 + + TotIndexedPlainTextBytes uint64 + + TotBytesReadAtQueryTime uint64 + TotBytesWrittenAtIndexTime uint64 + + TotTermSearchersStarted uint64 + TotTermSearchersFinished uint64 + + TotEventTriggerStarted uint64 + TotEventTriggerCompleted uint64 + + TotIntroduceLoop uint64 + TotIntroduceSegmentBeg uint64 + TotIntroduceSegmentEnd uint64 + TotIntroducePersistBeg uint64 + TotIntroducePersistEnd uint64 + TotIntroduceMergeBeg uint64 + TotIntroduceMergeEnd uint64 + TotIntroduceRevertBeg uint64 + TotIntroduceRevertEnd uint64 + + TotIntroducedItems uint64 + TotIntroducedSegmentsBatch uint64 + TotIntroducedSegmentsMerge uint64 + + TotPersistLoopBeg uint64 + TotPersistLoopErr uint64 + TotPersistLoopProgress uint64 + TotPersistLoopWait uint64 + TotPersistLoopWaitNotified uint64 + TotPersistLoopEnd uint64 + + TotPersistedItems uint64 + TotItemsToPersist uint64 + TotPersistedSegments uint64 + + TotPersisterSlowMergerPause uint64 + TotPersisterSlowMergerResume uint64 + + TotPersisterNapPauseCompleted uint64 + TotPersisterMergerNapBreak uint64 + + TotFileMergeLoopBeg uint64 + TotFileMergeLoopErr uint64 + TotFileMergeLoopEnd uint64 + + TotFileMergeForceOpsStarted uint64 + TotFileMergeForceOpsCompleted uint64 + + TotFileMergePlan uint64 + TotFileMergePlanErr uint64 + TotFileMergePlanNone uint64 + TotFileMergePlanOk uint64 + + TotFileMergePlanTasks uint64 + TotFileMergePlanTasksDone uint64 + TotFileMergePlanTasksErr uint64 + TotFileMergePlanTasksSegments uint64 + TotFileMergePlanTasksSegmentsEmpty uint64 + + TotFileMergeSegmentsEmpty uint64 + TotFileMergeSegments uint64 + TotFileSegmentsAtRoot uint64 + TotFileMergeWrittenBytes uint64 + + TotFileMergeZapBeg uint64 + TotFileMergeZapEnd uint64 + TotFileMergeZapTime uint64 + MaxFileMergeZapTime uint64 + TotFileMergeZapIntroductionTime uint64 + MaxFileMergeZapIntroductionTime uint64 + + TotFileMergeIntroductions uint64 + TotFileMergeIntroductionsDone uint64 + TotFileMergeIntroductionsSkipped uint64 + TotFileMergeIntroductionsObsoleted uint64 + + CurFilesIneligibleForRemoval uint64 + TotSnapshotsRemovedFromMetaStore uint64 + + TotMemMergeBeg uint64 + TotMemMergeErr uint64 + TotMemMergeDone uint64 + TotMemMergeZapBeg uint64 + TotMemMergeZapEnd uint64 + TotMemMergeZapTime uint64 + MaxMemMergeZapTime uint64 + TotMemMergeSegments uint64 + TotMemorySegmentsAtRoot uint64 +} + +// atomically populates the returned map +func (s *Stats) ToMap() map[string]interface{} { + m := map[string]interface{}{} + sve := reflect.ValueOf(s).Elem() + svet := sve.Type() + for i := 0; i < svet.NumField(); i++ { + svef := sve.Field(i) + if svef.CanAddr() { + svefp := svef.Addr().Interface() + m[svet.Field(i).Name] = atomic.LoadUint64(svefp.(*uint64)) + } + } + return m +} + +// MarshalJSON implements json.Marshaler, and in contrast to standard +// json marshaling provides atomic safety +func (s *Stats) MarshalJSON() ([]byte, error) { + return util.MarshalJSON(s.ToMap()) +} diff --git a/vendor/github.com/blevesearch/bleve/v2/index/scorch/unadorned.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/unadorned.go new file mode 100644 index 00000000..8221b23e --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/unadorned.go @@ -0,0 +1,182 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "math" + "reflect" + + "github.com/RoaringBitmap/roaring" + segment "github.com/blevesearch/scorch_segment_api/v2" +) + +var reflectStaticSizeUnadornedPostingsIteratorBitmap int +var reflectStaticSizeUnadornedPostingsIterator1Hit int +var reflectStaticSizeUnadornedPosting int + +func init() { + var pib unadornedPostingsIteratorBitmap + reflectStaticSizeUnadornedPostingsIteratorBitmap = int(reflect.TypeOf(pib).Size()) + var pi1h unadornedPostingsIterator1Hit + reflectStaticSizeUnadornedPostingsIterator1Hit = int(reflect.TypeOf(pi1h).Size()) + var up UnadornedPosting + reflectStaticSizeUnadornedPosting = int(reflect.TypeOf(up).Size()) +} + +type unadornedPostingsIteratorBitmap struct { + actual roaring.IntPeekable + actualBM *roaring.Bitmap +} + +func (i *unadornedPostingsIteratorBitmap) Next() (segment.Posting, error) { + return i.nextAtOrAfter(0) +} + +func (i *unadornedPostingsIteratorBitmap) Advance(docNum uint64) (segment.Posting, error) { + return i.nextAtOrAfter(docNum) +} + +func (i *unadornedPostingsIteratorBitmap) nextAtOrAfter(atOrAfter uint64) (segment.Posting, error) { + docNum, exists := i.nextDocNumAtOrAfter(atOrAfter) + if !exists { + return nil, nil + } + return UnadornedPosting(docNum), nil +} + +func (i *unadornedPostingsIteratorBitmap) nextDocNumAtOrAfter(atOrAfter uint64) (uint64, bool) { + if i.actual == nil || !i.actual.HasNext() { + return 0, false + } + i.actual.AdvanceIfNeeded(uint32(atOrAfter)) + + if !i.actual.HasNext() { + return 0, false // couldn't find anything + } + + return uint64(i.actual.Next()), true +} + +func (i *unadornedPostingsIteratorBitmap) Size() int { + return reflectStaticSizeUnadornedPostingsIteratorBitmap +} + +func (i *unadornedPostingsIteratorBitmap) BytesRead() uint64 { + return 0 +} + +func (i *unadornedPostingsIteratorBitmap) BytesWritten() uint64 { + return 0 +} + +func (i *unadornedPostingsIteratorBitmap) ResetBytesRead(uint64) {} + +func (i *unadornedPostingsIteratorBitmap) ActualBitmap() *roaring.Bitmap { + return i.actualBM +} + +func (i *unadornedPostingsIteratorBitmap) DocNum1Hit() (uint64, bool) { + return 0, false +} + +func (i *unadornedPostingsIteratorBitmap) ReplaceActual(actual *roaring.Bitmap) { + i.actualBM = actual + i.actual = actual.Iterator() +} + +func newUnadornedPostingsIteratorFromBitmap(bm *roaring.Bitmap) segment.PostingsIterator { + return &unadornedPostingsIteratorBitmap{ + actualBM: bm, + actual: bm.Iterator(), + } +} + +const docNum1HitFinished = math.MaxUint64 + +type unadornedPostingsIterator1Hit struct { + docNum uint64 +} + +func (i *unadornedPostingsIterator1Hit) Next() (segment.Posting, error) { + return i.nextAtOrAfter(0) +} + +func (i *unadornedPostingsIterator1Hit) Advance(docNum uint64) (segment.Posting, error) { + return i.nextAtOrAfter(docNum) +} + +func (i *unadornedPostingsIterator1Hit) nextAtOrAfter(atOrAfter uint64) (segment.Posting, error) { + docNum, exists := i.nextDocNumAtOrAfter(atOrAfter) + if !exists { + return nil, nil + } + return UnadornedPosting(docNum), nil +} + +func (i *unadornedPostingsIterator1Hit) nextDocNumAtOrAfter(atOrAfter uint64) (uint64, bool) { + if i.docNum == docNum1HitFinished { + return 0, false + } + if i.docNum < atOrAfter { + // advanced past our 1-hit + i.docNum = docNum1HitFinished // consume our 1-hit docNum + return 0, false + } + docNum := i.docNum + i.docNum = docNum1HitFinished // consume our 1-hit docNum + return docNum, true +} + +func (i *unadornedPostingsIterator1Hit) Size() int { + return reflectStaticSizeUnadornedPostingsIterator1Hit +} + +func (i *unadornedPostingsIterator1Hit) BytesRead() uint64 { + return 0 +} + +func (i *unadornedPostingsIterator1Hit) BytesWritten() uint64 { + return 0 +} + +func (i *unadornedPostingsIterator1Hit) ResetBytesRead(uint64) {} + +func newUnadornedPostingsIteratorFrom1Hit(docNum1Hit uint64) segment.PostingsIterator { + return &unadornedPostingsIterator1Hit{ + docNum1Hit, + } +} + +type UnadornedPosting uint64 + +func (p UnadornedPosting) Number() uint64 { + return uint64(p) +} + +func (p UnadornedPosting) Frequency() uint64 { + return 0 +} + +func (p UnadornedPosting) Norm() float64 { + return 0 +} + +func (p UnadornedPosting) Locations() []segment.Location { + return nil +} + +func (p UnadornedPosting) Size() int { + return reflectStaticSizeUnadornedPosting +} diff --git a/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/analysis.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/analysis.go new file mode 100644 index 00000000..1ebd1918 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/analysis.go @@ -0,0 +1,129 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package upsidedown + +import ( + index "github.com/blevesearch/bleve_index_api" +) + +type IndexRow interface { + KeySize() int + KeyTo([]byte) (int, error) + Key() []byte + + ValueSize() int + ValueTo([]byte) (int, error) + Value() []byte +} + +type AnalysisResult struct { + DocID string + Rows []IndexRow +} + +func (udc *UpsideDownCouch) Analyze(d index.Document) *AnalysisResult { + return udc.analyze(d) +} + +func (udc *UpsideDownCouch) analyze(d index.Document) *AnalysisResult { + rv := &AnalysisResult{ + DocID: d.ID(), + Rows: make([]IndexRow, 0, 100), + } + + docIDBytes := []byte(d.ID()) + + // track our back index entries + backIndexStoredEntries := make([]*BackIndexStoreEntry, 0) + + // information we collate as we merge fields with same name + fieldTermFreqs := make(map[uint16]index.TokenFrequencies) + fieldLengths := make(map[uint16]int) + fieldIncludeTermVectors := make(map[uint16]bool) + fieldNames := make(map[uint16]string) + + analyzeField := func(field index.Field, storable bool) { + fieldIndex, newFieldRow := udc.fieldIndexOrNewRow(field.Name()) + if newFieldRow != nil { + rv.Rows = append(rv.Rows, newFieldRow) + } + fieldNames[fieldIndex] = field.Name() + + if field.Options().IsIndexed() { + field.Analyze() + fieldLength := field.AnalyzedLength() + tokenFreqs := field.AnalyzedTokenFrequencies() + existingFreqs := fieldTermFreqs[fieldIndex] + if existingFreqs == nil { + fieldTermFreqs[fieldIndex] = tokenFreqs + } else { + existingFreqs.MergeAll(field.Name(), tokenFreqs) + fieldTermFreqs[fieldIndex] = existingFreqs + } + fieldLengths[fieldIndex] += fieldLength + fieldIncludeTermVectors[fieldIndex] = field.Options().IncludeTermVectors() + } + + if storable && field.Options().IsStored() { + rv.Rows, backIndexStoredEntries = udc.storeField(docIDBytes, field, fieldIndex, rv.Rows, backIndexStoredEntries) + } + } + + // walk all the fields, record stored fields now + // place information about indexed fields into map + // this collates information across fields with + // same names (arrays) + d.VisitFields(func(field index.Field) { + analyzeField(field, true) + }) + + if d.HasComposite() { + for fieldIndex, tokenFreqs := range fieldTermFreqs { + // see if any of the composite fields need this + d.VisitComposite(func(field index.CompositeField) { + field.Compose(fieldNames[fieldIndex], fieldLengths[fieldIndex], tokenFreqs) + }) + } + + d.VisitComposite(func(field index.CompositeField) { + analyzeField(field, false) + }) + } + + rowsCapNeeded := len(rv.Rows) + 1 + for _, tokenFreqs := range fieldTermFreqs { + rowsCapNeeded += len(tokenFreqs) + } + + rv.Rows = append(make([]IndexRow, 0, rowsCapNeeded), rv.Rows...) + + backIndexTermsEntries := make([]*BackIndexTermsEntry, 0, len(fieldTermFreqs)) + + // walk through the collated information and process + // once for each indexed field (unique name) + for fieldIndex, tokenFreqs := range fieldTermFreqs { + fieldLength := fieldLengths[fieldIndex] + includeTermVectors := fieldIncludeTermVectors[fieldIndex] + + // encode this field + rv.Rows, backIndexTermsEntries = udc.indexField(docIDBytes, includeTermVectors, fieldIndex, fieldLength, tokenFreqs, rv.Rows, backIndexTermsEntries) + } + + // build the back index row + backIndexRow := NewBackIndexRow(docIDBytes, backIndexTermsEntries, backIndexStoredEntries) + rv.Rows = append(rv.Rows, backIndexRow) + + return rv +} diff --git a/vendor/github.com/blevesearch/bleve/index/upsidedown/benchmark_all.sh b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/benchmark_all.sh similarity index 100% rename from vendor/github.com/blevesearch/bleve/index/upsidedown/benchmark_all.sh rename to vendor/github.com/blevesearch/bleve/v2/index/upsidedown/benchmark_all.sh diff --git a/vendor/github.com/blevesearch/bleve/index/upsidedown/dump.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/dump.go similarity index 98% rename from vendor/github.com/blevesearch/bleve/index/upsidedown/dump.go rename to vendor/github.com/blevesearch/bleve/v2/index/upsidedown/dump.go index cb045d24..64ebb1b2 100644 --- a/vendor/github.com/blevesearch/bleve/index/upsidedown/dump.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/dump.go @@ -18,7 +18,7 @@ import ( "bytes" "sort" - "github.com/blevesearch/bleve/index/store" + "github.com/blevesearch/upsidedown_store_api" ) // the functions in this file are only intended to be used by diff --git a/vendor/github.com/blevesearch/bleve/index/field_cache.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/field_cache.go similarity index 99% rename from vendor/github.com/blevesearch/bleve/index/field_cache.go rename to vendor/github.com/blevesearch/bleve/v2/index/upsidedown/field_cache.go index 9354081f..1f68b71d 100644 --- a/vendor/github.com/blevesearch/bleve/index/field_cache.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/field_cache.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package index +package upsidedown import ( "sync" diff --git a/vendor/github.com/blevesearch/bleve/index/upsidedown/field_dict.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/field_dict.go similarity index 92% rename from vendor/github.com/blevesearch/bleve/index/upsidedown/field_dict.go rename to vendor/github.com/blevesearch/bleve/v2/index/upsidedown/field_dict.go index 20d4eb34..4875680c 100644 --- a/vendor/github.com/blevesearch/bleve/index/upsidedown/field_dict.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/field_dict.go @@ -17,8 +17,8 @@ package upsidedown import ( "fmt" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/store" + index "github.com/blevesearch/bleve_index_api" + store "github.com/blevesearch/upsidedown_store_api" ) type UpsideDownCouchFieldDict struct { @@ -51,6 +51,10 @@ func newUpsideDownCouchFieldDict(indexReader *IndexReader, field uint16, startTe } +func (r *UpsideDownCouchFieldDict) BytesRead() uint64 { + return 0 +} + func (r *UpsideDownCouchFieldDict) Next() (*index.DictEntry, error) { key, val, valid := r.iterator.Current() if !valid { diff --git a/vendor/github.com/blevesearch/bleve/index/upsidedown/index_reader.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/index_reader.go similarity index 84% rename from vendor/github.com/blevesearch/bleve/index/upsidedown/index_reader.go rename to vendor/github.com/blevesearch/bleve/v2/index/upsidedown/index_reader.go index ea7243ea..44ccf591 100644 --- a/vendor/github.com/blevesearch/bleve/index/upsidedown/index_reader.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/index_reader.go @@ -15,11 +15,12 @@ package upsidedown import ( + "context" "reflect" - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/store" + "github.com/blevesearch/bleve/v2/document" + index "github.com/blevesearch/bleve_index_api" + "github.com/blevesearch/upsidedown_store_api" ) var reflectStaticSizeIndexReader int @@ -35,7 +36,7 @@ type IndexReader struct { docCount uint64 } -func (i *IndexReader) TermFieldReader(term []byte, fieldName string, includeFreq, includeNorm, includeTermVectors bool) (index.TermFieldReader, error) { +func (i *IndexReader) TermFieldReader(ctx context.Context, term []byte, fieldName string, includeFreq, includeNorm, includeTermVectors bool) (index.TermFieldReader, error) { fieldIndex, fieldExists := i.index.fieldCache.FieldNamed(fieldName, false) if fieldExists { return newUpsideDownCouchTermFieldReader(i, term, uint16(fieldIndex), includeFreq, includeNorm, includeTermVectors) @@ -67,7 +68,7 @@ func (i *IndexReader) DocIDReaderOnly(ids []string) (index.DocIDReader, error) { return newUpsideDownCouchDocIDReaderOnly(i, ids) } -func (i *IndexReader) Document(id string) (doc *document.Document, err error) { +func (i *IndexReader) Document(id string) (doc index.Document, err error) { // first hit the back index to confirm doc exists var backIndexRow *BackIndexRow backIndexRow, err = backIndexRowForDoc(i.kvreader, []byte(id)) @@ -77,7 +78,7 @@ func (i *IndexReader) Document(id string) (doc *document.Document, err error) { if backIndexRow == nil { return } - doc = document.NewDocument(id) + rvd := document.NewDocument(id) storedRow := NewStoredRow([]byte(id), 0, []uint64{}, 'x', nil) storedRowScanPrefix := storedRow.ScanPrefixForDoc() it := i.kvreader.PrefixIterator(storedRowScanPrefix) @@ -93,24 +94,23 @@ func (i *IndexReader) Document(id string) (doc *document.Document, err error) { var row *StoredRow row, err = NewStoredRowKV(key, safeVal) if err != nil { - doc = nil - return + return nil, err } if row != nil { fieldName := i.index.fieldCache.FieldIndexed(row.field) field := decodeFieldType(row.typ, fieldName, row.arrayPositions, row.value) if field != nil { - doc.AddField(field) + rvd.AddField(field) } } it.Next() key, val, valid = it.Current() } - return + return rvd, nil } -func (i *IndexReader) DocumentVisitFieldTerms(id index.IndexInternalID, fields []string, visitor index.DocumentFieldTermVisitor) error { +func (i *IndexReader) documentVisitFieldTerms(id index.IndexInternalID, fields []string, visitor index.DocValueVisitor) error { fieldsMap := make(map[uint16]string, len(fields)) for _, f := range fields { id, ok := i.index.fieldCache.FieldNamed(f, false) @@ -124,16 +124,16 @@ func (i *IndexReader) DocumentVisitFieldTerms(id index.IndexInternalID, fields [ } keyBuf := GetRowBuffer() - if tempRow.KeySize() > len(keyBuf) { - keyBuf = make([]byte, 2*tempRow.KeySize()) + if tempRow.KeySize() > len(keyBuf.buf) { + keyBuf.buf = make([]byte, 2*tempRow.KeySize()) } defer PutRowBuffer(keyBuf) - keySize, err := tempRow.KeyTo(keyBuf) + keySize, err := tempRow.KeyTo(keyBuf.buf) if err != nil { return err } - value, err := i.kvreader.Get(keyBuf[:keySize]) + value, err := i.kvreader.Get(keyBuf.buf[:keySize]) if err != nil { return err } @@ -221,6 +221,8 @@ type DocValueReader struct { } func (dvr *DocValueReader) VisitDocValues(id index.IndexInternalID, - visitor index.DocumentFieldTermVisitor) error { - return dvr.i.DocumentVisitFieldTerms(id, dvr.fields, visitor) + visitor index.DocValueVisitor) error { + return dvr.i.documentVisitFieldTerms(id, dvr.fields, visitor) } + +func (dvr *DocValueReader) BytesRead() uint64 { return 0 } diff --git a/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/reader.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/reader.go new file mode 100644 index 00000000..68b15318 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/reader.go @@ -0,0 +1,376 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package upsidedown + +import ( + "bytes" + "reflect" + "sort" + "sync/atomic" + + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" + "github.com/blevesearch/upsidedown_store_api" +) + +var reflectStaticSizeUpsideDownCouchTermFieldReader int +var reflectStaticSizeUpsideDownCouchDocIDReader int + +func init() { + var tfr UpsideDownCouchTermFieldReader + reflectStaticSizeUpsideDownCouchTermFieldReader = + int(reflect.TypeOf(tfr).Size()) + var cdr UpsideDownCouchDocIDReader + reflectStaticSizeUpsideDownCouchDocIDReader = + int(reflect.TypeOf(cdr).Size()) +} + +type UpsideDownCouchTermFieldReader struct { + count uint64 + indexReader *IndexReader + iterator store.KVIterator + term []byte + tfrNext *TermFrequencyRow + tfrPrealloc TermFrequencyRow + keyBuf []byte + field uint16 + includeTermVectors bool +} + +func (r *UpsideDownCouchTermFieldReader) Size() int { + sizeInBytes := reflectStaticSizeUpsideDownCouchTermFieldReader + size.SizeOfPtr + + len(r.term) + + r.tfrPrealloc.Size() + + len(r.keyBuf) + + if r.tfrNext != nil { + sizeInBytes += r.tfrNext.Size() + } + + return sizeInBytes +} + +func newUpsideDownCouchTermFieldReader(indexReader *IndexReader, term []byte, field uint16, includeFreq, includeNorm, includeTermVectors bool) (*UpsideDownCouchTermFieldReader, error) { + bufNeeded := termFrequencyRowKeySize(term, nil) + if bufNeeded < dictionaryRowKeySize(term) { + bufNeeded = dictionaryRowKeySize(term) + } + buf := make([]byte, bufNeeded) + + bufUsed := dictionaryRowKeyTo(buf, field, term) + val, err := indexReader.kvreader.Get(buf[:bufUsed]) + if err != nil { + return nil, err + } + if val == nil { + atomic.AddUint64(&indexReader.index.stats.termSearchersStarted, uint64(1)) + rv := &UpsideDownCouchTermFieldReader{ + count: 0, + term: term, + field: field, + includeTermVectors: includeTermVectors, + } + rv.tfrNext = &rv.tfrPrealloc + return rv, nil + } + + count, err := dictionaryRowParseV(val) + if err != nil { + return nil, err + } + + bufUsed = termFrequencyRowKeyTo(buf, field, term, nil) + it := indexReader.kvreader.PrefixIterator(buf[:bufUsed]) + + atomic.AddUint64(&indexReader.index.stats.termSearchersStarted, uint64(1)) + return &UpsideDownCouchTermFieldReader{ + indexReader: indexReader, + iterator: it, + count: count, + term: term, + field: field, + includeTermVectors: includeTermVectors, + }, nil +} + +func (r *UpsideDownCouchTermFieldReader) Count() uint64 { + return r.count +} + +func (r *UpsideDownCouchTermFieldReader) Next(preAlloced *index.TermFieldDoc) (*index.TermFieldDoc, error) { + if r.iterator != nil { + // We treat tfrNext also like an initialization flag, which + // tells us whether we need to invoke the underlying + // iterator.Next(). The first time, don't call iterator.Next(). + if r.tfrNext != nil { + r.iterator.Next() + } else { + r.tfrNext = &r.tfrPrealloc + } + key, val, valid := r.iterator.Current() + if valid { + tfr := r.tfrNext + err := tfr.parseKDoc(key, r.term) + if err != nil { + return nil, err + } + err = tfr.parseV(val, r.includeTermVectors) + if err != nil { + return nil, err + } + rv := preAlloced + if rv == nil { + rv = &index.TermFieldDoc{} + } + rv.ID = append(rv.ID, tfr.doc...) + rv.Freq = tfr.freq + rv.Norm = float64(tfr.norm) + if tfr.vectors != nil { + rv.Vectors = r.indexReader.index.termFieldVectorsFromTermVectors(tfr.vectors) + } + return rv, nil + } + } + return nil, nil +} + +func (r *UpsideDownCouchTermFieldReader) Advance(docID index.IndexInternalID, preAlloced *index.TermFieldDoc) (rv *index.TermFieldDoc, err error) { + if r.iterator != nil { + if r.tfrNext == nil { + r.tfrNext = &TermFrequencyRow{} + } + tfr := InitTermFrequencyRow(r.tfrNext, r.term, r.field, docID, 0, 0) + r.keyBuf, err = tfr.KeyAppendTo(r.keyBuf[:0]) + if err != nil { + return nil, err + } + r.iterator.Seek(r.keyBuf) + key, val, valid := r.iterator.Current() + if valid { + err := tfr.parseKDoc(key, r.term) + if err != nil { + return nil, err + } + err = tfr.parseV(val, r.includeTermVectors) + if err != nil { + return nil, err + } + rv = preAlloced + if rv == nil { + rv = &index.TermFieldDoc{} + } + rv.ID = append(rv.ID, tfr.doc...) + rv.Freq = tfr.freq + rv.Norm = float64(tfr.norm) + if tfr.vectors != nil { + rv.Vectors = r.indexReader.index.termFieldVectorsFromTermVectors(tfr.vectors) + } + return rv, nil + } + } + return nil, nil +} + +func (r *UpsideDownCouchTermFieldReader) Close() error { + if r.indexReader != nil { + atomic.AddUint64(&r.indexReader.index.stats.termSearchersFinished, uint64(1)) + } + if r.iterator != nil { + return r.iterator.Close() + } + return nil +} + +type UpsideDownCouchDocIDReader struct { + indexReader *IndexReader + iterator store.KVIterator + only []string + onlyPos int + onlyMode bool +} + +func (r *UpsideDownCouchDocIDReader) Size() int { + sizeInBytes := reflectStaticSizeUpsideDownCouchDocIDReader + + reflectStaticSizeIndexReader + size.SizeOfPtr + + for _, entry := range r.only { + sizeInBytes += size.SizeOfString + len(entry) + } + + return sizeInBytes +} + +func newUpsideDownCouchDocIDReader(indexReader *IndexReader) (*UpsideDownCouchDocIDReader, error) { + startBytes := []byte{0x0} + endBytes := []byte{0xff} + + bisr := NewBackIndexRow(startBytes, nil, nil) + bier := NewBackIndexRow(endBytes, nil, nil) + it := indexReader.kvreader.RangeIterator(bisr.Key(), bier.Key()) + + return &UpsideDownCouchDocIDReader{ + indexReader: indexReader, + iterator: it, + }, nil +} + +func newUpsideDownCouchDocIDReaderOnly(indexReader *IndexReader, ids []string) (*UpsideDownCouchDocIDReader, error) { + // we don't actually own the list of ids, so if before we sort we must copy + idsCopy := make([]string, len(ids)) + copy(idsCopy, ids) + // ensure ids are sorted + sort.Strings(idsCopy) + startBytes := []byte{0x0} + if len(idsCopy) > 0 { + startBytes = []byte(idsCopy[0]) + } + endBytes := []byte{0xff} + if len(idsCopy) > 0 { + endBytes = incrementBytes([]byte(idsCopy[len(idsCopy)-1])) + } + bisr := NewBackIndexRow(startBytes, nil, nil) + bier := NewBackIndexRow(endBytes, nil, nil) + it := indexReader.kvreader.RangeIterator(bisr.Key(), bier.Key()) + + return &UpsideDownCouchDocIDReader{ + indexReader: indexReader, + iterator: it, + only: idsCopy, + onlyMode: true, + }, nil +} + +func (r *UpsideDownCouchDocIDReader) Next() (index.IndexInternalID, error) { + key, val, valid := r.iterator.Current() + + if r.onlyMode { + var rv index.IndexInternalID + for valid && r.onlyPos < len(r.only) { + br, err := NewBackIndexRowKV(key, val) + if err != nil { + return nil, err + } + if !bytes.Equal(br.doc, []byte(r.only[r.onlyPos])) { + ok := r.nextOnly() + if !ok { + return nil, nil + } + r.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key()) + key, val, valid = r.iterator.Current() + continue + } else { + rv = append([]byte(nil), br.doc...) + break + } + } + if valid && r.onlyPos < len(r.only) { + ok := r.nextOnly() + if ok { + r.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key()) + } + return rv, nil + } + + } else { + if valid { + br, err := NewBackIndexRowKV(key, val) + if err != nil { + return nil, err + } + rv := append([]byte(nil), br.doc...) + r.iterator.Next() + return rv, nil + } + } + return nil, nil +} + +func (r *UpsideDownCouchDocIDReader) Advance(docID index.IndexInternalID) (index.IndexInternalID, error) { + + if r.onlyMode { + r.onlyPos = sort.SearchStrings(r.only, string(docID)) + if r.onlyPos >= len(r.only) { + // advanced to key after our last only key + return nil, nil + } + r.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key()) + key, val, valid := r.iterator.Current() + + var rv index.IndexInternalID + for valid && r.onlyPos < len(r.only) { + br, err := NewBackIndexRowKV(key, val) + if err != nil { + return nil, err + } + if !bytes.Equal(br.doc, []byte(r.only[r.onlyPos])) { + // the only key we seek'd to didn't exist + // now look for the closest key that did exist in only + r.onlyPos = sort.SearchStrings(r.only, string(br.doc)) + if r.onlyPos >= len(r.only) { + // advanced to key after our last only key + return nil, nil + } + // now seek to this new only key + r.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key()) + key, val, valid = r.iterator.Current() + continue + } else { + rv = append([]byte(nil), br.doc...) + break + } + } + if valid && r.onlyPos < len(r.only) { + ok := r.nextOnly() + if ok { + r.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key()) + } + return rv, nil + } + } else { + bir := NewBackIndexRow(docID, nil, nil) + r.iterator.Seek(bir.Key()) + key, val, valid := r.iterator.Current() + if valid { + br, err := NewBackIndexRowKV(key, val) + if err != nil { + return nil, err + } + rv := append([]byte(nil), br.doc...) + r.iterator.Next() + return rv, nil + } + } + return nil, nil +} + +func (r *UpsideDownCouchDocIDReader) Close() error { + return r.iterator.Close() +} + +// move the r.only pos forward one, skipping duplicates +// return true if there is more data, or false if we got to the end of the list +func (r *UpsideDownCouchDocIDReader) nextOnly() bool { + + // advance 1 position, until we see a different key + // it's already sorted, so this skips duplicates + start := r.onlyPos + r.onlyPos++ + for r.onlyPos < len(r.only) && r.only[r.onlyPos] == r.only[start] { + start = r.onlyPos + r.onlyPos++ + } + // inidicate if we got to the end of the list + return r.onlyPos < len(r.only) +} diff --git a/vendor/github.com/blevesearch/bleve/index/upsidedown/row.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/row.go similarity index 99% rename from vendor/github.com/blevesearch/bleve/index/upsidedown/row.go rename to vendor/github.com/blevesearch/bleve/v2/index/upsidedown/row.go index 531e0a0d..fff6d067 100644 --- a/vendor/github.com/blevesearch/bleve/index/upsidedown/row.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/row.go @@ -22,7 +22,7 @@ import ( "math" "reflect" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/size" "github.com/golang/protobuf/proto" ) @@ -880,6 +880,9 @@ func NewStoredRowK(key []byte) (*StoredRow, error) { } rv.doc, err = buf.ReadBytes(ByteSeparator) + if err != nil { + return nil, err + } if len(rv.doc) < 2 { // 1 for min doc id length, 1 for separator err = fmt.Errorf("invalid doc length 0") return nil, err diff --git a/vendor/github.com/blevesearch/bleve/index/upsidedown/row_merge.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/row_merge.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/index/upsidedown/row_merge.go rename to vendor/github.com/blevesearch/bleve/v2/index/upsidedown/row_merge.go diff --git a/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/stats.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/stats.go new file mode 100644 index 00000000..5fea7c17 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/stats.go @@ -0,0 +1,55 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package upsidedown + +import ( + "sync/atomic" + + "github.com/blevesearch/bleve/v2/util" + "github.com/blevesearch/upsidedown_store_api" +) + +type indexStat struct { + updates, deletes, batches, errors uint64 + analysisTime, indexTime uint64 + termSearchersStarted uint64 + termSearchersFinished uint64 + numPlainTextBytesIndexed uint64 + i *UpsideDownCouch +} + +func (i *indexStat) statsMap() map[string]interface{} { + m := map[string]interface{}{} + m["updates"] = atomic.LoadUint64(&i.updates) + m["deletes"] = atomic.LoadUint64(&i.deletes) + m["batches"] = atomic.LoadUint64(&i.batches) + m["errors"] = atomic.LoadUint64(&i.errors) + m["analysis_time"] = atomic.LoadUint64(&i.analysisTime) + m["index_time"] = atomic.LoadUint64(&i.indexTime) + m["term_searchers_started"] = atomic.LoadUint64(&i.termSearchersStarted) + m["term_searchers_finished"] = atomic.LoadUint64(&i.termSearchersFinished) + m["num_plain_text_bytes_indexed"] = atomic.LoadUint64(&i.numPlainTextBytesIndexed) + + if o, ok := i.i.store.(store.KVStoreStats); ok { + m["kv"] = o.StatsMap() + } + + return m +} + +func (i *indexStat) MarshalJSON() ([]byte, error) { + m := i.statsMap() + return util.MarshalJSON(m) +} diff --git a/vendor/github.com/blevesearch/bleve/index/store/boltdb/iterator.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb/iterator.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/index/store/boltdb/iterator.go rename to vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb/iterator.go diff --git a/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb/reader.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb/reader.go new file mode 100644 index 00000000..79513f60 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb/reader.go @@ -0,0 +1,73 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package boltdb + +import ( + store "github.com/blevesearch/upsidedown_store_api" + bolt "go.etcd.io/bbolt" +) + +type Reader struct { + store *Store + tx *bolt.Tx + bucket *bolt.Bucket +} + +func (r *Reader) Get(key []byte) ([]byte, error) { + var rv []byte + v := r.bucket.Get(key) + if v != nil { + rv = make([]byte, len(v)) + copy(rv, v) + } + return rv, nil +} + +func (r *Reader) MultiGet(keys [][]byte) ([][]byte, error) { + return store.MultiGet(r, keys) +} + +func (r *Reader) PrefixIterator(prefix []byte) store.KVIterator { + cursor := r.bucket.Cursor() + + rv := &Iterator{ + store: r.store, + tx: r.tx, + cursor: cursor, + prefix: prefix, + } + + rv.Seek(prefix) + return rv +} + +func (r *Reader) RangeIterator(start, end []byte) store.KVIterator { + cursor := r.bucket.Cursor() + + rv := &Iterator{ + store: r.store, + tx: r.tx, + cursor: cursor, + start: start, + end: end, + } + + rv.Seek(start) + return rv +} + +func (r *Reader) Close() error { + return r.tx.Rollback() +} diff --git a/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb/stats.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb/stats.go new file mode 100644 index 00000000..d8961832 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb/stats.go @@ -0,0 +1,28 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package boltdb + +import ( + "github.com/blevesearch/bleve/v2/util" +) + +type stats struct { + s *Store +} + +func (s *stats) MarshalJSON() ([]byte, error) { + bs := s.s.db.Stats() + return util.MarshalJSON(bs) +} diff --git a/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb/store.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb/store.go new file mode 100644 index 00000000..bc99275e --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb/store.go @@ -0,0 +1,181 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package boltdb implements a store.KVStore on top of BoltDB. It supports the +// following options: +// +// "bucket" (string): the name of BoltDB bucket to use, defaults to "bleve". +// +// "nosync" (bool): if true, set boltdb.DB.NoSync to true. It speeds up index +// operations in exchange of losing integrity guarantees if indexation aborts +// without closing the index. Use it when rebuilding indexes from zero. +package boltdb + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + + "github.com/blevesearch/bleve/v2/registry" + store "github.com/blevesearch/upsidedown_store_api" + bolt "go.etcd.io/bbolt" +) + +const ( + Name = "boltdb" + defaultCompactBatchSize = 100 +) + +type Store struct { + path string + bucket string + db *bolt.DB + noSync bool + fillPercent float64 + mo store.MergeOperator +} + +func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) { + path, ok := config["path"].(string) + if !ok { + return nil, fmt.Errorf("must specify path") + } + if path == "" { + return nil, os.ErrInvalid + } + + bucket, ok := config["bucket"].(string) + if !ok { + bucket = "bleve" + } + + noSync, _ := config["nosync"].(bool) + + fillPercent, ok := config["fillPercent"].(float64) + if !ok { + fillPercent = bolt.DefaultFillPercent + } + + bo := &bolt.Options{} + ro, ok := config["read_only"].(bool) + if ok { + bo.ReadOnly = ro + } + + if initialMmapSize, ok := config["initialMmapSize"].(int); ok { + bo.InitialMmapSize = initialMmapSize + } else if initialMmapSize, ok := config["initialMmapSize"].(float64); ok { + bo.InitialMmapSize = int(initialMmapSize) + } + + db, err := bolt.Open(path, 0600, bo) + if err != nil { + return nil, err + } + db.NoSync = noSync + + if !bo.ReadOnly { + err = db.Update(func(tx *bolt.Tx) error { + _, err := tx.CreateBucketIfNotExists([]byte(bucket)) + + return err + }) + if err != nil { + return nil, err + } + } + + rv := Store{ + path: path, + bucket: bucket, + db: db, + mo: mo, + noSync: noSync, + fillPercent: fillPercent, + } + return &rv, nil +} + +func (bs *Store) Close() error { + return bs.db.Close() +} + +func (bs *Store) Reader() (store.KVReader, error) { + tx, err := bs.db.Begin(false) + if err != nil { + return nil, err + } + return &Reader{ + store: bs, + tx: tx, + bucket: tx.Bucket([]byte(bs.bucket)), + }, nil +} + +func (bs *Store) Writer() (store.KVWriter, error) { + return &Writer{ + store: bs, + }, nil +} + +func (bs *Store) Stats() json.Marshaler { + return &stats{ + s: bs, + } +} + +// CompactWithBatchSize removes DictionaryTerm entries with a count of zero (in batchSize batches) +// Removing entries is a workaround for github issue #374. +func (bs *Store) CompactWithBatchSize(batchSize int) error { + for { + cnt := 0 + err := bs.db.Batch(func(tx *bolt.Tx) error { + c := tx.Bucket([]byte(bs.bucket)).Cursor() + prefix := []byte("d") + + for k, v := c.Seek(prefix); bytes.HasPrefix(k, prefix); k, v = c.Next() { + if bytes.Equal(v, []byte{0}) { + cnt++ + if err := c.Delete(); err != nil { + return err + } + if cnt == batchSize { + break + } + } + + } + return nil + }) + if err != nil { + return err + } + + if cnt == 0 { + break + } + } + return nil +} + +// Compact calls CompactWithBatchSize with a default batch size of 100. This is a workaround +// for github issue #374. +func (bs *Store) Compact() error { + return bs.CompactWithBatchSize(defaultCompactBatchSize) +} + +func init() { + registry.RegisterKVStore(Name, New) +} diff --git a/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb/writer.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb/writer.go new file mode 100644 index 00000000..c25583ca --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb/writer.go @@ -0,0 +1,95 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package boltdb + +import ( + "fmt" + + store "github.com/blevesearch/upsidedown_store_api" +) + +type Writer struct { + store *Store +} + +func (w *Writer) NewBatch() store.KVBatch { + return store.NewEmulatedBatch(w.store.mo) +} + +func (w *Writer) NewBatchEx(options store.KVBatchOptions) ([]byte, store.KVBatch, error) { + return make([]byte, options.TotalBytes), w.NewBatch(), nil +} + +func (w *Writer) ExecuteBatch(batch store.KVBatch) (err error) { + + emulatedBatch, ok := batch.(*store.EmulatedBatch) + if !ok { + return fmt.Errorf("wrong type of batch") + } + + tx, err := w.store.db.Begin(true) + if err != nil { + return + } + // defer function to ensure that once started, + // we either Commit tx or Rollback + defer func() { + // if nothing went wrong, commit + if err == nil { + // careful to catch error here too + err = tx.Commit() + } else { + // caller should see error that caused abort, + // not success or failure of Rollback itself + _ = tx.Rollback() + } + }() + + bucket := tx.Bucket([]byte(w.store.bucket)) + bucket.FillPercent = w.store.fillPercent + + for k, mergeOps := range emulatedBatch.Merger.Merges { + kb := []byte(k) + existingVal := bucket.Get(kb) + mergedVal, fullMergeOk := w.store.mo.FullMerge(kb, existingVal, mergeOps) + if !fullMergeOk { + err = fmt.Errorf("merge operator returned failure") + return + } + err = bucket.Put(kb, mergedVal) + if err != nil { + return + } + } + + for _, op := range emulatedBatch.Ops { + if op.V != nil { + err = bucket.Put(op.K, op.V) + if err != nil { + return + } + } else { + err = bucket.Delete(op.K) + if err != nil { + return + } + } + } + return +} + +func (w *Writer) Close() error { + return nil +} diff --git a/vendor/github.com/blevesearch/bleve/index/store/gtreap/iterator.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap/iterator.go similarity index 98% rename from vendor/github.com/blevesearch/bleve/index/store/gtreap/iterator.go rename to vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap/iterator.go index 092ccf24..c03f75c7 100644 --- a/vendor/github.com/blevesearch/bleve/index/store/gtreap/iterator.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap/iterator.go @@ -21,7 +21,7 @@ import ( "bytes" "sync" - "github.com/steveyen/gtreap" + "github.com/blevesearch/gtreap" ) type Iterator struct { diff --git a/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap/reader.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap/reader.go new file mode 100644 index 00000000..52b05d78 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap/reader.go @@ -0,0 +1,66 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package gtreap provides an in-memory implementation of the +// KVStore interfaces using the gtreap balanced-binary treap, +// copy-on-write data structure. +package gtreap + +import ( + "github.com/blevesearch/upsidedown_store_api" + + "github.com/blevesearch/gtreap" +) + +type Reader struct { + t *gtreap.Treap +} + +func (w *Reader) Get(k []byte) (v []byte, err error) { + var rv []byte + itm := w.t.Get(&Item{k: k}) + if itm != nil { + rv = make([]byte, len(itm.(*Item).v)) + copy(rv, itm.(*Item).v) + return rv, nil + } + return nil, nil +} + +func (r *Reader) MultiGet(keys [][]byte) ([][]byte, error) { + return store.MultiGet(r, keys) +} + +func (w *Reader) PrefixIterator(k []byte) store.KVIterator { + rv := Iterator{ + t: w.t, + prefix: k, + } + rv.restart(&Item{k: k}) + return &rv +} + +func (w *Reader) RangeIterator(start, end []byte) store.KVIterator { + rv := Iterator{ + t: w.t, + start: start, + end: end, + } + rv.restart(&Item{k: start}) + return &rv +} + +func (w *Reader) Close() error { + return nil +} diff --git a/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap/store.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap/store.go new file mode 100644 index 00000000..3cc7eb9a --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap/store.go @@ -0,0 +1,82 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package gtreap provides an in-memory implementation of the +// KVStore interfaces using the gtreap balanced-binary treap, +// copy-on-write data structure. + +package gtreap + +import ( + "bytes" + "fmt" + "os" + "sync" + + "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/gtreap" + "github.com/blevesearch/upsidedown_store_api" +) + +const Name = "gtreap" + +type Store struct { + m sync.Mutex + t *gtreap.Treap + mo store.MergeOperator +} + +type Item struct { + k []byte + v []byte +} + +func itemCompare(a, b interface{}) int { + return bytes.Compare(a.(*Item).k, b.(*Item).k) +} + +func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) { + path, ok := config["path"].(string) + if !ok { + return nil, fmt.Errorf("must specify path") + } + if path != "" { + return nil, os.ErrInvalid + } + + rv := Store{ + t: gtreap.NewTreap(itemCompare), + mo: mo, + } + return &rv, nil +} + +func (s *Store) Close() error { + return nil +} + +func (s *Store) Reader() (store.KVReader, error) { + s.m.Lock() + t := s.t + s.m.Unlock() + return &Reader{t: t}, nil +} + +func (s *Store) Writer() (store.KVWriter, error) { + return &Writer{s: s}, nil +} + +func init() { + registry.RegisterKVStore(Name, New) +} diff --git a/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap/writer.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap/writer.go new file mode 100644 index 00000000..80aa15b1 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap/writer.go @@ -0,0 +1,76 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package gtreap provides an in-memory implementation of the +// KVStore interfaces using the gtreap balanced-binary treap, +// copy-on-write data structure. +package gtreap + +import ( + "fmt" + "math/rand" + + "github.com/blevesearch/upsidedown_store_api" +) + +type Writer struct { + s *Store +} + +func (w *Writer) NewBatch() store.KVBatch { + return store.NewEmulatedBatch(w.s.mo) +} + +func (w *Writer) NewBatchEx(options store.KVBatchOptions) ([]byte, store.KVBatch, error) { + return make([]byte, options.TotalBytes), w.NewBatch(), nil +} + +func (w *Writer) ExecuteBatch(batch store.KVBatch) error { + + emulatedBatch, ok := batch.(*store.EmulatedBatch) + if !ok { + return fmt.Errorf("wrong type of batch") + } + + w.s.m.Lock() + for k, mergeOps := range emulatedBatch.Merger.Merges { + kb := []byte(k) + var existingVal []byte + existingItem := w.s.t.Get(&Item{k: kb}) + if existingItem != nil { + existingVal = w.s.t.Get(&Item{k: kb}).(*Item).v + } + mergedVal, fullMergeOk := w.s.mo.FullMerge(kb, existingVal, mergeOps) + if !fullMergeOk { + return fmt.Errorf("merge operator returned failure") + } + w.s.t = w.s.t.Upsert(&Item{k: kb, v: mergedVal}, rand.Int()) + } + + for _, op := range emulatedBatch.Ops { + if op.V != nil { + w.s.t = w.s.t.Upsert(&Item{k: op.K, v: op.V}, rand.Int()) + } else { + w.s.t = w.s.t.Delete(&Item{k: op.K}) + } + } + w.s.m.Unlock() + + return nil +} + +func (w *Writer) Close() error { + w.s = nil + return nil +} diff --git a/vendor/github.com/blevesearch/bleve/index/upsidedown/upsidedown.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/upsidedown.go similarity index 87% rename from vendor/github.com/blevesearch/bleve/index/upsidedown/upsidedown.go rename to vendor/github.com/blevesearch/bleve/v2/index/upsidedown/upsidedown.go index 8e915c6a..3756422d 100644 --- a/vendor/github.com/blevesearch/bleve/index/upsidedown/upsidedown.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/upsidedown.go @@ -25,11 +25,10 @@ import ( "sync/atomic" "time" - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/store" - "github.com/blevesearch/bleve/registry" + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/registry" + index "github.com/blevesearch/bleve_index_api" + store "github.com/blevesearch/upsidedown_store_api" "github.com/golang/protobuf/proto" ) @@ -49,13 +48,15 @@ const Version uint8 = 7 var IncompatibleVersion = fmt.Errorf("incompatible version, %d is supported", Version) +var ErrorUnknownStorageType = fmt.Errorf("unknown storage type") + type UpsideDownCouch struct { version uint8 path string storeName string storeConfig map[string]interface{} store store.KVStore - fieldCache *index.FieldCache + fieldCache *FieldCache analysisQueue *index.AnalysisQueue stats *indexStat @@ -68,14 +69,14 @@ type UpsideDownCouch struct { type docBackIndexRow struct { docID string - doc *document.Document // If deletion, doc will be nil. + doc index.Document // If deletion, doc will be nil. backIndexRow *BackIndexRow } func NewUpsideDownCouch(storeName string, storeConfig map[string]interface{}, analysisQueue *index.AnalysisQueue) (index.Index, error) { rv := &UpsideDownCouch{ version: Version, - fieldCache: index.NewFieldCache(), + fieldCache: NewFieldCache(), storeName: storeName, storeConfig: storeConfig, analysisQueue: analysisQueue, @@ -133,18 +134,23 @@ func (udc *UpsideDownCouch) loadSchema(kvreader store.KVReader) (err error) { return } +type rowBuffer struct { + buf []byte +} + var rowBufferPool sync.Pool -func GetRowBuffer() []byte { - if rb, ok := rowBufferPool.Get().([]byte); ok { +func GetRowBuffer() *rowBuffer { + if rb, ok := rowBufferPool.Get().(*rowBuffer); ok { return rb } else { - return make([]byte, RowBufferSize) + buf := make([]byte, RowBufferSize) + return &rowBuffer{buf: buf} } } -func PutRowBuffer(buf []byte) { - rowBufferPool.Put(buf) +func PutRowBuffer(rb *rowBuffer) { + rowBufferPool.Put(rb) } func (udc *UpsideDownCouch) batchRows(writer store.KVWriter, addRowsAll [][]UpsideDownCouchRow, updateRowsAll [][]UpsideDownCouchRow, deleteRowsAll [][]UpsideDownCouchRow) (err error) { @@ -168,14 +174,14 @@ func (udc *UpsideDownCouch) batchRows(writer store.KVWriter, addRowsAll [][]Upsi for _, row := range addRows { tfr, ok := row.(*TermFrequencyRow) if ok { - if tfr.DictionaryRowKeySize() > len(rowBuf) { - rowBuf = make([]byte, tfr.DictionaryRowKeySize()) + if tfr.DictionaryRowKeySize() > len(rowBuf.buf) { + rowBuf.buf = make([]byte, tfr.DictionaryRowKeySize()) } - dictKeySize, err := tfr.DictionaryRowKeyTo(rowBuf) + dictKeySize, err := tfr.DictionaryRowKeyTo(rowBuf.buf) if err != nil { return err } - dictionaryDeltas[string(rowBuf[:dictKeySize])] += 1 + dictionaryDeltas[string(rowBuf.buf[:dictKeySize])] += 1 } addKeyBytes += row.KeySize() addValBytes += row.ValueSize() @@ -196,14 +202,14 @@ func (udc *UpsideDownCouch) batchRows(writer store.KVWriter, addRowsAll [][]Upsi tfr, ok := row.(*TermFrequencyRow) if ok { // need to decrement counter - if tfr.DictionaryRowKeySize() > len(rowBuf) { - rowBuf = make([]byte, tfr.DictionaryRowKeySize()) + if tfr.DictionaryRowKeySize() > len(rowBuf.buf) { + rowBuf.buf = make([]byte, tfr.DictionaryRowKeySize()) } - dictKeySize, err := tfr.DictionaryRowKeyTo(rowBuf) + dictKeySize, err := tfr.DictionaryRowKeyTo(rowBuf.buf) if err != nil { return err } - dictionaryDeltas[string(rowBuf[:dictKeySize])] -= 1 + dictionaryDeltas[string(rowBuf.buf[:dictKeySize])] -= 1 } deleteKeyBytes += row.KeySize() } @@ -300,7 +306,7 @@ func (udc *UpsideDownCouch) Open() (err error) { // open the kv store storeConstructor := registry.KVStoreConstructorByName(udc.storeName) if storeConstructor == nil { - err = index.ErrorUnknownStorageType + err = ErrorUnknownStorageType return } @@ -412,14 +418,16 @@ func (udc *UpsideDownCouch) Close() error { return udc.store.Close() } -func (udc *UpsideDownCouch) Update(doc *document.Document) (err error) { +func (udc *UpsideDownCouch) Update(doc index.Document) (err error) { // do analysis before acquiring write lock analysisStart := time.Now() - resultChan := make(chan *index.AnalysisResult) - aw := index.NewAnalysisWork(udc, doc, resultChan) + resultChan := make(chan *AnalysisResult) // put the work on the queue - udc.analysisQueue.Queue(aw) + udc.analysisQueue.Queue(func() { + ar := udc.analyze(doc) + resultChan <- ar + }) // wait for the result result := <-resultChan @@ -439,7 +447,7 @@ func (udc *UpsideDownCouch) Update(doc *document.Document) (err error) { // first we lookup the backindex row for the doc id if it exists // lookup the back index row var backIndexRow *BackIndexRow - backIndexRow, err = backIndexRowForDoc(kvreader, index.IndexInternalID(doc.ID)) + backIndexRow, err = backIndexRowForDoc(kvreader, index.IndexInternalID(doc.ID())) if err != nil { _ = kvreader.Close() atomic.AddUint64(&udc.stats.errors, 1) @@ -454,8 +462,8 @@ func (udc *UpsideDownCouch) Update(doc *document.Document) (err error) { return udc.UpdateWithAnalysis(doc, result, backIndexRow) } -func (udc *UpsideDownCouch) UpdateWithAnalysis(doc *document.Document, - result *index.AnalysisResult, backIndexRow *BackIndexRow) (err error) { +func (udc *UpsideDownCouch) UpdateWithAnalysis(doc index.Document, + result *AnalysisResult, backIndexRow *BackIndexRow) (err error) { // start a writer for this update indexStart := time.Now() var kvwriter store.KVWriter @@ -501,7 +509,7 @@ func (udc *UpsideDownCouch) UpdateWithAnalysis(doc *document.Document, return } -func (udc *UpsideDownCouch) mergeOldAndNew(backIndexRow *BackIndexRow, rows []index.IndexRow) (addRows []UpsideDownCouchRow, updateRows []UpsideDownCouchRow, deleteRows []UpsideDownCouchRow) { +func (udc *UpsideDownCouch) mergeOldAndNew(backIndexRow *BackIndexRow, rows []IndexRow) (addRows []UpsideDownCouchRow, updateRows []UpsideDownCouchRow, deleteRows []UpsideDownCouchRow) { addRows = make([]UpsideDownCouchRow, 0, len(rows)) if backIndexRow == nil { @@ -538,26 +546,26 @@ func (udc *UpsideDownCouch) mergeOldAndNew(backIndexRow *BackIndexRow, rows []in switch row := row.(type) { case *TermFrequencyRow: if existingTermKeys != nil { - if row.KeySize() > len(keyBuf) { - keyBuf = make([]byte, row.KeySize()) + if row.KeySize() > len(keyBuf.buf) { + keyBuf.buf = make([]byte, row.KeySize()) } - keySize, _ := row.KeyTo(keyBuf) - if _, ok := existingTermKeys[string(keyBuf[:keySize])]; ok { + keySize, _ := row.KeyTo(keyBuf.buf) + if _, ok := existingTermKeys[string(keyBuf.buf[:keySize])]; ok { updateRows = append(updateRows, row) - delete(existingTermKeys, string(keyBuf[:keySize])) + delete(existingTermKeys, string(keyBuf.buf[:keySize])) continue } } addRows = append(addRows, row) case *StoredRow: if existingStoredKeys != nil { - if row.KeySize() > len(keyBuf) { - keyBuf = make([]byte, row.KeySize()) + if row.KeySize() > len(keyBuf.buf) { + keyBuf.buf = make([]byte, row.KeySize()) } - keySize, _ := row.KeyTo(keyBuf) - if _, ok := existingStoredKeys[string(keyBuf[:keySize])]; ok { + keySize, _ := row.KeyTo(keyBuf.buf) + if _, ok := existingStoredKeys[string(keyBuf.buf[:keySize])]; ok { updateRows = append(updateRows, row) - delete(existingStoredKeys, string(keyBuf[:keySize])) + delete(existingStoredKeys, string(keyBuf.buf[:keySize])) continue } } @@ -587,8 +595,8 @@ func (udc *UpsideDownCouch) mergeOldAndNew(backIndexRow *BackIndexRow, rows []in return addRows, updateRows, deleteRows } -func (udc *UpsideDownCouch) storeField(docID []byte, field document.Field, fieldIndex uint16, rows []index.IndexRow, backIndexStoredEntries []*BackIndexStoreEntry) ([]index.IndexRow, []*BackIndexStoreEntry) { - fieldType := encodeFieldType(field) +func (udc *UpsideDownCouch) storeField(docID []byte, field index.Field, fieldIndex uint16, rows []IndexRow, backIndexStoredEntries []*BackIndexStoreEntry) ([]IndexRow, []*BackIndexStoreEntry) { + fieldType := field.EncodedFieldType() storedRow := NewStoredRow(docID, fieldIndex, field.ArrayPositions(), fieldType, field.Value()) // record the back index entry @@ -597,26 +605,7 @@ func (udc *UpsideDownCouch) storeField(docID []byte, field document.Field, field return append(rows, storedRow), append(backIndexStoredEntries, &backIndexStoredEntry) } -func encodeFieldType(f document.Field) byte { - fieldType := byte('x') - switch f.(type) { - case *document.TextField: - fieldType = 't' - case *document.NumericField: - fieldType = 'n' - case *document.DateTimeField: - fieldType = 'd' - case *document.BooleanField: - fieldType = 'b' - case *document.GeoPointField: - fieldType = 'g' - case *document.CompositeField: - fieldType = 'c' - } - return fieldType -} - -func (udc *UpsideDownCouch) indexField(docID []byte, includeTermVectors bool, fieldIndex uint16, fieldLength int, tokenFreqs analysis.TokenFrequencies, rows []index.IndexRow, backIndexTermsEntries []*BackIndexTermsEntry) ([]index.IndexRow, []*BackIndexTermsEntry) { +func (udc *UpsideDownCouch) indexField(docID []byte, includeTermVectors bool, fieldIndex uint16, fieldLength int, tokenFreqs index.TokenFrequencies, rows []IndexRow, backIndexTermsEntries []*BackIndexTermsEntry) ([]IndexRow, []*BackIndexTermsEntry) { fieldNorm := float32(1.0 / math.Sqrt(float64(fieldLength))) termFreqRows := make([]TermFrequencyRow, len(tokenFreqs)) @@ -743,15 +732,17 @@ func decodeFieldType(typ byte, name string, pos []uint64, value []byte) document return document.NewBooleanFieldFromBytes(name, pos, value) case 'g': return document.NewGeoPointFieldFromBytes(name, pos, value) + case 'i': + return document.NewIPFieldFromBytes(name, pos, value) } return nil } -func frequencyFromTokenFreq(tf *analysis.TokenFreq) int { +func frequencyFromTokenFreq(tf *index.TokenFreq) int { return tf.Frequency() } -func (udc *UpsideDownCouch) termVectorsFromTokenFreq(field uint16, tf *analysis.TokenFreq, rows []index.IndexRow) ([]*TermVector, []index.IndexRow) { +func (udc *UpsideDownCouch) termVectorsFromTokenFreq(field uint16, tf *index.TokenFreq, rows []IndexRow) ([]*TermVector, []IndexRow) { a := make([]TermVector, len(tf.Locations)) rv := make([]*TermVector, len(tf.Locations)) @@ -807,7 +798,7 @@ func (udc *UpsideDownCouch) Batch(batch *index.Batch) (err error) { } analysisStart := time.Now() - resultChan := make(chan *index.AnalysisResult, len(batch.IndexOps)) + resultChan := make(chan *AnalysisResult, len(batch.IndexOps)) var numUpdates uint64 var numPlainTextBytes uint64 @@ -823,9 +814,11 @@ func (udc *UpsideDownCouch) Batch(batch *index.Batch) (err error) { for k := range batch.IndexOps { doc := batch.IndexOps[k] if doc != nil { - aw := index.NewAnalysisWork(udc, doc, resultChan) // put the work on the queue - udc.analysisQueue.Queue(aw) + udc.analysisQueue.Queue(func() { + ar := udc.analyze(doc) + resultChan <- ar + }) } } }() @@ -866,7 +859,7 @@ func (udc *UpsideDownCouch) Batch(batch *index.Batch) (err error) { }() // wait for analysis result - newRowsMap := make(map[string][]index.IndexRow) + newRowsMap := make(map[string][]IndexRow) var itemsDeQueued uint64 for itemsDeQueued < numUpdates { result := <-resultChan @@ -1059,23 +1052,23 @@ func backIndexRowForDoc(kvreader store.KVReader, docID index.IndexInternalID) (* } keyBuf := GetRowBuffer() - if tempRow.KeySize() > len(keyBuf) { - keyBuf = make([]byte, 2*tempRow.KeySize()) + if tempRow.KeySize() > len(keyBuf.buf) { + keyBuf.buf = make([]byte, 2*tempRow.KeySize()) } defer PutRowBuffer(keyBuf) - keySize, err := tempRow.KeyTo(keyBuf) + keySize, err := tempRow.KeyTo(keyBuf.buf) if err != nil { return nil, err } - value, err := kvreader.Get(keyBuf[:keySize]) + value, err := kvreader.Get(keyBuf.buf[:keySize]) if err != nil { return nil, err } if value == nil { return nil, nil } - backIndexRow, err := NewBackIndexRowKV(keyBuf[:keySize], value) + backIndexRow, err := NewBackIndexRowKV(keyBuf.buf[:keySize], value) if err != nil { return nil, err } diff --git a/vendor/github.com/blevesearch/bleve/index/upsidedown/upsidedown.pb.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/upsidedown.pb.go similarity index 98% rename from vendor/github.com/blevesearch/bleve/index/upsidedown/upsidedown.pb.go rename to vendor/github.com/blevesearch/bleve/v2/index/upsidedown/upsidedown.pb.go index c161e1cc..f2080c9b 100644 --- a/vendor/github.com/blevesearch/bleve/index/upsidedown/upsidedown.pb.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/upsidedown.pb.go @@ -3,15 +3,17 @@ // DO NOT EDIT! /* - Package upsidedown is a generated protocol buffer package. +Package upsidedown is a generated protocol buffer package. - It is generated from these files: - upsidedown.proto +It is generated from these files: - It has these top-level messages: - BackIndexTermsEntry - BackIndexStoreEntry - BackIndexRowValue + upsidedown.proto + +It has these top-level messages: + + BackIndexTermsEntry + BackIndexStoreEntry + BackIndexRowValue */ package upsidedown diff --git a/vendor/github.com/blevesearch/bleve/index/upsidedown/upsidedown.proto b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/upsidedown.proto similarity index 100% rename from vendor/github.com/blevesearch/bleve/index/upsidedown/upsidedown.proto rename to vendor/github.com/blevesearch/bleve/v2/index/upsidedown/upsidedown.proto diff --git a/vendor/github.com/blevesearch/bleve/index_alias.go b/vendor/github.com/blevesearch/bleve/v2/index_alias.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/index_alias.go rename to vendor/github.com/blevesearch/bleve/v2/index_alias.go diff --git a/vendor/github.com/blevesearch/bleve/index_alias_impl.go b/vendor/github.com/blevesearch/bleve/v2/index_alias_impl.go similarity index 96% rename from vendor/github.com/blevesearch/bleve/index_alias_impl.go rename to vendor/github.com/blevesearch/bleve/v2/index_alias_impl.go index 5aa57d8a..a73dd6b8 100644 --- a/vendor/github.com/blevesearch/bleve/index_alias_impl.go +++ b/vendor/github.com/blevesearch/bleve/v2/index_alias_impl.go @@ -19,11 +19,9 @@ import ( "sync" "time" - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/store" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" ) type indexAliasImpl struct { @@ -110,7 +108,7 @@ func (i *indexAliasImpl) Batch(b *Batch) error { return i.indexes[0].Batch(b) } -func (i *indexAliasImpl) Document(id string) (*document.Document, error) { +func (i *indexAliasImpl) Document(id string) (index.Document, error) { i.mutex.RLock() defer i.mutex.RUnlock() @@ -369,17 +367,17 @@ func (i *indexAliasImpl) DeleteInternal(key []byte) error { return i.indexes[0].DeleteInternal(key) } -func (i *indexAliasImpl) Advanced() (index.Index, store.KVStore, error) { +func (i *indexAliasImpl) Advanced() (index.Index, error) { i.mutex.RLock() defer i.mutex.RUnlock() if !i.open { - return nil, nil, ErrorIndexClosed + return nil, ErrorIndexClosed } err := i.isAliasToSingleIndex() if err != nil { - return nil, nil, err + return nil, err } return i.indexes[0].Advanced() @@ -604,6 +602,10 @@ type indexAliasImplFieldDict struct { fieldDict index.FieldDict } +func (f *indexAliasImplFieldDict) BytesRead() uint64 { + return f.fieldDict.BytesRead() +} + func (f *indexAliasImplFieldDict) Next() (*index.DictEntry, error) { return f.fieldDict.Next() } diff --git a/vendor/github.com/blevesearch/bleve/index_impl.go b/vendor/github.com/blevesearch/bleve/v2/index_impl.go similarity index 79% rename from vendor/github.com/blevesearch/bleve/index_impl.go rename to vendor/github.com/blevesearch/bleve/v2/index_impl.go index 0520fe43..d5f34a2a 100644 --- a/vendor/github.com/blevesearch/bleve/index_impl.go +++ b/vendor/github.com/blevesearch/bleve/v2/index_impl.go @@ -16,23 +16,27 @@ package bleve import ( "context" - "encoding/json" "fmt" + "io" "os" + "path/filepath" + "strconv" "sync" "sync/atomic" "time" - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/store" - "github.com/blevesearch/bleve/index/upsidedown" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/registry" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/collector" - "github.com/blevesearch/bleve/search/facet" - "github.com/blevesearch/bleve/search/highlight" + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/index/scorch" + "github.com/blevesearch/bleve/v2/index/upsidedown" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/collector" + "github.com/blevesearch/bleve/v2/search/facet" + "github.com/blevesearch/bleve/v2/search/highlight" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" + "github.com/blevesearch/geo/s2" ) type indexImpl struct { @@ -107,9 +111,6 @@ func newIndexUsing(path string, mapping mapping.IndexMapping, indexType string, } err = rv.i.Open() if err != nil { - if err == index.ErrorUnknownStorageType { - return nil, ErrorUnknownStorageType - } return nil, err } defer func(rv *indexImpl) { @@ -119,7 +120,7 @@ func newIndexUsing(path string, mapping mapping.IndexMapping, indexType string, }(&rv) // now persist the mapping - mappingBytes, err := json.Marshal(mapping) + mappingBytes, err := util.MarshalJSON(mapping) if err != nil { return nil, err } @@ -177,9 +178,6 @@ func openIndexUsing(path string, runtimeConfig map[string]interface{}) (rv *inde } err = rv.i.Open() if err != nil { - if err == index.ErrorUnknownStorageType { - return nil, ErrorUnknownStorageType - } return nil, err } defer func(rv *indexImpl) { @@ -205,7 +203,7 @@ func openIndexUsing(path string, runtimeConfig map[string]interface{}) (rv *inde } var im *mapping.IndexMappingImpl - err = json.Unmarshal(mappingBytes, &im) + err = util.UnmarshalJSON(mappingBytes, &im) if err != nil { return nil, fmt.Errorf("error parsing mapping JSON: %v\nmapping contents:\n%s", err, string(mappingBytes)) } @@ -228,14 +226,9 @@ func openIndexUsing(path string, runtimeConfig map[string]interface{}) (rv *inde return rv, err } -// Advanced returns implementation internals -// necessary ONLY for advanced usage. -func (i *indexImpl) Advanced() (index.Index, store.KVStore, error) { - s, err := i.i.Advanced() - if err != nil { - return nil, nil, err - } - return i.i, s, nil +// Advanced returns internal index implementation +func (i *indexImpl) Advanced() (index.Index, error) { + return i.i, nil } // Mapping returns the IndexMapping in use by this @@ -271,7 +264,7 @@ func (i *indexImpl) Index(id string, data interface{}) (err error) { // IndexAdvanced takes a document.Document object // skips the mapping and indexes it. func (i *indexImpl) IndexAdvanced(doc *document.Document) (err error) { - if doc.ID == "" { + if doc.ID() == "" { return ErrorEmptyID } @@ -323,7 +316,7 @@ func (i *indexImpl) Batch(b *Batch) error { // stored fields for a document in the index. These // stored fields are put back into a Document object // and returned. -func (i *indexImpl) Document(id string) (doc *document.Document, err error) { +func (i *indexImpl) Document(id string) (doc index.Document, err error) { i.mutex.RLock() defer i.mutex.RUnlock() @@ -478,7 +471,32 @@ func (i *indexImpl) SearchInContext(ctx context.Context, req *SearchRequest) (sr } }() - searcher, err := req.Query.Searcher(indexReader, i.m, search.SearcherOptions{ + // This callback and variable handles the tracking of bytes read + // 1. as part of creation of tfr and its Next() calls which is + // accounted by invoking this callback when the TFR is closed. + // 2. the docvalues portion (accounted in collector) and the retrieval + // of stored fields bytes (by LoadAndHighlightFields) + var totalSearchCost uint64 + sendBytesRead := func(bytesRead uint64) { + totalSearchCost += bytesRead + } + + ctx = context.WithValue(ctx, search.SearchIOStatsCallbackKey, + search.SearchIOStatsCallbackFunc(sendBytesRead)) + + var bufPool *s2.GeoBufferPool + getBufferPool := func() *s2.GeoBufferPool { + if bufPool == nil { + bufPool = s2.NewGeoBufferPool(search.MaxGeoBufPoolSize, search.MinGeoBufPoolSize) + } + + return bufPool + } + + ctx = context.WithValue(ctx, search.GeoBufferPoolCallbackKey, + search.GeoBufferPoolCallbackFunc(getBufferPool)) + + searcher, err := req.Query.Searcher(ctx, indexReader, i.m, search.SearcherOptions{ Explain: req.Explain, IncludeTermVectors: req.IncludeLocations || req.Highlight != nil, Score: req.Score, @@ -490,6 +508,14 @@ func (i *indexImpl) SearchInContext(ctx context.Context, req *SearchRequest) (sr if serr := searcher.Close(); err == nil && serr != nil { err = serr } + if sr != nil { + sr.Cost = totalSearchCost + } + if sr, ok := indexReader.(*scorch.IndexSnapshot); ok { + sr.UpdateIOStats(totalSearchCost) + } + + search.RecordSearchCost(ctx, search.DoneM, 0) }() if req.Facets != nil { @@ -505,10 +531,23 @@ func (i *indexImpl) SearchInContext(ctx context.Context, req *SearchRequest) (sr } else if facetRequest.DateTimeRanges != nil { // build date range facet facetBuilder := facet.NewDateTimeFacetBuilder(facetRequest.Field, facetRequest.Size) - dateTimeParser := i.m.DateTimeParserNamed("") for _, dr := range facetRequest.DateTimeRanges { - start, end := dr.ParseDates(dateTimeParser) - facetBuilder.AddRange(dr.Name, start, end) + dateTimeParserName := defaultDateTimeParser + if dr.DateTimeParser != "" { + dateTimeParserName = dr.DateTimeParser + } + dateTimeParser := i.m.DateTimeParserNamed(dateTimeParserName) + if dateTimeParser == nil { + return nil, fmt.Errorf("no date time parser named `%s` registered", dateTimeParserName) + } + start, end, startLayout, endLayout, err := dr.ParseDates(dateTimeParser) + if err != nil { + return nil, fmt.Errorf("ParseDates err: %v, using date time parser named %s", err, dateTimeParserName) + } + if start.IsZero() && end.IsZero() { + return nil, fmt.Errorf("date range query must specify either start, end or both for date range name '%s'", dr.Name) + } + facetBuilder.AddRange(dr.Name, start, end, startLayout, endLayout) } facetsBuilder.Add(facetName, facetBuilder) } else { @@ -564,16 +603,21 @@ func (i *indexImpl) SearchInContext(ctx context.Context, req *SearchRequest) (sr } } + var storedFieldsCost uint64 for _, hit := range hits { if i.name != "" { hit.Index = i.name } - err = LoadAndHighlightFields(hit, req, i.name, indexReader, highlighter) + err, storedFieldsBytes := LoadAndHighlightFields(hit, req, i.name, indexReader, highlighter) if err != nil { return nil, err } + storedFieldsCost += storedFieldsBytes } + totalSearchCost += storedFieldsCost + search.RecordSearchCost(ctx, search.AddM, storedFieldsCost) + atomic.AddUint64(&i.stats.searches, 1) searchDuration := time.Since(searchStart) atomic.AddUint64(&i.stats.searchTime, uint64(searchDuration)) @@ -610,35 +654,42 @@ func (i *indexImpl) SearchInContext(ctx context.Context, req *SearchRequest) (sr func LoadAndHighlightFields(hit *search.DocumentMatch, req *SearchRequest, indexName string, r index.IndexReader, - highlighter highlight.Highlighter) error { + highlighter highlight.Highlighter) (error, uint64) { + var totalStoredFieldsBytes uint64 if len(req.Fields) > 0 || highlighter != nil { doc, err := r.Document(hit.ID) + totalStoredFieldsBytes = doc.StoredFieldsBytes() if err == nil && doc != nil { if len(req.Fields) > 0 { fieldsToLoad := deDuplicate(req.Fields) for _, f := range fieldsToLoad { - for _, docF := range doc.Fields { + doc.VisitFields(func(docF index.Field) { if f == "*" || docF.Name() == f { var value interface{} switch docF := docF.(type) { - case *document.TextField: - value = string(docF.Value()) - case *document.NumericField: + case index.TextField: + value = docF.Text() + case index.NumericField: num, err := docF.Number() if err == nil { value = num } - case *document.DateTimeField: - datetime, err := docF.DateTime() + case index.DateTimeField: + datetime, layout, err := docF.DateTime() if err == nil { - value = datetime.Format(time.RFC3339) + if layout == "" { + // layout not set probably means it was indexed as a timestamp + value = strconv.FormatInt(datetime.UnixNano(), 10) + } else { + value = datetime.Format(layout) + } } - case *document.BooleanField: + case index.BooleanField: boolean, err := docF.Boolean() if err == nil { value = boolean } - case *document.GeoPointField: + case index.GeoPointField: lon, err := docF.Lon() if err == nil { lat, err := docF.Lat() @@ -646,12 +697,18 @@ func LoadAndHighlightFields(hit *search.DocumentMatch, req *SearchRequest, value = []float64{lon, lat} } } + case index.GeoShapeField: + v, err := docF.GeoShape() + if err == nil { + value = v + } } + if value != nil { hit.AddFieldValue(docF.Name(), value) } } - } + }) } } if highlighter != nil { @@ -670,11 +727,11 @@ func LoadAndHighlightFields(hit *search.DocumentMatch, req *SearchRequest, } else if doc == nil { // unexpected case, a doc ID that was found as a search hit // was unable to be found during document lookup - return ErrorIndexReadInconsistency + return ErrorIndexReadInconsistency, 0 } } - return nil + return nil, totalStoredFieldsBytes } // Fields returns the name of all the fields this @@ -874,6 +931,10 @@ type indexImplFieldDict struct { fieldDict index.FieldDict } +func (f *indexImplFieldDict) BytesRead() uint64 { + return f.fieldDict.BytesRead() +} + func (f *indexImplFieldDict) Next() (*index.DictEntry, error) { return f.fieldDict.Next() } @@ -922,3 +983,49 @@ func (m *searchHitSorter) Less(i, j int) bool { c := m.sort.Compare(m.cachedScoring, m.cachedDesc, m.hits[i], m.hits[j]) return c < 0 } + +func (i *indexImpl) CopyTo(d index.Directory) (err error) { + i.mutex.RLock() + defer i.mutex.RUnlock() + + if !i.open { + return ErrorIndexClosed + } + + indexReader, err := i.i.Reader() + if err != nil { + return err + } + defer func() { + if cerr := indexReader.Close(); err == nil && cerr != nil { + err = cerr + } + }() + + irc, ok := indexReader.(IndexCopyable) + if !ok { + return fmt.Errorf("index implementation does not support copy") + } + + err = irc.CopyTo(d) + if err != nil { + return fmt.Errorf("error copying index metadata: %v", err) + } + + // copy the metadata + return i.meta.CopyTo(d) +} + +func (f FileSystemDirectory) GetWriter(filePath string) (io.WriteCloser, + error) { + dir, file := filepath.Split(filePath) + if dir != "" { + err := os.MkdirAll(filepath.Join(string(f), dir), os.ModePerm) + if err != nil { + return nil, err + } + } + + return os.OpenFile(filepath.Join(string(f), dir, file), + os.O_RDWR|os.O_CREATE, 0600) +} diff --git a/vendor/github.com/blevesearch/bleve/index_meta.go b/vendor/github.com/blevesearch/bleve/v2/index_meta.go similarity index 78% rename from vendor/github.com/blevesearch/bleve/index_meta.go rename to vendor/github.com/blevesearch/bleve/v2/index_meta.go index d814799a..14b88dcb 100644 --- a/vendor/github.com/blevesearch/bleve/index_meta.go +++ b/vendor/github.com/blevesearch/bleve/v2/index_meta.go @@ -15,12 +15,13 @@ package bleve import ( - "encoding/json" - "io/ioutil" + "fmt" "os" "path/filepath" - "github.com/blevesearch/bleve/index/upsidedown" + "github.com/blevesearch/bleve/v2/index/upsidedown" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" ) const metaFilename = "index_meta.json" @@ -44,12 +45,12 @@ func openIndexMeta(path string) (*indexMeta, error) { return nil, ErrorIndexPathDoesNotExist } indexMetaPath := indexMetaPath(path) - metaBytes, err := ioutil.ReadFile(indexMetaPath) + metaBytes, err := os.ReadFile(indexMetaPath) if err != nil { return nil, ErrorIndexMetaMissing } var im indexMeta - err = json.Unmarshal(metaBytes, &im) + err = util.UnmarshalJSON(metaBytes, &im) if err != nil { return nil, ErrorIndexMetaCorrupt } @@ -69,7 +70,7 @@ func (i *indexMeta) Save(path string) (err error) { } return err } - metaBytes, err := json.Marshal(i) + metaBytes, err := util.MarshalJSON(i) if err != nil { return err } @@ -92,6 +93,23 @@ func (i *indexMeta) Save(path string) (err error) { return nil } +func (i *indexMeta) CopyTo(d index.Directory) (err error) { + metaBytes, err := util.MarshalJSON(i) + if err != nil { + return err + } + + w, err := d.GetWriter(metaFilename) + if w == nil || err != nil { + return fmt.Errorf("invalid writer for file: %s, err: %v", + metaFilename, err) + } + defer w.Close() + + _, err = w.Write(metaBytes) + return err +} + func indexMetaPath(path string) string { return filepath.Join(path, metaFilename) } diff --git a/vendor/github.com/blevesearch/bleve/index_stats.go b/vendor/github.com/blevesearch/bleve/v2/index_stats.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/index_stats.go rename to vendor/github.com/blevesearch/bleve/v2/index_stats.go diff --git a/vendor/github.com/blevesearch/bleve/v2/mapping.go b/vendor/github.com/blevesearch/bleve/v2/mapping.go new file mode 100644 index 00000000..723105a2 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/mapping.go @@ -0,0 +1,79 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bleve + +import "github.com/blevesearch/bleve/v2/mapping" + +// NewIndexMapping creates a new IndexMapping that will use all the default indexing rules +func NewIndexMapping() *mapping.IndexMappingImpl { + return mapping.NewIndexMapping() +} + +// NewDocumentMapping returns a new document mapping +// with all the default values. +func NewDocumentMapping() *mapping.DocumentMapping { + return mapping.NewDocumentMapping() +} + +// NewDocumentStaticMapping returns a new document +// mapping that will not automatically index parts +// of a document without an explicit mapping. +func NewDocumentStaticMapping() *mapping.DocumentMapping { + return mapping.NewDocumentStaticMapping() +} + +// NewDocumentDisabledMapping returns a new document +// mapping that will not perform any indexing. +func NewDocumentDisabledMapping() *mapping.DocumentMapping { + return mapping.NewDocumentDisabledMapping() +} + +// NewTextFieldMapping returns a default field mapping for text +func NewTextFieldMapping() *mapping.FieldMapping { + return mapping.NewTextFieldMapping() +} + +// NewKeywordFieldMapping returns a field mapping for text using the keyword +// analyzer, which essentially doesn't apply any specific text analysis. +func NewKeywordFieldMapping() *mapping.FieldMapping { + return mapping.NewKeywordFieldMapping() +} + +// NewNumericFieldMapping returns a default field mapping for numbers +func NewNumericFieldMapping() *mapping.FieldMapping { + return mapping.NewNumericFieldMapping() +} + +// NewDateTimeFieldMapping returns a default field mapping for dates +func NewDateTimeFieldMapping() *mapping.FieldMapping { + return mapping.NewDateTimeFieldMapping() +} + +// NewBooleanFieldMapping returns a default field mapping for booleans +func NewBooleanFieldMapping() *mapping.FieldMapping { + return mapping.NewBooleanFieldMapping() +} + +func NewGeoPointFieldMapping() *mapping.FieldMapping { + return mapping.NewGeoPointFieldMapping() +} + +func NewGeoShapeFieldMapping() *mapping.FieldMapping { + return mapping.NewGeoShapeFieldMapping() +} + +func NewIPFieldMapping() *mapping.FieldMapping { + return mapping.NewIPFieldMapping() +} diff --git a/vendor/github.com/blevesearch/bleve/mapping/analysis.go b/vendor/github.com/blevesearch/bleve/v2/mapping/analysis.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/mapping/analysis.go rename to vendor/github.com/blevesearch/bleve/v2/mapping/analysis.go diff --git a/vendor/github.com/blevesearch/bleve/v2/mapping/document.go b/vendor/github.com/blevesearch/bleve/v2/mapping/document.go new file mode 100644 index 00000000..aacaa0a5 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/mapping/document.go @@ -0,0 +1,553 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mapping + +import ( + "encoding" + "encoding/json" + "fmt" + "net" + "reflect" + "time" + + "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/bleve/v2/util" +) + +// A DocumentMapping describes how a type of document +// should be indexed. +// As documents can be hierarchical, named sub-sections +// of documents are mapped using the same structure in +// the Properties field. +// Each value inside a document can be indexed 0 or more +// ways. These index entries are called fields and +// are stored in the Fields field. +// Entire sections of a document can be ignored or +// excluded by setting Enabled to false. +// If not explicitly mapped, default mapping operations +// are used. To disable this automatic handling, set +// Dynamic to false. +type DocumentMapping struct { + Enabled bool `json:"enabled"` + Dynamic bool `json:"dynamic"` + Properties map[string]*DocumentMapping `json:"properties,omitempty"` + Fields []*FieldMapping `json:"fields,omitempty"` + DefaultAnalyzer string `json:"default_analyzer,omitempty"` + + // StructTagKey overrides "json" when looking for field names in struct tags + StructTagKey string `json:"struct_tag_key,omitempty"` +} + +func (dm *DocumentMapping) Validate(cache *registry.Cache) error { + var err error + if dm.DefaultAnalyzer != "" { + _, err := cache.AnalyzerNamed(dm.DefaultAnalyzer) + if err != nil { + return err + } + } + for _, property := range dm.Properties { + err = property.Validate(cache) + if err != nil { + return err + } + } + for _, field := range dm.Fields { + if field.Analyzer != "" { + _, err = cache.AnalyzerNamed(field.Analyzer) + if err != nil { + return err + } + } + if field.DateFormat != "" { + _, err = cache.DateTimeParserNamed(field.DateFormat) + if err != nil { + return err + } + } + switch field.Type { + case "text", "datetime", "number", "boolean", "geopoint", "geoshape", "IP": + default: + return fmt.Errorf("unknown field type: '%s'", field.Type) + } + } + return nil +} + +// analyzerNameForPath attempts to first find the field +// described by this path, then returns the analyzer +// configured for that field +func (dm *DocumentMapping) analyzerNameForPath(path string) string { + field := dm.fieldDescribedByPath(path) + if field != nil { + return field.Analyzer + } + return "" +} + +func (dm *DocumentMapping) fieldDescribedByPath(path string) *FieldMapping { + pathElements := decodePath(path) + if len(pathElements) > 1 { + // easy case, there is more than 1 path element remaining + // the next path element must match a property name + // at this level + for propName, subDocMapping := range dm.Properties { + if propName == pathElements[0] { + return subDocMapping.fieldDescribedByPath(encodePath(pathElements[1:])) + } + } + } + + // either the path just had one element + // or it had multiple, but no match for the first element at this level + // look for match with full path + + // first look for property name with empty field + for propName, subDocMapping := range dm.Properties { + if propName == path { + // found property name match, now look at its fields + for _, field := range subDocMapping.Fields { + if field.Name == "" || field.Name == path { + // match + return field + } + } + } + } + // next, walk the properties again, looking for field overriding the name + for propName, subDocMapping := range dm.Properties { + if propName != path { + // property name isn't a match, but field name could override it + for _, field := range subDocMapping.Fields { + if field.Name == path { + return field + } + } + } + } + + return nil +} + +// documentMappingForPath returns the EXACT and closest matches for a sub +// document or for an explicitly mapped field; the closest most specific +// document mapping could be one that matches part of the provided path. +func (dm *DocumentMapping) documentMappingForPath(path string) ( + *DocumentMapping, *DocumentMapping) { + pathElements := decodePath(path) + current := dm +OUTER: + for i, pathElement := range pathElements { + if subDocMapping, exists := current.Properties[pathElement]; exists { + current = subDocMapping + continue OUTER + } + + // no subDocMapping matches this pathElement + // only if this is the last element check for field name + if i == len(pathElements)-1 { + for _, field := range current.Fields { + if field.Name == pathElement { + break + } + } + } + + return nil, current + } + return current, current +} + +// NewDocumentMapping returns a new document mapping +// with all the default values. +func NewDocumentMapping() *DocumentMapping { + return &DocumentMapping{ + Enabled: true, + Dynamic: true, + } +} + +// NewDocumentStaticMapping returns a new document +// mapping that will not automatically index parts +// of a document without an explicit mapping. +func NewDocumentStaticMapping() *DocumentMapping { + return &DocumentMapping{ + Enabled: true, + } +} + +// NewDocumentDisabledMapping returns a new document +// mapping that will not perform any indexing. +func NewDocumentDisabledMapping() *DocumentMapping { + return &DocumentMapping{} +} + +// AddSubDocumentMapping adds the provided DocumentMapping as a sub-mapping +// for the specified named subsection. +func (dm *DocumentMapping) AddSubDocumentMapping(property string, sdm *DocumentMapping) { + if dm.Properties == nil { + dm.Properties = make(map[string]*DocumentMapping) + } + dm.Properties[property] = sdm +} + +// AddFieldMappingsAt adds one or more FieldMappings +// at the named sub-document. If the named sub-document +// doesn't yet exist it is created for you. +// This is a convenience function to make most common +// mappings more concise. +// Otherwise, you would: +// +// subMapping := NewDocumentMapping() +// subMapping.AddFieldMapping(fieldMapping) +// parentMapping.AddSubDocumentMapping(property, subMapping) +func (dm *DocumentMapping) AddFieldMappingsAt(property string, fms ...*FieldMapping) { + if dm.Properties == nil { + dm.Properties = make(map[string]*DocumentMapping) + } + sdm, ok := dm.Properties[property] + if !ok { + sdm = NewDocumentMapping() + } + for _, fm := range fms { + sdm.AddFieldMapping(fm) + } + dm.Properties[property] = sdm +} + +// AddFieldMapping adds the provided FieldMapping for this section +// of the document. +func (dm *DocumentMapping) AddFieldMapping(fm *FieldMapping) { + if dm.Fields == nil { + dm.Fields = make([]*FieldMapping, 0) + } + dm.Fields = append(dm.Fields, fm) +} + +// UnmarshalJSON offers custom unmarshaling with optional strict validation +func (dm *DocumentMapping) UnmarshalJSON(data []byte) error { + var tmp map[string]json.RawMessage + err := util.UnmarshalJSON(data, &tmp) + if err != nil { + return err + } + + // set defaults for fields which might have been omitted + dm.Enabled = true + dm.Dynamic = true + + var invalidKeys []string + for k, v := range tmp { + switch k { + case "enabled": + err := util.UnmarshalJSON(v, &dm.Enabled) + if err != nil { + return err + } + case "dynamic": + err := util.UnmarshalJSON(v, &dm.Dynamic) + if err != nil { + return err + } + case "default_analyzer": + err := util.UnmarshalJSON(v, &dm.DefaultAnalyzer) + if err != nil { + return err + } + case "properties": + err := util.UnmarshalJSON(v, &dm.Properties) + if err != nil { + return err + } + case "fields": + err := util.UnmarshalJSON(v, &dm.Fields) + if err != nil { + return err + } + case "struct_tag_key": + err := util.UnmarshalJSON(v, &dm.StructTagKey) + if err != nil { + return err + } + default: + invalidKeys = append(invalidKeys, k) + } + } + + if MappingJSONStrict && len(invalidKeys) > 0 { + return fmt.Errorf("document mapping contains invalid keys: %v", invalidKeys) + } + + return nil +} + +func (dm *DocumentMapping) defaultAnalyzerName(path []string) string { + current := dm + rv := current.DefaultAnalyzer + for _, pathElement := range path { + var ok bool + current, ok = current.Properties[pathElement] + if !ok { + break + } + if current.DefaultAnalyzer != "" { + rv = current.DefaultAnalyzer + } + } + return rv +} + +func (dm *DocumentMapping) walkDocument(data interface{}, path []string, indexes []uint64, context *walkContext) { + // allow default "json" tag to be overridden + structTagKey := dm.StructTagKey + if structTagKey == "" { + structTagKey = "json" + } + + val := reflect.ValueOf(data) + if !val.IsValid() { + return + } + + typ := val.Type() + switch typ.Kind() { + case reflect.Map: + // FIXME can add support for other map keys in the future + if typ.Key().Kind() == reflect.String { + for _, key := range val.MapKeys() { + fieldName := key.String() + fieldVal := val.MapIndex(key).Interface() + dm.processProperty(fieldVal, append(path, fieldName), indexes, context) + } + } + case reflect.Struct: + for i := 0; i < val.NumField(); i++ { + field := typ.Field(i) + fieldName := field.Name + // anonymous fields of type struct can elide the type name + if field.Anonymous && field.Type.Kind() == reflect.Struct { + fieldName = "" + } + + // if the field has a name under the specified tag, prefer that + tag := field.Tag.Get(structTagKey) + tagFieldName := parseTagName(tag) + if tagFieldName == "-" { + continue + } + // allow tag to set field name to empty, only if anonymous + if field.Tag != "" && (tagFieldName != "" || field.Anonymous) { + fieldName = tagFieldName + } + + if val.Field(i).CanInterface() { + fieldVal := val.Field(i).Interface() + newpath := path + if fieldName != "" { + newpath = append(path, fieldName) + } + dm.processProperty(fieldVal, newpath, indexes, context) + } + } + case reflect.Slice, reflect.Array: + for i := 0; i < val.Len(); i++ { + if val.Index(i).CanInterface() { + fieldVal := val.Index(i).Interface() + dm.processProperty(fieldVal, path, append(indexes, uint64(i)), context) + } + } + case reflect.Ptr: + ptrElem := val.Elem() + if ptrElem.IsValid() && ptrElem.CanInterface() { + dm.processProperty(ptrElem.Interface(), path, indexes, context) + } + case reflect.String: + dm.processProperty(val.String(), path, indexes, context) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + dm.processProperty(float64(val.Int()), path, indexes, context) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + dm.processProperty(float64(val.Uint()), path, indexes, context) + case reflect.Float32, reflect.Float64: + dm.processProperty(float64(val.Float()), path, indexes, context) + case reflect.Bool: + dm.processProperty(val.Bool(), path, indexes, context) + } + +} + +func (dm *DocumentMapping) processProperty(property interface{}, path []string, indexes []uint64, context *walkContext) { + pathString := encodePath(path) + // look to see if there is a mapping for this field + subDocMapping, closestDocMapping := dm.documentMappingForPath(pathString) + + // check to see if we even need to do further processing + if subDocMapping != nil && !subDocMapping.Enabled { + return + } + + propertyValue := reflect.ValueOf(property) + if !propertyValue.IsValid() { + // cannot do anything with the zero value + return + } + propertyType := propertyValue.Type() + switch propertyType.Kind() { + case reflect.String: + propertyValueString := propertyValue.String() + if subDocMapping != nil { + // index by explicit mapping + for _, fieldMapping := range subDocMapping.Fields { + if fieldMapping.Type == "geoshape" { + fieldMapping.processGeoShape(property, pathString, path, indexes, context) + } else if fieldMapping.Type == "geopoint" { + fieldMapping.processGeoPoint(property, pathString, path, indexes, context) + } else { + fieldMapping.processString(propertyValueString, pathString, path, indexes, context) + } + } + } else if closestDocMapping.Dynamic { + // automatic indexing behavior + + // first see if it can be parsed by the default date parser + dateTimeParser := context.im.DateTimeParserNamed(context.im.DefaultDateTimeParser) + if dateTimeParser != nil { + parsedDateTime, layout, err := dateTimeParser.ParseDateTime(propertyValueString) + if err != nil { + // index as text + fieldMapping := newTextFieldMappingDynamic(context.im) + fieldMapping.processString(propertyValueString, pathString, path, indexes, context) + } else { + // index as datetime + fieldMapping := newDateTimeFieldMappingDynamic(context.im) + fieldMapping.processTime(parsedDateTime, layout, pathString, path, indexes, context) + } + } + } + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + dm.processProperty(float64(propertyValue.Int()), path, indexes, context) + return + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + dm.processProperty(float64(propertyValue.Uint()), path, indexes, context) + return + case reflect.Float64, reflect.Float32: + propertyValFloat := propertyValue.Float() + if subDocMapping != nil { + // index by explicit mapping + for _, fieldMapping := range subDocMapping.Fields { + fieldMapping.processFloat64(propertyValFloat, pathString, path, indexes, context) + } + } else if closestDocMapping.Dynamic { + // automatic indexing behavior + fieldMapping := newNumericFieldMappingDynamic(context.im) + fieldMapping.processFloat64(propertyValFloat, pathString, path, indexes, context) + } + case reflect.Bool: + propertyValBool := propertyValue.Bool() + if subDocMapping != nil { + // index by explicit mapping + for _, fieldMapping := range subDocMapping.Fields { + fieldMapping.processBoolean(propertyValBool, pathString, path, indexes, context) + } + } else if closestDocMapping.Dynamic { + // automatic indexing behavior + fieldMapping := newBooleanFieldMappingDynamic(context.im) + fieldMapping.processBoolean(propertyValBool, pathString, path, indexes, context) + } + case reflect.Struct: + switch property := property.(type) { + case time.Time: + // don't descend into the time struct + if subDocMapping != nil { + // index by explicit mapping + for _, fieldMapping := range subDocMapping.Fields { + fieldMapping.processTime(property, time.RFC3339, pathString, path, indexes, context) + } + } else if closestDocMapping.Dynamic { + fieldMapping := newDateTimeFieldMappingDynamic(context.im) + fieldMapping.processTime(property, time.RFC3339, pathString, path, indexes, context) + } + case encoding.TextMarshaler: + txt, err := property.MarshalText() + if err == nil && subDocMapping != nil { + // index by explicit mapping + for _, fieldMapping := range subDocMapping.Fields { + if fieldMapping.Type == "text" { + fieldMapping.processString(string(txt), pathString, path, indexes, context) + } + } + } + dm.walkDocument(property, path, indexes, context) + default: + if subDocMapping != nil { + for _, fieldMapping := range subDocMapping.Fields { + if fieldMapping.Type == "geopoint" { + fieldMapping.processGeoPoint(property, pathString, path, indexes, context) + } else if fieldMapping.Type == "geoshape" { + fieldMapping.processGeoShape(property, pathString, path, indexes, context) + } + } + } + dm.walkDocument(property, path, indexes, context) + } + case reflect.Map, reflect.Slice: + if subDocMapping != nil { + for _, fieldMapping := range subDocMapping.Fields { + switch fieldMapping.Type { + case "geopoint": + fieldMapping.processGeoPoint(property, pathString, path, indexes, context) + case "IP": + ip, ok := property.(net.IP) + if ok { + fieldMapping.processIP(ip, pathString, path, indexes, context) + } + case "geoshape": + fieldMapping.processGeoShape(property, pathString, path, indexes, context) + } + } + } + dm.walkDocument(property, path, indexes, context) + case reflect.Ptr: + if !propertyValue.IsNil() { + switch property := property.(type) { + case encoding.TextMarshaler: + // ONLY process TextMarshaler if there is an explicit mapping + // AND all of the fields are of type text + // OTHERWISE process field without TextMarshaler + if subDocMapping != nil { + allFieldsText := true + for _, fieldMapping := range subDocMapping.Fields { + if fieldMapping.Type != "text" { + allFieldsText = false + break + } + } + txt, err := property.MarshalText() + if err == nil && allFieldsText { + txtStr := string(txt) + for _, fieldMapping := range subDocMapping.Fields { + fieldMapping.processString(txtStr, pathString, path, indexes, context) + } + return + } + } + dm.walkDocument(property, path, indexes, context) + default: + dm.walkDocument(property, path, indexes, context) + } + } + default: + dm.walkDocument(property, path, indexes, context) + } +} diff --git a/vendor/github.com/blevesearch/bleve/v2/mapping/field.go b/vendor/github.com/blevesearch/bleve/v2/mapping/field.go new file mode 100644 index 00000000..82d51f31 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/mapping/field.go @@ -0,0 +1,461 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mapping + +import ( + "encoding/json" + "fmt" + "net" + "time" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" +) + +// control the default behavior for dynamic fields (those not explicitly mapped) +var ( + IndexDynamic = true + StoreDynamic = true + DocValuesDynamic = true // TODO revisit default? +) + +// A FieldMapping describes how a specific item +// should be put into the index. +type FieldMapping struct { + Name string `json:"name,omitempty"` + Type string `json:"type,omitempty"` + + // Analyzer specifies the name of the analyzer to use for this field. If + // Analyzer is empty, traverse the DocumentMapping tree toward the root and + // pick the first non-empty DefaultAnalyzer found. If there is none, use + // the IndexMapping.DefaultAnalyzer. + Analyzer string `json:"analyzer,omitempty"` + + // Store indicates whether to store field values in the index. Stored + // values can be retrieved from search results using SearchRequest.Fields. + Store bool `json:"store,omitempty"` + Index bool `json:"index,omitempty"` + + // IncludeTermVectors, if true, makes terms occurrences to be recorded for + // this field. It includes the term position within the terms sequence and + // the term offsets in the source document field. Term vectors are required + // to perform phrase queries or terms highlighting in source documents. + IncludeTermVectors bool `json:"include_term_vectors,omitempty"` + IncludeInAll bool `json:"include_in_all,omitempty"` + DateFormat string `json:"date_format,omitempty"` + + // DocValues, if true makes the index uninverting possible for this field + // It is useful for faceting and sorting queries. + DocValues bool `json:"docvalues,omitempty"` + + // SkipFreqNorm, if true, avoids the indexing of frequency and norm values + // of the tokens for this field. This option would be useful for saving + // the processing of freq/norm details when the default score based relevancy + // isn't needed. + SkipFreqNorm bool `json:"skip_freq_norm,omitempty"` +} + +// NewTextFieldMapping returns a default field mapping for text +func NewTextFieldMapping() *FieldMapping { + return &FieldMapping{ + Type: "text", + Store: true, + Index: true, + IncludeTermVectors: true, + IncludeInAll: true, + DocValues: true, + } +} + +func newTextFieldMappingDynamic(im *IndexMappingImpl) *FieldMapping { + rv := NewTextFieldMapping() + rv.Store = im.StoreDynamic + rv.Index = im.IndexDynamic + rv.DocValues = im.DocValuesDynamic + return rv +} + +// NewKeyworFieldMapping returns a default field mapping for text with analyzer "keyword". +func NewKeywordFieldMapping() *FieldMapping { + return &FieldMapping{ + Type: "text", + Analyzer: keyword.Name, + Store: true, + Index: true, + IncludeTermVectors: true, + IncludeInAll: true, + DocValues: true, + } +} + +// NewNumericFieldMapping returns a default field mapping for numbers +func NewNumericFieldMapping() *FieldMapping { + return &FieldMapping{ + Type: "number", + Store: true, + Index: true, + IncludeInAll: true, + DocValues: true, + } +} + +func newNumericFieldMappingDynamic(im *IndexMappingImpl) *FieldMapping { + rv := NewNumericFieldMapping() + rv.Store = im.StoreDynamic + rv.Index = im.IndexDynamic + rv.DocValues = im.DocValuesDynamic + return rv +} + +// NewDateTimeFieldMapping returns a default field mapping for dates +func NewDateTimeFieldMapping() *FieldMapping { + return &FieldMapping{ + Type: "datetime", + Store: true, + Index: true, + IncludeInAll: true, + DocValues: true, + } +} + +func newDateTimeFieldMappingDynamic(im *IndexMappingImpl) *FieldMapping { + rv := NewDateTimeFieldMapping() + rv.Store = im.StoreDynamic + rv.Index = im.IndexDynamic + rv.DocValues = im.DocValuesDynamic + return rv +} + +// NewBooleanFieldMapping returns a default field mapping for booleans +func NewBooleanFieldMapping() *FieldMapping { + return &FieldMapping{ + Type: "boolean", + Store: true, + Index: true, + IncludeInAll: true, + DocValues: true, + } +} + +func newBooleanFieldMappingDynamic(im *IndexMappingImpl) *FieldMapping { + rv := NewBooleanFieldMapping() + rv.Store = im.StoreDynamic + rv.Index = im.IndexDynamic + rv.DocValues = im.DocValuesDynamic + return rv +} + +// NewGeoPointFieldMapping returns a default field mapping for geo points +func NewGeoPointFieldMapping() *FieldMapping { + return &FieldMapping{ + Type: "geopoint", + Store: true, + Index: true, + IncludeInAll: true, + DocValues: true, + } +} + +// NewGeoShapeFieldMapping returns a default field mapping +// for geoshapes +func NewGeoShapeFieldMapping() *FieldMapping { + return &FieldMapping{ + Type: "geoshape", + Store: true, + Index: true, + IncludeInAll: true, + DocValues: true, + } +} + +// NewIPFieldMapping returns a default field mapping for IP points +func NewIPFieldMapping() *FieldMapping { + return &FieldMapping{ + Type: "IP", + Store: true, + Index: true, + IncludeInAll: true, + } +} + +// Options returns the indexing options for this field. +func (fm *FieldMapping) Options() index.FieldIndexingOptions { + var rv index.FieldIndexingOptions + if fm.Store { + rv |= index.StoreField + } + if fm.Index { + rv |= index.IndexField + } + if fm.IncludeTermVectors { + rv |= index.IncludeTermVectors + } + if fm.DocValues { + rv |= index.DocValues + } + if fm.SkipFreqNorm { + rv |= index.SkipFreqNorm + } + return rv +} + +func (fm *FieldMapping) processString(propertyValueString string, pathString string, path []string, indexes []uint64, context *walkContext) { + fieldName := getFieldName(pathString, path, fm) + options := fm.Options() + if fm.Type == "text" { + analyzer := fm.analyzerForField(path, context) + field := document.NewTextFieldCustom(fieldName, indexes, []byte(propertyValueString), options, analyzer) + context.doc.AddField(field) + + if !fm.IncludeInAll { + context.excludedFromAll = append(context.excludedFromAll, fieldName) + } + } else if fm.Type == "datetime" { + dateTimeFormat := context.im.DefaultDateTimeParser + if fm.DateFormat != "" { + dateTimeFormat = fm.DateFormat + } + dateTimeParser := context.im.DateTimeParserNamed(dateTimeFormat) + if dateTimeParser != nil { + parsedDateTime, layout, err := dateTimeParser.ParseDateTime(propertyValueString) + if err == nil { + fm.processTime(parsedDateTime, layout, pathString, path, indexes, context) + } + } + } else if fm.Type == "IP" { + ip := net.ParseIP(propertyValueString) + if ip != nil { + fm.processIP(ip, pathString, path, indexes, context) + } + } +} + +func (fm *FieldMapping) processFloat64(propertyValFloat float64, pathString string, path []string, indexes []uint64, context *walkContext) { + fieldName := getFieldName(pathString, path, fm) + if fm.Type == "number" { + options := fm.Options() + field := document.NewNumericFieldWithIndexingOptions(fieldName, indexes, propertyValFloat, options) + context.doc.AddField(field) + + if !fm.IncludeInAll { + context.excludedFromAll = append(context.excludedFromAll, fieldName) + } + } +} + +func (fm *FieldMapping) processTime(propertyValueTime time.Time, layout string, pathString string, path []string, indexes []uint64, context *walkContext) { + fieldName := getFieldName(pathString, path, fm) + if fm.Type == "datetime" { + options := fm.Options() + field, err := document.NewDateTimeFieldWithIndexingOptions(fieldName, indexes, propertyValueTime, layout, options) + if err == nil { + context.doc.AddField(field) + } else { + logger.Printf("could not build date %v", err) + } + + if !fm.IncludeInAll { + context.excludedFromAll = append(context.excludedFromAll, fieldName) + } + } +} + +func (fm *FieldMapping) processBoolean(propertyValueBool bool, pathString string, path []string, indexes []uint64, context *walkContext) { + fieldName := getFieldName(pathString, path, fm) + if fm.Type == "boolean" { + options := fm.Options() + field := document.NewBooleanFieldWithIndexingOptions(fieldName, indexes, propertyValueBool, options) + context.doc.AddField(field) + + if !fm.IncludeInAll { + context.excludedFromAll = append(context.excludedFromAll, fieldName) + } + } +} + +func (fm *FieldMapping) processGeoPoint(propertyMightBeGeoPoint interface{}, pathString string, path []string, indexes []uint64, context *walkContext) { + lon, lat, found := geo.ExtractGeoPoint(propertyMightBeGeoPoint) + if found { + fieldName := getFieldName(pathString, path, fm) + options := fm.Options() + field := document.NewGeoPointFieldWithIndexingOptions(fieldName, indexes, lon, lat, options) + context.doc.AddField(field) + + if !fm.IncludeInAll { + context.excludedFromAll = append(context.excludedFromAll, fieldName) + } + } +} + +func (fm *FieldMapping) processIP(ip net.IP, pathString string, path []string, indexes []uint64, context *walkContext) { + fieldName := getFieldName(pathString, path, fm) + options := fm.Options() + field := document.NewIPFieldWithIndexingOptions(fieldName, indexes, ip, options) + context.doc.AddField(field) + + if !fm.IncludeInAll { + context.excludedFromAll = append(context.excludedFromAll, fieldName) + } +} + +func (fm *FieldMapping) processGeoShape(propertyMightBeGeoShape interface{}, + pathString string, path []string, indexes []uint64, context *walkContext) { + coordValue, shape, err := geo.ParseGeoShapeField(propertyMightBeGeoShape) + if err != nil { + return + } + + if shape == geo.CircleType { + center, radius, found := geo.ExtractCircle(propertyMightBeGeoShape) + if found { + fieldName := getFieldName(pathString, path, fm) + options := fm.Options() + field := document.NewGeoCircleFieldWithIndexingOptions(fieldName, + indexes, center, radius, options) + context.doc.AddField(field) + + if !fm.IncludeInAll { + context.excludedFromAll = append(context.excludedFromAll, fieldName) + } + } + } else if shape == geo.GeometryCollectionType { + coordinates, shapes, found := geo.ExtractGeometryCollection(propertyMightBeGeoShape) + if found { + fieldName := getFieldName(pathString, path, fm) + options := fm.Options() + field := document.NewGeometryCollectionFieldWithIndexingOptions(fieldName, + indexes, coordinates, shapes, options) + context.doc.AddField(field) + + if !fm.IncludeInAll { + context.excludedFromAll = append(context.excludedFromAll, fieldName) + } + } + } else { + coordinates, shape, found := geo.ExtractGeoShapeCoordinates(coordValue, shape) + if found { + fieldName := getFieldName(pathString, path, fm) + options := fm.Options() + field := document.NewGeoShapeFieldWithIndexingOptions(fieldName, + indexes, coordinates, shape, options) + context.doc.AddField(field) + + if !fm.IncludeInAll { + context.excludedFromAll = append(context.excludedFromAll, fieldName) + } + } + } +} + +func (fm *FieldMapping) analyzerForField(path []string, context *walkContext) analysis.Analyzer { + analyzerName := fm.Analyzer + if analyzerName == "" { + analyzerName = context.dm.defaultAnalyzerName(path) + if analyzerName == "" { + analyzerName = context.im.DefaultAnalyzer + } + } + return context.im.AnalyzerNamed(analyzerName) +} + +func getFieldName(pathString string, path []string, fieldMapping *FieldMapping) string { + fieldName := pathString + if fieldMapping.Name != "" { + parentName := "" + if len(path) > 1 { + parentName = encodePath(path[:len(path)-1]) + pathSeparator + } + fieldName = parentName + fieldMapping.Name + } + return fieldName +} + +// UnmarshalJSON offers custom unmarshaling with optional strict validation +func (fm *FieldMapping) UnmarshalJSON(data []byte) error { + + var tmp map[string]json.RawMessage + err := util.UnmarshalJSON(data, &tmp) + if err != nil { + return err + } + + var invalidKeys []string + for k, v := range tmp { + switch k { + case "name": + err := util.UnmarshalJSON(v, &fm.Name) + if err != nil { + return err + } + case "type": + err := util.UnmarshalJSON(v, &fm.Type) + if err != nil { + return err + } + case "analyzer": + err := util.UnmarshalJSON(v, &fm.Analyzer) + if err != nil { + return err + } + case "store": + err := util.UnmarshalJSON(v, &fm.Store) + if err != nil { + return err + } + case "index": + err := util.UnmarshalJSON(v, &fm.Index) + if err != nil { + return err + } + case "include_term_vectors": + err := util.UnmarshalJSON(v, &fm.IncludeTermVectors) + if err != nil { + return err + } + case "include_in_all": + err := util.UnmarshalJSON(v, &fm.IncludeInAll) + if err != nil { + return err + } + case "date_format": + err := util.UnmarshalJSON(v, &fm.DateFormat) + if err != nil { + return err + } + case "docvalues": + err := util.UnmarshalJSON(v, &fm.DocValues) + if err != nil { + return err + } + case "skip_freq_norm": + err := util.UnmarshalJSON(v, &fm.SkipFreqNorm) + if err != nil { + return err + } + default: + invalidKeys = append(invalidKeys, k) + } + } + + if MappingJSONStrict && len(invalidKeys) > 0 { + return fmt.Errorf("field mapping contains invalid keys: %v", invalidKeys) + } + + return nil +} diff --git a/vendor/github.com/blevesearch/bleve/v2/mapping/index.go b/vendor/github.com/blevesearch/bleve/v2/mapping/index.go new file mode 100644 index 00000000..0de4147a --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/mapping/index.go @@ -0,0 +1,438 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mapping + +import ( + "encoding/json" + "fmt" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/analyzer/standard" + "github.com/blevesearch/bleve/v2/analysis/datetime/optional" + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" +) + +var MappingJSONStrict = false + +const defaultTypeField = "_type" +const defaultType = "_default" +const defaultField = "_all" +const defaultAnalyzer = standard.Name +const defaultDateTimeParser = optional.Name + +// An IndexMappingImpl controls how objects are placed +// into an index. +// First the type of the object is determined. +// Once the type is know, the appropriate +// DocumentMapping is selected by the type. +// If no mapping was determined for that type, +// a DefaultMapping will be used. +type IndexMappingImpl struct { + TypeMapping map[string]*DocumentMapping `json:"types,omitempty"` + DefaultMapping *DocumentMapping `json:"default_mapping"` + TypeField string `json:"type_field"` + DefaultType string `json:"default_type"` + DefaultAnalyzer string `json:"default_analyzer"` + DefaultDateTimeParser string `json:"default_datetime_parser"` + DefaultField string `json:"default_field"` + StoreDynamic bool `json:"store_dynamic"` + IndexDynamic bool `json:"index_dynamic"` + DocValuesDynamic bool `json:"docvalues_dynamic"` + CustomAnalysis *customAnalysis `json:"analysis,omitempty"` + cache *registry.Cache +} + +// AddCustomCharFilter defines a custom char filter for use in this mapping +func (im *IndexMappingImpl) AddCustomCharFilter(name string, config map[string]interface{}) error { + _, err := im.cache.DefineCharFilter(name, config) + if err != nil { + return err + } + im.CustomAnalysis.CharFilters[name] = config + return nil +} + +// AddCustomTokenizer defines a custom tokenizer for use in this mapping +func (im *IndexMappingImpl) AddCustomTokenizer(name string, config map[string]interface{}) error { + _, err := im.cache.DefineTokenizer(name, config) + if err != nil { + return err + } + im.CustomAnalysis.Tokenizers[name] = config + return nil +} + +// AddCustomTokenMap defines a custom token map for use in this mapping +func (im *IndexMappingImpl) AddCustomTokenMap(name string, config map[string]interface{}) error { + _, err := im.cache.DefineTokenMap(name, config) + if err != nil { + return err + } + im.CustomAnalysis.TokenMaps[name] = config + return nil +} + +// AddCustomTokenFilter defines a custom token filter for use in this mapping +func (im *IndexMappingImpl) AddCustomTokenFilter(name string, config map[string]interface{}) error { + _, err := im.cache.DefineTokenFilter(name, config) + if err != nil { + return err + } + im.CustomAnalysis.TokenFilters[name] = config + return nil +} + +// AddCustomAnalyzer defines a custom analyzer for use in this mapping. The +// config map must have a "type" string entry to resolve the analyzer +// constructor. The constructor is invoked with the remaining entries and +// returned analyzer is registered in the IndexMapping. +// +// bleve comes with predefined analyzers, like +// github.com/blevesearch/bleve/analysis/analyzer/custom. They are +// available only if their package is imported by client code. To achieve this, +// use their metadata to fill configuration entries: +// +// import ( +// "github.com/blevesearch/bleve/v2/analysis/analyzer/custom" +// "github.com/blevesearch/bleve/v2/analysis/char/html" +// "github.com/blevesearch/bleve/v2/analysis/token/lowercase" +// "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" +// ) +// +// m := bleve.NewIndexMapping() +// err := m.AddCustomAnalyzer("html", map[string]interface{}{ +// "type": custom.Name, +// "char_filters": []string{ +// html.Name, +// }, +// "tokenizer": unicode.Name, +// "token_filters": []string{ +// lowercase.Name, +// ... +// }, +// }) +func (im *IndexMappingImpl) AddCustomAnalyzer(name string, config map[string]interface{}) error { + _, err := im.cache.DefineAnalyzer(name, config) + if err != nil { + return err + } + im.CustomAnalysis.Analyzers[name] = config + return nil +} + +// AddCustomDateTimeParser defines a custom date time parser for use in this mapping +func (im *IndexMappingImpl) AddCustomDateTimeParser(name string, config map[string]interface{}) error { + _, err := im.cache.DefineDateTimeParser(name, config) + if err != nil { + return err + } + im.CustomAnalysis.DateTimeParsers[name] = config + return nil +} + +// NewIndexMapping creates a new IndexMapping that will use all the default indexing rules +func NewIndexMapping() *IndexMappingImpl { + return &IndexMappingImpl{ + TypeMapping: make(map[string]*DocumentMapping), + DefaultMapping: NewDocumentMapping(), + TypeField: defaultTypeField, + DefaultType: defaultType, + DefaultAnalyzer: defaultAnalyzer, + DefaultDateTimeParser: defaultDateTimeParser, + DefaultField: defaultField, + IndexDynamic: IndexDynamic, + StoreDynamic: StoreDynamic, + DocValuesDynamic: DocValuesDynamic, + CustomAnalysis: newCustomAnalysis(), + cache: registry.NewCache(), + } +} + +// Validate will walk the entire structure ensuring the following +// explicitly named and default analyzers can be built +func (im *IndexMappingImpl) Validate() error { + _, err := im.cache.AnalyzerNamed(im.DefaultAnalyzer) + if err != nil { + return err + } + _, err = im.cache.DateTimeParserNamed(im.DefaultDateTimeParser) + if err != nil { + return err + } + err = im.DefaultMapping.Validate(im.cache) + if err != nil { + return err + } + for _, docMapping := range im.TypeMapping { + err = docMapping.Validate(im.cache) + if err != nil { + return err + } + } + return nil +} + +// AddDocumentMapping sets a custom document mapping for the specified type +func (im *IndexMappingImpl) AddDocumentMapping(doctype string, dm *DocumentMapping) { + im.TypeMapping[doctype] = dm +} + +func (im *IndexMappingImpl) mappingForType(docType string) *DocumentMapping { + docMapping := im.TypeMapping[docType] + if docMapping == nil { + docMapping = im.DefaultMapping + } + return docMapping +} + +// UnmarshalJSON offers custom unmarshaling with optional strict validation +func (im *IndexMappingImpl) UnmarshalJSON(data []byte) error { + + var tmp map[string]json.RawMessage + err := util.UnmarshalJSON(data, &tmp) + if err != nil { + return err + } + + // set defaults for fields which might have been omitted + im.cache = registry.NewCache() + im.CustomAnalysis = newCustomAnalysis() + im.TypeField = defaultTypeField + im.DefaultType = defaultType + im.DefaultAnalyzer = defaultAnalyzer + im.DefaultDateTimeParser = defaultDateTimeParser + im.DefaultField = defaultField + im.DefaultMapping = NewDocumentMapping() + im.TypeMapping = make(map[string]*DocumentMapping) + im.StoreDynamic = StoreDynamic + im.IndexDynamic = IndexDynamic + im.DocValuesDynamic = DocValuesDynamic + + var invalidKeys []string + for k, v := range tmp { + switch k { + case "analysis": + err := util.UnmarshalJSON(v, &im.CustomAnalysis) + if err != nil { + return err + } + case "type_field": + err := util.UnmarshalJSON(v, &im.TypeField) + if err != nil { + return err + } + case "default_type": + err := util.UnmarshalJSON(v, &im.DefaultType) + if err != nil { + return err + } + case "default_analyzer": + err := util.UnmarshalJSON(v, &im.DefaultAnalyzer) + if err != nil { + return err + } + case "default_datetime_parser": + err := util.UnmarshalJSON(v, &im.DefaultDateTimeParser) + if err != nil { + return err + } + case "default_field": + err := util.UnmarshalJSON(v, &im.DefaultField) + if err != nil { + return err + } + case "default_mapping": + err := util.UnmarshalJSON(v, &im.DefaultMapping) + if err != nil { + return err + } + case "types": + err := util.UnmarshalJSON(v, &im.TypeMapping) + if err != nil { + return err + } + case "store_dynamic": + err := util.UnmarshalJSON(v, &im.StoreDynamic) + if err != nil { + return err + } + case "index_dynamic": + err := util.UnmarshalJSON(v, &im.IndexDynamic) + if err != nil { + return err + } + case "docvalues_dynamic": + err := util.UnmarshalJSON(v, &im.DocValuesDynamic) + if err != nil { + return err + } + default: + invalidKeys = append(invalidKeys, k) + } + } + + if MappingJSONStrict && len(invalidKeys) > 0 { + return fmt.Errorf("index mapping contains invalid keys: %v", invalidKeys) + } + + err = im.CustomAnalysis.registerAll(im) + if err != nil { + return err + } + + return nil +} + +func (im *IndexMappingImpl) determineType(data interface{}) string { + // first see if the object implements bleveClassifier + bleveClassifier, ok := data.(bleveClassifier) + if ok { + return bleveClassifier.BleveType() + } + // next see if the object implements Classifier + classifier, ok := data.(Classifier) + if ok { + return classifier.Type() + } + + // now see if we can find a type using the mapping + typ, ok := mustString(lookupPropertyPath(data, im.TypeField)) + if ok { + return typ + } + + return im.DefaultType +} + +func (im *IndexMappingImpl) MapDocument(doc *document.Document, data interface{}) error { + docType := im.determineType(data) + docMapping := im.mappingForType(docType) + if docMapping.Enabled { + walkContext := im.newWalkContext(doc, docMapping) + docMapping.walkDocument(data, []string{}, []uint64{}, walkContext) + + // see if the _all field was disabled + allMapping, _ := docMapping.documentMappingForPath("_all") + if allMapping == nil || allMapping.Enabled { + field := document.NewCompositeFieldWithIndexingOptions("_all", true, []string{}, walkContext.excludedFromAll, index.IndexField|index.IncludeTermVectors) + doc.AddField(field) + } + } + + return nil +} + +type walkContext struct { + doc *document.Document + im *IndexMappingImpl + dm *DocumentMapping + excludedFromAll []string +} + +func (im *IndexMappingImpl) newWalkContext(doc *document.Document, dm *DocumentMapping) *walkContext { + return &walkContext{ + doc: doc, + im: im, + dm: dm, + excludedFromAll: []string{"_id"}, + } +} + +// AnalyzerNameForPath attempts to find the best analyzer to use with only a +// field name will walk all the document types, look for field mappings at the +// provided path, if one exists and it has an explicit analyzer that is +// returned. +func (im *IndexMappingImpl) AnalyzerNameForPath(path string) string { + // first we look for explicit mapping on the field + for _, docMapping := range im.TypeMapping { + analyzerName := docMapping.analyzerNameForPath(path) + if analyzerName != "" { + return analyzerName + } + } + + // now try the default mapping + pathMapping, _ := im.DefaultMapping.documentMappingForPath(path) + if pathMapping != nil { + if len(pathMapping.Fields) > 0 { + if pathMapping.Fields[0].Analyzer != "" { + return pathMapping.Fields[0].Analyzer + } + } + } + + // next we will try default analyzers for the path + pathDecoded := decodePath(path) + for _, docMapping := range im.TypeMapping { + if docMapping.Enabled { + rv := docMapping.defaultAnalyzerName(pathDecoded) + if rv != "" { + return rv + } + } + } + // now the default analyzer for the default mapping + if im.DefaultMapping.Enabled { + rv := im.DefaultMapping.defaultAnalyzerName(pathDecoded) + if rv != "" { + return rv + } + } + + return im.DefaultAnalyzer +} + +func (im *IndexMappingImpl) AnalyzerNamed(name string) analysis.Analyzer { + analyzer, err := im.cache.AnalyzerNamed(name) + if err != nil { + logger.Printf("error using analyzer named: %s", name) + return nil + } + return analyzer +} + +func (im *IndexMappingImpl) DateTimeParserNamed(name string) analysis.DateTimeParser { + if name == "" { + name = im.DefaultDateTimeParser + } + dateTimeParser, err := im.cache.DateTimeParserNamed(name) + if err != nil { + logger.Printf("error using datetime parser named: %s", name) + return nil + } + return dateTimeParser +} + +func (im *IndexMappingImpl) AnalyzeText(analyzerName string, text []byte) (analysis.TokenStream, error) { + analyzer, err := im.cache.AnalyzerNamed(analyzerName) + if err != nil { + return nil, err + } + return analyzer.Analyze(text), nil +} + +// FieldAnalyzer returns the name of the analyzer used on a field. +func (im *IndexMappingImpl) FieldAnalyzer(field string) string { + return im.AnalyzerNameForPath(field) +} + +// wrapper to satisfy new interface + +func (im *IndexMappingImpl) DefaultSearchField() string { + return im.DefaultField +} diff --git a/vendor/github.com/blevesearch/bleve/v2/mapping/mapping.go b/vendor/github.com/blevesearch/bleve/v2/mapping/mapping.go new file mode 100644 index 00000000..a3e5a54e --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/mapping/mapping.go @@ -0,0 +1,58 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mapping + +import ( + "io" + "log" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/document" +) + +// A Classifier is an interface describing any object which knows how to +// identify its own type. Alternatively, if a struct already has a Type +// field or method in conflict, one can use BleveType instead. +type Classifier interface { + Type() string +} + +// A bleveClassifier is an interface describing any object which knows how +// to identify its own type. This is introduced as an alternative to the +// Classifier interface which often has naming conflicts with existing +// structures. +type bleveClassifier interface { + BleveType() string +} + +var logger = log.New(io.Discard, "bleve mapping ", log.LstdFlags) + +// SetLog sets the logger used for logging +// by default log messages are sent to io.Discard +func SetLog(l *log.Logger) { + logger = l +} + +type IndexMapping interface { + MapDocument(doc *document.Document, data interface{}) error + Validate() error + + DateTimeParserNamed(name string) analysis.DateTimeParser + + DefaultSearchField() string + + AnalyzerNameForPath(path string) string + AnalyzerNamed(name string) analysis.Analyzer +} diff --git a/vendor/github.com/blevesearch/bleve/mapping/reflect.go b/vendor/github.com/blevesearch/bleve/v2/mapping/reflect.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/mapping/reflect.go rename to vendor/github.com/blevesearch/bleve/v2/mapping/reflect.go diff --git a/vendor/github.com/blevesearch/bleve/numeric/bin.go b/vendor/github.com/blevesearch/bleve/v2/numeric/bin.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/numeric/bin.go rename to vendor/github.com/blevesearch/bleve/v2/numeric/bin.go diff --git a/vendor/github.com/blevesearch/bleve/numeric/float.go b/vendor/github.com/blevesearch/bleve/v2/numeric/float.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/numeric/float.go rename to vendor/github.com/blevesearch/bleve/v2/numeric/float.go diff --git a/vendor/github.com/blevesearch/bleve/numeric/prefix_coded.go b/vendor/github.com/blevesearch/bleve/v2/numeric/prefix_coded.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/numeric/prefix_coded.go rename to vendor/github.com/blevesearch/bleve/v2/numeric/prefix_coded.go diff --git a/vendor/github.com/blevesearch/bleve/v2/query.go b/vendor/github.com/blevesearch/bleve/v2/query.go new file mode 100644 index 00000000..3af750a0 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/query.go @@ -0,0 +1,290 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bleve + +import ( + "time" + + "github.com/blevesearch/bleve/v2/search/query" +) + +// NewBoolFieldQuery creates a new Query for boolean fields +func NewBoolFieldQuery(val bool) *query.BoolFieldQuery { + return query.NewBoolFieldQuery(val) +} + +// NewBooleanQuery creates a compound Query composed +// of several other Query objects. +// These other query objects are added using the +// AddMust() AddShould() and AddMustNot() methods. +// Result documents must satisfy ALL of the +// must Queries. +// Result documents must satisfy NONE of the must not +// Queries. +// Result documents that ALSO satisfy any of the should +// Queries will score higher. +func NewBooleanQuery() *query.BooleanQuery { + return query.NewBooleanQuery(nil, nil, nil) +} + +// NewConjunctionQuery creates a new compound Query. +// Result documents must satisfy all of the queries. +func NewConjunctionQuery(conjuncts ...query.Query) *query.ConjunctionQuery { + return query.NewConjunctionQuery(conjuncts) +} + +// NewDateRangeQuery creates a new Query for ranges +// of date values. +// Date strings are parsed using the DateTimeParser configured in the +// +// top-level config.QueryDateTimeParser +// +// Either, but not both endpoints can be nil. +func NewDateRangeQuery(start, end time.Time) *query.DateRangeQuery { + return query.NewDateRangeQuery(start, end) +} + +// NewDateRangeInclusiveQuery creates a new Query for ranges +// of date values. +// Date strings are parsed using the DateTimeParser configured in the +// +// top-level config.QueryDateTimeParser +// +// Either, but not both endpoints can be nil. +// startInclusive and endInclusive control inclusion of the endpoints. +func NewDateRangeInclusiveQuery(start, end time.Time, startInclusive, endInclusive *bool) *query.DateRangeQuery { + return query.NewDateRangeInclusiveQuery(start, end, startInclusive, endInclusive) +} + +// NewDateRangeStringQuery creates a new Query for ranges +// of date values. +// Date strings are parsed using the DateTimeParser set using +// +// the DateRangeStringQuery.SetDateTimeParser() method. +// +// If no DateTimeParser is set, then the +// +// top-level config.QueryDateTimeParser +// +// is used. +func NewDateRangeStringQuery(start, end string) *query.DateRangeStringQuery { + return query.NewDateRangeStringQuery(start, end) +} + +// NewDateRangeStringQuery creates a new Query for ranges +// of date values. +// Date strings are parsed using the DateTimeParser set using +// +// the DateRangeStringQuery.SetDateTimeParser() method. +// +// this DateTimeParser is a custom date time parser defined in the index mapping, +// using AddCustomDateTimeParser() method. +// If no DateTimeParser is set, then the +// +// top-level config.QueryDateTimeParser +// +// is used. +// Either, but not both endpoints can be nil. +// startInclusive and endInclusive control inclusion of the endpoints. +func NewDateRangeInclusiveStringQuery(start, end string, startInclusive, endInclusive *bool) *query.DateRangeStringQuery { + return query.NewDateRangeStringInclusiveQuery(start, end, startInclusive, endInclusive) +} + +// NewDisjunctionQuery creates a new compound Query. +// Result documents satisfy at least one Query. +func NewDisjunctionQuery(disjuncts ...query.Query) *query.DisjunctionQuery { + return query.NewDisjunctionQuery(disjuncts) +} + +// NewDocIDQuery creates a new Query object returning indexed documents among +// the specified set. Combine it with ConjunctionQuery to restrict the scope of +// other queries output. +func NewDocIDQuery(ids []string) *query.DocIDQuery { + return query.NewDocIDQuery(ids) +} + +// NewFuzzyQuery creates a new Query which finds +// documents containing terms within a specific +// fuzziness of the specified term. +// The default fuzziness is 1. +// +// The current implementation uses Levenshtein edit +// distance as the fuzziness metric. +func NewFuzzyQuery(term string) *query.FuzzyQuery { + return query.NewFuzzyQuery(term) +} + +// NewMatchAllQuery creates a Query which will +// match all documents in the index. +func NewMatchAllQuery() *query.MatchAllQuery { + return query.NewMatchAllQuery() +} + +// NewMatchNoneQuery creates a Query which will not +// match any documents in the index. +func NewMatchNoneQuery() *query.MatchNoneQuery { + return query.NewMatchNoneQuery() +} + +// NewMatchPhraseQuery creates a new Query object +// for matching phrases in the index. +// An Analyzer is chosen based on the field. +// Input text is analyzed using this analyzer. +// Token terms resulting from this analysis are +// used to build a search phrase. Result documents +// must match this phrase. Queried field must have been indexed with +// IncludeTermVectors set to true. +func NewMatchPhraseQuery(matchPhrase string) *query.MatchPhraseQuery { + return query.NewMatchPhraseQuery(matchPhrase) +} + +// NewMatchQuery creates a Query for matching text. +// An Analyzer is chosen based on the field. +// Input text is analyzed using this analyzer. +// Token terms resulting from this analysis are +// used to perform term searches. Result documents +// must satisfy at least one of these term searches. +func NewMatchQuery(match string) *query.MatchQuery { + return query.NewMatchQuery(match) +} + +// NewNumericRangeQuery creates a new Query for ranges +// of numeric values. +// Either, but not both endpoints can be nil. +// The minimum value is inclusive. +// The maximum value is exclusive. +func NewNumericRangeQuery(min, max *float64) *query.NumericRangeQuery { + return query.NewNumericRangeQuery(min, max) +} + +// NewNumericRangeInclusiveQuery creates a new Query for ranges +// of numeric values. +// Either, but not both endpoints can be nil. +// Control endpoint inclusion with inclusiveMin, inclusiveMax. +func NewNumericRangeInclusiveQuery(min, max *float64, minInclusive, maxInclusive *bool) *query.NumericRangeQuery { + return query.NewNumericRangeInclusiveQuery(min, max, minInclusive, maxInclusive) +} + +// NewTermRangeQuery creates a new Query for ranges +// of text terms. +// Either, but not both endpoints can be "". +// The minimum value is inclusive. +// The maximum value is exclusive. +func NewTermRangeQuery(min, max string) *query.TermRangeQuery { + return query.NewTermRangeQuery(min, max) +} + +// NewTermRangeInclusiveQuery creates a new Query for ranges +// of text terms. +// Either, but not both endpoints can be "". +// Control endpoint inclusion with inclusiveMin, inclusiveMax. +func NewTermRangeInclusiveQuery(min, max string, minInclusive, maxInclusive *bool) *query.TermRangeQuery { + return query.NewTermRangeInclusiveQuery(min, max, minInclusive, maxInclusive) +} + +// NewPhraseQuery creates a new Query for finding +// exact term phrases in the index. +// The provided terms must exist in the correct +// order, at the correct index offsets, in the +// specified field. Queried field must have been indexed with +// IncludeTermVectors set to true. +func NewPhraseQuery(terms []string, field string) *query.PhraseQuery { + return query.NewPhraseQuery(terms, field) +} + +// NewPrefixQuery creates a new Query which finds +// documents containing terms that start with the +// specified prefix. +func NewPrefixQuery(prefix string) *query.PrefixQuery { + return query.NewPrefixQuery(prefix) +} + +// NewRegexpQuery creates a new Query which finds +// documents containing terms that match the +// specified regular expression. +func NewRegexpQuery(regexp string) *query.RegexpQuery { + return query.NewRegexpQuery(regexp) +} + +// NewQueryStringQuery creates a new Query used for +// finding documents that satisfy a query string. The +// query string is a small query language for humans. +func NewQueryStringQuery(q string) *query.QueryStringQuery { + return query.NewQueryStringQuery(q) +} + +// NewTermQuery creates a new Query for finding an +// exact term match in the index. +func NewTermQuery(term string) *query.TermQuery { + return query.NewTermQuery(term) +} + +// NewWildcardQuery creates a new Query which finds +// documents containing terms that match the +// specified wildcard. In the wildcard pattern '*' +// will match any sequence of 0 or more characters, +// and '?' will match any single character. +func NewWildcardQuery(wildcard string) *query.WildcardQuery { + return query.NewWildcardQuery(wildcard) +} + +// NewGeoBoundingBoxQuery creates a new Query for performing geo bounding +// box searches. The arguments describe the position of the box and documents +// which have an indexed geo point inside the box will be returned. +func NewGeoBoundingBoxQuery(topLeftLon, topLeftLat, bottomRightLon, bottomRightLat float64) *query.GeoBoundingBoxQuery { + return query.NewGeoBoundingBoxQuery(topLeftLon, topLeftLat, bottomRightLon, bottomRightLat) +} + +// NewGeoDistanceQuery creates a new Query for performing geo distance +// searches. The arguments describe a position and a distance. Documents +// which have an indexed geo point which is less than or equal to the provided +// distance from the given position will be returned. +func NewGeoDistanceQuery(lon, lat float64, distance string) *query.GeoDistanceQuery { + return query.NewGeoDistanceQuery(lon, lat, distance) +} + +// NewIPRangeQuery creates a new Query for matching IP addresses. +// If the argument is in CIDR format, then the query will match all +// IP addresses in the network specified. If the argument is an IP address, +// then the query will return documents which contain that IP. +// Both ipv4 and ipv6 are supported. +func NewIPRangeQuery(cidr string) *query.IPRangeQuery { + return query.NewIPRangeQuery(cidr) +} + +// NewGeoShapeQuery creates a new Query for matching the given geo shape. +// This method can be used for creating geoshape queries for shape types +// like: point, linestring, polygon, multipoint, multilinestring, +// multipolygon and envelope. +func NewGeoShapeQuery(coordinates [][][][]float64, typ, relation string) (*query.GeoShapeQuery, error) { + return query.NewGeoShapeQuery(coordinates, typ, relation) +} + +// NewGeoShapeCircleQuery creates a new query for a geoshape that is a +// circle given center point and the radius. Radius formats supported: +// "5in" "5inch" "7yd" "7yards" "9ft" "9feet" "11km" "11kilometers" +// "3nm" "3nauticalmiles" "13mm" "13millimeters" "15cm" "15centimeters" +// "17mi" "17miles" "19m" "19meters" If the unit cannot be determined, +// the entire string is parsed and the unit of meters is assumed. +func NewGeoShapeCircleQuery(coordinates []float64, radius, relation string) (*query.GeoShapeQuery, error) { + return query.NewGeoShapeCircleQuery(coordinates, radius, relation) +} + +// NewGeometryCollectionQuery creates a new query for the provided +// geometrycollection coordinates and types, which could contain +// multiple geo shapes. +func NewGeometryCollectionQuery(coordinates [][][][][]float64, types []string, relation string) (*query.GeoShapeQuery, error) { + return query.NewGeometryCollectionQuery(coordinates, types, relation) +} diff --git a/vendor/github.com/blevesearch/bleve/registry/analyzer.go b/vendor/github.com/blevesearch/bleve/v2/registry/analyzer.go similarity index 89% rename from vendor/github.com/blevesearch/bleve/registry/analyzer.go rename to vendor/github.com/blevesearch/bleve/v2/registry/analyzer.go index 340e349d..f4753bc1 100644 --- a/vendor/github.com/blevesearch/bleve/registry/analyzer.go +++ b/vendor/github.com/blevesearch/bleve/v2/registry/analyzer.go @@ -17,7 +17,7 @@ package registry import ( "fmt" - "github.com/blevesearch/bleve/analysis" + "github.com/blevesearch/bleve/v2/analysis" ) func RegisterAnalyzer(name string, constructor AnalyzerConstructor) { @@ -28,7 +28,7 @@ func RegisterAnalyzer(name string, constructor AnalyzerConstructor) { analyzers[name] = constructor } -type AnalyzerConstructor func(config map[string]interface{}, cache *Cache) (*analysis.Analyzer, error) +type AnalyzerConstructor func(config map[string]interface{}, cache *Cache) (analysis.Analyzer, error) type AnalyzerRegistry map[string]AnalyzerConstructor type AnalyzerCache struct { @@ -53,15 +53,15 @@ func AnalyzerBuild(name string, config map[string]interface{}, cache *Cache) (in return analyzer, nil } -func (c *AnalyzerCache) AnalyzerNamed(name string, cache *Cache) (*analysis.Analyzer, error) { +func (c *AnalyzerCache) AnalyzerNamed(name string, cache *Cache) (analysis.Analyzer, error) { item, err := c.ItemNamed(name, cache, AnalyzerBuild) if err != nil { return nil, err } - return item.(*analysis.Analyzer), nil + return item.(analysis.Analyzer), nil } -func (c *AnalyzerCache) DefineAnalyzer(name string, typ string, config map[string]interface{}, cache *Cache) (*analysis.Analyzer, error) { +func (c *AnalyzerCache) DefineAnalyzer(name string, typ string, config map[string]interface{}, cache *Cache) (analysis.Analyzer, error) { item, err := c.DefineItem(name, typ, config, cache, AnalyzerBuild) if err != nil { if err == ErrAlreadyDefined { @@ -69,7 +69,7 @@ func (c *AnalyzerCache) DefineAnalyzer(name string, typ string, config map[strin } return nil, err } - return item.(*analysis.Analyzer), nil + return item.(analysis.Analyzer), nil } func AnalyzerTypesAndInstances() ([]string, []string) { diff --git a/vendor/github.com/blevesearch/bleve/registry/cache.go b/vendor/github.com/blevesearch/bleve/v2/registry/cache.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/registry/cache.go rename to vendor/github.com/blevesearch/bleve/v2/registry/cache.go diff --git a/vendor/github.com/blevesearch/bleve/registry/char_filter.go b/vendor/github.com/blevesearch/bleve/v2/registry/char_filter.go similarity index 98% rename from vendor/github.com/blevesearch/bleve/registry/char_filter.go rename to vendor/github.com/blevesearch/bleve/v2/registry/char_filter.go index 4696713c..aa400be6 100644 --- a/vendor/github.com/blevesearch/bleve/registry/char_filter.go +++ b/vendor/github.com/blevesearch/bleve/v2/registry/char_filter.go @@ -17,7 +17,7 @@ package registry import ( "fmt" - "github.com/blevesearch/bleve/analysis" + "github.com/blevesearch/bleve/v2/analysis" ) func RegisterCharFilter(name string, constructor CharFilterConstructor) { diff --git a/vendor/github.com/blevesearch/bleve/registry/datetime_parser.go b/vendor/github.com/blevesearch/bleve/v2/registry/datetime_parser.go similarity index 98% rename from vendor/github.com/blevesearch/bleve/registry/datetime_parser.go rename to vendor/github.com/blevesearch/bleve/v2/registry/datetime_parser.go index 2cd46e5a..a2d8ac24 100644 --- a/vendor/github.com/blevesearch/bleve/registry/datetime_parser.go +++ b/vendor/github.com/blevesearch/bleve/v2/registry/datetime_parser.go @@ -17,7 +17,7 @@ package registry import ( "fmt" - "github.com/blevesearch/bleve/analysis" + "github.com/blevesearch/bleve/v2/analysis" ) func RegisterDateTimeParser(name string, constructor DateTimeParserConstructor) { diff --git a/vendor/github.com/blevesearch/bleve/registry/fragment_formatter.go b/vendor/github.com/blevesearch/bleve/v2/registry/fragment_formatter.go similarity index 98% rename from vendor/github.com/blevesearch/bleve/registry/fragment_formatter.go rename to vendor/github.com/blevesearch/bleve/v2/registry/fragment_formatter.go index d0121d96..6699f53b 100644 --- a/vendor/github.com/blevesearch/bleve/registry/fragment_formatter.go +++ b/vendor/github.com/blevesearch/bleve/v2/registry/fragment_formatter.go @@ -17,7 +17,7 @@ package registry import ( "fmt" - "github.com/blevesearch/bleve/search/highlight" + "github.com/blevesearch/bleve/v2/search/highlight" ) func RegisterFragmentFormatter(name string, constructor FragmentFormatterConstructor) { diff --git a/vendor/github.com/blevesearch/bleve/registry/fragmenter.go b/vendor/github.com/blevesearch/bleve/v2/registry/fragmenter.go similarity index 98% rename from vendor/github.com/blevesearch/bleve/registry/fragmenter.go rename to vendor/github.com/blevesearch/bleve/v2/registry/fragmenter.go index 18ab2ac0..cd1e32d2 100644 --- a/vendor/github.com/blevesearch/bleve/registry/fragmenter.go +++ b/vendor/github.com/blevesearch/bleve/v2/registry/fragmenter.go @@ -17,7 +17,7 @@ package registry import ( "fmt" - "github.com/blevesearch/bleve/search/highlight" + "github.com/blevesearch/bleve/v2/search/highlight" ) func RegisterFragmenter(name string, constructor FragmenterConstructor) { diff --git a/vendor/github.com/blevesearch/bleve/v2/registry/highlighter.go b/vendor/github.com/blevesearch/bleve/v2/registry/highlighter.go new file mode 100644 index 00000000..8eb210fb --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/registry/highlighter.go @@ -0,0 +1,89 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry + +import ( + "fmt" + + "github.com/blevesearch/bleve/v2/search/highlight" +) + +func RegisterHighlighter(name string, constructor HighlighterConstructor) { + _, exists := highlighters[name] + if exists { + panic(fmt.Errorf("attempted to register duplicate highlighter named '%s'", name)) + } + highlighters[name] = constructor +} + +type HighlighterConstructor func(config map[string]interface{}, cache *Cache) (highlight.Highlighter, error) +type HighlighterRegistry map[string]HighlighterConstructor + +type HighlighterCache struct { + *ConcurrentCache +} + +func NewHighlighterCache() *HighlighterCache { + return &HighlighterCache{ + NewConcurrentCache(), + } +} + +func HighlighterBuild(name string, config map[string]interface{}, cache *Cache) (interface{}, error) { + cons, registered := highlighters[name] + if !registered { + return nil, fmt.Errorf("no highlighter with name or type '%s' registered", name) + } + highlighter, err := cons(config, cache) + if err != nil { + return nil, fmt.Errorf("error building highlighter: %v", err) + } + return highlighter, nil +} + +func (c *HighlighterCache) HighlighterNamed(name string, cache *Cache) (highlight.Highlighter, error) { + item, err := c.ItemNamed(name, cache, HighlighterBuild) + if err != nil { + return nil, err + } + return item.(highlight.Highlighter), nil +} + +func (c *HighlighterCache) DefineHighlighter(name string, typ string, config map[string]interface{}, cache *Cache) (highlight.Highlighter, error) { + item, err := c.DefineItem(name, typ, config, cache, HighlighterBuild) + if err != nil { + if err == ErrAlreadyDefined { + return nil, fmt.Errorf("highlighter named '%s' already defined", name) + } + return nil, err + } + return item.(highlight.Highlighter), nil +} + +func HighlighterTypesAndInstances() ([]string, []string) { + emptyConfig := map[string]interface{}{} + emptyCache := NewCache() + var types []string + var instances []string + for name, cons := range highlighters { + _, err := cons(emptyConfig, emptyCache) + if err == nil { + instances = append(instances, name) + } else { + types = append(types, name) + } + } + return types, instances +} diff --git a/vendor/github.com/blevesearch/bleve/registry/index_type.go b/vendor/github.com/blevesearch/bleve/v2/registry/index_type.go similarity index 96% rename from vendor/github.com/blevesearch/bleve/registry/index_type.go rename to vendor/github.com/blevesearch/bleve/v2/registry/index_type.go index 4da07c82..67938c4a 100644 --- a/vendor/github.com/blevesearch/bleve/registry/index_type.go +++ b/vendor/github.com/blevesearch/bleve/v2/registry/index_type.go @@ -17,7 +17,7 @@ package registry import ( "fmt" - "github.com/blevesearch/bleve/index" + index "github.com/blevesearch/bleve_index_api" ) func RegisterIndexType(name string, constructor IndexTypeConstructor) { diff --git a/vendor/github.com/blevesearch/bleve/registry/registry.go b/vendor/github.com/blevesearch/bleve/v2/registry/registry.go similarity index 96% rename from vendor/github.com/blevesearch/bleve/registry/registry.go rename to vendor/github.com/blevesearch/bleve/v2/registry/registry.go index a0ea69c8..1954d089 100644 --- a/vendor/github.com/blevesearch/bleve/registry/registry.go +++ b/vendor/github.com/blevesearch/bleve/v2/registry/registry.go @@ -17,8 +17,8 @@ package registry import ( "fmt" - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/search/highlight" + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/search/highlight" ) var stores = make(KVStoreRegistry, 0) @@ -123,11 +123,11 @@ func (c *Cache) DefineTokenFilter(name string, config map[string]interface{}) (a return c.TokenFilters.DefineTokenFilter(name, typ, config, c) } -func (c *Cache) AnalyzerNamed(name string) (*analysis.Analyzer, error) { +func (c *Cache) AnalyzerNamed(name string) (analysis.Analyzer, error) { return c.Analyzers.AnalyzerNamed(name, c) } -func (c *Cache) DefineAnalyzer(name string, config map[string]interface{}) (*analysis.Analyzer, error) { +func (c *Cache) DefineAnalyzer(name string, config map[string]interface{}) (analysis.Analyzer, error) { typ, err := typeFromConfig(config) if err != nil { return nil, err diff --git a/vendor/github.com/blevesearch/bleve/v2/registry/store.go b/vendor/github.com/blevesearch/bleve/v2/registry/store.go new file mode 100644 index 00000000..02ebd888 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/registry/store.go @@ -0,0 +1,51 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry + +import ( + "fmt" + + "github.com/blevesearch/upsidedown_store_api" +) + +func RegisterKVStore(name string, constructor KVStoreConstructor) { + _, exists := stores[name] + if exists { + panic(fmt.Errorf("attempted to register duplicate store named '%s'", name)) + } + stores[name] = constructor +} + +// KVStoreConstructor is used to build a KVStore of a specific type when +// specificied by the index configuration. In addition to meeting the +// store.KVStore interface, KVStores must also support this constructor. +// Note that currently the values of config must +// be able to be marshaled and unmarshaled using the encoding/json library (used +// when reading/writing the index metadata file). +type KVStoreConstructor func(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) +type KVStoreRegistry map[string]KVStoreConstructor + +func KVStoreConstructorByName(name string) KVStoreConstructor { + return stores[name] +} + +func KVStoreTypesAndInstances() ([]string, []string) { + var types []string + var instances []string + for name := range stores { + types = append(types, name) + } + return types, instances +} diff --git a/vendor/github.com/blevesearch/bleve/registry/token_filter.go b/vendor/github.com/blevesearch/bleve/v2/registry/token_filter.go similarity index 98% rename from vendor/github.com/blevesearch/bleve/registry/token_filter.go rename to vendor/github.com/blevesearch/bleve/v2/registry/token_filter.go index e202e15f..df39411a 100644 --- a/vendor/github.com/blevesearch/bleve/registry/token_filter.go +++ b/vendor/github.com/blevesearch/bleve/v2/registry/token_filter.go @@ -17,7 +17,7 @@ package registry import ( "fmt" - "github.com/blevesearch/bleve/analysis" + "github.com/blevesearch/bleve/v2/analysis" ) func RegisterTokenFilter(name string, constructor TokenFilterConstructor) { diff --git a/vendor/github.com/blevesearch/bleve/registry/token_maps.go b/vendor/github.com/blevesearch/bleve/v2/registry/token_maps.go similarity index 98% rename from vendor/github.com/blevesearch/bleve/registry/token_maps.go rename to vendor/github.com/blevesearch/bleve/v2/registry/token_maps.go index 66ca08fd..08c9956e 100644 --- a/vendor/github.com/blevesearch/bleve/registry/token_maps.go +++ b/vendor/github.com/blevesearch/bleve/v2/registry/token_maps.go @@ -17,7 +17,7 @@ package registry import ( "fmt" - "github.com/blevesearch/bleve/analysis" + "github.com/blevesearch/bleve/v2/analysis" ) func RegisterTokenMap(name string, constructor TokenMapConstructor) { diff --git a/vendor/github.com/blevesearch/bleve/registry/tokenizer.go b/vendor/github.com/blevesearch/bleve/v2/registry/tokenizer.go similarity index 98% rename from vendor/github.com/blevesearch/bleve/registry/tokenizer.go rename to vendor/github.com/blevesearch/bleve/v2/registry/tokenizer.go index cb9af643..eb954287 100644 --- a/vendor/github.com/blevesearch/bleve/registry/tokenizer.go +++ b/vendor/github.com/blevesearch/bleve/v2/registry/tokenizer.go @@ -17,7 +17,7 @@ package registry import ( "fmt" - "github.com/blevesearch/bleve/analysis" + "github.com/blevesearch/bleve/v2/analysis" ) func RegisterTokenizer(name string, constructor TokenizerConstructor) { diff --git a/vendor/github.com/blevesearch/bleve/v2/search.go b/vendor/github.com/blevesearch/bleve/v2/search.go new file mode 100644 index 00000000..8ca0310f --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/search.go @@ -0,0 +1,695 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bleve + +import ( + "encoding/json" + "fmt" + "reflect" + "sort" + "time" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/datetime/optional" + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/collector" + "github.com/blevesearch/bleve/v2/search/query" + "github.com/blevesearch/bleve/v2/size" + "github.com/blevesearch/bleve/v2/util" +) + +const defaultDateTimeParser = optional.Name + +var cache = registry.NewCache() + +var ( + reflectStaticSizeSearchResult int + reflectStaticSizeSearchStatus int +) + +func init() { + reflectStaticSizeSearchResult = int(reflect.TypeOf(SearchResult{}).Size()) + reflectStaticSizeSearchStatus = int(reflect.TypeOf(SearchStatus{}).Size()) +} + +type dateTimeRange struct { + Name string `json:"name,omitempty"` + Start time.Time `json:"start,omitempty"` + End time.Time `json:"end,omitempty"` + DateTimeParser string `json:"datetime_parser,omitempty"` + startString *string + endString *string +} + +func (dr *dateTimeRange) ParseDates(dateTimeParser analysis.DateTimeParser) (start, end time.Time, startLayout, endLayout string, err error) { + start = dr.Start + startLayout = time.RFC3339Nano + if dr.Start.IsZero() && dr.startString != nil { + s, layout, parseError := dateTimeParser.ParseDateTime(*dr.startString) + if parseError != nil { + return start, end, startLayout, endLayout, fmt.Errorf("error parsing start date '%s' for date range name '%s': %v", *dr.startString, dr.Name, parseError) + } + start = s + startLayout = layout + } + end = dr.End + endLayout = time.RFC3339Nano + if dr.End.IsZero() && dr.endString != nil { + e, layout, parseError := dateTimeParser.ParseDateTime(*dr.endString) + if parseError != nil { + return start, end, startLayout, endLayout, fmt.Errorf("error parsing end date '%s' for date range name '%s': %v", *dr.endString, dr.Name, parseError) + } + end = e + endLayout = layout + } + return start, end, startLayout, endLayout, err +} + +func (dr *dateTimeRange) UnmarshalJSON(input []byte) error { + var temp struct { + Name string `json:"name,omitempty"` + Start *string `json:"start,omitempty"` + End *string `json:"end,omitempty"` + DateTimeParser string `json:"datetime_parser,omitempty"` + } + + if err := util.UnmarshalJSON(input, &temp); err != nil { + return err + } + + dr.Name = temp.Name + if temp.Start != nil { + dr.startString = temp.Start + } + if temp.End != nil { + dr.endString = temp.End + } + if temp.DateTimeParser != "" { + dr.DateTimeParser = temp.DateTimeParser + } + + return nil +} + +func (dr *dateTimeRange) MarshalJSON() ([]byte, error) { + rv := map[string]interface{}{ + "name": dr.Name, + } + + if !dr.Start.IsZero() { + rv["start"] = dr.Start + } else if dr.startString != nil { + rv["start"] = dr.startString + } + + if !dr.End.IsZero() { + rv["end"] = dr.End + } else if dr.endString != nil { + rv["end"] = dr.endString + } + + if dr.DateTimeParser != "" { + rv["datetime_parser"] = dr.DateTimeParser + } + return util.MarshalJSON(rv) +} + +type numericRange struct { + Name string `json:"name,omitempty"` + Min *float64 `json:"min,omitempty"` + Max *float64 `json:"max,omitempty"` +} + +// A FacetRequest describes a facet or aggregation +// of the result document set you would like to be +// built. +type FacetRequest struct { + Size int `json:"size"` + Field string `json:"field"` + NumericRanges []*numericRange `json:"numeric_ranges,omitempty"` + DateTimeRanges []*dateTimeRange `json:"date_ranges,omitempty"` +} + +// NewFacetRequest creates a facet on the specified +// field that limits the number of entries to the +// specified size. +func NewFacetRequest(field string, size int) *FacetRequest { + return &FacetRequest{ + Field: field, + Size: size, + } +} + +func (fr *FacetRequest) Validate() error { + nrCount := len(fr.NumericRanges) + drCount := len(fr.DateTimeRanges) + if nrCount > 0 && drCount > 0 { + return fmt.Errorf("facet can only contain numeric ranges or date ranges, not both") + } + + if nrCount > 0 { + nrNames := map[string]interface{}{} + for _, nr := range fr.NumericRanges { + if _, ok := nrNames[nr.Name]; ok { + return fmt.Errorf("numeric ranges contains duplicate name '%s'", nr.Name) + } + nrNames[nr.Name] = struct{}{} + if nr.Min == nil && nr.Max == nil { + return fmt.Errorf("numeric range query must specify either min, max or both for range name '%s'", nr.Name) + } + } + + } else { + dateTimeParser, err := cache.DateTimeParserNamed(defaultDateTimeParser) + if err != nil { + return err + } + drNames := map[string]interface{}{} + for _, dr := range fr.DateTimeRanges { + if _, ok := drNames[dr.Name]; ok { + return fmt.Errorf("date ranges contains duplicate name '%s'", dr.Name) + } + drNames[dr.Name] = struct{}{} + if dr.DateTimeParser == "" { + // cannot parse the date range dates as the defaultDateTimeParser is overridden + // so perform this validation at query time + start, end, _, _, err := dr.ParseDates(dateTimeParser) + if err != nil { + return fmt.Errorf("ParseDates err: %v, using date time parser named %s", err, defaultDateTimeParser) + } + if start.IsZero() && end.IsZero() { + return fmt.Errorf("date range query must specify either start, end or both for range name '%s'", dr.Name) + } + } + } + } + + return nil +} + +// AddDateTimeRange adds a bucket to a field +// containing date values. Documents with a +// date value falling into this range are tabulated +// as part of this bucket/range. +func (fr *FacetRequest) AddDateTimeRange(name string, start, end time.Time) { + if fr.DateTimeRanges == nil { + fr.DateTimeRanges = make([]*dateTimeRange, 0, 1) + } + fr.DateTimeRanges = append(fr.DateTimeRanges, &dateTimeRange{Name: name, Start: start, End: end}) +} + +// AddDateTimeRangeString adds a bucket to a field +// containing date values. Uses defaultDateTimeParser to parse the date strings. +func (fr *FacetRequest) AddDateTimeRangeString(name string, start, end *string) { + if fr.DateTimeRanges == nil { + fr.DateTimeRanges = make([]*dateTimeRange, 0, 1) + } + fr.DateTimeRanges = append(fr.DateTimeRanges, + &dateTimeRange{Name: name, startString: start, endString: end}) +} + +// AddDateTimeRangeString adds a bucket to a field +// containing date values. Uses the specified parser to parse the date strings. +// provided the parser is registered in the index mapping. +func (fr *FacetRequest) AddDateTimeRangeStringWithParser(name string, start, end *string, parser string) { + if fr.DateTimeRanges == nil { + fr.DateTimeRanges = make([]*dateTimeRange, 0, 1) + } + fr.DateTimeRanges = append(fr.DateTimeRanges, + &dateTimeRange{Name: name, startString: start, endString: end, DateTimeParser: parser}) +} + +// AddNumericRange adds a bucket to a field +// containing numeric values. Documents with a +// numeric value falling into this range are +// tabulated as part of this bucket/range. +func (fr *FacetRequest) AddNumericRange(name string, min, max *float64) { + if fr.NumericRanges == nil { + fr.NumericRanges = make([]*numericRange, 0, 1) + } + fr.NumericRanges = append(fr.NumericRanges, &numericRange{Name: name, Min: min, Max: max}) +} + +// FacetsRequest groups together all the +// FacetRequest objects for a single query. +type FacetsRequest map[string]*FacetRequest + +func (fr FacetsRequest) Validate() error { + for _, v := range fr { + if err := v.Validate(); err != nil { + return err + } + } + return nil +} + +// HighlightRequest describes how field matches +// should be highlighted. +type HighlightRequest struct { + Style *string `json:"style"` + Fields []string `json:"fields"` +} + +// NewHighlight creates a default +// HighlightRequest. +func NewHighlight() *HighlightRequest { + return &HighlightRequest{} +} + +// NewHighlightWithStyle creates a HighlightRequest +// with an alternate style. +func NewHighlightWithStyle(style string) *HighlightRequest { + return &HighlightRequest{ + Style: &style, + } +} + +func (h *HighlightRequest) AddField(field string) { + if h.Fields == nil { + h.Fields = make([]string, 0, 1) + } + h.Fields = append(h.Fields, field) +} + +// A SearchRequest describes all the parameters +// needed to search the index. +// Query is required. +// Size/From describe how much and which part of the +// result set to return. +// Highlight describes optional search result +// highlighting. +// Fields describes a list of field values which +// should be retrieved for result documents, provided they +// were stored while indexing. +// Facets describe the set of facets to be computed. +// Explain triggers inclusion of additional search +// result score explanations. +// Sort describes the desired order for the results to be returned. +// Score controls the kind of scoring performed +// SearchAfter supports deep paging by providing a minimum sort key +// SearchBefore supports deep paging by providing a maximum sort key +// sortFunc specifies the sort implementation to use for sorting results. +// +// A special field named "*" can be used to return all fields. +type SearchRequest struct { + ClientContextID string `json:"client_context_id,omitempty"` + Query query.Query `json:"query"` + Size int `json:"size"` + From int `json:"from"` + Highlight *HighlightRequest `json:"highlight"` + Fields []string `json:"fields"` + Facets FacetsRequest `json:"facets"` + Explain bool `json:"explain"` + Sort search.SortOrder `json:"sort"` + IncludeLocations bool `json:"includeLocations"` + Score string `json:"score,omitempty"` + SearchAfter []string `json:"search_after"` + SearchBefore []string `json:"search_before"` + + sortFunc func(sort.Interface) +} + +func (r *SearchRequest) SetClientContextID(id string) { + r.ClientContextID = id +} + +func (r *SearchRequest) Validate() error { + if srq, ok := r.Query.(query.ValidatableQuery); ok { + if err := srq.Validate(); err != nil { + return err + } + } + + if r.SearchAfter != nil && r.SearchBefore != nil { + return fmt.Errorf("cannot use search after and search before together") + } + + if r.SearchAfter != nil { + if r.From != 0 { + return fmt.Errorf("cannot use search after with from !=0") + } + if len(r.SearchAfter) != len(r.Sort) { + return fmt.Errorf("search after must have same size as sort order") + } + } + if r.SearchBefore != nil { + if r.From != 0 { + return fmt.Errorf("cannot use search before with from !=0") + } + if len(r.SearchBefore) != len(r.Sort) { + return fmt.Errorf("search before must have same size as sort order") + } + } + + return r.Facets.Validate() +} + +// AddFacet adds a FacetRequest to this SearchRequest +func (r *SearchRequest) AddFacet(facetName string, f *FacetRequest) { + if r.Facets == nil { + r.Facets = make(FacetsRequest, 1) + } + r.Facets[facetName] = f +} + +// SortBy changes the request to use the requested sort order +// this form uses the simplified syntax with an array of strings +// each string can either be a field name +// or the magic value _id and _score which refer to the doc id and search score +// any of these values can optionally be prefixed with - to reverse the order +func (r *SearchRequest) SortBy(order []string) { + so := search.ParseSortOrderStrings(order) + r.Sort = so +} + +// SortByCustom changes the request to use the requested sort order +func (r *SearchRequest) SortByCustom(order search.SortOrder) { + r.Sort = order +} + +// SetSearchAfter sets the request to skip over hits with a sort +// value less than the provided sort after key +func (r *SearchRequest) SetSearchAfter(after []string) { + r.SearchAfter = after +} + +// SetSearchBefore sets the request to skip over hits with a sort +// value greater than the provided sort before key +func (r *SearchRequest) SetSearchBefore(before []string) { + r.SearchBefore = before +} + +// UnmarshalJSON deserializes a JSON representation of +// a SearchRequest +func (r *SearchRequest) UnmarshalJSON(input []byte) error { + var ( + temp struct { + ClientContextID string `json:"client_context_id"` + Q json.RawMessage `json:"query"` + Size *int `json:"size"` + From int `json:"from"` + Highlight *HighlightRequest `json:"highlight"` + Fields []string `json:"fields"` + Facets FacetsRequest `json:"facets"` + Explain bool `json:"explain"` + Sort []json.RawMessage `json:"sort"` + IncludeLocations bool `json:"includeLocations"` + Score string `json:"score"` + SearchAfter []string `json:"search_after"` + SearchBefore []string `json:"search_before"` + } + err error + ) + + if err = util.UnmarshalJSON(input, &temp); err != nil { + return err + } + + if temp.Size == nil { + r.Size = 10 + } else { + r.Size = *temp.Size + } + if temp.Sort == nil { + r.Sort = search.SortOrder{&search.SortScore{Desc: true}} + } else { + if r.Sort, err = search.ParseSortOrderJSON(temp.Sort); err != nil { + return err + } + } + r.ClientContextID = temp.ClientContextID + r.From = temp.From + r.Explain = temp.Explain + r.Highlight = temp.Highlight + r.Fields = temp.Fields + r.Facets = temp.Facets + r.IncludeLocations = temp.IncludeLocations + r.Score = temp.Score + r.SearchAfter = temp.SearchAfter + r.SearchBefore = temp.SearchBefore + if r.Query, err = query.ParseQuery(temp.Q); err != nil { + return err + } + + if r.Size < 0 { + r.Size = 10 + } + if r.From < 0 { + r.From = 0 + } + + return nil + +} + +// NewSearchRequest creates a new SearchRequest +// for the Query, using default values for all +// other search parameters. +func NewSearchRequest(q query.Query) *SearchRequest { + return NewSearchRequestOptions(q, 10, 0, false) +} + +// NewSearchRequestOptions creates a new SearchRequest +// for the Query, with the requested size, from +// and explanation search parameters. +// By default results are ordered by score, descending. +func NewSearchRequestOptions(q query.Query, size, from int, explain bool) *SearchRequest { + return &SearchRequest{ + Query: q, + Size: size, + From: from, + Explain: explain, + Sort: search.SortOrder{&search.SortScore{Desc: true}}, + } +} + +// IndexErrMap tracks errors with the name of the index where it occurred +type IndexErrMap map[string]error + +// MarshalJSON seralizes the error into a string for JSON consumption +func (iem IndexErrMap) MarshalJSON() ([]byte, error) { + tmp := make(map[string]string, len(iem)) + for k, v := range iem { + tmp[k] = v.Error() + } + return util.MarshalJSON(tmp) +} + +func (iem IndexErrMap) UnmarshalJSON(data []byte) error { + var tmp map[string]string + if err := util.UnmarshalJSON(data, &tmp); err != nil { + return err + } + for k, v := range tmp { + iem[k] = fmt.Errorf("%s", v) + } + return nil +} + +// SearchStatus is a secion in the SearchResult reporting how many +// underlying indexes were queried, how many were successful/failed +// and a map of any errors that were encountered +type SearchStatus struct { + Total int `json:"total"` + Failed int `json:"failed"` + Successful int `json:"successful"` + Errors IndexErrMap `json:"errors,omitempty"` +} + +// Merge will merge together multiple SearchStatuses during a MultiSearch +func (ss *SearchStatus) Merge(other *SearchStatus) { + ss.Total += other.Total + ss.Failed += other.Failed + ss.Successful += other.Successful + if len(other.Errors) > 0 { + if ss.Errors == nil { + ss.Errors = make(map[string]error) + } + for otherIndex, otherError := range other.Errors { + ss.Errors[otherIndex] = otherError + } + } +} + +// A SearchResult describes the results of executing +// a SearchRequest. +// +// Status - Whether the search was executed on the underlying indexes successfully +// or failed, and the corresponding errors. +// Request - The SearchRequest that was executed. +// Hits - The list of documents that matched the query and their corresponding +// scores, score explanation, location info and so on. +// Total - The total number of documents that matched the query. +// Cost - indicates how expensive was the query with respect to bytes read +// from the mmaped index files. +// MaxScore - The maximum score seen across all document hits seen for this query. +// Took - The time taken to execute the search. +// Facets - The facet results for the search. +type SearchResult struct { + Status *SearchStatus `json:"status"` + Request *SearchRequest `json:"request"` + Hits search.DocumentMatchCollection `json:"hits"` + Total uint64 `json:"total_hits"` + Cost uint64 `json:"cost"` + MaxScore float64 `json:"max_score"` + Took time.Duration `json:"took"` + Facets search.FacetResults `json:"facets"` +} + +func (sr *SearchResult) Size() int { + sizeInBytes := reflectStaticSizeSearchResult + size.SizeOfPtr + + reflectStaticSizeSearchStatus + + for _, entry := range sr.Hits { + if entry != nil { + sizeInBytes += entry.Size() + } + } + + for k, v := range sr.Facets { + sizeInBytes += size.SizeOfString + len(k) + + v.Size() + } + + return sizeInBytes +} + +func (sr *SearchResult) String() string { + rv := "" + if sr.Total > 0 { + if sr.Request.Size > 0 { + rv = fmt.Sprintf("%d matches, showing %d through %d, took %s\n", sr.Total, sr.Request.From+1, sr.Request.From+len(sr.Hits), sr.Took) + for i, hit := range sr.Hits { + rv += fmt.Sprintf("%5d. %s (%f)\n", i+sr.Request.From+1, hit.ID, hit.Score) + for fragmentField, fragments := range hit.Fragments { + rv += fmt.Sprintf("\t%s\n", fragmentField) + for _, fragment := range fragments { + rv += fmt.Sprintf("\t\t%s\n", fragment) + } + } + for otherFieldName, otherFieldValue := range hit.Fields { + if _, ok := hit.Fragments[otherFieldName]; !ok { + rv += fmt.Sprintf("\t%s\n", otherFieldName) + rv += fmt.Sprintf("\t\t%v\n", otherFieldValue) + } + } + } + } else { + rv = fmt.Sprintf("%d matches, took %s\n", sr.Total, sr.Took) + } + } else { + rv = "No matches" + } + if len(sr.Facets) > 0 { + rv += fmt.Sprintf("Facets:\n") + for fn, f := range sr.Facets { + rv += fmt.Sprintf("%s(%d)\n", fn, f.Total) + for _, t := range f.Terms.Terms() { + rv += fmt.Sprintf("\t%s(%d)\n", t.Term, t.Count) + } + for _, n := range f.NumericRanges { + rv += fmt.Sprintf("\t%s(%d)\n", n.Name, n.Count) + } + for _, d := range f.DateRanges { + rv += fmt.Sprintf("\t%s(%d)\n", d.Name, d.Count) + } + if f.Other != 0 { + rv += fmt.Sprintf("\tOther(%d)\n", f.Other) + } + } + } + return rv +} + +// Merge will merge together multiple SearchResults during a MultiSearch +func (sr *SearchResult) Merge(other *SearchResult) { + sr.Status.Merge(other.Status) + sr.Hits = append(sr.Hits, other.Hits...) + sr.Total += other.Total + sr.Cost += other.Cost + if other.MaxScore > sr.MaxScore { + sr.MaxScore = other.MaxScore + } + if sr.Facets == nil && len(other.Facets) != 0 { + sr.Facets = other.Facets + return + } + + sr.Facets.Merge(other.Facets) +} + +// MemoryNeededForSearchResult is an exported helper function to determine the RAM +// needed to accommodate the results for a given search request. +func MemoryNeededForSearchResult(req *SearchRequest) uint64 { + if req == nil { + return 0 + } + + numDocMatches := req.Size + req.From + if req.Size+req.From > collector.PreAllocSizeSkipCap { + numDocMatches = collector.PreAllocSizeSkipCap + } + + estimate := 0 + + // overhead from the SearchResult structure + var sr SearchResult + estimate += sr.Size() + + var dm search.DocumentMatch + sizeOfDocumentMatch := dm.Size() + + // overhead from results + estimate += numDocMatches * sizeOfDocumentMatch + + // overhead from facet results + if req.Facets != nil { + var fr search.FacetResult + estimate += len(req.Facets) * fr.Size() + } + + // overhead from fields, highlighting + var d document.Document + if len(req.Fields) > 0 || req.Highlight != nil { + numDocsApplicable := req.Size + if numDocsApplicable > collector.PreAllocSizeSkipCap { + numDocsApplicable = collector.PreAllocSizeSkipCap + } + estimate += numDocsApplicable * d.Size() + } + + return uint64(estimate) +} + +// SetSortFunc sets the sort implementation to use when sorting hits. +// +// SearchRequests can specify a custom sort implementation to meet +// their needs. For instance, by specifying a parallel sort +// that uses all available cores. +func (r *SearchRequest) SetSortFunc(s func(sort.Interface)) { + r.sortFunc = s +} + +// SortFunc returns the sort implementation to use when sorting hits. +// Defaults to sort.Sort. +func (r *SearchRequest) SortFunc() func(data sort.Interface) { + if r.sortFunc != nil { + return r.sortFunc + } + + return sort.Sort +} diff --git a/vendor/github.com/blevesearch/bleve/search/collector.go b/vendor/github.com/blevesearch/bleve/v2/search/collector.go similarity index 97% rename from vendor/github.com/blevesearch/bleve/search/collector.go rename to vendor/github.com/blevesearch/bleve/v2/search/collector.go index df3ff9c5..38e34fe7 100644 --- a/vendor/github.com/blevesearch/bleve/search/collector.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/collector.go @@ -18,7 +18,7 @@ import ( "context" "time" - "github.com/blevesearch/bleve/index" + index "github.com/blevesearch/bleve_index_api" ) type Collector interface { diff --git a/vendor/github.com/blevesearch/bleve/search/collector/heap.go b/vendor/github.com/blevesearch/bleve/v2/search/collector/heap.go similarity index 98% rename from vendor/github.com/blevesearch/bleve/search/collector/heap.go rename to vendor/github.com/blevesearch/bleve/v2/search/collector/heap.go index 05502d5d..9503f006 100644 --- a/vendor/github.com/blevesearch/bleve/search/collector/heap.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/collector/heap.go @@ -17,7 +17,7 @@ package collector import ( "container/heap" - "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/v2/search" ) type collectStoreHeap struct { diff --git a/vendor/github.com/blevesearch/bleve/search/collector/list.go b/vendor/github.com/blevesearch/bleve/v2/search/collector/list.go similarity index 98% rename from vendor/github.com/blevesearch/bleve/search/collector/list.go rename to vendor/github.com/blevesearch/bleve/v2/search/collector/list.go index f01d205c..20d4c9d0 100644 --- a/vendor/github.com/blevesearch/bleve/search/collector/list.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/collector/list.go @@ -17,7 +17,7 @@ package collector import ( "container/list" - "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/v2/search" ) type collectStoreList struct { diff --git a/vendor/github.com/blevesearch/bleve/search/collector/slice.go b/vendor/github.com/blevesearch/bleve/v2/search/collector/slice.go similarity index 97% rename from vendor/github.com/blevesearch/bleve/search/collector/slice.go rename to vendor/github.com/blevesearch/bleve/v2/search/collector/slice.go index 85fe73c4..b38d9abc 100644 --- a/vendor/github.com/blevesearch/bleve/search/collector/slice.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/collector/slice.go @@ -14,7 +14,7 @@ package collector -import "github.com/blevesearch/bleve/search" +import "github.com/blevesearch/bleve/v2/search" type collectStoreSlice struct { slice search.DocumentMatchCollection diff --git a/vendor/github.com/blevesearch/bleve/search/collector/topn.go b/vendor/github.com/blevesearch/bleve/v2/search/collector/topn.go similarity index 90% rename from vendor/github.com/blevesearch/bleve/search/collector/topn.go rename to vendor/github.com/blevesearch/bleve/v2/search/collector/topn.go index 8d4afb63..270d5f92 100644 --- a/vendor/github.com/blevesearch/bleve/search/collector/topn.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/collector/topn.go @@ -20,9 +20,9 @@ import ( "strconv" "time" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" ) var reflectStaticSizeTopNCollector int @@ -54,6 +54,7 @@ type TopNCollector struct { size int skip int total uint64 + bytesRead uint64 maxScore float64 took time.Duration sort search.SortOrder @@ -68,7 +69,7 @@ type TopNCollector struct { cachedDesc []bool lowestMatchOutsideResults *search.DocumentMatch - updateFieldVisitor index.DocumentFieldTermVisitor + updateFieldVisitor index.DocValueVisitor dvReader index.DocValueReader searchAfter *search.DocumentMatch } @@ -83,7 +84,7 @@ func NewTopNCollector(size int, skip int, sort search.SortOrder) *TopNCollector return newTopNCollector(size, skip, sort) } -// NewTopNCollector builds a collector to find the top 'size' hits +// NewTopNCollectorAfter builds a collector to find the top 'size' hits // skipping over the first 'skip' hits // ordering hits by the provided sort order func NewTopNCollectorAfter(size int, sort search.SortOrder, after []string) *TopNCollector { @@ -197,9 +198,9 @@ func (hc *TopNCollector) Collect(ctx context.Context, searcher search.Searcher, } hc.needDocIds = hc.needDocIds || loadID - select { case <-ctx.Done(): + search.RecordSearchCost(ctx, search.AbortM, 0) return ctx.Err() default: next, err = searcher.Next(searchContext) @@ -208,6 +209,7 @@ func (hc *TopNCollector) Collect(ctx context.Context, searcher search.Searcher, if hc.total%CheckDoneEvery == 0 { select { case <-ctx.Done(): + search.RecordSearchCost(ctx, search.AbortM, 0) return ctx.Err() default: } @@ -226,6 +228,16 @@ func (hc *TopNCollector) Collect(ctx context.Context, searcher search.Searcher, next, err = searcher.Next(searchContext) } + statsCallbackFn := ctx.Value(search.SearchIOStatsCallbackKey) + if statsCallbackFn != nil { + // hc.bytesRead corresponds to the + // total bytes read as part of docValues being read every hit + // which must be accounted by invoking the callback. + statsCallbackFn.(search.SearchIOStatsCallbackFunc)(hc.bytesRead) + + search.RecordSearchCost(ctx, search.AddM, hc.bytesRead) + } + // help finalize/flush the results in case // of custom document match handlers. err = dmHandler(nil) @@ -235,9 +247,7 @@ func (hc *TopNCollector) Collect(ctx context.Context, searcher search.Searcher, // compute search duration hc.took = time.Since(startTime) - if err != nil { - return err - } + // finalize actual results err = hc.finalizeResults(reader) if err != nil { @@ -353,13 +363,28 @@ func (hc *TopNCollector) visitFieldTerms(reader index.IndexReader, d *search.Doc hc.facetsBuilder.EndDoc() } + hc.bytesRead += hc.dvReader.BytesRead() + return err } // SetFacetsBuilder registers a facet builder for this collector func (hc *TopNCollector) SetFacetsBuilder(facetsBuilder *search.FacetsBuilder) { hc.facetsBuilder = facetsBuilder - hc.neededFields = append(hc.neededFields, hc.facetsBuilder.RequiredFields()...) + fieldsRequiredForFaceting := facetsBuilder.RequiredFields() + // for each of these fields, append only if not already there in hc.neededFields. + for _, field := range fieldsRequiredForFaceting { + found := false + for _, neededField := range hc.neededFields { + if field == neededField { + found = true + break + } + } + if !found { + hc.neededFields = append(hc.neededFields, field) + } + } } // finalizeResults starts with the heap containing the final top size+skip diff --git a/vendor/github.com/blevesearch/bleve/search/explanation.go b/vendor/github.com/blevesearch/bleve/v2/search/explanation.go similarity index 97% rename from vendor/github.com/blevesearch/bleve/search/explanation.go rename to vendor/github.com/blevesearch/bleve/v2/search/explanation.go index 3b81737b..b1ac29aa 100644 --- a/vendor/github.com/blevesearch/bleve/search/explanation.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/explanation.go @@ -19,7 +19,7 @@ import ( "fmt" "reflect" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/size" ) var reflectStaticSizeExplanation int diff --git a/vendor/github.com/blevesearch/bleve/search/facet/benchmark_data.txt b/vendor/github.com/blevesearch/bleve/v2/search/facet/benchmark_data.txt similarity index 100% rename from vendor/github.com/blevesearch/bleve/search/facet/benchmark_data.txt rename to vendor/github.com/blevesearch/bleve/v2/search/facet/benchmark_data.txt diff --git a/vendor/github.com/blevesearch/bleve/v2/search/facet/facet_builder_datetime.go b/vendor/github.com/blevesearch/bleve/v2/search/facet/facet_builder_datetime.go new file mode 100644 index 00000000..c272396b --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/search/facet/facet_builder_datetime.go @@ -0,0 +1,178 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package facet + +import ( + "reflect" + "sort" + "strconv" + "time" + + "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/size" +) + +var reflectStaticSizeDateTimeFacetBuilder int +var reflectStaticSizedateTimeRange int + +func init() { + var dtfb DateTimeFacetBuilder + reflectStaticSizeDateTimeFacetBuilder = int(reflect.TypeOf(dtfb).Size()) + var dtr dateTimeRange + reflectStaticSizedateTimeRange = int(reflect.TypeOf(dtr).Size()) +} + +type dateTimeRange struct { + start time.Time + end time.Time + startLayout string + endLayout string +} + +type DateTimeFacetBuilder struct { + size int + field string + termsCount map[string]int + total int + missing int + ranges map[string]*dateTimeRange + sawValue bool +} + +func NewDateTimeFacetBuilder(field string, size int) *DateTimeFacetBuilder { + return &DateTimeFacetBuilder{ + size: size, + field: field, + termsCount: make(map[string]int), + ranges: make(map[string]*dateTimeRange, 0), + } +} + +func (fb *DateTimeFacetBuilder) Size() int { + sizeInBytes := reflectStaticSizeDateTimeFacetBuilder + size.SizeOfPtr + + len(fb.field) + + for k, _ := range fb.termsCount { + sizeInBytes += size.SizeOfString + len(k) + + size.SizeOfInt + } + + for k, _ := range fb.ranges { + sizeInBytes += size.SizeOfString + len(k) + + size.SizeOfPtr + reflectStaticSizedateTimeRange + } + + return sizeInBytes +} + +func (fb *DateTimeFacetBuilder) AddRange(name string, start, end time.Time, startLayout string, endLayout string) { + r := dateTimeRange{ + start: start, + end: end, + startLayout: startLayout, + endLayout: endLayout, + } + fb.ranges[name] = &r +} + +func (fb *DateTimeFacetBuilder) Field() string { + return fb.field +} + +func (fb *DateTimeFacetBuilder) UpdateVisitor(term []byte) { + fb.sawValue = true + // only consider the values which are shifted 0 + prefixCoded := numeric.PrefixCoded(term) + shift, err := prefixCoded.Shift() + if err == nil && shift == 0 { + i64, err := prefixCoded.Int64() + if err == nil { + t := time.Unix(0, i64) + + // look at each of the ranges for a match + for rangeName, r := range fb.ranges { + if (r.start.IsZero() || t.After(r.start) || t.Equal(r.start)) && (r.end.IsZero() || t.Before(r.end)) { + fb.termsCount[rangeName] = fb.termsCount[rangeName] + 1 + fb.total++ + } + } + } + } +} + +func (fb *DateTimeFacetBuilder) StartDoc() { + fb.sawValue = false +} + +func (fb *DateTimeFacetBuilder) EndDoc() { + if !fb.sawValue { + fb.missing++ + } +} + +func (fb *DateTimeFacetBuilder) Result() *search.FacetResult { + rv := search.FacetResult{ + Field: fb.field, + Total: fb.total, + Missing: fb.missing, + } + + rv.DateRanges = make([]*search.DateRangeFacet, 0, len(fb.termsCount)) + + for term, count := range fb.termsCount { + dateRange := fb.ranges[term] + tf := &search.DateRangeFacet{ + Name: term, + Count: count, + } + if !dateRange.start.IsZero() { + var start string + if dateRange.startLayout == "" { + // layout not set probably means it is probably a timestamp + start = strconv.FormatInt(dateRange.start.UnixNano(), 10) + } else { + start = dateRange.start.Format(dateRange.startLayout) + } + tf.Start = &start + } + if !dateRange.end.IsZero() { + var end string + if dateRange.endLayout == "" { + // layout not set probably means it is probably a timestamp + end = strconv.FormatInt(dateRange.end.UnixNano(), 10) + } else { + end = dateRange.end.Format(dateRange.endLayout) + } + tf.End = &end + } + rv.DateRanges = append(rv.DateRanges, tf) + } + + sort.Sort(rv.DateRanges) + + // we now have the list of the top N facets + if fb.size < len(rv.DateRanges) { + rv.DateRanges = rv.DateRanges[:fb.size] + } + + notOther := 0 + for _, nr := range rv.DateRanges { + notOther += nr.Count + } + rv.Other = fb.total - notOther + + return &rv +} diff --git a/vendor/github.com/blevesearch/bleve/search/facet/facet_builder_numeric.go b/vendor/github.com/blevesearch/bleve/v2/search/facet/facet_builder_numeric.go similarity index 80% rename from vendor/github.com/blevesearch/bleve/search/facet/facet_builder_numeric.go rename to vendor/github.com/blevesearch/bleve/v2/search/facet/facet_builder_numeric.go index c1692b54..f19634d7 100644 --- a/vendor/github.com/blevesearch/bleve/search/facet/facet_builder_numeric.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/facet/facet_builder_numeric.go @@ -18,9 +18,9 @@ import ( "reflect" "sort" - "github.com/blevesearch/bleve/numeric" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/size" ) var reflectStaticSizeNumericFacetBuilder int @@ -86,23 +86,21 @@ func (fb *NumericFacetBuilder) Field() string { return fb.field } -func (fb *NumericFacetBuilder) UpdateVisitor(field string, term []byte) { - if field == fb.field { - fb.sawValue = true - // only consider the values which are shifted 0 - prefixCoded := numeric.PrefixCoded(term) - shift, err := prefixCoded.Shift() - if err == nil && shift == 0 { - i64, err := prefixCoded.Int64() - if err == nil { - f64 := numeric.Int64ToFloat64(i64) - - // look at each of the ranges for a match - for rangeName, r := range fb.ranges { - if (r.min == nil || f64 >= *r.min) && (r.max == nil || f64 < *r.max) { - fb.termsCount[rangeName] = fb.termsCount[rangeName] + 1 - fb.total++ - } +func (fb *NumericFacetBuilder) UpdateVisitor(term []byte) { + fb.sawValue = true + // only consider the values which are shifted 0 + prefixCoded := numeric.PrefixCoded(term) + shift, err := prefixCoded.Shift() + if err == nil && shift == 0 { + i64, err := prefixCoded.Int64() + if err == nil { + f64 := numeric.Int64ToFloat64(i64) + + // look at each of the ranges for a match + for rangeName, r := range fb.ranges { + if (r.min == nil || f64 >= *r.min) && (r.max == nil || f64 < *r.max) { + fb.termsCount[rangeName] = fb.termsCount[rangeName] + 1 + fb.total++ } } } diff --git a/vendor/github.com/blevesearch/bleve/search/facet/facet_builder_terms.go b/vendor/github.com/blevesearch/bleve/v2/search/facet/facet_builder_terms.go similarity index 80% rename from vendor/github.com/blevesearch/bleve/search/facet/facet_builder_terms.go rename to vendor/github.com/blevesearch/bleve/v2/search/facet/facet_builder_terms.go index 5b5901e0..c5a1c831 100644 --- a/vendor/github.com/blevesearch/bleve/search/facet/facet_builder_terms.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/facet/facet_builder_terms.go @@ -18,8 +18,8 @@ import ( "reflect" "sort" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/size" ) var reflectStaticSizeTermsFacetBuilder int @@ -62,12 +62,10 @@ func (fb *TermsFacetBuilder) Field() string { return fb.field } -func (fb *TermsFacetBuilder) UpdateVisitor(field string, term []byte) { - if field == fb.field { - fb.sawValue = true - fb.termsCount[string(term)] = fb.termsCount[string(term)] + 1 - fb.total++ - } +func (fb *TermsFacetBuilder) UpdateVisitor(term []byte) { + fb.sawValue = true + fb.termsCount[string(term)] = fb.termsCount[string(term)] + 1 + fb.total++ } func (fb *TermsFacetBuilder) StartDoc() { @@ -87,7 +85,7 @@ func (fb *TermsFacetBuilder) Result() *search.FacetResult { Missing: fb.missing, } - rv.Terms = make([]*search.TermFacet, 0, len(fb.termsCount)) + rv.Terms = &search.TermFacets{} for term, count := range fb.termsCount { tf := &search.TermFacet{ @@ -95,20 +93,20 @@ func (fb *TermsFacetBuilder) Result() *search.FacetResult { Count: count, } - rv.Terms = append(rv.Terms, tf) + rv.Terms.Add(tf) } sort.Sort(rv.Terms) // we now have the list of the top N facets trimTopN := fb.size - if trimTopN > len(rv.Terms) { - trimTopN = len(rv.Terms) + if trimTopN > rv.Terms.Len() { + trimTopN = rv.Terms.Len() } - rv.Terms = rv.Terms[:trimTopN] + rv.Terms.TrimToTopN(trimTopN) notOther := 0 - for _, tf := range rv.Terms { + for _, tf := range rv.Terms.Terms() { notOther += tf.Count } rv.Other = fb.total - notOther diff --git a/vendor/github.com/blevesearch/bleve/v2/search/facets_builder.go b/vendor/github.com/blevesearch/bleve/v2/search/facets_builder.go new file mode 100644 index 00000000..ebe785c0 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/search/facets_builder.go @@ -0,0 +1,399 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package search + +import ( + "reflect" + "sort" + + "github.com/blevesearch/bleve/v2/size" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" +) + +var reflectStaticSizeFacetsBuilder int +var reflectStaticSizeFacetResult int +var reflectStaticSizeTermFacet int +var reflectStaticSizeNumericRangeFacet int +var reflectStaticSizeDateRangeFacet int + +func init() { + var fb FacetsBuilder + reflectStaticSizeFacetsBuilder = int(reflect.TypeOf(fb).Size()) + var fr FacetResult + reflectStaticSizeFacetResult = int(reflect.TypeOf(fr).Size()) + var tf TermFacet + reflectStaticSizeTermFacet = int(reflect.TypeOf(tf).Size()) + var nrf NumericRangeFacet + reflectStaticSizeNumericRangeFacet = int(reflect.TypeOf(nrf).Size()) + var drf DateRangeFacet + reflectStaticSizeDateRangeFacet = int(reflect.TypeOf(drf).Size()) +} + +type FacetBuilder interface { + StartDoc() + UpdateVisitor(term []byte) + EndDoc() + + Result() *FacetResult + Field() string + + Size() int +} + +type FacetsBuilder struct { + indexReader index.IndexReader + facetNames []string + facets []FacetBuilder + facetsByField map[string][]FacetBuilder + fields []string +} + +func NewFacetsBuilder(indexReader index.IndexReader) *FacetsBuilder { + return &FacetsBuilder{ + indexReader: indexReader, + } +} + +func (fb *FacetsBuilder) Size() int { + sizeInBytes := reflectStaticSizeFacetsBuilder + size.SizeOfPtr + + for k, v := range fb.facets { + sizeInBytes += size.SizeOfString + v.Size() + len(fb.facetNames[k]) + } + + for _, entry := range fb.fields { + sizeInBytes += size.SizeOfString + len(entry) + } + + return sizeInBytes +} + +func (fb *FacetsBuilder) Add(name string, facetBuilder FacetBuilder) { + if fb.facetsByField == nil { + fb.facetsByField = map[string][]FacetBuilder{} + } + + fb.facetNames = append(fb.facetNames, name) + fb.facets = append(fb.facets, facetBuilder) + fb.facetsByField[facetBuilder.Field()] = append(fb.facetsByField[facetBuilder.Field()], facetBuilder) + fb.fields = append(fb.fields, facetBuilder.Field()) +} + +func (fb *FacetsBuilder) RequiredFields() []string { + return fb.fields +} + +func (fb *FacetsBuilder) StartDoc() { + for _, facetBuilder := range fb.facets { + facetBuilder.StartDoc() + } +} + +func (fb *FacetsBuilder) EndDoc() { + for _, facetBuilder := range fb.facets { + facetBuilder.EndDoc() + } +} + +func (fb *FacetsBuilder) UpdateVisitor(field string, term []byte) { + if facetBuilders, ok := fb.facetsByField[field]; ok { + for _, facetBuilder := range facetBuilders { + facetBuilder.UpdateVisitor(term) + } + } +} + +type TermFacet struct { + Term string `json:"term"` + Count int `json:"count"` +} + +type TermFacets struct { + termFacets []*TermFacet + termLookup map[string]*TermFacet +} + +func (tf *TermFacets) Terms() []*TermFacet { + if tf == nil { + return []*TermFacet{} + } + return tf.termFacets +} + +func (tf *TermFacets) TrimToTopN(n int) { + tf.termFacets = tf.termFacets[:n] +} + +func (tf *TermFacets) Add(termFacets ...*TermFacet) { + for _, termFacet := range termFacets { + if tf.termLookup == nil { + tf.termLookup = map[string]*TermFacet{} + } + + if term, ok := tf.termLookup[termFacet.Term]; ok { + term.Count += termFacet.Count + return + } + + // if we got here it wasn't already in the existing terms + tf.termFacets = append(tf.termFacets, termFacet) + tf.termLookup[termFacet.Term] = termFacet + } +} + +func (tf *TermFacets) Len() int { + // Handle case where *TermFacets is not fully initialized in index_impl.go.init() + if tf == nil { + return 0 + } + + return len(tf.termFacets) +} +func (tf *TermFacets) Swap(i, j int) { + tf.termFacets[i], tf.termFacets[j] = tf.termFacets[j], tf.termFacets[i] +} +func (tf *TermFacets) Less(i, j int) bool { + if tf.termFacets[i].Count == tf.termFacets[j].Count { + return tf.termFacets[i].Term < tf.termFacets[j].Term + } + return tf.termFacets[i].Count > tf.termFacets[j].Count +} + +// TermFacets used to be a type alias for []*TermFacet. +// To maintain backwards compatibility, we have to implement custom +// JSON marshalling. +func (tf *TermFacets) MarshalJSON() ([]byte, error) { + return util.MarshalJSON(tf.termFacets) +} + +func (tf *TermFacets) UnmarshalJSON(b []byte) error { + termFacets := []*TermFacet{} + err := util.UnmarshalJSON(b, &termFacets) + if err != nil { + return err + } + + for _, termFacet := range termFacets { + tf.Add(termFacet) + } + + return nil +} + +type NumericRangeFacet struct { + Name string `json:"name"` + Min *float64 `json:"min,omitempty"` + Max *float64 `json:"max,omitempty"` + Count int `json:"count"` +} + +func (nrf *NumericRangeFacet) Same(other *NumericRangeFacet) bool { + if nrf.Min == nil && other.Min != nil { + return false + } + if nrf.Min != nil && other.Min == nil { + return false + } + if nrf.Min != nil && other.Min != nil && *nrf.Min != *other.Min { + return false + } + if nrf.Max == nil && other.Max != nil { + return false + } + if nrf.Max != nil && other.Max == nil { + return false + } + if nrf.Max != nil && other.Max != nil && *nrf.Max != *other.Max { + return false + } + + return true +} + +type NumericRangeFacets []*NumericRangeFacet + +func (nrf NumericRangeFacets) Add(numericRangeFacet *NumericRangeFacet) NumericRangeFacets { + for _, existingNr := range nrf { + if numericRangeFacet.Same(existingNr) { + existingNr.Count += numericRangeFacet.Count + return nrf + } + } + // if we got here it wasn't already in the existing terms + nrf = append(nrf, numericRangeFacet) + return nrf +} + +func (nrf NumericRangeFacets) Len() int { return len(nrf) } +func (nrf NumericRangeFacets) Swap(i, j int) { nrf[i], nrf[j] = nrf[j], nrf[i] } +func (nrf NumericRangeFacets) Less(i, j int) bool { + if nrf[i].Count == nrf[j].Count { + return nrf[i].Name < nrf[j].Name + } + return nrf[i].Count > nrf[j].Count +} + +type DateRangeFacet struct { + Name string `json:"name"` + Start *string `json:"start,omitempty"` + End *string `json:"end,omitempty"` + Count int `json:"count"` +} + +func (drf *DateRangeFacet) Same(other *DateRangeFacet) bool { + if drf.Start == nil && other.Start != nil { + return false + } + if drf.Start != nil && other.Start == nil { + return false + } + if drf.Start != nil && other.Start != nil && *drf.Start != *other.Start { + return false + } + if drf.End == nil && other.End != nil { + return false + } + if drf.End != nil && other.End == nil { + return false + } + if drf.End != nil && other.End != nil && *drf.End != *other.End { + return false + } + + return true +} + +type DateRangeFacets []*DateRangeFacet + +func (drf DateRangeFacets) Add(dateRangeFacet *DateRangeFacet) DateRangeFacets { + for _, existingDr := range drf { + if dateRangeFacet.Same(existingDr) { + existingDr.Count += dateRangeFacet.Count + return drf + } + } + // if we got here it wasn't already in the existing terms + drf = append(drf, dateRangeFacet) + return drf +} + +func (drf DateRangeFacets) Len() int { return len(drf) } +func (drf DateRangeFacets) Swap(i, j int) { drf[i], drf[j] = drf[j], drf[i] } +func (drf DateRangeFacets) Less(i, j int) bool { + if drf[i].Count == drf[j].Count { + return drf[i].Name < drf[j].Name + } + return drf[i].Count > drf[j].Count +} + +type FacetResult struct { + Field string `json:"field"` + Total int `json:"total"` + Missing int `json:"missing"` + Other int `json:"other"` + Terms *TermFacets `json:"terms,omitempty"` + NumericRanges NumericRangeFacets `json:"numeric_ranges,omitempty"` + DateRanges DateRangeFacets `json:"date_ranges,omitempty"` +} + +func (fr *FacetResult) Size() int { + return reflectStaticSizeFacetResult + size.SizeOfPtr + + len(fr.Field) + + fr.Terms.Len()*(reflectStaticSizeTermFacet+size.SizeOfPtr) + + len(fr.NumericRanges)*(reflectStaticSizeNumericRangeFacet+size.SizeOfPtr) + + len(fr.DateRanges)*(reflectStaticSizeDateRangeFacet+size.SizeOfPtr) +} + +func (fr *FacetResult) Merge(other *FacetResult) { + fr.Total += other.Total + fr.Missing += other.Missing + fr.Other += other.Other + if fr.Terms != nil && other.Terms != nil { + for _, term := range other.Terms.termFacets { + fr.Terms.Add(term) + } + } + if fr.NumericRanges != nil && other.NumericRanges != nil { + for _, nr := range other.NumericRanges { + fr.NumericRanges = fr.NumericRanges.Add(nr) + } + } + if fr.DateRanges != nil && other.DateRanges != nil { + for _, dr := range other.DateRanges { + fr.DateRanges = fr.DateRanges.Add(dr) + } + } +} + +func (fr *FacetResult) Fixup(size int) { + if fr.Terms != nil { + sort.Sort(fr.Terms) + if fr.Terms.Len() > size { + moveToOther := fr.Terms.termFacets[size:] + for _, mto := range moveToOther { + fr.Other += mto.Count + } + fr.Terms.termFacets = fr.Terms.termFacets[0:size] + } + } else if fr.NumericRanges != nil { + sort.Sort(fr.NumericRanges) + if len(fr.NumericRanges) > size { + moveToOther := fr.NumericRanges[size:] + for _, mto := range moveToOther { + fr.Other += mto.Count + } + fr.NumericRanges = fr.NumericRanges[0:size] + } + } else if fr.DateRanges != nil { + sort.Sort(fr.DateRanges) + if len(fr.DateRanges) > size { + moveToOther := fr.DateRanges[size:] + for _, mto := range moveToOther { + fr.Other += mto.Count + } + fr.DateRanges = fr.DateRanges[0:size] + } + } +} + +type FacetResults map[string]*FacetResult + +func (fr FacetResults) Merge(other FacetResults) { + for name, oFacetResult := range other { + facetResult, ok := fr[name] + if ok { + facetResult.Merge(oFacetResult) + } else { + fr[name] = oFacetResult + } + } +} + +func (fr FacetResults) Fixup(name string, size int) { + facetResult, ok := fr[name] + if ok { + facetResult.Fixup(size) + } +} + +func (fb *FacetsBuilder) Results() FacetResults { + fr := make(FacetResults) + for i, facetBuilder := range fb.facets { + facetResult := facetBuilder.Result() + fr[fb.facetNames[i]] = facetResult + } + return fr +} diff --git a/vendor/github.com/blevesearch/bleve/v2/search/highlight/format/html/html.go b/vendor/github.com/blevesearch/bleve/v2/search/highlight/format/html/html.go new file mode 100644 index 00000000..a0658d9c --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/search/highlight/format/html/html.go @@ -0,0 +1,91 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package html + +import ( + "html" + + "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/bleve/v2/search/highlight" +) + +const Name = "html" + +const defaultHTMLHighlightBefore = "" +const defaultHTMLHighlightAfter = "" + +type FragmentFormatter struct { + before string + after string +} + +func NewFragmentFormatter(before, after string) *FragmentFormatter { + return &FragmentFormatter{ + before: before, + after: after, + } +} + +func (a *FragmentFormatter) Format(f *highlight.Fragment, orderedTermLocations highlight.TermLocations) string { + rv := "" + curr := f.Start + for _, termLocation := range orderedTermLocations { + if termLocation == nil { + continue + } + // make sure the array positions match + if !termLocation.ArrayPositions.Equals(f.ArrayPositions) { + continue + } + if termLocation.Start < curr { + continue + } + if termLocation.End > f.End { + break + } + // add the stuff before this location + rv += html.EscapeString(string(f.Orig[curr:termLocation.Start])) + // start the tag + rv += a.before + // add the term itself + rv += html.EscapeString(string(f.Orig[termLocation.Start:termLocation.End])) + // end the tag + rv += a.after + // update current + curr = termLocation.End + } + // add any remaining text after the last token + rv += html.EscapeString(string(f.Orig[curr:f.End])) + + return rv +} + +func Constructor(config map[string]interface{}, cache *registry.Cache) (highlight.FragmentFormatter, error) { + before := defaultHTMLHighlightBefore + beforeVal, ok := config["before"].(string) + if ok { + before = beforeVal + } + after := defaultHTMLHighlightAfter + afterVal, ok := config["after"].(string) + if ok { + after = afterVal + } + return NewFragmentFormatter(before, after), nil +} + +func init() { + registry.RegisterFragmentFormatter(Name, Constructor) +} diff --git a/vendor/github.com/blevesearch/bleve/search/highlight/fragmenter/simple/simple.go b/vendor/github.com/blevesearch/bleve/v2/search/highlight/fragmenter/simple/simple.go similarity index 92% rename from vendor/github.com/blevesearch/bleve/search/highlight/fragmenter/simple/simple.go rename to vendor/github.com/blevesearch/bleve/v2/search/highlight/fragmenter/simple/simple.go index 9c63f7fb..34e5c959 100644 --- a/vendor/github.com/blevesearch/bleve/search/highlight/fragmenter/simple/simple.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/highlight/fragmenter/simple/simple.go @@ -17,8 +17,8 @@ package simple import ( "unicode/utf8" - "github.com/blevesearch/bleve/registry" - "github.com/blevesearch/bleve/search/highlight" + "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/bleve/v2/search/highlight" ) const Name = "simple" @@ -123,9 +123,15 @@ OUTER: // if there were no terms to highlight // produce a single fragment from the beginning start := 0 - end := start + s.fragmentSize - if end > len(orig) { - end = len(orig) + end := start + used := 0 + for end < len(orig) && used < s.fragmentSize { + r, size := utf8.DecodeRune(orig[end:]) + if r == utf8.RuneError { + break + } + end += size + used++ } rv = append(rv, &highlight.Fragment{Orig: orig, Start: start, End: end}) } diff --git a/vendor/github.com/blevesearch/bleve/v2/search/highlight/highlighter.go b/vendor/github.com/blevesearch/bleve/v2/search/highlight/highlighter.go new file mode 100644 index 00000000..3dd9ce05 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/search/highlight/highlighter.go @@ -0,0 +1,64 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package highlight + +import ( + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +type Fragment struct { + Orig []byte + ArrayPositions []uint64 + Start int + End int + Score float64 + Index int // used by heap +} + +func (f *Fragment) Overlaps(other *Fragment) bool { + if other.Start >= f.Start && other.Start < f.End { + return true + } else if f.Start >= other.Start && f.Start < other.End { + return true + } + return false +} + +type Fragmenter interface { + Fragment([]byte, TermLocations) []*Fragment +} + +type FragmentFormatter interface { + Format(f *Fragment, orderedTermLocations TermLocations) string +} + +type FragmentScorer interface { + Score(f *Fragment) float64 +} + +type Highlighter interface { + Fragmenter() Fragmenter + SetFragmenter(Fragmenter) + + FragmentFormatter() FragmentFormatter + SetFragmentFormatter(FragmentFormatter) + + Separator() string + SetSeparator(string) + + BestFragmentInField(*search.DocumentMatch, index.Document, string) string + BestFragmentsInField(*search.DocumentMatch, index.Document, string, int) []string +} diff --git a/vendor/github.com/blevesearch/bleve/v2/search/highlight/highlighter/html/html.go b/vendor/github.com/blevesearch/bleve/v2/search/highlight/highlighter/html/html.go new file mode 100644 index 00000000..ceb686dc --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/search/highlight/highlighter/html/html.go @@ -0,0 +1,50 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package html + +import ( + "fmt" + + "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/bleve/v2/search/highlight" + htmlFormatter "github.com/blevesearch/bleve/v2/search/highlight/format/html" + simpleFragmenter "github.com/blevesearch/bleve/v2/search/highlight/fragmenter/simple" + simpleHighlighter "github.com/blevesearch/bleve/v2/search/highlight/highlighter/simple" +) + +const Name = "html" + +func Constructor(config map[string]interface{}, cache *registry.Cache) (highlight.Highlighter, error) { + + fragmenter, err := cache.FragmenterNamed(simpleFragmenter.Name) + if err != nil { + return nil, fmt.Errorf("error building fragmenter: %v", err) + } + + formatter, err := cache.FragmentFormatterNamed(htmlFormatter.Name) + if err != nil { + return nil, fmt.Errorf("error building fragment formatter: %v", err) + } + + return simpleHighlighter.NewHighlighter( + fragmenter, + formatter, + simpleHighlighter.DefaultSeparator), + nil +} + +func init() { + registry.RegisterHighlighter(Name, Constructor) +} diff --git a/vendor/github.com/blevesearch/bleve/search/highlight/highlighter/simple/fragment_scorer_simple.go b/vendor/github.com/blevesearch/bleve/v2/search/highlight/highlighter/simple/fragment_scorer_simple.go similarity index 93% rename from vendor/github.com/blevesearch/bleve/search/highlight/highlighter/simple/fragment_scorer_simple.go rename to vendor/github.com/blevesearch/bleve/v2/search/highlight/highlighter/simple/fragment_scorer_simple.go index 3ec4c3d2..786e33cb 100644 --- a/vendor/github.com/blevesearch/bleve/search/highlight/highlighter/simple/fragment_scorer_simple.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/highlight/highlighter/simple/fragment_scorer_simple.go @@ -15,8 +15,8 @@ package simple import ( - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/highlight" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/highlight" ) // FragmentScorer will score fragments by how many diff --git a/vendor/github.com/blevesearch/bleve/search/highlight/highlighter/simple/highlighter_simple.go b/vendor/github.com/blevesearch/bleve/v2/search/highlight/highlighter/simple/highlighter_simple.go similarity index 93% rename from vendor/github.com/blevesearch/bleve/search/highlight/highlighter/simple/highlighter_simple.go rename to vendor/github.com/blevesearch/bleve/v2/search/highlight/highlighter/simple/highlighter_simple.go index 4849516b..19949687 100644 --- a/vendor/github.com/blevesearch/bleve/search/highlight/highlighter/simple/highlighter_simple.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/highlight/highlighter/simple/highlighter_simple.go @@ -17,11 +17,11 @@ package simple import ( "container/heap" "fmt" + index "github.com/blevesearch/bleve_index_api" - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/registry" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/highlight" + "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/highlight" ) const Name = "simple" @@ -65,7 +65,7 @@ func (s *Highlighter) SetSeparator(sep string) { s.sep = sep } -func (s *Highlighter) BestFragmentInField(dm *search.DocumentMatch, doc *document.Document, field string) string { +func (s *Highlighter) BestFragmentInField(dm *search.DocumentMatch, doc index.Document, field string) string { fragments := s.BestFragmentsInField(dm, doc, field, 1) if len(fragments) > 0 { return fragments[0] @@ -73,7 +73,7 @@ func (s *Highlighter) BestFragmentInField(dm *search.DocumentMatch, doc *documen return "" } -func (s *Highlighter) BestFragmentsInField(dm *search.DocumentMatch, doc *document.Document, field string, num int) []string { +func (s *Highlighter) BestFragmentsInField(dm *search.DocumentMatch, doc index.Document, field string, num int) []string { tlm := dm.Locations[field] orderedTermLocations := highlight.OrderTermLocations(tlm) scorer := NewFragmentScorer(tlm) @@ -81,9 +81,9 @@ func (s *Highlighter) BestFragmentsInField(dm *search.DocumentMatch, doc *docume // score the fragments and put them into a priority queue ordered by score fq := make(FragmentQueue, 0) heap.Init(&fq) - for _, f := range doc.Fields { + doc.VisitFields(func(f index.Field) { if f.Name() == field { - _, ok := f.(*document.TextField) + _, ok := f.(index.TextField) if ok { termLocationsSameArrayPosition := make(highlight.TermLocations, 0) for _, otl := range orderedTermLocations { @@ -101,7 +101,7 @@ func (s *Highlighter) BestFragmentsInField(dm *search.DocumentMatch, doc *docume } } } - } + }) // now find the N best non-overlapping fragments var bestFragments []*highlight.Fragment diff --git a/vendor/github.com/blevesearch/bleve/search/highlight/term_locations.go b/vendor/github.com/blevesearch/bleve/v2/search/highlight/term_locations.go similarity index 98% rename from vendor/github.com/blevesearch/bleve/search/highlight/term_locations.go rename to vendor/github.com/blevesearch/bleve/v2/search/highlight/term_locations.go index 6d2cb133..6bf385c0 100644 --- a/vendor/github.com/blevesearch/bleve/search/highlight/term_locations.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/highlight/term_locations.go @@ -18,7 +18,7 @@ import ( "reflect" "sort" - "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/v2/search" ) type TermLocation struct { diff --git a/vendor/github.com/blevesearch/bleve/search/levenshtein.go b/vendor/github.com/blevesearch/bleve/v2/search/levenshtein.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/search/levenshtein.go rename to vendor/github.com/blevesearch/bleve/v2/search/levenshtein.go diff --git a/vendor/github.com/blevesearch/bleve/search/pool.go b/vendor/github.com/blevesearch/bleve/v2/search/pool.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/search/pool.go rename to vendor/github.com/blevesearch/bleve/v2/search/pool.go diff --git a/vendor/github.com/blevesearch/bleve/search/query/bool_field.go b/vendor/github.com/blevesearch/bleve/v2/search/query/bool_field.go similarity index 75% rename from vendor/github.com/blevesearch/bleve/search/query/bool_field.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/bool_field.go index b7b5a3d3..5aa7bb8a 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/bool_field.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/bool_field.go @@ -15,10 +15,12 @@ package query import ( - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searcher" + "context" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" ) type BoolFieldQuery struct { @@ -51,7 +53,7 @@ func (q *BoolFieldQuery) Field() string { return q.FieldVal } -func (q *BoolFieldQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { +func (q *BoolFieldQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() @@ -60,5 +62,5 @@ func (q *BoolFieldQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, o if q.Bool { term = "T" } - return searcher.NewTermSearcher(i, term, field, q.BoostVal.Value(), options) + return searcher.NewTermSearcher(ctx, i, term, field, q.BoostVal.Value(), options) } diff --git a/vendor/github.com/blevesearch/bleve/search/query/boolean.go b/vendor/github.com/blevesearch/bleve/v2/search/query/boolean.go similarity index 88% rename from vendor/github.com/blevesearch/bleve/search/query/boolean.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/boolean.go index 3cfa1d99..026a5868 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/boolean.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/boolean.go @@ -15,13 +15,15 @@ package query import ( + "context" "encoding/json" "fmt" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searcher" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" ) type BooleanQuery struct { @@ -113,11 +115,11 @@ func (q *BooleanQuery) Boost() float64 { return q.BoostVal.Value() } -func (q *BooleanQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { +func (q *BooleanQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { var err error var mustNotSearcher search.Searcher if q.MustNot != nil { - mustNotSearcher, err = q.MustNot.Searcher(i, m, options) + mustNotSearcher, err = q.MustNot.Searcher(ctx, i, m, options) if err != nil { return nil, err } @@ -129,7 +131,7 @@ func (q *BooleanQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, opt var mustSearcher search.Searcher if q.Must != nil { - mustSearcher, err = q.Must.Searcher(i, m, options) + mustSearcher, err = q.Must.Searcher(ctx, i, m, options) if err != nil { return nil, err } @@ -141,7 +143,7 @@ func (q *BooleanQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, opt var shouldSearcher search.Searcher if q.Should != nil { - shouldSearcher, err = q.Should.Searcher(i, m, options) + shouldSearcher, err = q.Should.Searcher(ctx, i, m, options) if err != nil { return nil, err } @@ -158,7 +160,7 @@ func (q *BooleanQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, opt // if only mustNotSearcher, start with MatchAll if mustSearcher == nil && shouldSearcher == nil && mustNotSearcher != nil { - mustSearcher, err = searcher.NewMatchAllSearcher(i, 1.0, options) + mustSearcher, err = searcher.NewMatchAllSearcher(ctx, i, 1.0, options) if err != nil { return nil, err } @@ -169,7 +171,7 @@ func (q *BooleanQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, opt return shouldSearcher, nil } - return searcher.NewBooleanSearcher(i, mustSearcher, shouldSearcher, mustNotSearcher, options) + return searcher.NewBooleanSearcher(ctx, i, mustSearcher, shouldSearcher, mustNotSearcher, options) } func (q *BooleanQuery) Validate() error { @@ -204,7 +206,7 @@ func (q *BooleanQuery) UnmarshalJSON(data []byte) error { MustNot json.RawMessage `json:"must_not,omitempty"` Boost *Boost `json:"boost,omitempty"` }{} - err := json.Unmarshal(data, &tmp) + err := util.UnmarshalJSON(data, &tmp) if err != nil { return err } diff --git a/vendor/github.com/blevesearch/bleve/search/query/boost.go b/vendor/github.com/blevesearch/bleve/v2/search/query/boost.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/search/query/boost.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/boost.go diff --git a/vendor/github.com/blevesearch/bleve/search/query/conjunction.go b/vendor/github.com/blevesearch/bleve/v2/search/query/conjunction.go similarity index 81% rename from vendor/github.com/blevesearch/bleve/search/query/conjunction.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/conjunction.go index 1a7ed1bc..0565e18f 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/conjunction.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/conjunction.go @@ -15,12 +15,14 @@ package query import ( + "context" "encoding/json" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searcher" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" ) type ConjunctionQuery struct { @@ -52,10 +54,10 @@ func (q *ConjunctionQuery) AddQuery(aq ...Query) { } } -func (q *ConjunctionQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { +func (q *ConjunctionQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { ss := make([]search.Searcher, 0, len(q.Conjuncts)) for _, conjunct := range q.Conjuncts { - sr, err := conjunct.Searcher(i, m, options) + sr, err := conjunct.Searcher(ctx, i, m, options) if err != nil { for _, searcher := range ss { if searcher != nil { @@ -75,7 +77,7 @@ func (q *ConjunctionQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, return searcher.NewMatchNoneSearcher(i) } - return searcher.NewConjunctionSearcher(i, ss, options) + return searcher.NewConjunctionSearcher(ctx, i, ss, options) } func (q *ConjunctionQuery) Validate() error { @@ -95,7 +97,7 @@ func (q *ConjunctionQuery) UnmarshalJSON(data []byte) error { Conjuncts []json.RawMessage `json:"conjuncts"` Boost *Boost `json:"boost,omitempty"` }{} - err := json.Unmarshal(data, &tmp) + err := util.UnmarshalJSON(data, &tmp) if err != nil { return err } diff --git a/vendor/github.com/blevesearch/bleve/search/query/date_range.go b/vendor/github.com/blevesearch/bleve/v2/search/query/date_range.go similarity index 83% rename from vendor/github.com/blevesearch/bleve/search/query/date_range.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/date_range.go index 3ac0322f..47012fb1 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/date_range.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/date_range.go @@ -15,24 +15,25 @@ package query import ( - "encoding/json" + "context" "fmt" "math" "time" - "github.com/blevesearch/bleve/analysis/datetime/optional" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/numeric" - "github.com/blevesearch/bleve/registry" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searcher" + "github.com/blevesearch/bleve/v2/analysis/datetime/optional" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" ) -// QueryDateTimeParser controls the default query date time parser +// QueryDateTimeParser controls the default query date time parser. var QueryDateTimeParser = optional.Name -// QueryDateTimeFormat controls the format when Marshaling to JSON +// QueryDateTimeFormat controls the format when Marshaling to JSON. var QueryDateTimeFormat = time.RFC3339 var cache = registry.NewCache() @@ -54,7 +55,7 @@ func queryTimeFromString(t string) (time.Time, error) { if err != nil { return time.Time{}, err } - rv, err := dateTimeParser.ParseDateTime(t) + rv, _, err := dateTimeParser.ParseDateTime(t) if err != nil { return time.Time{}, err } @@ -68,7 +69,7 @@ func (t *BleveQueryTime) MarshalJSON() ([]byte, error) { func (t *BleveQueryTime) UnmarshalJSON(data []byte) error { var timeString string - err := json.Unmarshal(data, &timeString) + err := util.UnmarshalJSON(data, &timeString) if err != nil { return err } @@ -76,7 +77,7 @@ func (t *BleveQueryTime) UnmarshalJSON(data []byte) error { if err != nil { return err } - t.Time, err = dateTimeParser.ParseDateTime(timeString) + t.Time, _, err = dateTimeParser.ParseDateTime(timeString) if err != nil { return err } @@ -95,7 +96,7 @@ type DateRangeQuery struct { // NewDateRangeQuery creates a new Query for ranges // of date values. // Date strings are parsed using the DateTimeParser configured in the -// top-level config.QueryDateTimeParser +// top-level config.QueryDateTimeParser // Either, but not both endpoints can be nil. func NewDateRangeQuery(start, end time.Time) *DateRangeQuery { return NewDateRangeInclusiveQuery(start, end, nil, nil) @@ -104,7 +105,7 @@ func NewDateRangeQuery(start, end time.Time) *DateRangeQuery { // NewDateRangeInclusiveQuery creates a new Query for ranges // of date values. // Date strings are parsed using the DateTimeParser configured in the -// top-level config.QueryDateTimeParser +// top-level config.QueryDateTimeParser // Either, but not both endpoints can be nil. // startInclusive and endInclusive control inclusion of the endpoints. func NewDateRangeInclusiveQuery(start, end time.Time, startInclusive, endInclusive *bool) *DateRangeQuery { @@ -133,7 +134,7 @@ func (q *DateRangeQuery) Field() string { return q.FieldVal } -func (q *DateRangeQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { +func (q *DateRangeQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { min, max, err := q.parseEndpoints() if err != nil { return nil, err @@ -144,7 +145,7 @@ func (q *DateRangeQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, o field = m.DefaultSearchField() } - return searcher.NewNumericRangeSearcher(i, min, max, q.InclusiveStart, q.InclusiveEnd, field, q.BoostVal.Value(), options) + return searcher.NewNumericRangeSearcher(ctx, i, min, max, q.InclusiveStart, q.InclusiveEnd, field, q.BoostVal.Value(), options) } func (q *DateRangeQuery) parseEndpoints() (*float64, *float64, error) { diff --git a/vendor/github.com/blevesearch/bleve/v2/search/query/date_range_string.go b/vendor/github.com/blevesearch/bleve/v2/search/query/date_range_string.go new file mode 100644 index 00000000..b5e5c170 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/date_range_string.go @@ -0,0 +1,176 @@ +// Copyright (c) 2023 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + "fmt" + "math" + "time" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" +) + +// DateRangeStringQuery represents a query for a range of date values. +// Start and End are the range endpoints, as strings. +// Start and End are parsed using DateTimeParser, which is a custom date time parser +// defined in the index mapping. If DateTimeParser is not specified, then the +// top-level config.QueryDateTimeParser is used. +type DateRangeStringQuery struct { + Start string `json:"start,omitempty"` + End string `json:"end,omitempty"` + InclusiveStart *bool `json:"inclusive_start,omitempty"` + InclusiveEnd *bool `json:"inclusive_end,omitempty"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` + DateTimeParser string `json:"datetime_parser,omitempty"` +} + +// NewDateRangeStringQuery creates a new Query for ranges +// of date values. +// Date strings are parsed using the DateTimeParser field of the query struct, +// which is a custom date time parser defined in the index mapping. +// if DateTimeParser is not specified, then the +// top-level config.QueryDateTimeParser is used. +// Either, but not both endpoints can be nil. +func NewDateRangeStringQuery(start, end string) *DateRangeStringQuery { + return NewDateRangeStringInclusiveQuery(start, end, nil, nil) +} + +// NewDateRangeStringQuery creates a new Query for ranges +// of date values. +// Date strings are parsed using the DateTimeParser field of the query struct, +// which is a custom date time parser defined in the index mapping. +// if DateTimeParser is not specified, then the +// top-level config.QueryDateTimeParser is used. +// Either, but not both endpoints can be nil. +// startInclusive and endInclusive control inclusion of the endpoints. +func NewDateRangeStringInclusiveQuery(start, end string, startInclusive, endInclusive *bool) *DateRangeStringQuery { + return &DateRangeStringQuery{ + Start: start, + End: end, + InclusiveStart: startInclusive, + InclusiveEnd: endInclusive, + } +} + +func (q *DateRangeStringQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *DateRangeStringQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *DateRangeStringQuery) SetField(f string) { + q.FieldVal = f +} + +func (q *DateRangeStringQuery) Field() string { + return q.FieldVal +} + +func (q *DateRangeStringQuery) SetDateTimeParser(d string) { + q.DateTimeParser = d +} + +func (q *DateRangeStringQuery) DateTimeParserName() string { + return q.DateTimeParser +} + +func (q *DateRangeStringQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + field := q.FieldVal + if q.FieldVal == "" { + field = m.DefaultSearchField() + } + + dateTimeParserName := QueryDateTimeParser + if q.DateTimeParser != "" { + dateTimeParserName = q.DateTimeParser + } + dateTimeParser := m.DateTimeParserNamed(dateTimeParserName) + if dateTimeParser == nil { + return nil, fmt.Errorf("no dateTimeParser named '%s' registered", dateTimeParserName) + } + + var startTime, endTime time.Time + var err error + if q.Start != "" { + startTime, _, err = dateTimeParser.ParseDateTime(q.Start) + if err != nil { + return nil, fmt.Errorf("%v, date time parser name: %s", err, dateTimeParserName) + } + } + if q.End != "" { + endTime, _, err = dateTimeParser.ParseDateTime(q.End) + if err != nil { + return nil, fmt.Errorf("%v, date time parser name: %s", err, dateTimeParserName) + } + } + + min, max, err := q.parseEndpoints(startTime, endTime) + if err != nil { + return nil, err + } + return searcher.NewNumericRangeSearcher(ctx, i, min, max, q.InclusiveStart, q.InclusiveEnd, field, q.BoostVal.Value(), options) +} + +func (q *DateRangeStringQuery) parseEndpoints(startTime, endTime time.Time) (*float64, *float64, error) { + min := math.Inf(-1) + max := math.Inf(1) + + if startTime.IsZero() && endTime.IsZero() { + return nil, nil, fmt.Errorf("date range query must specify at least one of start/end") + } + + if !startTime.IsZero() { + if !isDateTimeWithinRange(startTime) { + // overflow + return nil, nil, fmt.Errorf("invalid/unsupported date range, start: %v", q.Start) + } + startInt64 := startTime.UnixNano() + min = numeric.Int64ToFloat64(startInt64) + } + if !endTime.IsZero() { + if !isDateTimeWithinRange(endTime) { + // overflow + return nil, nil, fmt.Errorf("invalid/unsupported date range, end: %v", q.End) + } + endInt64 := endTime.UnixNano() + max = numeric.Int64ToFloat64(endInt64) + } + + return &min, &max, nil +} + +func (q *DateRangeStringQuery) Validate() error { + // either start or end must be specified + if q.Start == "" && q.End == "" { + return fmt.Errorf("date range query must specify at least one of start/end") + } + return nil +} + +func isDateTimeWithinRange(t time.Time) bool { + if t.Before(MinRFC3339CompatibleTime) || t.After(MaxRFC3339CompatibleTime) { + return false + } + return true +} diff --git a/vendor/github.com/blevesearch/bleve/search/query/disjunction.go b/vendor/github.com/blevesearch/bleve/v2/search/query/disjunction.go similarity index 84% rename from vendor/github.com/blevesearch/bleve/search/query/disjunction.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/disjunction.go index a1fc1439..f8573d08 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/disjunction.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/disjunction.go @@ -15,13 +15,15 @@ package query import ( + "context" "encoding/json" "fmt" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searcher" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" ) type DisjunctionQuery struct { @@ -58,11 +60,11 @@ func (q *DisjunctionQuery) SetMin(m float64) { q.Min = m } -func (q *DisjunctionQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, +func (q *DisjunctionQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { ss := make([]search.Searcher, 0, len(q.Disjuncts)) for _, disjunct := range q.Disjuncts { - sr, err := disjunct.Searcher(i, m, options) + sr, err := disjunct.Searcher(ctx, i, m, options) if err != nil { for _, searcher := range ss { if searcher != nil { @@ -82,7 +84,7 @@ func (q *DisjunctionQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, return searcher.NewMatchNoneSearcher(i) } - return searcher.NewDisjunctionSearcher(i, ss, q.Min, options) + return searcher.NewDisjunctionSearcher(ctx, i, ss, q.Min, options) } func (q *DisjunctionQuery) Validate() error { @@ -106,7 +108,7 @@ func (q *DisjunctionQuery) UnmarshalJSON(data []byte) error { Boost *Boost `json:"boost,omitempty"` Min float64 `json:"min"` }{} - err := json.Unmarshal(data, &tmp) + err := util.UnmarshalJSON(data, &tmp) if err != nil { return err } diff --git a/vendor/github.com/blevesearch/bleve/v2/search/query/docid.go b/vendor/github.com/blevesearch/bleve/v2/search/query/docid.go new file mode 100644 index 00000000..7116f391 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/docid.go @@ -0,0 +1,51 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" +) + +type DocIDQuery struct { + IDs []string `json:"ids"` + BoostVal *Boost `json:"boost,omitempty"` +} + +// NewDocIDQuery creates a new Query object returning indexed documents among +// the specified set. Combine it with ConjunctionQuery to restrict the scope of +// other queries output. +func NewDocIDQuery(ids []string) *DocIDQuery { + return &DocIDQuery{ + IDs: ids, + } +} + +func (q *DocIDQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *DocIDQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *DocIDQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + return searcher.NewDocIDSearcher(ctx, i, q.IDs, q.BoostVal.Value(), options) +} diff --git a/vendor/github.com/blevesearch/bleve/search/query/fuzzy.go b/vendor/github.com/blevesearch/bleve/v2/search/query/fuzzy.go similarity index 78% rename from vendor/github.com/blevesearch/bleve/search/query/fuzzy.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/fuzzy.go index f18982d4..f24eb0c2 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/fuzzy.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/fuzzy.go @@ -15,10 +15,12 @@ package query import ( - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searcher" + "context" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" ) type FuzzyQuery struct { @@ -68,10 +70,10 @@ func (q *FuzzyQuery) SetPrefix(p int) { q.Prefix = p } -func (q *FuzzyQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { +func (q *FuzzyQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() } - return searcher.NewFuzzySearcher(i, q.Term, q.Prefix, q.Fuzziness, field, q.BoostVal.Value(), options) + return searcher.NewFuzzySearcher(ctx, i, q.Term, q.Prefix, q.Fuzziness, field, q.BoostVal.Value(), options) } diff --git a/vendor/github.com/blevesearch/bleve/v2/search/query/geo_boundingbox.go b/vendor/github.com/blevesearch/bleve/v2/search/query/geo_boundingbox.go new file mode 100644 index 00000000..feb45d31 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/geo_boundingbox.go @@ -0,0 +1,116 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + "fmt" + + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" +) + +type GeoBoundingBoxQuery struct { + TopLeft []float64 `json:"top_left,omitempty"` + BottomRight []float64 `json:"bottom_right,omitempty"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` +} + +func NewGeoBoundingBoxQuery(topLeftLon, topLeftLat, bottomRightLon, bottomRightLat float64) *GeoBoundingBoxQuery { + return &GeoBoundingBoxQuery{ + TopLeft: []float64{topLeftLon, topLeftLat}, + BottomRight: []float64{bottomRightLon, bottomRightLat}, + } +} + +func (q *GeoBoundingBoxQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *GeoBoundingBoxQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *GeoBoundingBoxQuery) SetField(f string) { + q.FieldVal = f +} + +func (q *GeoBoundingBoxQuery) Field() string { + return q.FieldVal +} + +func (q *GeoBoundingBoxQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + field := q.FieldVal + if q.FieldVal == "" { + field = m.DefaultSearchField() + } + + ctx = context.WithValue(ctx, search.QueryTypeKey, search.Geo) + + if q.BottomRight[0] < q.TopLeft[0] { + // cross date line, rewrite as two parts + + leftSearcher, err := searcher.NewGeoBoundingBoxSearcher(ctx, i, -180, q.BottomRight[1], q.BottomRight[0], q.TopLeft[1], field, q.BoostVal.Value(), options, true) + if err != nil { + return nil, err + } + rightSearcher, err := searcher.NewGeoBoundingBoxSearcher(ctx, i, q.TopLeft[0], q.BottomRight[1], 180, q.TopLeft[1], field, q.BoostVal.Value(), options, true) + if err != nil { + _ = leftSearcher.Close() + return nil, err + } + + return searcher.NewDisjunctionSearcher(ctx, i, []search.Searcher{leftSearcher, rightSearcher}, 0, options) + } + + return searcher.NewGeoBoundingBoxSearcher(ctx, i, q.TopLeft[0], q.BottomRight[1], q.BottomRight[0], q.TopLeft[1], field, q.BoostVal.Value(), options, true) +} + +func (q *GeoBoundingBoxQuery) Validate() error { + return nil +} + +func (q *GeoBoundingBoxQuery) UnmarshalJSON(data []byte) error { + tmp := struct { + TopLeft interface{} `json:"top_left,omitempty"` + BottomRight interface{} `json:"bottom_right,omitempty"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` + }{} + err := util.UnmarshalJSON(data, &tmp) + if err != nil { + return err + } + // now use our generic point parsing code from the geo package + lon, lat, found := geo.ExtractGeoPoint(tmp.TopLeft) + if !found { + return fmt.Errorf("geo location top_left not in a valid format") + } + q.TopLeft = []float64{lon, lat} + lon, lat, found = geo.ExtractGeoPoint(tmp.BottomRight) + if !found { + return fmt.Errorf("geo location bottom_right not in a valid format") + } + q.BottomRight = []float64{lon, lat} + q.FieldVal = tmp.FieldVal + q.BoostVal = tmp.BoostVal + return nil +} diff --git a/vendor/github.com/blevesearch/bleve/search/query/geo_boundingpolygon.go b/vendor/github.com/blevesearch/bleve/v2/search/query/geo_boundingpolygon.go similarity index 79% rename from vendor/github.com/blevesearch/bleve/search/query/geo_boundingpolygon.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/geo_boundingpolygon.go index 41c7f7f3..7f81a7ca 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/geo_boundingpolygon.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/geo_boundingpolygon.go @@ -15,14 +15,15 @@ package query import ( - "encoding/json" + "context" "fmt" - "github.com/blevesearch/bleve/geo" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searcher" + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" ) type GeoBoundingPolygonQuery struct { @@ -53,14 +54,16 @@ func (q *GeoBoundingPolygonQuery) Field() string { return q.FieldVal } -func (q *GeoBoundingPolygonQuery) Searcher(i index.IndexReader, +func (q *GeoBoundingPolygonQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() } - return searcher.NewGeoBoundedPolygonSearcher(i, q.Points, field, q.BoostVal.Value(), options) + ctx = context.WithValue(ctx, search.QueryTypeKey, search.Geo) + + return searcher.NewGeoBoundedPolygonSearcher(ctx, i, q.Points, field, q.BoostVal.Value(), options) } func (q *GeoBoundingPolygonQuery) Validate() error { @@ -73,7 +76,7 @@ func (q *GeoBoundingPolygonQuery) UnmarshalJSON(data []byte) error { FieldVal string `json:"field,omitempty"` BoostVal *Boost `json:"boost,omitempty"` }{} - err := json.Unmarshal(data, &tmp) + err := util.UnmarshalJSON(data, &tmp) if err != nil { return err } diff --git a/vendor/github.com/blevesearch/bleve/search/query/geo_distance.go b/vendor/github.com/blevesearch/bleve/v2/search/query/geo_distance.go similarity index 80% rename from vendor/github.com/blevesearch/bleve/search/query/geo_distance.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/geo_distance.go index ef3aa88c..2ca09642 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/geo_distance.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/geo_distance.go @@ -15,14 +15,15 @@ package query import ( - "encoding/json" + "context" "fmt" - "github.com/blevesearch/bleve/geo" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searcher" + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" ) type GeoDistanceQuery struct { @@ -56,19 +57,21 @@ func (q *GeoDistanceQuery) Field() string { return q.FieldVal } -func (q *GeoDistanceQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, +func (q *GeoDistanceQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() } + ctx = context.WithValue(ctx, search.QueryTypeKey, search.Geo) + dist, err := geo.ParseDistance(q.Distance) if err != nil { return nil, err } - return searcher.NewGeoPointDistanceSearcher(i, q.Location[0], q.Location[1], + return searcher.NewGeoPointDistanceSearcher(ctx, i, q.Location[0], q.Location[1], dist, field, q.BoostVal.Value(), options) } @@ -83,7 +86,7 @@ func (q *GeoDistanceQuery) UnmarshalJSON(data []byte) error { FieldVal string `json:"field,omitempty"` BoostVal *Boost `json:"boost,omitempty"` }{} - err := json.Unmarshal(data, &tmp) + err := util.UnmarshalJSON(data, &tmp) if err != nil { return err } diff --git a/vendor/github.com/blevesearch/bleve/v2/search/query/geo_shape.go b/vendor/github.com/blevesearch/bleve/v2/search/query/geo_shape.go new file mode 100644 index 00000000..686f486f --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/geo_shape.go @@ -0,0 +1,138 @@ +// Copyright (c) 2022 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + "encoding/json" + + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" +) + +type Geometry struct { + Shape index.GeoJSON `json:"shape"` + Relation string `json:"relation"` +} + +type GeoShapeQuery struct { + Geometry Geometry `json:"geometry"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` +} + +// NewGeoShapeQuery creates a geoshape query for the +// given shape type. This method can be used for +// creating geoshape queries for shape types like: point, +// linestring, polygon, multipoint, multilinestring, +// multipolygon and envelope. +func NewGeoShapeQuery(coordinates [][][][]float64, typ, + relation string) (*GeoShapeQuery, error) { + s, _, err := geo.NewGeoJsonShape(coordinates, typ) + if err != nil { + return nil, err + } + + return &GeoShapeQuery{Geometry: Geometry{Shape: s, + Relation: relation}}, nil +} + +// NewGeoShapeCircleQuery creates a geoshape query for the +// given center point and the radius. Radius formats supported: +// "5in" "5inch" "7yd" "7yards" "9ft" "9feet" "11km" "11kilometers" +// "3nm" "3nauticalmiles" "13mm" "13millimeters" "15cm" "15centimeters" +// "17mi" "17miles" "19m" "19meters" If the unit cannot be determined, +// the entire string is parsed and the unit of meters is assumed. +func NewGeoShapeCircleQuery(coordinates []float64, radius, + relation string) (*GeoShapeQuery, error) { + + s, _, err := geo.NewGeoCircleShape(coordinates, radius) + if err != nil { + return nil, err + } + + return &GeoShapeQuery{Geometry: Geometry{Shape: s, + Relation: relation}}, nil +} + +// NewGeometryCollectionQuery creates a geoshape query for the +// given geometrycollection coordinates and types. +func NewGeometryCollectionQuery(coordinates [][][][][]float64, types []string, + relation string) (*GeoShapeQuery, error) { + s, _, err := geo.NewGeometryCollection(coordinates, types) + if err != nil { + return nil, err + } + + return &GeoShapeQuery{Geometry: Geometry{Shape: s, + Relation: relation}}, nil +} + +func (q *GeoShapeQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *GeoShapeQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *GeoShapeQuery) SetField(f string) { + q.FieldVal = f +} + +func (q *GeoShapeQuery) Field() string { + return q.FieldVal +} + +func (q *GeoShapeQuery) Searcher(ctx context.Context, i index.IndexReader, + m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + field := q.FieldVal + if q.FieldVal == "" { + field = m.DefaultSearchField() + } + + ctx = context.WithValue(ctx, search.QueryTypeKey, search.Geo) + + return searcher.NewGeoShapeSearcher(ctx, i, q.Geometry.Shape, q.Geometry.Relation, field, + q.BoostVal.Value(), options) +} + +func (q *GeoShapeQuery) Validate() error { + return nil +} + +func (q *Geometry) UnmarshalJSON(data []byte) error { + tmp := struct { + Shape json.RawMessage `json:"shape"` + Relation string `json:"relation"` + }{} + + err := util.UnmarshalJSON(data, &tmp) + if err != nil { + return err + } + + q.Shape, err = geo.ParseGeoJSONShape(tmp.Shape) + if err != nil { + return err + } + q.Relation = tmp.Relation + return nil +} diff --git a/vendor/github.com/blevesearch/bleve/v2/search/query/ip_range.go b/vendor/github.com/blevesearch/bleve/v2/search/query/ip_range.go new file mode 100644 index 00000000..ba46f0b2 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/ip_range.go @@ -0,0 +1,85 @@ +// Copyright (c) 2021 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + "fmt" + "net" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" +) + +type IPRangeQuery struct { + CIDR string `json:"cidr, omitempty"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` +} + +func NewIPRangeQuery(cidr string) *IPRangeQuery { + return &IPRangeQuery{ + CIDR: cidr, + } +} + +func (q *IPRangeQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *IPRangeQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *IPRangeQuery) SetField(f string) { + q.FieldVal = f +} + +func (q *IPRangeQuery) Field() string { + return q.FieldVal +} + +func (q *IPRangeQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + field := q.FieldVal + if q.FieldVal == "" { + field = m.DefaultSearchField() + } + _, ipNet, err := net.ParseCIDR(q.CIDR) + if err != nil { + ip := net.ParseIP(q.CIDR) + if ip == nil { + return nil, err + } + // If we are searching for a specific ip rather than members of a network, just use a term search. + return searcher.NewTermSearcherBytes(ctx, i, ip.To16(), field, q.BoostVal.Value(), options) + } + return searcher.NewIPRangeSearcher(ctx, i, ipNet, field, q.BoostVal.Value(), options) +} + +func (q *IPRangeQuery) Validate() error { + _, _, err := net.ParseCIDR(q.CIDR) + if err == nil { + return nil + } + // We also allow search for a specific IP. + ip := net.ParseIP(q.CIDR) + if ip != nil { + return nil // we have a valid ip + } + return fmt.Errorf("IPRangeQuery must be for a network or ip address, %q", q.CIDR) +} diff --git a/vendor/github.com/blevesearch/bleve/v2/search/query/match.go b/vendor/github.com/blevesearch/bleve/v2/search/query/match.go new file mode 100644 index 00000000..074d11d3 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/match.go @@ -0,0 +1,177 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + "fmt" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" +) + +type MatchQuery struct { + Match string `json:"match"` + FieldVal string `json:"field,omitempty"` + Analyzer string `json:"analyzer,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` + Prefix int `json:"prefix_length"` + Fuzziness int `json:"fuzziness"` + Operator MatchQueryOperator `json:"operator,omitempty"` +} + +type MatchQueryOperator int + +const ( + // Document must satisfy AT LEAST ONE of term searches. + MatchQueryOperatorOr = MatchQueryOperator(0) + // Document must satisfy ALL of term searches. + MatchQueryOperatorAnd = MatchQueryOperator(1) +) + +func (o MatchQueryOperator) MarshalJSON() ([]byte, error) { + switch o { + case MatchQueryOperatorOr: + return util.MarshalJSON("or") + case MatchQueryOperatorAnd: + return util.MarshalJSON("and") + default: + return nil, fmt.Errorf("cannot marshal match operator %d to JSON", o) + } +} + +func (o *MatchQueryOperator) UnmarshalJSON(data []byte) error { + var operatorString string + err := util.UnmarshalJSON(data, &operatorString) + if err != nil { + return err + } + + switch operatorString { + case "or": + *o = MatchQueryOperatorOr + return nil + case "and": + *o = MatchQueryOperatorAnd + return nil + default: + return fmt.Errorf("cannot unmarshal match operator '%v' from JSON", o) + } +} + +// NewMatchQuery creates a Query for matching text. +// An Analyzer is chosen based on the field. +// Input text is analyzed using this analyzer. +// Token terms resulting from this analysis are +// used to perform term searches. Result documents +// must satisfy at least one of these term searches. +func NewMatchQuery(match string) *MatchQuery { + return &MatchQuery{ + Match: match, + Operator: MatchQueryOperatorOr, + } +} + +func (q *MatchQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *MatchQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *MatchQuery) SetField(f string) { + q.FieldVal = f +} + +func (q *MatchQuery) Field() string { + return q.FieldVal +} + +func (q *MatchQuery) SetFuzziness(f int) { + q.Fuzziness = f +} + +func (q *MatchQuery) SetPrefix(p int) { + q.Prefix = p +} + +func (q *MatchQuery) SetOperator(operator MatchQueryOperator) { + q.Operator = operator +} + +func (q *MatchQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + + field := q.FieldVal + if q.FieldVal == "" { + field = m.DefaultSearchField() + } + + analyzerName := "" + if q.Analyzer != "" { + analyzerName = q.Analyzer + } else { + analyzerName = m.AnalyzerNameForPath(field) + } + analyzer := m.AnalyzerNamed(analyzerName) + + if analyzer == nil { + return nil, fmt.Errorf("no analyzer named '%s' registered", q.Analyzer) + } + + tokens := analyzer.Analyze([]byte(q.Match)) + if len(tokens) > 0 { + + tqs := make([]Query, len(tokens)) + if q.Fuzziness != 0 { + for i, token := range tokens { + query := NewFuzzyQuery(string(token.Term)) + query.SetFuzziness(q.Fuzziness) + query.SetPrefix(q.Prefix) + query.SetField(field) + query.SetBoost(q.BoostVal.Value()) + tqs[i] = query + } + } else { + for i, token := range tokens { + tq := NewTermQuery(string(token.Term)) + tq.SetField(field) + tq.SetBoost(q.BoostVal.Value()) + tqs[i] = tq + } + } + + switch q.Operator { + case MatchQueryOperatorOr: + shouldQuery := NewDisjunctionQuery(tqs) + shouldQuery.SetMin(1) + shouldQuery.SetBoost(q.BoostVal.Value()) + return shouldQuery.Searcher(ctx, i, m, options) + + case MatchQueryOperatorAnd: + mustQuery := NewConjunctionQuery(tqs) + mustQuery.SetBoost(q.BoostVal.Value()) + return mustQuery.Searcher(ctx, i, m, options) + + default: + return nil, fmt.Errorf("unhandled operator %d", q.Operator) + } + } + noneQuery := NewMatchNoneQuery() + return noneQuery.Searcher(ctx, i, m, options) +} diff --git a/vendor/github.com/blevesearch/bleve/v2/search/query/match_all.go b/vendor/github.com/blevesearch/bleve/v2/search/query/match_all.go new file mode 100644 index 00000000..e88825ae --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/match_all.go @@ -0,0 +1,56 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + "encoding/json" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" +) + +type MatchAllQuery struct { + BoostVal *Boost `json:"boost,omitempty"` +} + +// NewMatchAllQuery creates a Query which will +// match all documents in the index. +func NewMatchAllQuery() *MatchAllQuery { + return &MatchAllQuery{} +} + +func (q *MatchAllQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *MatchAllQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *MatchAllQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + return searcher.NewMatchAllSearcher(ctx, i, q.BoostVal.Value(), options) +} + +func (q *MatchAllQuery) MarshalJSON() ([]byte, error) { + tmp := map[string]interface{}{ + "boost": q.BoostVal, + "match_all": map[string]interface{}{}, + } + return json.Marshal(tmp) +} diff --git a/vendor/github.com/blevesearch/bleve/search/query/match_none.go b/vendor/github.com/blevesearch/bleve/v2/search/query/match_none.go similarity index 78% rename from vendor/github.com/blevesearch/bleve/search/query/match_none.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/match_none.go index dc2ea780..cb65a725 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/match_none.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/match_none.go @@ -15,12 +15,13 @@ package query import ( + "context" "encoding/json" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searcher" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" ) type MatchNoneQuery struct { @@ -42,7 +43,7 @@ func (q *MatchNoneQuery) Boost() float64 { return q.BoostVal.Value() } -func (q *MatchNoneQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { +func (q *MatchNoneQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { return searcher.NewMatchNoneSearcher(i) } diff --git a/vendor/github.com/blevesearch/bleve/search/query/match_phrase.go b/vendor/github.com/blevesearch/bleve/v2/search/query/match_phrase.go similarity index 82% rename from vendor/github.com/blevesearch/bleve/search/query/match_phrase.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/match_phrase.go index 51be3552..63a16a53 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/match_phrase.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/match_phrase.go @@ -15,12 +15,13 @@ package query import ( + "context" "fmt" - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" ) type MatchPhraseQuery struct { @@ -28,6 +29,7 @@ type MatchPhraseQuery struct { FieldVal string `json:"field,omitempty"` Analyzer string `json:"analyzer,omitempty"` BoostVal *Boost `json:"boost,omitempty"` + Fuzziness int `json:"fuzziness"` } // NewMatchPhraseQuery creates a new Query object @@ -57,11 +59,15 @@ func (q *MatchPhraseQuery) SetField(f string) { q.FieldVal = f } +func (q *MatchPhraseQuery) SetFuzziness(f int) { + q.Fuzziness = f +} + func (q *MatchPhraseQuery) Field() string { return q.FieldVal } -func (q *MatchPhraseQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { +func (q *MatchPhraseQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() @@ -83,10 +89,11 @@ func (q *MatchPhraseQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, phrase := tokenStreamToPhrase(tokens) phraseQuery := NewMultiPhraseQuery(phrase, field) phraseQuery.SetBoost(q.BoostVal.Value()) - return phraseQuery.Searcher(i, m, options) + phraseQuery.SetFuzziness(q.Fuzziness) + return phraseQuery.Searcher(ctx, i, m, options) } noneQuery := NewMatchNoneQuery() - return noneQuery.Searcher(i, m, options) + return noneQuery.Searcher(ctx, i, m, options) } func tokenStreamToPhrase(tokens analysis.TokenStream) [][]string { diff --git a/vendor/github.com/blevesearch/bleve/v2/search/query/multi_phrase.go b/vendor/github.com/blevesearch/bleve/v2/search/query/multi_phrase.go new file mode 100644 index 00000000..d1144d90 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/multi_phrase.go @@ -0,0 +1,87 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + "fmt" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" +) + +type MultiPhraseQuery struct { + Terms [][]string `json:"terms"` + Field string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` + Fuzziness int `json:"fuzziness"` +} + +// NewMultiPhraseQuery creates a new Query for finding +// term phrases in the index. +// It is like PhraseQuery, but each position in the +// phrase may be satisfied by a list of terms +// as opposed to just one. +// At least one of the terms must exist in the correct +// order, at the correct index offsets, in the +// specified field. Queried field must have been indexed with +// IncludeTermVectors set to true. +func NewMultiPhraseQuery(terms [][]string, field string) *MultiPhraseQuery { + return &MultiPhraseQuery{ + Terms: terms, + Field: field, + } +} + +func (q *MultiPhraseQuery) SetFuzziness(f int) { + q.Fuzziness = f +} + +func (q *MultiPhraseQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *MultiPhraseQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *MultiPhraseQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + return searcher.NewMultiPhraseSearcher(ctx, i, q.Terms, q.Fuzziness, q.Field, q.BoostVal.Value(), options) +} + +func (q *MultiPhraseQuery) Validate() error { + if len(q.Terms) < 1 { + return fmt.Errorf("phrase query must contain at least one term") + } + return nil +} + +func (q *MultiPhraseQuery) UnmarshalJSON(data []byte) error { + type _mphraseQuery MultiPhraseQuery + tmp := _mphraseQuery{} + err := util.UnmarshalJSON(data, &tmp) + if err != nil { + return err + } + q.Terms = tmp.Terms + q.Field = tmp.Field + q.BoostVal = tmp.BoostVal + q.Fuzziness = tmp.Fuzziness + return nil +} diff --git a/vendor/github.com/blevesearch/bleve/search/query/numeric_range.go b/vendor/github.com/blevesearch/bleve/v2/search/query/numeric_range.go similarity index 80% rename from vendor/github.com/blevesearch/bleve/search/query/numeric_range.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/numeric_range.go index ea3f0688..205ceecf 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/numeric_range.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/numeric_range.go @@ -15,12 +15,13 @@ package query import ( + "context" "fmt" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searcher" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" ) type NumericRangeQuery struct { @@ -71,12 +72,13 @@ func (q *NumericRangeQuery) Field() string { return q.FieldVal } -func (q *NumericRangeQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { +func (q *NumericRangeQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() } - return searcher.NewNumericRangeSearcher(i, q.Min, q.Max, q.InclusiveMin, q.InclusiveMax, field, q.BoostVal.Value(), options) + ctx = context.WithValue(ctx, search.QueryTypeKey, search.Numeric) + return searcher.NewNumericRangeSearcher(ctx, i, q.Min, q.Max, q.InclusiveMin, q.InclusiveMax, field, q.BoostVal.Value(), options) } func (q *NumericRangeQuery) Validate() error { diff --git a/vendor/github.com/blevesearch/bleve/v2/search/query/phrase.go b/vendor/github.com/blevesearch/bleve/v2/search/query/phrase.go new file mode 100644 index 00000000..9092e72d --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/phrase.go @@ -0,0 +1,84 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + "fmt" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" +) + +type PhraseQuery struct { + Terms []string `json:"terms"` + Field string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` + Fuzziness int `json:"fuzziness"` +} + +// NewPhraseQuery creates a new Query for finding +// exact term phrases in the index. +// The provided terms must exist in the correct +// order, at the correct index offsets, in the +// specified field. Queried field must have been indexed with +// IncludeTermVectors set to true. +func NewPhraseQuery(terms []string, field string) *PhraseQuery { + return &PhraseQuery{ + Terms: terms, + Field: field, + } +} + +func (q *PhraseQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *PhraseQuery) SetFuzziness(f int) { + q.Fuzziness = f +} + +func (q *PhraseQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *PhraseQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + return searcher.NewPhraseSearcher(ctx, i, q.Terms, q.Fuzziness, q.Field, q.BoostVal.Value(), options) +} + +func (q *PhraseQuery) Validate() error { + if len(q.Terms) < 1 { + return fmt.Errorf("phrase query must contain at least one term") + } + return nil +} + +func (q *PhraseQuery) UnmarshalJSON(data []byte) error { + type _phraseQuery PhraseQuery + tmp := _phraseQuery{} + err := util.UnmarshalJSON(data, &tmp) + if err != nil { + return err + } + q.Terms = tmp.Terms + q.Field = tmp.Field + q.BoostVal = tmp.BoostVal + q.Fuzziness = tmp.Fuzziness + return nil +} diff --git a/vendor/github.com/blevesearch/bleve/search/query/prefix.go b/vendor/github.com/blevesearch/bleve/v2/search/query/prefix.go similarity index 75% rename from vendor/github.com/blevesearch/bleve/search/query/prefix.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/prefix.go index 4f5be2b2..debbbc1e 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/prefix.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/prefix.go @@ -15,10 +15,12 @@ package query import ( - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searcher" + "context" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" ) type PrefixQuery struct { @@ -53,10 +55,10 @@ func (q *PrefixQuery) Field() string { return q.FieldVal } -func (q *PrefixQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { +func (q *PrefixQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() } - return searcher.NewTermPrefixSearcher(i, q.Prefix, field, q.BoostVal.Value(), options) + return searcher.NewTermPrefixSearcher(ctx, i, q.Prefix, field, q.BoostVal.Value(), options) } diff --git a/vendor/github.com/blevesearch/bleve/v2/search/query/query.go b/vendor/github.com/blevesearch/bleve/v2/search/query/query.go new file mode 100644 index 00000000..eb7b34ad --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/query.go @@ -0,0 +1,384 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" +) + +var logger = log.New(io.Discard, "bleve mapping ", log.LstdFlags) + +// SetLog sets the logger used for logging +// by default log messages are sent to io.Discard +func SetLog(l *log.Logger) { + logger = l +} + +// A Query represents a description of the type +// and parameters for a query into the index. +type Query interface { + Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, + options search.SearcherOptions) (search.Searcher, error) +} + +// A BoostableQuery represents a Query which can be boosted +// relative to other queries. +type BoostableQuery interface { + Query + SetBoost(b float64) + Boost() float64 +} + +// A FieldableQuery represents a Query which can be restricted +// to a single field. +type FieldableQuery interface { + Query + SetField(f string) + Field() string +} + +// A ValidatableQuery represents a Query which can be validated +// prior to execution. +type ValidatableQuery interface { + Query + Validate() error +} + +// ParseQuery deserializes a JSON representation of +// a Query object. +func ParseQuery(input []byte) (Query, error) { + var tmp map[string]interface{} + err := util.UnmarshalJSON(input, &tmp) + if err != nil { + return nil, err + } + _, hasFuzziness := tmp["fuzziness"] + _, isMatchQuery := tmp["match"] + _, isMatchPhraseQuery := tmp["match_phrase"] + _, hasTerms := tmp["terms"] + if hasFuzziness && !isMatchQuery && !isMatchPhraseQuery && !hasTerms { + var rv FuzzyQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + if isMatchQuery { + var rv MatchQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + if isMatchPhraseQuery { + var rv MatchPhraseQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + if hasTerms { + var rv PhraseQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + // now try multi-phrase + var rv2 MultiPhraseQuery + err = util.UnmarshalJSON(input, &rv2) + if err != nil { + return nil, err + } + return &rv2, nil + } + return &rv, nil + } + _, isTermQuery := tmp["term"] + if isTermQuery { + var rv TermQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasMust := tmp["must"] + _, hasShould := tmp["should"] + _, hasMustNot := tmp["must_not"] + if hasMust || hasShould || hasMustNot { + var rv BooleanQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasConjuncts := tmp["conjuncts"] + if hasConjuncts { + var rv ConjunctionQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasDisjuncts := tmp["disjuncts"] + if hasDisjuncts { + var rv DisjunctionQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + + _, hasSyntaxQuery := tmp["query"] + if hasSyntaxQuery { + var rv QueryStringQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasMin := tmp["min"].(float64) + _, hasMax := tmp["max"].(float64) + if hasMin || hasMax { + var rv NumericRangeQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasMinStr := tmp["min"].(string) + _, hasMaxStr := tmp["max"].(string) + if hasMinStr || hasMaxStr { + var rv TermRangeQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasStart := tmp["start"] + _, hasEnd := tmp["end"] + if hasStart || hasEnd { + var rv DateRangeStringQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasPrefix := tmp["prefix"] + if hasPrefix { + var rv PrefixQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasRegexp := tmp["regexp"] + if hasRegexp { + var rv RegexpQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasWildcard := tmp["wildcard"] + if hasWildcard { + var rv WildcardQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasMatchAll := tmp["match_all"] + if hasMatchAll { + var rv MatchAllQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasMatchNone := tmp["match_none"] + if hasMatchNone { + var rv MatchNoneQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasDocIds := tmp["ids"] + if hasDocIds { + var rv DocIDQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasBool := tmp["bool"] + if hasBool { + var rv BoolFieldQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasTopLeft := tmp["top_left"] + _, hasBottomRight := tmp["bottom_right"] + if hasTopLeft && hasBottomRight { + var rv GeoBoundingBoxQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasDistance := tmp["distance"] + if hasDistance { + var rv GeoDistanceQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasPoints := tmp["polygon_points"] + if hasPoints { + var rv GeoBoundingPolygonQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + + _, hasGeo := tmp["geometry"] + if hasGeo { + var rv GeoShapeQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + + _, hasCIDR := tmp["cidr"] + if hasCIDR { + var rv IPRangeQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + + return nil, fmt.Errorf("unknown query type") +} + +// expandQuery traverses the input query tree and returns a new tree where +// query string queries have been expanded into base queries. Returned tree may +// reference queries from the input tree or new queries. +func expandQuery(m mapping.IndexMapping, query Query) (Query, error) { + var expand func(query Query) (Query, error) + var expandSlice func(queries []Query) ([]Query, error) + + expandSlice = func(queries []Query) ([]Query, error) { + expanded := []Query{} + for _, q := range queries { + exp, err := expand(q) + if err != nil { + return nil, err + } + expanded = append(expanded, exp) + } + return expanded, nil + } + + expand = func(query Query) (Query, error) { + switch q := query.(type) { + case *QueryStringQuery: + parsed, err := parseQuerySyntax(q.Query) + if err != nil { + return nil, fmt.Errorf("could not parse '%s': %s", q.Query, err) + } + return expand(parsed) + case *ConjunctionQuery: + children, err := expandSlice(q.Conjuncts) + if err != nil { + return nil, err + } + q.Conjuncts = children + return q, nil + case *DisjunctionQuery: + children, err := expandSlice(q.Disjuncts) + if err != nil { + return nil, err + } + q.Disjuncts = children + return q, nil + case *BooleanQuery: + var err error + q.Must, err = expand(q.Must) + if err != nil { + return nil, err + } + q.Should, err = expand(q.Should) + if err != nil { + return nil, err + } + q.MustNot, err = expand(q.MustNot) + if err != nil { + return nil, err + } + return q, nil + default: + return query, nil + } + } + return expand(query) +} + +// DumpQuery returns a string representation of the query tree, where query +// string queries have been expanded into base queries. The output format is +// meant for debugging purpose and may change in the future. +func DumpQuery(m mapping.IndexMapping, query Query) (string, error) { + q, err := expandQuery(m, query) + if err != nil { + return "", err + } + data, err := json.MarshalIndent(q, "", " ") + return string(data), err +} diff --git a/vendor/github.com/blevesearch/bleve/search/query/query_string.go b/vendor/github.com/blevesearch/bleve/v2/search/query/query_string.go similarity index 81% rename from vendor/github.com/blevesearch/bleve/search/query/query_string.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/query_string.go index ecafe6bc..42bb598b 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/query_string.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/query_string.go @@ -15,9 +15,11 @@ package query import ( - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" + "context" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" ) type QueryStringQuery struct { @@ -47,12 +49,12 @@ func (q *QueryStringQuery) Parse() (Query, error) { return parseQuerySyntax(q.Query) } -func (q *QueryStringQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { +func (q *QueryStringQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { newQuery, err := parseQuerySyntax(q.Query) if err != nil { return nil, err } - return newQuery.Searcher(i, m, options) + return newQuery.Searcher(ctx, i, m, options) } func (q *QueryStringQuery) Validate() error { diff --git a/vendor/github.com/blevesearch/bleve/search/query/query_string.y b/vendor/github.com/blevesearch/bleve/v2/search/query/query_string.y similarity index 92% rename from vendor/github.com/blevesearch/bleve/search/query/query_string.y rename to vendor/github.com/blevesearch/bleve/v2/search/query/query_string.y index d3e5ac9d..aeec8560 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/query_string.y +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/query_string.y @@ -28,10 +28,11 @@ tEQUAL tTILDE %type tPHRASE %type tNUMBER %type posOrNegNumber +%type fieldName %type tTILDE %type tBOOST %type searchBase -%type searchSuffix +%type searchSuffix %type searchPrefix %% @@ -111,7 +112,7 @@ tSTRING tTILDE { $$ = q } | -tSTRING tCOLON tSTRING tTILDE { +fieldName tCOLON tSTRING tTILDE { field := $1 str := $3 fuzziness, err := strconv.ParseFloat($4, 64) @@ -147,7 +148,7 @@ tPHRASE { $$ = q } | -tSTRING tCOLON tSTRING { +fieldName tCOLON tSTRING { field := $1 str := $3 logDebugGrammar("FIELD - %s STRING - %s", field, str) @@ -163,7 +164,7 @@ tSTRING tCOLON tSTRING { $$ = q } | -tSTRING tCOLON posOrNegNumber { +fieldName tCOLON posOrNegNumber { field := $1 str := $3 logDebugGrammar("FIELD - %s STRING - %s", field, str) @@ -181,7 +182,7 @@ tSTRING tCOLON posOrNegNumber { $$ = q } | -tSTRING tCOLON tPHRASE { +fieldName tCOLON tPHRASE { field := $1 phrase := $3 logDebugGrammar("FIELD - %s PHRASE - %s", field, phrase) @@ -190,7 +191,7 @@ tSTRING tCOLON tPHRASE { $$ = q } | -tSTRING tCOLON tGREATER posOrNegNumber { +fieldName tCOLON tGREATER posOrNegNumber { field := $1 min, err := strconv.ParseFloat($4, 64) if err != nil { @@ -203,7 +204,7 @@ tSTRING tCOLON tGREATER posOrNegNumber { $$ = q } | -tSTRING tCOLON tGREATER tEQUAL posOrNegNumber { +fieldName tCOLON tGREATER tEQUAL posOrNegNumber { field := $1 min, err := strconv.ParseFloat($5, 64) if err != nil { @@ -216,7 +217,7 @@ tSTRING tCOLON tGREATER tEQUAL posOrNegNumber { $$ = q } | -tSTRING tCOLON tLESS posOrNegNumber { +fieldName tCOLON tLESS posOrNegNumber { field := $1 max, err := strconv.ParseFloat($4, 64) if err != nil { @@ -229,7 +230,7 @@ tSTRING tCOLON tLESS posOrNegNumber { $$ = q } | -tSTRING tCOLON tLESS tEQUAL posOrNegNumber { +fieldName tCOLON tLESS tEQUAL posOrNegNumber { field := $1 max, err := strconv.ParseFloat($5, 64) if err != nil { @@ -242,7 +243,7 @@ tSTRING tCOLON tLESS tEQUAL posOrNegNumber { $$ = q } | -tSTRING tCOLON tGREATER tPHRASE { +fieldName tCOLON tGREATER tPHRASE { field := $1 minInclusive := false phrase := $4 @@ -257,7 +258,7 @@ tSTRING tCOLON tGREATER tPHRASE { $$ = q } | -tSTRING tCOLON tGREATER tEQUAL tPHRASE { +fieldName tCOLON tGREATER tEQUAL tPHRASE { field := $1 minInclusive := true phrase := $5 @@ -272,7 +273,7 @@ tSTRING tCOLON tGREATER tEQUAL tPHRASE { $$ = q } | -tSTRING tCOLON tLESS tPHRASE { +fieldName tCOLON tLESS tPHRASE { field := $1 maxInclusive := false phrase := $4 @@ -287,7 +288,7 @@ tSTRING tCOLON tLESS tPHRASE { $$ = q } | -tSTRING tCOLON tLESS tEQUAL tPHRASE { +fieldName tCOLON tLESS tEQUAL tPHRASE { field := $1 maxInclusive := true phrase := $5 @@ -326,3 +327,12 @@ tNUMBER { tMINUS tNUMBER { $$ = "-" + $2 }; + +fieldName: +tPHRASE { + $$ = $1 +} +| +tSTRING { + $$ = $1 +}; diff --git a/vendor/github.com/blevesearch/bleve/search/query/query_string.y.go b/vendor/github.com/blevesearch/bleve/v2/search/query/query_string.y.go similarity index 89% rename from vendor/github.com/blevesearch/bleve/search/query/query_string.y.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/query_string.y.go index ac2d3222..3a2abc13 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/query_string.y.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/query_string.y.go @@ -1,3 +1,6 @@ +// Code generated by goyacc -o query_string.y.go query_string.y. DO NOT EDIT. + +//line query_string.y:2 package query import __yyfmt__ "fmt" @@ -54,6 +57,7 @@ var yyToknames = [...]string{ "tEQUAL", "tTILDE", } + var yyStatenames = [...]string{} const yyEofCode = 1 @@ -68,70 +72,72 @@ var yyExca = [...]int{ -1, 3, 1, 3, -2, 5, + -1, 9, + 8, 29, + -2, 8, + -1, 12, + 8, 28, + -2, 12, } -const yyNprod = 28 const yyPrivate = 57344 -var yyTokenNames []string -var yyStates []string - -const yyLast = 42 +const yyLast = 43 var yyAct = [...]int{ - - 17, 16, 18, 23, 22, 30, 3, 21, 19, 20, - 29, 26, 22, 22, 1, 21, 21, 15, 28, 25, - 24, 27, 34, 14, 22, 13, 31, 21, 32, 33, - 22, 9, 11, 21, 5, 6, 2, 10, 4, 12, - 7, 8, + 18, 17, 19, 24, 23, 15, 31, 22, 20, 21, + 30, 27, 23, 23, 3, 22, 22, 14, 29, 26, + 16, 25, 28, 35, 33, 23, 23, 32, 22, 22, + 34, 9, 12, 1, 5, 6, 2, 11, 4, 13, + 7, 8, 10, } -var yyPact = [...]int{ - 28, -1000, -1000, 28, 27, -1000, -1000, -1000, 16, 9, - -1000, -1000, -1000, -1000, -1000, -3, -11, -1000, -1000, 6, - 5, -1000, -5, -1000, -1000, 23, -1000, -1000, 17, -1000, - -1000, -1000, -1000, -1000, -1000, +var yyPact = [...]int{ + 28, -1000, -1000, 28, 27, -1000, -1000, -1000, 8, -9, + 12, -1000, -1000, -1000, -1000, -1000, -3, -11, -1000, -1000, + 6, 5, -1000, -4, -1000, -1000, 19, -1000, -1000, 18, + -1000, -1000, -1000, -1000, -1000, -1000, } -var yyPgo = [...]int{ - 0, 0, 41, 39, 38, 14, 36, 6, +var yyPgo = [...]int{ + 0, 0, 42, 41, 39, 38, 33, 36, 14, } -var yyR1 = [...]int{ - 0, 5, 6, 6, 7, 4, 4, 4, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 3, 3, 1, 1, +var yyR1 = [...]int{ + 0, 6, 7, 7, 8, 5, 5, 5, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 4, 4, 1, 1, 2, 2, } -var yyR2 = [...]int{ +var yyR2 = [...]int{ 0, 1, 2, 1, 3, 0, 1, 1, 1, 2, 4, 1, 1, 3, 3, 3, 4, 5, 4, 5, - 4, 5, 4, 5, 0, 1, 1, 2, + 4, 5, 4, 5, 0, 1, 1, 2, 1, 1, } -var yyChk = [...]int{ - -1000, -5, -6, -7, -4, 6, 7, -6, -2, 4, - 10, 5, -3, 9, 14, 8, 4, -1, 5, 11, - 12, 10, 7, 14, -1, 13, 5, -1, 13, 5, - 10, -1, 5, -1, 5, +var yyChk = [...]int{ + -1000, -6, -7, -8, -5, 6, 7, -7, -3, 4, + -2, 10, 5, -4, 9, 14, 8, 4, -1, 5, + 11, 12, 10, 7, 14, -1, 13, 5, -1, 13, + 5, 10, -1, 5, -1, 5, } -var yyDef = [...]int{ - 5, -2, 1, -2, 0, 6, 7, 2, 24, 8, - 11, 12, 4, 25, 9, 0, 13, 14, 15, 0, - 0, 26, 0, 10, 16, 0, 20, 18, 0, 22, - 27, 17, 21, 19, 23, +var yyDef = [...]int{ + 5, -2, 1, -2, 0, 6, 7, 2, 24, -2, + 0, 11, -2, 4, 25, 9, 0, 13, 14, 15, + 0, 0, 26, 0, 10, 16, 0, 20, 18, 0, + 22, 27, 17, 21, 19, 23, } -var yyTok1 = [...]int{ +var yyTok1 = [...]int{ 1, } -var yyTok2 = [...]int{ +var yyTok2 = [...]int{ 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, } + var yyTok3 = [...]int{ 0, } @@ -475,25 +481,25 @@ yydefault: case 1: yyDollar = yyS[yypt-1 : yypt+1] - //line query_string.y:40 +//line query_string.y:41 { logDebugGrammar("INPUT") } case 2: yyDollar = yyS[yypt-2 : yypt+1] - //line query_string.y:45 +//line query_string.y:46 { logDebugGrammar("SEARCH PARTS") } case 3: yyDollar = yyS[yypt-1 : yypt+1] - //line query_string.y:49 +//line query_string.y:50 { logDebugGrammar("SEARCH PART") } case 4: yyDollar = yyS[yypt-3 : yypt+1] - //line query_string.y:54 +//line query_string.y:55 { query := yyDollar[2].q if yyDollar[3].pf != nil { @@ -512,27 +518,27 @@ yydefault: } case 5: yyDollar = yyS[yypt-0 : yypt+1] - //line query_string.y:73 +//line query_string.y:74 { yyVAL.n = queryShould } case 6: yyDollar = yyS[yypt-1 : yypt+1] - //line query_string.y:77 +//line query_string.y:78 { logDebugGrammar("PLUS") yyVAL.n = queryMust } case 7: yyDollar = yyS[yypt-1 : yypt+1] - //line query_string.y:82 +//line query_string.y:83 { logDebugGrammar("MINUS") yyVAL.n = queryMustNot } case 8: yyDollar = yyS[yypt-1 : yypt+1] - //line query_string.y:88 +//line query_string.y:89 { str := yyDollar[1].s logDebugGrammar("STRING - %s", str) @@ -548,7 +554,7 @@ yydefault: } case 9: yyDollar = yyS[yypt-2 : yypt+1] - //line query_string.y:102 +//line query_string.y:103 { str := yyDollar[1].s fuzziness, err := strconv.ParseFloat(yyDollar[2].s, 64) @@ -562,7 +568,7 @@ yydefault: } case 10: yyDollar = yyS[yypt-4 : yypt+1] - //line query_string.y:114 +//line query_string.y:115 { field := yyDollar[1].s str := yyDollar[3].s @@ -578,7 +584,7 @@ yydefault: } case 11: yyDollar = yyS[yypt-1 : yypt+1] - //line query_string.y:128 +//line query_string.y:129 { str := yyDollar[1].s logDebugGrammar("STRING - %s", str) @@ -595,7 +601,7 @@ yydefault: } case 12: yyDollar = yyS[yypt-1 : yypt+1] - //line query_string.y:143 +//line query_string.y:144 { phrase := yyDollar[1].s logDebugGrammar("PHRASE - %s", phrase) @@ -604,7 +610,7 @@ yydefault: } case 13: yyDollar = yyS[yypt-3 : yypt+1] - //line query_string.y:150 +//line query_string.y:151 { field := yyDollar[1].s str := yyDollar[3].s @@ -622,7 +628,7 @@ yydefault: } case 14: yyDollar = yyS[yypt-3 : yypt+1] - //line query_string.y:166 +//line query_string.y:167 { field := yyDollar[1].s str := yyDollar[3].s @@ -642,7 +648,7 @@ yydefault: } case 15: yyDollar = yyS[yypt-3 : yypt+1] - //line query_string.y:184 +//line query_string.y:185 { field := yyDollar[1].s phrase := yyDollar[3].s @@ -653,7 +659,7 @@ yydefault: } case 16: yyDollar = yyS[yypt-4 : yypt+1] - //line query_string.y:193 +//line query_string.y:194 { field := yyDollar[1].s min, err := strconv.ParseFloat(yyDollar[4].s, 64) @@ -668,7 +674,7 @@ yydefault: } case 17: yyDollar = yyS[yypt-5 : yypt+1] - //line query_string.y:206 +//line query_string.y:207 { field := yyDollar[1].s min, err := strconv.ParseFloat(yyDollar[5].s, 64) @@ -683,7 +689,7 @@ yydefault: } case 18: yyDollar = yyS[yypt-4 : yypt+1] - //line query_string.y:219 +//line query_string.y:220 { field := yyDollar[1].s max, err := strconv.ParseFloat(yyDollar[4].s, 64) @@ -698,7 +704,7 @@ yydefault: } case 19: yyDollar = yyS[yypt-5 : yypt+1] - //line query_string.y:232 +//line query_string.y:233 { field := yyDollar[1].s max, err := strconv.ParseFloat(yyDollar[5].s, 64) @@ -713,7 +719,7 @@ yydefault: } case 20: yyDollar = yyS[yypt-4 : yypt+1] - //line query_string.y:245 +//line query_string.y:246 { field := yyDollar[1].s minInclusive := false @@ -730,7 +736,7 @@ yydefault: } case 21: yyDollar = yyS[yypt-5 : yypt+1] - //line query_string.y:260 +//line query_string.y:261 { field := yyDollar[1].s minInclusive := true @@ -747,7 +753,7 @@ yydefault: } case 22: yyDollar = yyS[yypt-4 : yypt+1] - //line query_string.y:275 +//line query_string.y:276 { field := yyDollar[1].s maxInclusive := false @@ -764,7 +770,7 @@ yydefault: } case 23: yyDollar = yyS[yypt-5 : yypt+1] - //line query_string.y:290 +//line query_string.y:291 { field := yyDollar[1].s maxInclusive := true @@ -781,13 +787,13 @@ yydefault: } case 24: yyDollar = yyS[yypt-0 : yypt+1] - //line query_string.y:306 +//line query_string.y:307 { yyVAL.pf = nil } case 25: yyDollar = yyS[yypt-1 : yypt+1] - //line query_string.y:310 +//line query_string.y:311 { yyVAL.pf = nil boost, err := strconv.ParseFloat(yyDollar[1].s, 64) @@ -800,16 +806,28 @@ yydefault: } case 26: yyDollar = yyS[yypt-1 : yypt+1] - //line query_string.y:322 +//line query_string.y:323 { yyVAL.s = yyDollar[1].s } case 27: yyDollar = yyS[yypt-2 : yypt+1] - //line query_string.y:326 +//line query_string.y:327 { yyVAL.s = "-" + yyDollar[2].s } + case 28: + yyDollar = yyS[yypt-1 : yypt+1] +//line query_string.y:332 + { + yyVAL.s = yyDollar[1].s + } + case 29: + yyDollar = yyS[yypt-1 : yypt+1] +//line query_string.y:336 + { + yyVAL.s = yyDollar[1].s + } } goto yystack /* stack new state and value */ } diff --git a/vendor/github.com/blevesearch/bleve/search/query/query_string_lex.go b/vendor/github.com/blevesearch/bleve/v2/search/query/query_string_lex.go similarity index 95% rename from vendor/github.com/blevesearch/bleve/search/query/query_string_lex.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/query_string_lex.go index 3a9cf239..c01fa6fc 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/query_string_lex.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/query_string_lex.go @@ -248,8 +248,8 @@ func inTildeState(l *queryStringLex, next rune, eof bool) (lexState, bool) { } func inNumOrStrState(l *queryStringLex, next rune, eof bool) (lexState, bool) { - // only a non-escaped space ends the tilde (or eof) - if eof || (!l.inEscape && next == ' ') { + // end on non-escaped space, colon, tilde, boost (or eof) + if eof || (!l.inEscape && (next == ' ' || next == ':' || next == '^' || next == '~')) { // end number l.nextTokenType = tNUMBER l.nextToken = &yySymType{ @@ -257,7 +257,13 @@ func inNumOrStrState(l *queryStringLex, next rune, eof bool) (lexState, bool) { } logDebugTokens("NUMBER - '%s'", l.nextToken.s) l.reset() - return startState, true + + consumed := true + if !eof && (next == ':' || next == '^' || next == '~') { + consumed = false + } + + return startState, consumed } else if !l.inEscape && next == '\\' { l.inEscape = true return inNumOrStrState, true @@ -287,7 +293,7 @@ func inNumOrStrState(l *queryStringLex, next rune, eof bool) (lexState, bool) { } func inStrState(l *queryStringLex, next rune, eof bool) (lexState, bool) { - // end on non-escped space, colon, tilde, boost (or eof) + // end on non-escaped space, colon, tilde, boost (or eof) if eof || (!l.inEscape && (next == ' ' || next == ':' || next == '^' || next == '~')) { // end string l.nextTokenType = tSTRING diff --git a/vendor/github.com/blevesearch/bleve/search/query/query_string_parser.go b/vendor/github.com/blevesearch/bleve/v2/search/query/query_string_parser.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/search/query/query_string_parser.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/query_string_parser.go diff --git a/vendor/github.com/blevesearch/bleve/v2/search/query/regexp.go b/vendor/github.com/blevesearch/bleve/v2/search/query/regexp.go new file mode 100644 index 00000000..6b3da955 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/regexp.go @@ -0,0 +1,82 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + "strings" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" +) + +type RegexpQuery struct { + Regexp string `json:"regexp"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` +} + +// NewRegexpQuery creates a new Query which finds +// documents containing terms that match the +// specified regular expression. The regexp pattern +// SHOULD NOT include ^ or $ modifiers, the search +// will only match entire terms even without them. +func NewRegexpQuery(regexp string) *RegexpQuery { + return &RegexpQuery{ + Regexp: regexp, + } +} + +func (q *RegexpQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *RegexpQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *RegexpQuery) SetField(f string) { + q.FieldVal = f +} + +func (q *RegexpQuery) Field() string { + return q.FieldVal +} + +func (q *RegexpQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + field := q.FieldVal + if q.FieldVal == "" { + field = m.DefaultSearchField() + } + + // require that pattern NOT be anchored to start and end of term. + // do not attempt to remove trailing $, its presence is not + // known to interfere with LiteralPrefix() the way ^ does + // and removing $ introduces possible ambiguities with escaped \$, \\$, etc + actualRegexp := q.Regexp + if strings.HasPrefix(actualRegexp, "^") { + actualRegexp = actualRegexp[1:] // remove leading ^ + } + + return searcher.NewRegexpStringSearcher(ctx, i, actualRegexp, field, + q.BoostVal.Value(), options) +} + +func (q *RegexpQuery) Validate() error { + return nil // real validation delayed until searcher constructor +} diff --git a/vendor/github.com/blevesearch/bleve/v2/search/query/term.go b/vendor/github.com/blevesearch/bleve/v2/search/query/term.go new file mode 100644 index 00000000..5c6af396 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/term.go @@ -0,0 +1,63 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" +) + +type TermQuery struct { + Term string `json:"term"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` +} + +// NewTermQuery creates a new Query for finding an +// exact term match in the index. +func NewTermQuery(term string) *TermQuery { + return &TermQuery{ + Term: term, + } +} + +func (q *TermQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *TermQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *TermQuery) SetField(f string) { + q.FieldVal = f +} + +func (q *TermQuery) Field() string { + return q.FieldVal +} + +func (q *TermQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + field := q.FieldVal + if q.FieldVal == "" { + field = m.DefaultSearchField() + } + return searcher.NewTermSearcher(ctx, i, q.Term, field, q.BoostVal.Value(), options) +} diff --git a/vendor/github.com/blevesearch/bleve/search/query/term_range.go b/vendor/github.com/blevesearch/bleve/v2/search/query/term_range.go similarity index 82% rename from vendor/github.com/blevesearch/bleve/search/query/term_range.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/term_range.go index 8f8ca844..4dc3a34b 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/term_range.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/term_range.go @@ -15,12 +15,13 @@ package query import ( + "context" "fmt" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searcher" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" ) type TermRangeQuery struct { @@ -71,7 +72,7 @@ func (q *TermRangeQuery) Field() string { return q.FieldVal } -func (q *TermRangeQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { +func (q *TermRangeQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() @@ -84,7 +85,7 @@ func (q *TermRangeQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, o if q.Max != "" { maxTerm = []byte(q.Max) } - return searcher.NewTermRangeSearcher(i, minTerm, maxTerm, q.InclusiveMin, q.InclusiveMax, field, q.BoostVal.Value(), options) + return searcher.NewTermRangeSearcher(ctx, i, minTerm, maxTerm, q.InclusiveMin, q.InclusiveMax, field, q.BoostVal.Value(), options) } func (q *TermRangeQuery) Validate() error { diff --git a/vendor/github.com/blevesearch/bleve/search/query/wildcard.go b/vendor/github.com/blevesearch/bleve/v2/search/query/wildcard.go similarity index 82% rename from vendor/github.com/blevesearch/bleve/search/query/wildcard.go rename to vendor/github.com/blevesearch/bleve/v2/search/query/wildcard.go index 747dfe76..f04f3f2e 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/wildcard.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/wildcard.go @@ -15,12 +15,13 @@ package query import ( + "context" "strings" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/mapping" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searcher" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" ) var wildcardRegexpReplacer = strings.NewReplacer( @@ -76,7 +77,7 @@ func (q *WildcardQuery) Field() string { return q.FieldVal } -func (q *WildcardQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { +func (q *WildcardQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() @@ -84,7 +85,7 @@ func (q *WildcardQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, op regexpString := wildcardRegexpReplacer.Replace(q.Wildcard) - return searcher.NewRegexpStringSearcher(i, regexpString, field, + return searcher.NewRegexpStringSearcher(ctx, i, regexpString, field, q.BoostVal.Value(), options) } diff --git a/vendor/github.com/blevesearch/bleve/search/scorer/scorer_conjunction.go b/vendor/github.com/blevesearch/bleve/v2/search/scorer/scorer_conjunction.go similarity index 96% rename from vendor/github.com/blevesearch/bleve/search/scorer/scorer_conjunction.go rename to vendor/github.com/blevesearch/bleve/v2/search/scorer/scorer_conjunction.go index 48cdf3ae..f3c81a78 100644 --- a/vendor/github.com/blevesearch/bleve/search/scorer/scorer_conjunction.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/scorer/scorer_conjunction.go @@ -17,8 +17,8 @@ package scorer import ( "reflect" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/size" ) var reflectStaticSizeConjunctionQueryScorer int diff --git a/vendor/github.com/blevesearch/bleve/search/scorer/scorer_constant.go b/vendor/github.com/blevesearch/bleve/v2/search/scorer/scorer_constant.go similarity index 95% rename from vendor/github.com/blevesearch/bleve/search/scorer/scorer_constant.go rename to vendor/github.com/blevesearch/bleve/v2/search/scorer/scorer_constant.go index dc10fdaa..fc36fd5b 100644 --- a/vendor/github.com/blevesearch/bleve/search/scorer/scorer_constant.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/scorer/scorer_constant.go @@ -18,9 +18,9 @@ import ( "fmt" "reflect" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" ) var reflectStaticSizeConstantScorer int diff --git a/vendor/github.com/blevesearch/bleve/search/scorer/scorer_disjunction.go b/vendor/github.com/blevesearch/bleve/v2/search/scorer/scorer_disjunction.go similarity index 96% rename from vendor/github.com/blevesearch/bleve/search/scorer/scorer_disjunction.go rename to vendor/github.com/blevesearch/bleve/v2/search/scorer/scorer_disjunction.go index 7a955e16..054e76fd 100644 --- a/vendor/github.com/blevesearch/bleve/search/scorer/scorer_disjunction.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/scorer/scorer_disjunction.go @@ -18,8 +18,8 @@ import ( "fmt" "reflect" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/size" ) var reflectStaticSizeDisjunctionQueryScorer int diff --git a/vendor/github.com/blevesearch/bleve/search/scorer/scorer_term.go b/vendor/github.com/blevesearch/bleve/v2/search/scorer/scorer_term.go similarity index 97% rename from vendor/github.com/blevesearch/bleve/search/scorer/scorer_term.go rename to vendor/github.com/blevesearch/bleve/v2/search/scorer/scorer_term.go index 718de2ea..7b60eda4 100644 --- a/vendor/github.com/blevesearch/bleve/search/scorer/scorer_term.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/scorer/scorer_term.go @@ -19,9 +19,9 @@ import ( "math" "reflect" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" ) var reflectStaticSizeTermQueryScorer int @@ -198,6 +198,5 @@ func (s *TermQueryScorer) Score(ctx *search.SearchContext, termMatch *index.Term }) } } - return rv } diff --git a/vendor/github.com/blevesearch/bleve/search/scorer/sqrt_cache.go b/vendor/github.com/blevesearch/bleve/v2/search/scorer/sqrt_cache.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/search/scorer/sqrt_cache.go rename to vendor/github.com/blevesearch/bleve/v2/search/scorer/sqrt_cache.go diff --git a/vendor/github.com/blevesearch/bleve/v2/search/search.go b/vendor/github.com/blevesearch/bleve/v2/search/search.go new file mode 100644 index 00000000..b7a3c42a --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/search/search.go @@ -0,0 +1,385 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package search + +import ( + "fmt" + "reflect" + "sort" + + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" +) + +var reflectStaticSizeDocumentMatch int +var reflectStaticSizeSearchContext int +var reflectStaticSizeLocation int + +func init() { + var dm DocumentMatch + reflectStaticSizeDocumentMatch = int(reflect.TypeOf(dm).Size()) + var sc SearchContext + reflectStaticSizeSearchContext = int(reflect.TypeOf(sc).Size()) + var l Location + reflectStaticSizeLocation = int(reflect.TypeOf(l).Size()) +} + +type ArrayPositions []uint64 + +func (ap ArrayPositions) Equals(other ArrayPositions) bool { + if len(ap) != len(other) { + return false + } + for i := range ap { + if ap[i] != other[i] { + return false + } + } + return true +} + +func (ap ArrayPositions) Compare(other ArrayPositions) int { + for i, p := range ap { + if i >= len(other) { + return 1 + } + if p < other[i] { + return -1 + } + if p > other[i] { + return 1 + } + } + if len(ap) < len(other) { + return -1 + } + return 0 +} + +type Location struct { + // Pos is the position of the term within the field, starting at 1 + Pos uint64 `json:"pos"` + + // Start and End are the byte offsets of the term in the field + Start uint64 `json:"start"` + End uint64 `json:"end"` + + // ArrayPositions contains the positions of the term within any elements. + ArrayPositions ArrayPositions `json:"array_positions"` +} + +func (l *Location) Size() int { + return reflectStaticSizeLocation + size.SizeOfPtr + + len(l.ArrayPositions)*size.SizeOfUint64 +} + +type Locations []*Location + +func (p Locations) Len() int { return len(p) } +func (p Locations) Swap(i, j int) { p[i], p[j] = p[j], p[i] } + +func (p Locations) Less(i, j int) bool { + c := p[i].ArrayPositions.Compare(p[j].ArrayPositions) + if c < 0 { + return true + } + if c > 0 { + return false + } + return p[i].Pos < p[j].Pos +} + +func (p Locations) Dedupe() Locations { // destructive! + if len(p) <= 1 { + return p + } + + sort.Sort(p) + + slow := 0 + + for _, pfast := range p { + pslow := p[slow] + if pslow.Pos == pfast.Pos && + pslow.Start == pfast.Start && + pslow.End == pfast.End && + pslow.ArrayPositions.Equals(pfast.ArrayPositions) { + continue // duplicate, so only move fast ahead + } + + slow++ + + p[slow] = pfast + } + + return p[:slow+1] +} + +type TermLocationMap map[string]Locations + +func (t TermLocationMap) AddLocation(term string, location *Location) { + t[term] = append(t[term], location) +} + +type FieldTermLocationMap map[string]TermLocationMap + +type FieldTermLocation struct { + Field string + Term string + Location Location +} + +type FieldFragmentMap map[string][]string + +type DocumentMatch struct { + Index string `json:"index,omitempty"` + ID string `json:"id"` + IndexInternalID index.IndexInternalID `json:"-"` + Score float64 `json:"score"` + Expl *Explanation `json:"explanation,omitempty"` + Locations FieldTermLocationMap `json:"locations,omitempty"` + Fragments FieldFragmentMap `json:"fragments,omitempty"` + Sort []string `json:"sort,omitempty"` + + // Fields contains the values for document fields listed in + // SearchRequest.Fields. Text fields are returned as strings, numeric + // fields as float64s and date fields as strings. + Fields map[string]interface{} `json:"fields,omitempty"` + + // used to maintain natural index order + HitNumber uint64 `json:"-"` + + // used to temporarily hold field term location information during + // search processing in an efficient, recycle-friendly manner, to + // be later incorporated into the Locations map when search + // results are completed + FieldTermLocations []FieldTermLocation `json:"-"` + + // used to indicate if this match is a partial match + // in the case of a disjunction search + // this means that the match is partial because + // not all sub-queries matched + // if false, all the sub-queries matched + PartialMatch bool `json:"partial_match,omitempty"` +} + +func (dm *DocumentMatch) AddFieldValue(name string, value interface{}) { + if dm.Fields == nil { + dm.Fields = make(map[string]interface{}) + } + existingVal, ok := dm.Fields[name] + if !ok { + dm.Fields[name] = value + return + } + + valSlice, ok := existingVal.([]interface{}) + if ok { + // already a slice, append to it + valSlice = append(valSlice, value) + } else { + // create a slice + valSlice = []interface{}{existingVal, value} + } + dm.Fields[name] = valSlice +} + +// Reset allows an already allocated DocumentMatch to be reused +func (dm *DocumentMatch) Reset() *DocumentMatch { + // remember the []byte used for the IndexInternalID + indexInternalID := dm.IndexInternalID + // remember the []interface{} used for sort + sort := dm.Sort + // remember the FieldTermLocations backing array + ftls := dm.FieldTermLocations + for i := range ftls { // recycle the ArrayPositions of each location + ftls[i].Location.ArrayPositions = ftls[i].Location.ArrayPositions[:0] + } + // idiom to copy over from empty DocumentMatch (0 allocations) + *dm = DocumentMatch{} + // reuse the []byte already allocated (and reset len to 0) + dm.IndexInternalID = indexInternalID[:0] + // reuse the []interface{} already allocated (and reset len to 0) + dm.Sort = sort[:0] + // reuse the FieldTermLocations already allocated (and reset len to 0) + dm.FieldTermLocations = ftls[:0] + return dm +} + +func (dm *DocumentMatch) Size() int { + sizeInBytes := reflectStaticSizeDocumentMatch + size.SizeOfPtr + + len(dm.Index) + + len(dm.ID) + + len(dm.IndexInternalID) + + if dm.Expl != nil { + sizeInBytes += dm.Expl.Size() + } + + for k, v := range dm.Locations { + sizeInBytes += size.SizeOfString + len(k) + for k1, v1 := range v { + sizeInBytes += size.SizeOfString + len(k1) + + size.SizeOfSlice + for _, entry := range v1 { + sizeInBytes += entry.Size() + } + } + } + + for k, v := range dm.Fragments { + sizeInBytes += size.SizeOfString + len(k) + + size.SizeOfSlice + + for _, entry := range v { + sizeInBytes += size.SizeOfString + len(entry) + } + } + + for _, entry := range dm.Sort { + sizeInBytes += size.SizeOfString + len(entry) + } + + for k, _ := range dm.Fields { + sizeInBytes += size.SizeOfString + len(k) + + size.SizeOfPtr + } + + return sizeInBytes +} + +// Complete performs final preparation & transformation of the +// DocumentMatch at the end of search processing, also allowing the +// caller to provide an optional preallocated locations slice +func (dm *DocumentMatch) Complete(prealloc []Location) []Location { + // transform the FieldTermLocations slice into the Locations map + nlocs := len(dm.FieldTermLocations) + if nlocs > 0 { + if cap(prealloc) < nlocs { + prealloc = make([]Location, nlocs) + } + prealloc = prealloc[:nlocs] + + var lastField string + var tlm TermLocationMap + var needsDedupe bool + + for i, ftl := range dm.FieldTermLocations { + if i == 0 || lastField != ftl.Field { + lastField = ftl.Field + + if dm.Locations == nil { + dm.Locations = make(FieldTermLocationMap) + } + + tlm = dm.Locations[ftl.Field] + if tlm == nil { + tlm = make(TermLocationMap) + dm.Locations[ftl.Field] = tlm + } + } + + loc := &prealloc[i] + *loc = ftl.Location + + if len(loc.ArrayPositions) > 0 { // copy + loc.ArrayPositions = append(ArrayPositions(nil), loc.ArrayPositions...) + } + + locs := tlm[ftl.Term] + + // if the loc is before or at the last location, then there + // might be duplicates that need to be deduplicated + if !needsDedupe && len(locs) > 0 { + last := locs[len(locs)-1] + cmp := loc.ArrayPositions.Compare(last.ArrayPositions) + if cmp < 0 || (cmp == 0 && loc.Pos <= last.Pos) { + needsDedupe = true + } + } + + tlm[ftl.Term] = append(locs, loc) + + dm.FieldTermLocations[i] = FieldTermLocation{ // recycle + Location: Location{ + ArrayPositions: ftl.Location.ArrayPositions[:0], + }, + } + } + + if needsDedupe { + for _, tlm := range dm.Locations { + for term, locs := range tlm { + tlm[term] = locs.Dedupe() + } + } + } + } + + dm.FieldTermLocations = dm.FieldTermLocations[:0] // recycle + + return prealloc +} + +func (dm *DocumentMatch) String() string { + return fmt.Sprintf("[%s-%f]", string(dm.IndexInternalID), dm.Score) +} + +type DocumentMatchCollection []*DocumentMatch + +func (c DocumentMatchCollection) Len() int { return len(c) } +func (c DocumentMatchCollection) Swap(i, j int) { c[i], c[j] = c[j], c[i] } +func (c DocumentMatchCollection) Less(i, j int) bool { return c[i].Score > c[j].Score } + +type Searcher interface { + Next(ctx *SearchContext) (*DocumentMatch, error) + Advance(ctx *SearchContext, ID index.IndexInternalID) (*DocumentMatch, error) + Close() error + Weight() float64 + SetQueryNorm(float64) + Count() uint64 + Min() int + Size() int + + DocumentMatchPoolSize() int +} + +type SearcherOptions struct { + Explain bool + IncludeTermVectors bool + Score string +} + +// SearchContext represents the context around a single search +type SearchContext struct { + DocumentMatchPool *DocumentMatchPool + Collector Collector + IndexReader index.IndexReader +} + +func (sc *SearchContext) Size() int { + sizeInBytes := reflectStaticSizeSearchContext + size.SizeOfPtr + + reflectStaticSizeDocumentMatchPool + size.SizeOfPtr + + if sc.DocumentMatchPool != nil { + for _, entry := range sc.DocumentMatchPool.avail { + if entry != nil { + sizeInBytes += entry.Size() + } + } + } + + return sizeInBytes +} diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/ordered_searchers_list.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/ordered_searchers_list.go similarity index 95% rename from vendor/github.com/blevesearch/bleve/search/searcher/ordered_searchers_list.go rename to vendor/github.com/blevesearch/bleve/v2/search/searcher/ordered_searchers_list.go index 536c593c..f3e646e9 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/ordered_searchers_list.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/ordered_searchers_list.go @@ -15,7 +15,7 @@ package searcher import ( - "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/v2/search" ) type OrderedSearcherList []search.Searcher diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_boolean.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_boolean.go similarity index 96% rename from vendor/github.com/blevesearch/bleve/search/searcher/search_boolean.go rename to vendor/github.com/blevesearch/bleve/v2/search/searcher/search_boolean.go index 7f0bfa42..bf207f81 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_boolean.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_boolean.go @@ -15,13 +15,14 @@ package searcher import ( + "context" "math" "reflect" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/scorer" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/scorer" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" ) var reflectStaticSizeBooleanSearcher int @@ -48,7 +49,7 @@ type BooleanSearcher struct { done bool } -func NewBooleanSearcher(indexReader index.IndexReader, mustSearcher search.Searcher, shouldSearcher search.Searcher, mustNotSearcher search.Searcher, options search.SearcherOptions) (*BooleanSearcher, error) { +func NewBooleanSearcher(ctx context.Context, indexReader index.IndexReader, mustSearcher search.Searcher, shouldSearcher search.Searcher, mustNotSearcher search.Searcher, options search.SearcherOptions) (*BooleanSearcher, error) { // build our searcher rv := BooleanSearcher{ indexReader: indexReader, diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_conjunction.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_conjunction.go similarity index 94% rename from vendor/github.com/blevesearch/bleve/search/searcher/search_conjunction.go rename to vendor/github.com/blevesearch/bleve/v2/search/searcher/search_conjunction.go index ac737bcc..19ef199a 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_conjunction.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_conjunction.go @@ -15,14 +15,15 @@ package searcher import ( + "context" "math" "reflect" "sort" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/scorer" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/scorer" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" ) var reflectStaticSizeConjunctionSearcher int @@ -41,9 +42,10 @@ type ConjunctionSearcher struct { scorer *scorer.ConjunctionQueryScorer initialized bool options search.SearcherOptions + bytesRead uint64 } -func NewConjunctionSearcher(indexReader index.IndexReader, +func NewConjunctionSearcher(ctx context.Context, indexReader index.IndexReader, qsearchers []search.Searcher, options search.SearcherOptions) ( search.Searcher, error) { // build the sorted downstream searchers @@ -57,7 +59,7 @@ func NewConjunctionSearcher(indexReader index.IndexReader, // do not need extra information like freq-norm's or term vectors if len(searchers) > 1 && options.Score == "none" && !options.IncludeTermVectors { - rv, err := optimizeCompositeSearcher("conjunction:unadorned", + rv, err := optimizeCompositeSearcher(ctx, "conjunction:unadorned", indexReader, searchers, options) if err != nil || rv != nil { return rv, err @@ -76,7 +78,7 @@ func NewConjunctionSearcher(indexReader index.IndexReader, // attempt push-down conjunction optimization when there's >1 searchers if len(searchers) > 1 { - rv, err := optimizeCompositeSearcher("conjunction", + rv, err := optimizeCompositeSearcher(ctx, "conjunction", indexReader, searchers, options) if err != nil || rv != nil { return rv, err diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_disjunction.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_disjunction.go similarity index 82% rename from vendor/github.com/blevesearch/bleve/search/searcher/search_disjunction.go rename to vendor/github.com/blevesearch/bleve/v2/search/searcher/search_disjunction.go index f47da27c..606a157a 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_disjunction.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_disjunction.go @@ -15,9 +15,11 @@ package searcher import ( + "context" "fmt" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" + + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" ) // DisjunctionMaxClauseCount is a compile time setting that applications can @@ -30,10 +32,10 @@ var DisjunctionMaxClauseCount = 0 // slice implementation to a heap implementation. var DisjunctionHeapTakeover = 10 -func NewDisjunctionSearcher(indexReader index.IndexReader, +func NewDisjunctionSearcher(ctx context.Context, indexReader index.IndexReader, qsearchers []search.Searcher, min float64, options search.SearcherOptions) ( search.Searcher, error) { - return newDisjunctionSearcher(indexReader, qsearchers, min, options, true) + return newDisjunctionSearcher(ctx, indexReader, qsearchers, min, options, true) } func optionsDisjunctionOptimizable(options search.SearcherOptions) bool { @@ -41,7 +43,7 @@ func optionsDisjunctionOptimizable(options search.SearcherOptions) bool { return rv } -func newDisjunctionSearcher(indexReader index.IndexReader, +func newDisjunctionSearcher(ctx context.Context, indexReader index.IndexReader, qsearchers []search.Searcher, min float64, options search.SearcherOptions, limit bool) (search.Searcher, error) { // attempt the "unadorned" disjunction optimization only when we @@ -49,7 +51,7 @@ func newDisjunctionSearcher(indexReader index.IndexReader, // and the requested min is simple if len(qsearchers) > 1 && min <= 1 && optionsDisjunctionOptimizable(options) { - rv, err := optimizeCompositeSearcher("disjunction:unadorned", + rv, err := optimizeCompositeSearcher(ctx, "disjunction:unadorned", indexReader, qsearchers, options) if err != nil || rv != nil { return rv, err @@ -57,14 +59,14 @@ func newDisjunctionSearcher(indexReader index.IndexReader, } if len(qsearchers) > DisjunctionHeapTakeover { - return newDisjunctionHeapSearcher(indexReader, qsearchers, min, options, + return newDisjunctionHeapSearcher(ctx, indexReader, qsearchers, min, options, limit) } - return newDisjunctionSliceSearcher(indexReader, qsearchers, min, options, + return newDisjunctionSliceSearcher(ctx, indexReader, qsearchers, min, options, limit) } -func optimizeCompositeSearcher(optimizationKind string, +func optimizeCompositeSearcher(ctx context.Context, optimizationKind string, indexReader index.IndexReader, qsearchers []search.Searcher, options search.SearcherOptions) (search.Searcher, error) { var octx index.OptimizableContext diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_disjunction_heap.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_disjunction_heap.go similarity index 95% rename from vendor/github.com/blevesearch/bleve/search/searcher/search_disjunction_heap.go rename to vendor/github.com/blevesearch/bleve/v2/search/searcher/search_disjunction_heap.go index 7f0a5a00..d36e3013 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_disjunction_heap.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_disjunction_heap.go @@ -17,13 +17,14 @@ package searcher import ( "bytes" "container/heap" + "context" "math" "reflect" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/scorer" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/scorer" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" ) var reflectStaticSizeDisjunctionHeapSearcher int @@ -55,9 +56,11 @@ type DisjunctionHeapSearcher struct { matching []*search.DocumentMatch matchingCurrs []*SearcherCurr + + bytesRead uint64 } -func newDisjunctionHeapSearcher(indexReader index.IndexReader, +func newDisjunctionHeapSearcher(ctx context.Context, indexReader index.IndexReader, searchers []search.Searcher, min float64, options search.SearcherOptions, limit bool) ( *DisjunctionHeapSearcher, error) { @@ -194,8 +197,10 @@ func (s *DisjunctionHeapSearcher) Next(ctx *search.SearchContext) ( for !found && len(s.matching) > 0 { if len(s.matching) >= s.min { found = true + partialMatch := len(s.matching) != len(s.searchers) // score this match rv = s.scorer.Score(ctx, s.matching, len(s.matching), s.numSearchers) + rv.PartialMatch = partialMatch } // invoke next on all the matching searchers diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_disjunction_slice.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_disjunction_slice.go similarity index 94% rename from vendor/github.com/blevesearch/bleve/search/searcher/search_disjunction_slice.go rename to vendor/github.com/blevesearch/bleve/v2/search/searcher/search_disjunction_slice.go index dc566ade..0969c8cf 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_disjunction_slice.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_disjunction_slice.go @@ -15,14 +15,15 @@ package searcher import ( + "context" "math" "reflect" "sort" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/scorer" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/scorer" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" ) var reflectStaticSizeDisjunctionSliceSearcher int @@ -43,9 +44,10 @@ type DisjunctionSliceSearcher struct { matching []*search.DocumentMatch matchingIdxs []int initialized bool + bytesRead uint64 } -func newDisjunctionSliceSearcher(indexReader index.IndexReader, +func newDisjunctionSliceSearcher(ctx context.Context, indexReader index.IndexReader, qsearchers []search.Searcher, min float64, options search.SearcherOptions, limit bool) ( *DisjunctionSliceSearcher, error) { @@ -156,7 +158,6 @@ func (s *DisjunctionSliceSearcher) updateMatches() error { matchingIdxs = matchingIdxs[:0] } } - matching = append(matching, curr) matchingIdxs = append(matchingIdxs, i) } @@ -196,8 +197,10 @@ func (s *DisjunctionSliceSearcher) Next(ctx *search.SearchContext) ( for !found && len(s.matching) > 0 { if len(s.matching) >= s.min { found = true + partialMatch := len(s.matching) != len(s.searchers) // score this match rv = s.scorer.Score(ctx, s.matching, len(s.matching), s.numSearchers) + rv.PartialMatch = partialMatch } // invoke next on all the matching searchers diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_docid.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_docid.go similarity index 89% rename from vendor/github.com/blevesearch/bleve/search/searcher/search_docid.go rename to vendor/github.com/blevesearch/bleve/v2/search/searcher/search_docid.go index 3b258a58..720fd323 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_docid.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_docid.go @@ -15,12 +15,13 @@ package searcher import ( + "context" "reflect" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/scorer" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/scorer" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" ) var reflectStaticSizeDocIDSearcher int @@ -37,7 +38,7 @@ type DocIDSearcher struct { count int } -func NewDocIDSearcher(indexReader index.IndexReader, ids []string, boost float64, +func NewDocIDSearcher(ctx context.Context, indexReader index.IndexReader, ids []string, boost float64, options search.SearcherOptions) (searcher *DocIDSearcher, err error) { reader, err := indexReader.DocIDReaderOnly(ids) diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_filter.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_filter.go similarity index 90% rename from vendor/github.com/blevesearch/bleve/search/searcher/search_filter.go rename to vendor/github.com/blevesearch/bleve/v2/search/searcher/search_filter.go index 7c95fb41..4e4dd5ea 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_filter.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_filter.go @@ -15,11 +15,12 @@ package searcher import ( + "context" "reflect" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" ) var reflectStaticSizeFilteringSearcher int @@ -41,7 +42,7 @@ type FilteringSearcher struct { accept FilterFunc } -func NewFilteringSearcher(s search.Searcher, filter FilterFunc) *FilteringSearcher { +func NewFilteringSearcher(ctx context.Context, s search.Searcher, filter FilterFunc) *FilteringSearcher { return &FilteringSearcher{ child: s, accept: filter, diff --git a/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_fuzzy.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_fuzzy.go new file mode 100644 index 00000000..1957168b --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_fuzzy.go @@ -0,0 +1,156 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "fmt" + + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +var MaxFuzziness = 2 + +func NewFuzzySearcher(ctx context.Context, indexReader index.IndexReader, term string, + prefix, fuzziness int, field string, boost float64, + options search.SearcherOptions) (search.Searcher, error) { + + if fuzziness > MaxFuzziness { + return nil, fmt.Errorf("fuzziness exceeds max (%d)", MaxFuzziness) + } + + if fuzziness < 0 { + return nil, fmt.Errorf("invalid fuzziness, negative") + } + + // Note: we don't byte slice the term for a prefix because of runes. + prefixTerm := "" + for i, r := range term { + if i < prefix { + prefixTerm += string(r) + } else { + break + } + } + fuzzyCandidates, err := findFuzzyCandidateTerms(indexReader, term, fuzziness, + field, prefixTerm) + if err != nil { + return nil, err + } + + var candidates []string + var dictBytesRead uint64 + if fuzzyCandidates != nil { + candidates = fuzzyCandidates.candidates + dictBytesRead = fuzzyCandidates.bytesRead + } + + if ctx != nil { + reportIOStats(ctx, dictBytesRead) + search.RecordSearchCost(ctx, search.AddM, dictBytesRead) + fuzzyTermMatches := ctx.Value(search.FuzzyMatchPhraseKey) + if fuzzyTermMatches != nil { + fuzzyTermMatches.(map[string][]string)[term] = candidates + } + } + + return NewMultiTermSearcher(ctx, indexReader, candidates, field, + boost, options, true) +} + +type fuzzyCandidates struct { + candidates []string + bytesRead uint64 +} + +func reportIOStats(ctx context.Context, bytesRead uint64) { + // The fuzzy, regexp like queries essentially load a dictionary, + // which potentially incurs a cost that must be accounted by + // using the callback to report the value. + if ctx != nil { + statsCallbackFn := ctx.Value(search.SearchIOStatsCallbackKey) + if statsCallbackFn != nil { + statsCallbackFn.(search.SearchIOStatsCallbackFunc)(bytesRead) + } + } +} + +func findFuzzyCandidateTerms(indexReader index.IndexReader, term string, + fuzziness int, field, prefixTerm string) (rv *fuzzyCandidates, err error) { + rv = &fuzzyCandidates{ + candidates: make([]string, 0), + } + + // in case of advanced reader implementations directly call + // the levenshtein automaton based iterator to collect the + // candidate terms + if ir, ok := indexReader.(index.IndexReaderFuzzy); ok { + fieldDict, err := ir.FieldDictFuzzy(field, term, fuzziness, prefixTerm) + if err != nil { + return nil, err + } + defer func() { + if cerr := fieldDict.Close(); cerr != nil && err == nil { + err = cerr + } + }() + tfd, err := fieldDict.Next() + for err == nil && tfd != nil { + rv.candidates = append(rv.candidates, tfd.Term) + if tooManyClauses(len(rv.candidates)) { + return nil, tooManyClausesErr(field, len(rv.candidates)) + } + tfd, err = fieldDict.Next() + } + + rv.bytesRead = fieldDict.BytesRead() + return rv, err + } + + var fieldDict index.FieldDict + if len(prefixTerm) > 0 { + fieldDict, err = indexReader.FieldDictPrefix(field, []byte(prefixTerm)) + } else { + fieldDict, err = indexReader.FieldDict(field) + } + if err != nil { + return nil, err + } + defer func() { + if cerr := fieldDict.Close(); cerr != nil && err == nil { + err = cerr + } + }() + + // enumerate terms and check levenshtein distance + var reuse []int + tfd, err := fieldDict.Next() + for err == nil && tfd != nil { + var ld int + var exceeded bool + ld, exceeded, reuse = search.LevenshteinDistanceMaxReuseSlice(term, tfd.Term, fuzziness, reuse) + if !exceeded && ld <= fuzziness { + rv.candidates = append(rv.candidates, tfd.Term) + if tooManyClauses(len(rv.candidates)) { + return nil, tooManyClausesErr(field, len(rv.candidates)) + } + } + tfd, err = fieldDict.Next() + } + + rv.bytesRead = fieldDict.BytesRead() + return rv, err +} diff --git a/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_geoboundingbox.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_geoboundingbox.go new file mode 100644 index 00000000..c889ddce --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_geoboundingbox.go @@ -0,0 +1,302 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +type filterFunc func(key []byte) bool + +var GeoBitsShift1 = geo.GeoBits << 1 +var GeoBitsShift1Minus1 = GeoBitsShift1 - 1 + +func NewGeoBoundingBoxSearcher(ctx context.Context, indexReader index.IndexReader, minLon, minLat, + maxLon, maxLat float64, field string, boost float64, + options search.SearcherOptions, checkBoundaries bool) ( + search.Searcher, error) { + if tp, ok := indexReader.(index.SpatialIndexPlugin); ok { + sp, err := tp.GetSpatialAnalyzerPlugin("s2") + if err == nil { + terms := sp.GetQueryTokens(geo.NewBoundedRectangle(minLat, + minLon, maxLat, maxLon)) + boxSearcher, err := NewMultiTermSearcher(ctx, indexReader, + terms, field, boost, options, false) + if err != nil { + return nil, err + } + + dvReader, err := indexReader.DocValueReader([]string{field}) + if err != nil { + return nil, err + } + + return NewFilteringSearcher(ctx, boxSearcher, buildRectFilter(ctx, dvReader, + field, minLon, minLat, maxLon, maxLat)), nil + } + } + + // indexes without the spatial plugin override would continue here. + + // track list of opened searchers, for cleanup on early exit + var openedSearchers []search.Searcher + cleanupOpenedSearchers := func() { + for _, s := range openedSearchers { + _ = s.Close() + } + } + + // do math to produce list of terms needed for this search + onBoundaryTerms, notOnBoundaryTerms, err := ComputeGeoRange(nil, 0, GeoBitsShift1Minus1, + minLon, minLat, maxLon, maxLat, checkBoundaries, indexReader, field) + if err != nil { + return nil, err + } + + var onBoundarySearcher search.Searcher + dvReader, err := indexReader.DocValueReader([]string{field}) + if err != nil { + return nil, err + } + + if len(onBoundaryTerms) > 0 { + rawOnBoundarySearcher, err := NewMultiTermSearcherBytes(ctx, indexReader, + onBoundaryTerms, field, boost, options, false) + if err != nil { + return nil, err + } + // add filter to check points near the boundary + onBoundarySearcher = NewFilteringSearcher(ctx, rawOnBoundarySearcher, + buildRectFilter(ctx, dvReader, field, minLon, minLat, maxLon, maxLat)) + openedSearchers = append(openedSearchers, onBoundarySearcher) + } + + var notOnBoundarySearcher search.Searcher + if len(notOnBoundaryTerms) > 0 { + var err error + notOnBoundarySearcher, err = NewMultiTermSearcherBytes(ctx, indexReader, + notOnBoundaryTerms, field, boost, options, false) + if err != nil { + cleanupOpenedSearchers() + return nil, err + } + openedSearchers = append(openedSearchers, notOnBoundarySearcher) + } + + if onBoundarySearcher != nil && notOnBoundarySearcher != nil { + rv, err := NewDisjunctionSearcher(ctx, indexReader, + []search.Searcher{ + onBoundarySearcher, + notOnBoundarySearcher, + }, + 0, options) + if err != nil { + cleanupOpenedSearchers() + return nil, err + } + return rv, nil + } else if onBoundarySearcher != nil { + return onBoundarySearcher, nil + } else if notOnBoundarySearcher != nil { + return notOnBoundarySearcher, nil + } + + return NewMatchNoneSearcher(indexReader) +} + +var geoMaxShift = document.GeoPrecisionStep * 4 +var geoDetailLevel = ((geo.GeoBits << 1) - geoMaxShift) / 2 + +type closeFunc func() error + +func ComputeGeoRange(ctx context.Context, term uint64, shift uint, + sminLon, sminLat, smaxLon, smaxLat float64, checkBoundaries bool, + indexReader index.IndexReader, field string) ( + onBoundary [][]byte, notOnBoundary [][]byte, err error) { + + isIndexed, closeF, err := buildIsIndexedFunc(ctx, indexReader, field) + if closeF != nil { + defer func() { + cerr := closeF() + if cerr != nil { + err = cerr + } + }() + } + + grc := &geoRangeCompute{ + preallocBytesLen: 32, + preallocBytes: make([]byte, 32), + sminLon: sminLon, + sminLat: sminLat, + smaxLon: smaxLon, + smaxLat: smaxLat, + checkBoundaries: checkBoundaries, + isIndexed: isIndexed, + } + + grc.computeGeoRange(term, shift) + + return grc.onBoundary, grc.notOnBoundary, nil +} + +func buildIsIndexedFunc(ctx context.Context, indexReader index.IndexReader, field string) (isIndexed filterFunc, closeF closeFunc, err error) { + if irr, ok := indexReader.(index.IndexReaderContains); ok { + fieldDict, err := irr.FieldDictContains(field) + if err != nil { + return nil, nil, err + } + + isIndexed = func(term []byte) bool { + found, err := fieldDict.Contains(term) + return err == nil && found + } + + closeF = func() error { + if fd, ok := fieldDict.(index.FieldDict); ok { + err := fd.Close() + if err != nil { + return err + } + } + return nil + } + } else if indexReader != nil { + isIndexed = func(term []byte) bool { + reader, err := indexReader.TermFieldReader(ctx, term, field, false, false, false) + if err != nil || reader == nil { + return false + } + if reader.Count() == 0 { + _ = reader.Close() + return false + } + _ = reader.Close() + return true + } + + } else { + isIndexed = func([]byte) bool { + return true + } + } + return isIndexed, closeF, err +} + +func buildRectFilter(ctx context.Context, dvReader index.DocValueReader, field string, + minLon, minLat, maxLon, maxLat float64) FilterFunc { + return func(d *search.DocumentMatch) bool { + // check geo matches against all numeric type terms indexed + var lons, lats []float64 + var found bool + err := dvReader.VisitDocValues(d.IndexInternalID, func(field string, term []byte) { + // only consider the values which are shifted 0 + prefixCoded := numeric.PrefixCoded(term) + shift, err := prefixCoded.Shift() + if err == nil && shift == 0 { + var i64 int64 + i64, err = prefixCoded.Int64() + if err == nil { + lons = append(lons, geo.MortonUnhashLon(uint64(i64))) + lats = append(lats, geo.MortonUnhashLat(uint64(i64))) + found = true + } + } + }) + if err == nil && found { + bytes := dvReader.BytesRead() + if bytes > 0 { + reportIOStats(ctx, bytes) + search.RecordSearchCost(ctx, search.AddM, bytes) + } + for i := range lons { + if geo.BoundingBoxContains(lons[i], lats[i], + minLon, minLat, maxLon, maxLat) { + return true + } + } + } + return false + } +} + +type geoRangeCompute struct { + preallocBytesLen int + preallocBytes []byte + sminLon, sminLat, smaxLon, smaxLat float64 + checkBoundaries bool + onBoundary, notOnBoundary [][]byte + isIndexed func(term []byte) bool +} + +func (grc *geoRangeCompute) makePrefixCoded(in int64, shift uint) (rv numeric.PrefixCoded) { + if len(grc.preallocBytes) <= 0 { + grc.preallocBytesLen = grc.preallocBytesLen * 2 + grc.preallocBytes = make([]byte, grc.preallocBytesLen) + } + + rv, grc.preallocBytes, _ = + numeric.NewPrefixCodedInt64Prealloc(in, shift, grc.preallocBytes) + + return rv +} + +func (grc *geoRangeCompute) computeGeoRange(term uint64, shift uint) { + split := term | uint64(0x1)<> 1 + + within := res%document.GeoPrecisionStep == 0 && + geo.RectWithin(minLon, minLat, maxLon, maxLat, + grc.sminLon, grc.sminLat, grc.smaxLon, grc.smaxLat) + if within || (level == geoDetailLevel && + geo.RectIntersects(minLon, minLat, maxLon, maxLat, + grc.sminLon, grc.sminLat, grc.smaxLon, grc.smaxLat)) { + codedTerm := grc.makePrefixCoded(int64(start), res) + if grc.isIndexed(codedTerm) { + if !within && grc.checkBoundaries { + grc.onBoundary = append(grc.onBoundary, codedTerm) + } else { + grc.notOnBoundary = append(grc.notOnBoundary, codedTerm) + } + } + } else if level < geoDetailLevel && + geo.RectIntersects(minLon, minLat, maxLon, maxLat, + grc.sminLon, grc.sminLat, grc.smaxLon, grc.smaxLat) { + grc.computeGeoRange(start, res-1) + } +} diff --git a/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_geopointdistance.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_geopointdistance.go new file mode 100644 index 00000000..fbe95895 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_geopointdistance.go @@ -0,0 +1,151 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +func NewGeoPointDistanceSearcher(ctx context.Context, indexReader index.IndexReader, centerLon, + centerLat, dist float64, field string, boost float64, + options search.SearcherOptions) (search.Searcher, error) { + var rectSearcher search.Searcher + if tp, ok := indexReader.(index.SpatialIndexPlugin); ok { + sp, err := tp.GetSpatialAnalyzerPlugin("s2") + if err == nil { + terms := sp.GetQueryTokens(geo.NewPointDistance(centerLat, + centerLon, dist)) + rectSearcher, err = NewMultiTermSearcher(ctx, indexReader, terms, + field, boost, options, false) + if err != nil { + return nil, err + } + } + } + + // indexes without the spatial plugin override would get + // initialized here. + if rectSearcher == nil { + // compute bounding box containing the circle + topLeftLon, topLeftLat, bottomRightLon, bottomRightLat, err := + geo.RectFromPointDistance(centerLon, centerLat, dist) + if err != nil { + return nil, err + } + + // build a searcher for the box + rectSearcher, err = boxSearcher(ctx, indexReader, + topLeftLon, topLeftLat, bottomRightLon, bottomRightLat, + field, boost, options, false) + if err != nil { + return nil, err + } + } + + dvReader, err := indexReader.DocValueReader([]string{field}) + if err != nil { + return nil, err + } + + // wrap it in a filtering searcher which checks the actual distance + return NewFilteringSearcher(ctx, rectSearcher, + buildDistFilter(ctx, dvReader, field, centerLon, centerLat, dist)), nil +} + +// boxSearcher builds a searcher for the described bounding box +// if the desired box crosses the dateline, it is automatically split into +// two boxes joined through a disjunction searcher +func boxSearcher(ctx context.Context, indexReader index.IndexReader, + topLeftLon, topLeftLat, bottomRightLon, bottomRightLat float64, + field string, boost float64, options search.SearcherOptions, checkBoundaries bool) ( + search.Searcher, error) { + if bottomRightLon < topLeftLon { + // cross date line, rewrite as two parts + + leftSearcher, err := NewGeoBoundingBoxSearcher(ctx, indexReader, + -180, bottomRightLat, bottomRightLon, topLeftLat, + field, boost, options, checkBoundaries) + if err != nil { + return nil, err + } + rightSearcher, err := NewGeoBoundingBoxSearcher(ctx, indexReader, + topLeftLon, bottomRightLat, 180, topLeftLat, field, boost, options, + checkBoundaries) + if err != nil { + _ = leftSearcher.Close() + return nil, err + } + + boxSearcher, err := NewDisjunctionSearcher(ctx, indexReader, + []search.Searcher{leftSearcher, rightSearcher}, 0, options) + if err != nil { + _ = leftSearcher.Close() + _ = rightSearcher.Close() + return nil, err + } + return boxSearcher, nil + } + + // build geoboundingbox searcher for that bounding box + boxSearcher, err := NewGeoBoundingBoxSearcher(ctx, indexReader, + topLeftLon, bottomRightLat, bottomRightLon, topLeftLat, field, boost, + options, checkBoundaries) + if err != nil { + return nil, err + } + return boxSearcher, nil +} + +func buildDistFilter(ctx context.Context, dvReader index.DocValueReader, field string, + centerLon, centerLat, maxDist float64) FilterFunc { + return func(d *search.DocumentMatch) bool { + // check geo matches against all numeric type terms indexed + var lons, lats []float64 + var found bool + + err := dvReader.VisitDocValues(d.IndexInternalID, func(field string, term []byte) { + // only consider the values which are shifted 0 + prefixCoded := numeric.PrefixCoded(term) + shift, err := prefixCoded.Shift() + if err == nil && shift == 0 { + i64, err := prefixCoded.Int64() + if err == nil { + lons = append(lons, geo.MortonUnhashLon(uint64(i64))) + lats = append(lats, geo.MortonUnhashLat(uint64(i64))) + found = true + } + } + }) + if err == nil && found { + bytes := dvReader.BytesRead() + if bytes > 0 { + reportIOStats(ctx, bytes) + search.RecordSearchCost(ctx, search.AddM, bytes) + } + for i := range lons { + dist := geo.Haversin(lons[i], lats[i], centerLon, centerLat) + if dist <= maxDist/1000 { + return true + } + } + } + return false + } +} diff --git a/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_geopolygon.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_geopolygon.go new file mode 100644 index 00000000..a43edafb --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_geopolygon.go @@ -0,0 +1,149 @@ +// Copyright (c) 2019 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "fmt" + "math" + + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +func NewGeoBoundedPolygonSearcher(ctx context.Context, indexReader index.IndexReader, + coordinates []geo.Point, field string, boost float64, + options search.SearcherOptions) (search.Searcher, error) { + if len(coordinates) < 3 { + return nil, fmt.Errorf("Too few points specified for the polygon boundary") + } + + var rectSearcher search.Searcher + if sr, ok := indexReader.(index.SpatialIndexPlugin); ok { + tp, err := sr.GetSpatialAnalyzerPlugin("s2") + if err == nil { + terms := tp.GetQueryTokens(geo.NewBoundedPolygon(coordinates)) + rectSearcher, err = NewMultiTermSearcher(ctx, indexReader, terms, + field, boost, options, false) + if err != nil { + return nil, err + } + } + } + + // indexes without the spatial plugin override would get + // initialized here. + if rectSearcher == nil { + // compute the bounding box enclosing the polygon + topLeftLon, topLeftLat, bottomRightLon, bottomRightLat, err := + geo.BoundingRectangleForPolygon(coordinates) + if err != nil { + return nil, err + } + + // build a searcher for the bounding box on the polygon + rectSearcher, err = boxSearcher(ctx, indexReader, + topLeftLon, topLeftLat, bottomRightLon, bottomRightLat, + field, boost, options, true) + if err != nil { + return nil, err + } + } + + dvReader, err := indexReader.DocValueReader([]string{field}) + if err != nil { + return nil, err + } + + // wrap it in a filtering searcher that checks for the polygon inclusivity + return NewFilteringSearcher(ctx, rectSearcher, + buildPolygonFilter(ctx, dvReader, field, coordinates)), nil +} + +const float64EqualityThreshold = 1e-6 + +func almostEqual(a, b float64) bool { + return math.Abs(a-b) <= float64EqualityThreshold +} + +// buildPolygonFilter returns true if the point lies inside the +// polygon. It is based on the ray-casting technique as referred +// here: https://wrf.ecse.rpi.edu/nikola/pubdetails/pnpoly.html +func buildPolygonFilter(ctx context.Context, dvReader index.DocValueReader, field string, + coordinates []geo.Point) FilterFunc { + return func(d *search.DocumentMatch) bool { + // check geo matches against all numeric type terms indexed + var lons, lats []float64 + var found bool + + err := dvReader.VisitDocValues(d.IndexInternalID, func(field string, term []byte) { + // only consider the values which are shifted 0 + prefixCoded := numeric.PrefixCoded(term) + shift, err := prefixCoded.Shift() + if err == nil && shift == 0 { + i64, err := prefixCoded.Int64() + if err == nil { + lons = append(lons, geo.MortonUnhashLon(uint64(i64))) + lats = append(lats, geo.MortonUnhashLat(uint64(i64))) + found = true + } + } + }) + + // Note: this approach works for points which are strictly inside + // the polygon. ie it might fail for certain points on the polygon boundaries. + if err == nil && found { + bytes := dvReader.BytesRead() + if bytes > 0 { + reportIOStats(ctx, bytes) + search.RecordSearchCost(ctx, search.AddM, bytes) + } + nVertices := len(coordinates) + if len(coordinates) < 3 { + return false + } + rayIntersectsSegment := func(point, a, b geo.Point) bool { + return (a.Lat > point.Lat) != (b.Lat > point.Lat) && + point.Lon < (b.Lon-a.Lon)*(point.Lat-a.Lat)/(b.Lat-a.Lat)+a.Lon + } + + for i := range lons { + pt := geo.Point{Lon: lons[i], Lat: lats[i]} + inside := rayIntersectsSegment(pt, coordinates[len(coordinates)-1], coordinates[0]) + // check for a direct vertex match + if almostEqual(coordinates[0].Lat, lats[i]) && + almostEqual(coordinates[0].Lon, lons[i]) { + return true + } + + for j := 1; j < nVertices; j++ { + if almostEqual(coordinates[j].Lat, lats[i]) && + almostEqual(coordinates[j].Lon, lons[i]) { + return true + } + if rayIntersectsSegment(pt, coordinates[j-1], coordinates[j]) { + inside = !inside + } + } + if inside { + return true + } + } + } + return false + } +} diff --git a/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_geoshape.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_geoshape.go new file mode 100644 index 00000000..ae113107 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_geoshape.go @@ -0,0 +1,136 @@ +// Copyright (c) 2022 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "bytes" + "context" + + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" + "github.com/blevesearch/geo/geojson" + "github.com/blevesearch/geo/s2" +) + +func NewGeoShapeSearcher(ctx context.Context, indexReader index.IndexReader, shape index.GeoJSON, + relation string, field string, boost float64, + options search.SearcherOptions) (search.Searcher, error) { + var err error + var spatialPlugin index.SpatialAnalyzerPlugin + + // check for the spatial plugin from the index. + if sr, ok := indexReader.(index.SpatialIndexPlugin); ok { + spatialPlugin, _ = sr.GetSpatialAnalyzerPlugin("s2") + } + + if spatialPlugin == nil { + // fallback to the default spatial plugin(s2). + spatialPlugin = geo.GetSpatialAnalyzerPlugin("s2") + } + + // obtain the query tokens. + terms := spatialPlugin.GetQueryTokens(shape) + mSearcher, err := NewMultiTermSearcher(ctx, indexReader, terms, + field, boost, options, false) + if err != nil { + return nil, err + } + + dvReader, err := indexReader.DocValueReader([]string{field}) + if err != nil { + return nil, err + } + + return NewFilteringSearcher(ctx, mSearcher, + buildRelationFilterOnShapes(ctx, dvReader, field, relation, shape)), nil + +} + +// Using the same term splitter slice used in the doc values in zap. +// TODO: This needs to be revisited whenever we change the zap +// implementation of doc values. +var termSeparatorSplitSlice = []byte{0xff} + +func buildRelationFilterOnShapes(ctx context.Context, dvReader index.DocValueReader, field string, + relation string, shape index.GeoJSON) FilterFunc { + // this is for accumulating the shape's actual complete value + // spread across multiple docvalue visitor callbacks. + var dvShapeValue []byte + var startReading, finishReading bool + var reader *bytes.Reader + + var bufPool *s2.GeoBufferPool + if ctx != nil { + bufPool = ctx.Value(search.GeoBufferPoolCallbackKey).(search.GeoBufferPoolCallbackFunc)() + } + + return func(d *search.DocumentMatch) bool { + var found bool + + err := dvReader.VisitDocValues(d.IndexInternalID, + func(field string, term []byte) { + + // only consider the values which are GlueBytes prefixed or + // if it had already started reading the shape bytes from previous callbacks. + if startReading || len(term) > geo.GlueBytesOffset { + + if !startReading && bytes.Equal(geo.GlueBytes, term[:geo.GlueBytesOffset]) { + startReading = true + + if bytes.Equal(geo.GlueBytes, term[len(term)-geo.GlueBytesOffset:]) { + term = term[:len(term)-geo.GlueBytesOffset] + finishReading = true + } + + dvShapeValue = append(dvShapeValue, term[geo.GlueBytesOffset:]...) + + } else if startReading && !finishReading { + if len(term) > geo.GlueBytesOffset && + bytes.Equal(geo.GlueBytes, term[len(term)-geo.GlueBytesOffset:]) { + term = term[:len(term)-geo.GlueBytesOffset] + finishReading = true + } + + term = append(termSeparatorSplitSlice, term...) + dvShapeValue = append(dvShapeValue, term...) + } + + // apply the filter once the entire docvalue is finished reading. + if finishReading { + v, err := geojson.FilterGeoShapesOnRelation(shape, + dvShapeValue, relation, &reader, bufPool) + if err == nil && v { + found = true + } + dvShapeValue = dvShapeValue[:0] + startReading = false + finishReading = false + } + } + }) + + if err == nil && found { + bytes := dvReader.BytesRead() + if bytes > 0 { + reportIOStats(ctx, bytes) + search.RecordSearchCost(ctx, search.AddM, bytes) + } + return found + } + + return false + } +} diff --git a/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_ip_range.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_ip_range.go new file mode 100644 index 00000000..38266206 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_ip_range.go @@ -0,0 +1,68 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "net" + + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +// netLimits returns the lo and hi bounds inside the network. +func netLimits(n *net.IPNet) (lo net.IP, hi net.IP) { + ones, bits := n.Mask.Size() + netNum := n.IP + if bits == net.IPv4len*8 { + netNum = netNum.To16() + ones += 8 * (net.IPv6len - net.IPv4len) + } + mask := net.CIDRMask(ones, 8*net.IPv6len) + lo = make(net.IP, net.IPv6len) + hi = make(net.IP, net.IPv6len) + for i := 0; i < net.IPv6len; i++ { + lo[i] = netNum[i] & mask[i] + hi[i] = lo[i] | ^mask[i] + } + return lo, hi +} + +func NewIPRangeSearcher(ctx context.Context, indexReader index.IndexReader, ipNet *net.IPNet, + field string, boost float64, options search.SearcherOptions) ( + search.Searcher, error) { + + lo, hi := netLimits(ipNet) + fieldDict, err := indexReader.FieldDictRange(field, lo, hi) + if err != nil { + return nil, err + } + defer fieldDict.Close() + + var terms []string + tfd, err := fieldDict.Next() + for err == nil && tfd != nil { + terms = append(terms, tfd.Term) + if tooManyClauses(len(terms)) { + return nil, tooManyClausesErr(field, len(terms)) + } + tfd, err = fieldDict.Next() + } + if err != nil { + return nil, err + } + + return NewMultiTermSearcher(ctx, indexReader, terms, field, boost, options, true) +} diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_match_all.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_match_all.go similarity index 87% rename from vendor/github.com/blevesearch/bleve/search/searcher/search_match_all.go rename to vendor/github.com/blevesearch/bleve/v2/search/searcher/search_match_all.go index bb664012..57d8d072 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_match_all.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_match_all.go @@ -15,12 +15,13 @@ package searcher import ( + "context" "reflect" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/scorer" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/scorer" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" ) var reflectStaticSizeMatchAllSearcher int @@ -37,7 +38,7 @@ type MatchAllSearcher struct { count uint64 } -func NewMatchAllSearcher(indexReader index.IndexReader, boost float64, options search.SearcherOptions) (*MatchAllSearcher, error) { +func NewMatchAllSearcher(ctx context.Context, indexReader index.IndexReader, boost float64, options search.SearcherOptions) (*MatchAllSearcher, error) { reader, err := indexReader.DocIDReaderAll() if err != nil { return nil, err @@ -48,6 +49,7 @@ func NewMatchAllSearcher(indexReader index.IndexReader, boost float64, options s return nil, err } scorer := scorer.NewConstantScorer(1.0, boost, options) + return &MatchAllSearcher{ indexReader: indexReader, reader: reader, diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_match_none.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_match_none.go similarity index 93% rename from vendor/github.com/blevesearch/bleve/search/searcher/search_match_none.go rename to vendor/github.com/blevesearch/bleve/v2/search/searcher/search_match_none.go index a345e17f..b7f76941 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_match_none.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_match_none.go @@ -17,9 +17,9 @@ package searcher import ( "reflect" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/size" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" ) var reflectStaticSizeMatchNoneSearcher int diff --git a/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_multi_term.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_multi_term.go new file mode 100644 index 00000000..913f99f5 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_multi_term.go @@ -0,0 +1,217 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "fmt" + + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +func NewMultiTermSearcher(ctx context.Context, indexReader index.IndexReader, terms []string, + field string, boost float64, options search.SearcherOptions, limit bool) ( + search.Searcher, error) { + + if tooManyClauses(len(terms)) { + if optionsDisjunctionOptimizable(options) { + return optimizeMultiTermSearcher(ctx, indexReader, terms, field, boost, options) + } + if limit { + return nil, tooManyClausesErr(field, len(terms)) + } + } + + qsearchers, err := makeBatchSearchers(ctx, indexReader, terms, field, boost, options) + if err != nil { + return nil, err + } + + // build disjunction searcher of these ranges + return newMultiTermSearcherInternal(ctx, indexReader, qsearchers, field, boost, + options, limit) +} + +func NewMultiTermSearcherBytes(ctx context.Context, indexReader index.IndexReader, terms [][]byte, + field string, boost float64, options search.SearcherOptions, limit bool) ( + search.Searcher, error) { + + if tooManyClauses(len(terms)) { + if optionsDisjunctionOptimizable(options) { + return optimizeMultiTermSearcherBytes(ctx, indexReader, terms, field, boost, options) + } + + if limit { + return nil, tooManyClausesErr(field, len(terms)) + } + } + + qsearchers, err := makeBatchSearchersBytes(ctx, indexReader, terms, field, boost, options) + if err != nil { + return nil, err + } + + // build disjunction searcher of these ranges + return newMultiTermSearcherInternal(ctx, indexReader, qsearchers, field, boost, + options, limit) +} + +func newMultiTermSearcherInternal(ctx context.Context, indexReader index.IndexReader, + searchers []search.Searcher, field string, boost float64, + options search.SearcherOptions, limit bool) ( + search.Searcher, error) { + + // build disjunction searcher of these ranges + searcher, err := newDisjunctionSearcher(ctx, indexReader, searchers, 0, options, + limit) + if err != nil { + for _, s := range searchers { + _ = s.Close() + } + return nil, err + } + + return searcher, nil +} + +func optimizeMultiTermSearcher(ctx context.Context, indexReader index.IndexReader, terms []string, + field string, boost float64, options search.SearcherOptions) ( + search.Searcher, error) { + var finalSearcher search.Searcher + for len(terms) > 0 { + var batchTerms []string + if len(terms) > DisjunctionMaxClauseCount { + batchTerms = terms[:DisjunctionMaxClauseCount] + terms = terms[DisjunctionMaxClauseCount:] + } else { + batchTerms = terms + terms = nil + } + batch, err := makeBatchSearchers(ctx, indexReader, batchTerms, field, boost, options) + if err != nil { + return nil, err + } + if finalSearcher != nil { + batch = append(batch, finalSearcher) + } + cleanup := func() { + for _, searcher := range batch { + if searcher != nil { + _ = searcher.Close() + } + } + } + finalSearcher, err = optimizeCompositeSearcher(ctx, "disjunction:unadorned", + indexReader, batch, options) + // all searchers in batch should be closed, regardless of error or optimization failure + // either we're returning, or continuing and only finalSearcher is needed for next loop + cleanup() + if err != nil { + return nil, err + } + if finalSearcher == nil { + return nil, fmt.Errorf("unable to optimize") + } + } + return finalSearcher, nil +} + +func makeBatchSearchers(ctx context.Context, indexReader index.IndexReader, terms []string, field string, + boost float64, options search.SearcherOptions) ([]search.Searcher, error) { + + qsearchers := make([]search.Searcher, len(terms)) + qsearchersClose := func() { + for _, searcher := range qsearchers { + if searcher != nil { + _ = searcher.Close() + } + } + } + for i, term := range terms { + var err error + qsearchers[i], err = NewTermSearcher(ctx, indexReader, term, field, boost, options) + if err != nil { + qsearchersClose() + return nil, err + } + } + return qsearchers, nil +} + +func optimizeMultiTermSearcherBytes(ctx context.Context, indexReader index.IndexReader, terms [][]byte, + field string, boost float64, options search.SearcherOptions) ( + search.Searcher, error) { + + var finalSearcher search.Searcher + for len(terms) > 0 { + var batchTerms [][]byte + if len(terms) > DisjunctionMaxClauseCount { + batchTerms = terms[:DisjunctionMaxClauseCount] + terms = terms[DisjunctionMaxClauseCount:] + } else { + batchTerms = terms + terms = nil + } + batch, err := makeBatchSearchersBytes(ctx, indexReader, batchTerms, field, boost, options) + if err != nil { + return nil, err + } + if finalSearcher != nil { + batch = append(batch, finalSearcher) + } + cleanup := func() { + for _, searcher := range batch { + if searcher != nil { + _ = searcher.Close() + } + } + } + finalSearcher, err = optimizeCompositeSearcher(ctx, "disjunction:unadorned", + indexReader, batch, options) + // all searchers in batch should be closed, regardless of error or optimization failure + // either we're returning, or continuing and only finalSearcher is needed for next loop + cleanup() + if err != nil { + return nil, err + } + if finalSearcher == nil { + return nil, fmt.Errorf("unable to optimize") + } + } + return finalSearcher, nil +} + +func makeBatchSearchersBytes(ctx context.Context, indexReader index.IndexReader, terms [][]byte, field string, + boost float64, options search.SearcherOptions) ([]search.Searcher, error) { + + qsearchers := make([]search.Searcher, len(terms)) + qsearchersClose := func() { + for _, searcher := range qsearchers { + if searcher != nil { + _ = searcher.Close() + } + } + } + for i, term := range terms { + var err error + qsearchers[i], err = NewTermSearcherBytes(ctx, indexReader, term, field, boost, options) + if err != nil { + qsearchersClose() + return nil, err + } + } + return qsearchers, nil +} diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_numeric_range.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_numeric_range.go similarity index 87% rename from vendor/github.com/blevesearch/bleve/search/searcher/search_numeric_range.go rename to vendor/github.com/blevesearch/bleve/v2/search/searcher/search_numeric_range.go index 48d6226e..f086051c 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_numeric_range.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_numeric_range.go @@ -16,15 +16,16 @@ package searcher import ( "bytes" + "context" "math" "sort" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/numeric" - "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" ) -func NewNumericRangeSearcher(indexReader index.IndexReader, +func NewNumericRangeSearcher(ctx context.Context, indexReader index.IndexReader, min *float64, max *float64, inclusiveMin, inclusiveMax *bool, field string, boost float64, options search.SearcherOptions) (search.Searcher, error) { // account for unbounded edges @@ -55,6 +56,7 @@ func NewNumericRangeSearcher(indexReader index.IndexReader, } var fieldDict index.FieldDictContains + var dictBytesRead uint64 var isIndexed filterFunc var err error if irr, ok := indexReader.(index.IndexReaderContains); ok { @@ -67,6 +69,8 @@ func NewNumericRangeSearcher(indexReader index.IndexReader, found, err := fieldDict.Contains(term) return err == nil && found } + + dictBytesRead = fieldDict.BytesRead() } // FIXME hard-coded precision, should match field declaration @@ -81,10 +85,17 @@ func NewNumericRangeSearcher(indexReader index.IndexReader, } if len(terms) < 1 { + // reporting back the IO stats with respect to the dictionary + // loaded, using the context + if ctx != nil { + reportIOStats(ctx, dictBytesRead) + search.RecordSearchCost(ctx, search.AddM, dictBytesRead) + } + // cannot return MatchNoneSearcher because of interaction with // commit f391b991c20f02681bacd197afc6d8aed444e132 - return NewMultiTermSearcherBytes(indexReader, terms, field, boost, options, - true) + return NewMultiTermSearcherBytes(ctx, indexReader, terms, field, + boost, options, true) } // for upside_down @@ -99,31 +110,18 @@ func NewNumericRangeSearcher(indexReader index.IndexReader, return nil, tooManyClausesErr(field, len(terms)) } - return NewMultiTermSearcherBytes(indexReader, terms, field, boost, options, - true) + if ctx != nil { + reportIOStats(ctx, dictBytesRead) + search.RecordSearchCost(ctx, search.AddM, dictBytesRead) + } + + return NewMultiTermSearcherBytes(ctx, indexReader, terms, field, + boost, options, true) } func filterCandidateTerms(indexReader index.IndexReader, terms [][]byte, field string) (rv [][]byte, err error) { - if ir, ok := indexReader.(index.IndexReaderOnly); ok { - fieldDict, err := ir.FieldDictOnly(field, terms, false) - if err != nil { - return nil, err - } - // enumerate the terms (no need to check them again) - tfd, err := fieldDict.Next() - for err == nil && tfd != nil { - rv = append(rv, []byte(tfd.Term)) - tfd, err = fieldDict.Next() - } - if cerr := fieldDict.Close(); cerr != nil && err == nil { - err = cerr - } - - return rv, err - } - fieldDict, err := indexReader.FieldDictRange(field, terms[0], terms[len(terms)-1]) if err != nil { return nil, err diff --git a/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_phrase.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_phrase.go new file mode 100644 index 00000000..a7bdb2c8 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_phrase.go @@ -0,0 +1,496 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "fmt" + "math" + "reflect" + + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" +) + +var reflectStaticSizePhraseSearcher int + +func init() { + var ps PhraseSearcher + reflectStaticSizePhraseSearcher = int(reflect.TypeOf(ps).Size()) +} + +type PhraseSearcher struct { + mustSearcher search.Searcher + queryNorm float64 + currMust *search.DocumentMatch + terms [][]string + path phrasePath + paths []phrasePath + locations []search.Location + initialized bool + // map a term to a list of fuzzy terms that match it + fuzzyTermMatches map[string][]string +} + +func (s *PhraseSearcher) Size() int { + sizeInBytes := reflectStaticSizePhraseSearcher + size.SizeOfPtr + + if s.mustSearcher != nil { + sizeInBytes += s.mustSearcher.Size() + } + + if s.currMust != nil { + sizeInBytes += s.currMust.Size() + } + + for _, entry := range s.terms { + sizeInBytes += size.SizeOfSlice + for _, entry1 := range entry { + sizeInBytes += size.SizeOfString + len(entry1) + } + } + + return sizeInBytes +} + +func NewPhraseSearcher(ctx context.Context, indexReader index.IndexReader, terms []string, + fuzziness int, field string, boost float64, options search.SearcherOptions) (*PhraseSearcher, error) { + + // turn flat terms []string into [][]string + mterms := make([][]string, len(terms)) + for i, term := range terms { + mterms[i] = []string{term} + } + return NewMultiPhraseSearcher(ctx, indexReader, mterms, fuzziness, field, boost, options) +} + +func NewMultiPhraseSearcher(ctx context.Context, indexReader index.IndexReader, terms [][]string, + fuzziness int, field string, boost float64, options search.SearcherOptions) (*PhraseSearcher, error) { + + options.IncludeTermVectors = true + var termPositionSearchers []search.Searcher + var err error + var ts search.Searcher + var fuzzyTermMatches map[string][]string + if fuzziness > 0 { + fuzzyTermMatches = make(map[string][]string) + ctx = context.WithValue(ctx, search.FuzzyMatchPhraseKey, fuzzyTermMatches) + } + // in case of fuzzy multi-phrase, phrase and match-phrase queries we hardcode the + // prefix length to 0, as setting a per word matching prefix length would not + // make sense from a user perspective. + for _, termPos := range terms { + if len(termPos) == 1 && termPos[0] != "" { + // single term + if fuzziness > 0 { + // fuzzy + ts, err = NewFuzzySearcher(ctx, indexReader, termPos[0], 0, fuzziness, field, boost, options) + } else { + // non-fuzzy + ts, err = NewTermSearcher(ctx, indexReader, termPos[0], field, boost, options) + } + if err != nil { + // close any searchers already opened + for _, ts := range termPositionSearchers { + _ = ts.Close() + } + return nil, fmt.Errorf("phrase searcher error building term searcher: %v", err) + } + termPositionSearchers = append(termPositionSearchers, ts) + } else if len(termPos) > 1 { + // multiple terms + var termSearchers []search.Searcher + for _, term := range termPos { + if term == "" { + continue + } + if fuzziness > 0 { + // fuzzy + ts, err = NewFuzzySearcher(ctx, indexReader, term, 0, fuzziness, field, boost, options) + } else { + // non-fuzzy + ts, err = NewTermSearcher(ctx, indexReader, term, field, boost, options) + } + if err != nil { + // close any searchers already opened + for _, ts := range termPositionSearchers { + _ = ts.Close() + } + return nil, fmt.Errorf("phrase searcher error building term searcher: %v", err) + } + termSearchers = append(termSearchers, ts) + } + disjunction, err := NewDisjunctionSearcher(ctx, indexReader, termSearchers, 1, options) + if err != nil { + // close any searchers already opened + for _, ts := range termPositionSearchers { + _ = ts.Close() + } + return nil, fmt.Errorf("phrase searcher error building term position disjunction searcher: %v", err) + } + termPositionSearchers = append(termPositionSearchers, disjunction) + } + } + + mustSearcher, err := NewConjunctionSearcher(ctx, indexReader, termPositionSearchers, options) + if err != nil { + // close any searchers already opened + for _, ts := range termPositionSearchers { + _ = ts.Close() + } + return nil, fmt.Errorf("phrase searcher error building conjunction searcher: %v", err) + } + + // build our searcher + rv := PhraseSearcher{ + mustSearcher: mustSearcher, + terms: terms, + fuzzyTermMatches: fuzzyTermMatches, + } + rv.computeQueryNorm() + return &rv, nil +} + +func (s *PhraseSearcher) computeQueryNorm() { + // first calculate sum of squared weights + sumOfSquaredWeights := 0.0 + if s.mustSearcher != nil { + sumOfSquaredWeights += s.mustSearcher.Weight() + } + + // now compute query norm from this + s.queryNorm = 1.0 / math.Sqrt(sumOfSquaredWeights) + // finally tell all the downstream searchers the norm + if s.mustSearcher != nil { + s.mustSearcher.SetQueryNorm(s.queryNorm) + } +} + +func (s *PhraseSearcher) initSearchers(ctx *search.SearchContext) error { + err := s.advanceNextMust(ctx) + if err != nil { + return err + } + + s.initialized = true + return nil +} + +func (s *PhraseSearcher) advanceNextMust(ctx *search.SearchContext) error { + var err error + + if s.mustSearcher != nil { + if s.currMust != nil { + ctx.DocumentMatchPool.Put(s.currMust) + } + s.currMust, err = s.mustSearcher.Next(ctx) + if err != nil { + return err + } + } + + return nil +} + +func (s *PhraseSearcher) Weight() float64 { + return s.mustSearcher.Weight() +} + +func (s *PhraseSearcher) SetQueryNorm(qnorm float64) { + s.mustSearcher.SetQueryNorm(qnorm) +} + +func (s *PhraseSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { + if !s.initialized { + err := s.initSearchers(ctx) + if err != nil { + return nil, err + } + } + + for s.currMust != nil { + // check this match against phrase constraints + rv := s.checkCurrMustMatch(ctx) + + // prepare for next iteration (either loop or subsequent call to Next()) + err := s.advanceNextMust(ctx) + if err != nil { + return nil, err + } + + // if match satisfied phrase constraints return it as a hit + if rv != nil { + return rv, nil + } + } + + return nil, nil +} + +// checkCurrMustMatch is solely concerned with determining if the DocumentMatch +// pointed to by s.currMust (which satisifies the pre-condition searcher) +// also satisfies the phrase constraints. if so, it returns a DocumentMatch +// for this document, otherwise nil +func (s *PhraseSearcher) checkCurrMustMatch(ctx *search.SearchContext) *search.DocumentMatch { + s.locations = s.currMust.Complete(s.locations) + + locations := s.currMust.Locations + s.currMust.Locations = nil + + ftls := s.currMust.FieldTermLocations + + // typically we would expect there to only actually be results in + // one field, but we allow for this to not be the case + // but, we note that phrase constraints can only be satisfied within + // a single field, so we can check them each independently + for field, tlm := range locations { + ftls = s.checkCurrMustMatchField(ctx, field, tlm, ftls) + } + + if len(ftls) > 0 { + // return match + rv := s.currMust + s.currMust = nil + rv.FieldTermLocations = ftls + return rv + } + + return nil +} + +// checkCurrMustMatchField is solely concerned with determining if one +// particular field within the currMust DocumentMatch Locations +// satisfies the phrase constraints (possibly more than once). if so, +// the matching field term locations are appended to the provided +// slice +func (s *PhraseSearcher) checkCurrMustMatchField(ctx *search.SearchContext, + field string, tlm search.TermLocationMap, + ftls []search.FieldTermLocation) []search.FieldTermLocation { + if s.path == nil { + s.path = make(phrasePath, 0, len(s.terms)) + } + var tlmPtr *search.TermLocationMap = &tlm + if s.fuzzyTermMatches != nil { + // if fuzzy search, we need to expand the tlm to include all the fuzzy matches + // Example - term is "foo" and fuzzy matches are "foo", "fool", "food" + // the non expanded tlm will be: + // foo -> Locations[foo] + // fool -> Locations[fool] + // food -> Locations[food] + // the expanded tlm will be: + // foo -> [Locations[foo], Locations[fool], Locations[food]] + expandedTlm := make(search.TermLocationMap) + s.expandFuzzyMatches(tlm, expandedTlm) + tlmPtr = &expandedTlm + } + s.paths = findPhrasePaths(0, nil, s.terms, *tlmPtr, s.path[:0], 0, s.paths[:0]) + for _, p := range s.paths { + for _, pp := range p { + ftls = append(ftls, search.FieldTermLocation{ + Field: field, + Term: pp.term, + Location: search.Location{ + Pos: pp.loc.Pos, + Start: pp.loc.Start, + End: pp.loc.End, + ArrayPositions: pp.loc.ArrayPositions, + }, + }) + } + } + return ftls +} + +func (s *PhraseSearcher) expandFuzzyMatches(tlm search.TermLocationMap, expandedTlm search.TermLocationMap) { + for term, fuzzyMatches := range s.fuzzyTermMatches { + locations := tlm[term] + for _, fuzzyMatch := range fuzzyMatches { + locations = append(locations, tlm[fuzzyMatch]...) + } + expandedTlm[term] = locations + } +} + +type phrasePart struct { + term string + loc *search.Location +} + +func (p *phrasePart) String() string { + return fmt.Sprintf("[%s %v]", p.term, p.loc) +} + +type phrasePath []phrasePart + +func (p phrasePath) MergeInto(in search.TermLocationMap) { + for _, pp := range p { + in[pp.term] = append(in[pp.term], pp.loc) + } +} + +func (p phrasePath) String() string { + rv := "[" + for i, pp := range p { + if i > 0 { + rv += ", " + } + rv += pp.String() + } + rv += "]" + return rv +} + +// findPhrasePaths is a function to identify phrase matches from a set +// of known term locations. it recursive so care must be taken with +// arguments and return values. +// +// prevPos - the previous location, 0 on first invocation +// +// ap - array positions of the first candidate phrase part to +// which further recursive phrase parts must match, +// nil on initial invocation or when there are no array positions +// +// phraseTerms - slice containing the phrase terms, +// may contain empty string as placeholder (don't care) +// +// tlm - the Term Location Map containing all relevant term locations +// +// p - the current path being explored (appended to in recursive calls) +// this is the primary state being built during the traversal +// +// remainingSlop - amount of sloppiness that's allowed, which is the +// sum of the editDistances from each matching phrase part, where 0 means no +// sloppiness allowed (all editDistances must be 0), decremented during recursion +// +// rv - the final result being appended to by all the recursive calls +// +// returns slice of paths, or nil if invocation did not find any successful paths +func findPhrasePaths(prevPos uint64, ap search.ArrayPositions, phraseTerms [][]string, + tlm search.TermLocationMap, p phrasePath, remainingSlop int, rv []phrasePath) []phrasePath { + // no more terms + if len(phraseTerms) < 1 { + // snapshot or copy the recursively built phrasePath p and + // append it to the rv, also optimizing by checking if next + // phrasePath item in the rv (which we're about to overwrite) + // is available for reuse + var pcopy phrasePath + if len(rv) < cap(rv) { + pcopy = rv[:len(rv)+1][len(rv)][:0] + } + return append(rv, append(pcopy, p...)) + } + + car := phraseTerms[0] + cdr := phraseTerms[1:] + + // empty term is treated as match (continue) + if len(car) == 0 || (len(car) == 1 && car[0] == "") { + nextPos := prevPos + 1 + if prevPos == 0 { + // if prevPos was 0, don't set it to 1 (as thats not a real abs pos) + nextPos = 0 // don't advance nextPos if prevPos was 0 + } + return findPhrasePaths(nextPos, ap, cdr, tlm, p, remainingSlop, rv) + } + + // locations for this term + for _, carTerm := range car { + locations := tlm[carTerm] + LOCATIONS_LOOP: + for _, loc := range locations { + if prevPos != 0 && !loc.ArrayPositions.Equals(ap) { + // if the array positions are wrong, can't match, try next location + continue + } + + // compute distance from previous phrase term + dist := 0 + if prevPos != 0 { + dist = editDistance(prevPos+1, loc.Pos) + } + + // if enough slop remaining, continue recursively + if prevPos == 0 || (remainingSlop-dist) >= 0 { + // skip if we've already used this term+loc already + for _, ppart := range p { + if ppart.term == carTerm && ppart.loc == loc { + continue LOCATIONS_LOOP + } + } + + // this location works, add it to the path (but not for empty term) + px := append(p, phrasePart{term: carTerm, loc: loc}) + rv = findPhrasePaths(loc.Pos, loc.ArrayPositions, cdr, tlm, px, remainingSlop-dist, rv) + } + } + } + return rv +} + +func editDistance(p1, p2 uint64) int { + dist := int(p1 - p2) + if dist < 0 { + return -dist + } + return dist +} + +func (s *PhraseSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { + if !s.initialized { + err := s.initSearchers(ctx) + if err != nil { + return nil, err + } + } + if s.currMust != nil { + if s.currMust.IndexInternalID.Compare(ID) >= 0 { + return s.Next(ctx) + } + ctx.DocumentMatchPool.Put(s.currMust) + } + if s.currMust == nil { + return nil, nil + } + var err error + s.currMust, err = s.mustSearcher.Advance(ctx, ID) + if err != nil { + return nil, err + } + return s.Next(ctx) +} + +func (s *PhraseSearcher) Count() uint64 { + // for now return a worst case + return s.mustSearcher.Count() +} + +func (s *PhraseSearcher) Close() error { + if s.mustSearcher != nil { + err := s.mustSearcher.Close() + if err != nil { + return err + } + } + return nil +} + +func (s *PhraseSearcher) Min() int { + return 0 +} + +func (s *PhraseSearcher) DocumentMatchPoolSize() int { + return s.mustSearcher.DocumentMatchPoolSize() + 1 +} diff --git a/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_regexp.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_regexp.go new file mode 100644 index 00000000..b88133e3 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_regexp.go @@ -0,0 +1,149 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "regexp" + + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +// The Regexp interface defines the subset of the regexp.Regexp API +// methods that are used by bleve indexes, allowing callers to pass in +// alternate implementations. +type Regexp interface { + FindStringIndex(s string) (loc []int) + + LiteralPrefix() (prefix string, complete bool) + + String() string +} + +// NewRegexpStringSearcher is similar to NewRegexpSearcher, but +// additionally optimizes for index readers that handle regexp's. +func NewRegexpStringSearcher(ctx context.Context, indexReader index.IndexReader, pattern string, + field string, boost float64, options search.SearcherOptions) ( + search.Searcher, error) { + ir, ok := indexReader.(index.IndexReaderRegexp) + if !ok { + r, err := regexp.Compile(pattern) + if err != nil { + return nil, err + } + + return NewRegexpSearcher(ctx, indexReader, r, field, boost, options) + } + + fieldDict, err := ir.FieldDictRegexp(field, pattern) + if err != nil { + return nil, err + } + defer func() { + if cerr := fieldDict.Close(); cerr != nil && err == nil { + err = cerr + } + }() + + var candidateTerms []string + + tfd, err := fieldDict.Next() + for err == nil && tfd != nil { + candidateTerms = append(candidateTerms, tfd.Term) + tfd, err = fieldDict.Next() + } + if err != nil { + return nil, err + } + + return NewMultiTermSearcher(ctx, indexReader, candidateTerms, field, boost, + options, true) +} + +// NewRegexpSearcher creates a searcher which will match documents that +// contain terms which match the pattern regexp. The match must be EXACT +// matching the entire term. The provided regexp SHOULD NOT start with ^ +// or end with $ as this can intefere with the implementation. Separately, +// matches will be checked to ensure they match the entire term. +func NewRegexpSearcher(ctx context.Context, indexReader index.IndexReader, pattern Regexp, + field string, boost float64, options search.SearcherOptions) ( + search.Searcher, error) { + var candidateTerms []string + var regexpCandidates *regexpCandidates + prefixTerm, complete := pattern.LiteralPrefix() + if complete { + // there is no pattern + candidateTerms = []string{prefixTerm} + } else { + var err error + regexpCandidates, err = findRegexpCandidateTerms(indexReader, pattern, field, + prefixTerm) + if err != nil { + return nil, err + } + } + var dictBytesRead uint64 + if regexpCandidates != nil { + candidateTerms = regexpCandidates.candidates + dictBytesRead = regexpCandidates.bytesRead + } + + if ctx != nil { + reportIOStats(ctx, dictBytesRead) + search.RecordSearchCost(ctx, search.AddM, dictBytesRead) + } + + return NewMultiTermSearcher(ctx, indexReader, candidateTerms, field, boost, + options, true) +} + +type regexpCandidates struct { + candidates []string + bytesRead uint64 +} + +func findRegexpCandidateTerms(indexReader index.IndexReader, + pattern Regexp, field, prefixTerm string) (rv *regexpCandidates, err error) { + rv = ®expCandidates{ + candidates: make([]string, 0), + } + var fieldDict index.FieldDict + if len(prefixTerm) > 0 { + fieldDict, err = indexReader.FieldDictPrefix(field, []byte(prefixTerm)) + } else { + fieldDict, err = indexReader.FieldDict(field) + } + defer func() { + if cerr := fieldDict.Close(); cerr != nil && err == nil { + err = cerr + } + }() + + // enumerate the terms and check against regexp + tfd, err := fieldDict.Next() + for err == nil && tfd != nil { + matchPos := pattern.FindStringIndex(tfd.Term) + if matchPos != nil && matchPos[0] == 0 && matchPos[1] == len(tfd.Term) { + rv.candidates = append(rv.candidates, tfd.Term) + if tooManyClauses(len(rv.candidates)) { + return rv, tooManyClausesErr(field, len(rv.candidates)) + } + } + tfd, err = fieldDict.Next() + } + rv.bytesRead = fieldDict.BytesRead() + return rv, err +} diff --git a/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_term.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_term.go new file mode 100644 index 00000000..cd794ea3 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_term.go @@ -0,0 +1,156 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "reflect" + + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/scorer" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" +) + +var reflectStaticSizeTermSearcher int + +func init() { + var ts TermSearcher + reflectStaticSizeTermSearcher = int(reflect.TypeOf(ts).Size()) +} + +type TermSearcher struct { + indexReader index.IndexReader + reader index.TermFieldReader + scorer *scorer.TermQueryScorer + tfd index.TermFieldDoc +} + +func NewTermSearcher(ctx context.Context, indexReader index.IndexReader, term string, field string, boost float64, options search.SearcherOptions) (*TermSearcher, error) { + if isTermQuery(ctx) { + ctx = context.WithValue(ctx, search.QueryTypeKey, search.Term) + } + return NewTermSearcherBytes(ctx, indexReader, []byte(term), field, boost, options) +} + +func NewTermSearcherBytes(ctx context.Context, indexReader index.IndexReader, term []byte, field string, boost float64, options search.SearcherOptions) (*TermSearcher, error) { + needFreqNorm := options.Score != "none" + reader, err := indexReader.TermFieldReader(ctx, term, field, needFreqNorm, needFreqNorm, options.IncludeTermVectors) + if err != nil { + return nil, err + } + return newTermSearcherFromReader(indexReader, reader, term, field, boost, options) +} + +func newTermSearcherFromReader(indexReader index.IndexReader, reader index.TermFieldReader, + term []byte, field string, boost float64, options search.SearcherOptions) (*TermSearcher, error) { + count, err := indexReader.DocCount() + if err != nil { + _ = reader.Close() + return nil, err + } + scorer := scorer.NewTermQueryScorer(term, field, boost, count, reader.Count(), options) + return &TermSearcher{ + indexReader: indexReader, + reader: reader, + scorer: scorer, + }, nil +} + +func (s *TermSearcher) Size() int { + return reflectStaticSizeTermSearcher + size.SizeOfPtr + + s.reader.Size() + + s.tfd.Size() + + s.scorer.Size() +} + +func (s *TermSearcher) Count() uint64 { + return s.reader.Count() +} + +func (s *TermSearcher) Weight() float64 { + return s.scorer.Weight() +} + +func (s *TermSearcher) SetQueryNorm(qnorm float64) { + s.scorer.SetQueryNorm(qnorm) +} + +func (s *TermSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { + termMatch, err := s.reader.Next(s.tfd.Reset()) + if err != nil { + return nil, err + } + + if termMatch == nil { + return nil, nil + } + + // score match + docMatch := s.scorer.Score(ctx, termMatch) + // return doc match + return docMatch, nil + +} + +func (s *TermSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { + termMatch, err := s.reader.Advance(ID, s.tfd.Reset()) + if err != nil { + return nil, err + } + + if termMatch == nil { + return nil, nil + } + + // score match + docMatch := s.scorer.Score(ctx, termMatch) + + // return doc match + return docMatch, nil +} + +func (s *TermSearcher) Close() error { + return s.reader.Close() +} + +func (s *TermSearcher) Min() int { + return 0 +} + +func (s *TermSearcher) DocumentMatchPoolSize() int { + return 1 +} + +func (s *TermSearcher) Optimize(kind string, octx index.OptimizableContext) ( + index.OptimizableContext, error) { + o, ok := s.reader.(index.Optimizable) + if ok { + return o.Optimize(kind, octx) + } + + return nil, nil +} + +func isTermQuery(ctx context.Context) bool { + if ctx != nil { + // if the ctx already has a value set for query type + // it would've been done at a non term searcher level. + _, ok := ctx.Value(search.QueryTypeKey).(string) + return !ok + } + // if the context is nil, then don't set the query type + return false +} diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_term_prefix.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_term_prefix.go similarity index 75% rename from vendor/github.com/blevesearch/bleve/search/searcher/search_term_prefix.go rename to vendor/github.com/blevesearch/bleve/v2/search/searcher/search_term_prefix.go index 2a8f22cf..dc16e486 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_term_prefix.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_term_prefix.go @@ -15,11 +15,13 @@ package searcher import ( - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" + "context" + + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" ) -func NewTermPrefixSearcher(indexReader index.IndexReader, prefix string, +func NewTermPrefixSearcher(ctx context.Context, indexReader index.IndexReader, prefix string, field string, boost float64, options search.SearcherOptions) ( search.Searcher, error) { // find the terms with this prefix @@ -46,5 +48,10 @@ func NewTermPrefixSearcher(indexReader index.IndexReader, prefix string, return nil, err } - return NewMultiTermSearcher(indexReader, terms, field, boost, options, true) + if ctx != nil { + reportIOStats(ctx, fieldDict.BytesRead()) + search.RecordSearchCost(ctx, search.AddM, fieldDict.BytesRead()) + } + + return NewMultiTermSearcher(ctx, indexReader, terms, field, boost, options, true) } diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_term_range.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_term_range.go similarity index 83% rename from vendor/github.com/blevesearch/bleve/search/searcher/search_term_range.go rename to vendor/github.com/blevesearch/bleve/v2/search/searcher/search_term_range.go index 90be1e11..990c7386 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_term_range.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_term_range.go @@ -15,11 +15,13 @@ package searcher import ( - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" + "context" + + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" ) -func NewTermRangeSearcher(indexReader index.IndexReader, +func NewTermRangeSearcher(ctx context.Context, indexReader index.IndexReader, min, max []byte, inclusiveMin, inclusiveMax *bool, field string, boost float64, options search.SearcherOptions) (search.Searcher, error) { @@ -81,5 +83,10 @@ func NewTermRangeSearcher(indexReader index.IndexReader, terms = terms[:len(terms)-1] } - return NewMultiTermSearcher(indexReader, terms, field, boost, options, true) + if ctx != nil { + reportIOStats(ctx, fieldDict.BytesRead()) + search.RecordSearchCost(ctx, search.AddM, fieldDict.BytesRead()) + } + + return NewMultiTermSearcher(ctx, indexReader, terms, field, boost, options, true) } diff --git a/vendor/github.com/blevesearch/bleve/v2/search/sort.go b/vendor/github.com/blevesearch/bleve/v2/search/sort.go new file mode 100644 index 00000000..b13fa16c --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/search/sort.go @@ -0,0 +1,750 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package search + +import ( + "bytes" + "encoding/json" + "fmt" + "math" + "sort" + "strings" + "unicode/utf8" + + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/util" +) + +var HighTerm = strings.Repeat(string(utf8.MaxRune), 3) +var LowTerm = string([]byte{0x00}) + +type SearchSort interface { + UpdateVisitor(field string, term []byte) + Value(a *DocumentMatch) string + Descending() bool + + RequiresDocID() bool + RequiresScoring() bool + RequiresFields() []string + + Reverse() + + Copy() SearchSort +} + +func ParseSearchSortObj(input map[string]interface{}) (SearchSort, error) { + descending, ok := input["desc"].(bool) + by, ok := input["by"].(string) + if !ok { + return nil, fmt.Errorf("search sort must specify by") + } + switch by { + case "id": + return &SortDocID{ + Desc: descending, + }, nil + case "score": + return &SortScore{ + Desc: descending, + }, nil + case "geo_distance": + field, ok := input["field"].(string) + if !ok { + return nil, fmt.Errorf("search sort mode geo_distance must specify field") + } + lon, lat, foundLocation := geo.ExtractGeoPoint(input["location"]) + if !foundLocation { + return nil, fmt.Errorf("unable to parse geo_distance location") + } + rvd := &SortGeoDistance{ + Field: field, + Desc: descending, + Lon: lon, + Lat: lat, + unitMult: 1.0, + } + if distUnit, ok := input["unit"].(string); ok { + var err error + rvd.unitMult, err = geo.ParseDistanceUnit(distUnit) + if err != nil { + return nil, err + } + rvd.Unit = distUnit + } + return rvd, nil + case "field": + field, ok := input["field"].(string) + if !ok { + return nil, fmt.Errorf("search sort mode field must specify field") + } + rv := &SortField{ + Field: field, + Desc: descending, + } + typ, ok := input["type"].(string) + if ok { + switch typ { + case "auto": + rv.Type = SortFieldAuto + case "string": + rv.Type = SortFieldAsString + case "number": + rv.Type = SortFieldAsNumber + case "date": + rv.Type = SortFieldAsDate + default: + return nil, fmt.Errorf("unknown sort field type: %s", typ) + } + } + mode, ok := input["mode"].(string) + if ok { + switch mode { + case "default": + rv.Mode = SortFieldDefault + case "min": + rv.Mode = SortFieldMin + case "max": + rv.Mode = SortFieldMax + default: + return nil, fmt.Errorf("unknown sort field mode: %s", mode) + } + } + missing, ok := input["missing"].(string) + if ok { + switch missing { + case "first": + rv.Missing = SortFieldMissingFirst + case "last": + rv.Missing = SortFieldMissingLast + default: + return nil, fmt.Errorf("unknown sort field missing: %s", missing) + } + } + return rv, nil + } + + return nil, fmt.Errorf("unknown search sort by: %s", by) +} + +func ParseSearchSortString(input string) SearchSort { + descending := false + if strings.HasPrefix(input, "-") { + descending = true + input = input[1:] + } else if strings.HasPrefix(input, "+") { + input = input[1:] + } + if input == "_id" { + return &SortDocID{ + Desc: descending, + } + } else if input == "_score" { + return &SortScore{ + Desc: descending, + } + } + return &SortField{ + Field: input, + Desc: descending, + } +} + +func ParseSearchSortJSON(input json.RawMessage) (SearchSort, error) { + // first try to parse it as string + var sortString string + err := util.UnmarshalJSON(input, &sortString) + if err != nil { + var sortObj map[string]interface{} + err = util.UnmarshalJSON(input, &sortObj) + if err != nil { + return nil, err + } + return ParseSearchSortObj(sortObj) + } + return ParseSearchSortString(sortString), nil +} + +func ParseSortOrderStrings(in []string) SortOrder { + rv := make(SortOrder, 0, len(in)) + for _, i := range in { + ss := ParseSearchSortString(i) + rv = append(rv, ss) + } + return rv +} + +func ParseSortOrderJSON(in []json.RawMessage) (SortOrder, error) { + rv := make(SortOrder, 0, len(in)) + for _, i := range in { + ss, err := ParseSearchSortJSON(i) + if err != nil { + return nil, err + } + rv = append(rv, ss) + } + return rv, nil +} + +type SortOrder []SearchSort + +func (so SortOrder) Value(doc *DocumentMatch) { + for _, soi := range so { + doc.Sort = append(doc.Sort, soi.Value(doc)) + } +} + +func (so SortOrder) UpdateVisitor(field string, term []byte) { + for _, soi := range so { + soi.UpdateVisitor(field, term) + } +} + +func (so SortOrder) Copy() SortOrder { + rv := make(SortOrder, len(so)) + for i, soi := range so { + rv[i] = soi.Copy() + } + return rv +} + +// Compare will compare two document matches using the specified sort order +// if both are numbers, we avoid converting back to term +func (so SortOrder) Compare(cachedScoring, cachedDesc []bool, i, j *DocumentMatch) int { + // compare the documents on all search sorts until a differences is found + for x := range so { + c := 0 + if cachedScoring[x] { + if i.Score < j.Score { + c = -1 + } else if i.Score > j.Score { + c = 1 + } + } else { + iVal := i.Sort[x] + jVal := j.Sort[x] + if iVal < jVal { + c = -1 + } else if iVal > jVal { + c = 1 + } + } + + if c == 0 { + continue + } + if cachedDesc[x] { + c = -c + } + return c + } + // if they are the same at this point, impose order based on index natural sort order + if i.HitNumber == j.HitNumber { + return 0 + } else if i.HitNumber > j.HitNumber { + return 1 + } + return -1 +} + +func (so SortOrder) RequiresScore() bool { + for _, soi := range so { + if soi.RequiresScoring() { + return true + } + } + return false +} + +func (so SortOrder) RequiresDocID() bool { + for _, soi := range so { + if soi.RequiresDocID() { + return true + } + } + return false +} + +func (so SortOrder) RequiredFields() []string { + var rv []string + for _, soi := range so { + rv = append(rv, soi.RequiresFields()...) + } + return rv +} + +func (so SortOrder) CacheIsScore() []bool { + rv := make([]bool, 0, len(so)) + for _, soi := range so { + rv = append(rv, soi.RequiresScoring()) + } + return rv +} + +func (so SortOrder) CacheDescending() []bool { + rv := make([]bool, 0, len(so)) + for _, soi := range so { + rv = append(rv, soi.Descending()) + } + return rv +} + +func (so SortOrder) Reverse() { + for _, soi := range so { + soi.Reverse() + } +} + +// SortFieldType lets you control some internal sort behavior +// normally leaving this to the zero-value of SortFieldAuto is fine +type SortFieldType int + +const ( + // SortFieldAuto applies heuristics attempt to automatically sort correctly + SortFieldAuto SortFieldType = iota + // SortFieldAsString forces sort as string (no prefix coded terms removed) + SortFieldAsString + // SortFieldAsNumber forces sort as string (prefix coded terms with shift > 0 removed) + SortFieldAsNumber + // SortFieldAsDate forces sort as string (prefix coded terms with shift > 0 removed) + SortFieldAsDate +) + +// SortFieldMode describes the behavior if the field has multiple values +type SortFieldMode int + +const ( + // SortFieldDefault uses the first (or only) value, this is the default zero-value + SortFieldDefault SortFieldMode = iota // FIXME name is confusing + // SortFieldMin uses the minimum value + SortFieldMin + // SortFieldMax uses the maximum value + SortFieldMax +) + +// SortFieldMissing controls where documents missing a field value should be sorted +type SortFieldMissing int + +const ( + // SortFieldMissingLast sorts documents missing a field at the end + SortFieldMissingLast SortFieldMissing = iota + + // SortFieldMissingFirst sorts documents missing a field at the beginning + SortFieldMissingFirst +) + +// SortField will sort results by the value of a stored field +// +// Field is the name of the field +// Descending reverse the sort order (default false) +// Type allows forcing of string/number/date behavior (default auto) +// Mode controls behavior for multi-values fields (default first) +// Missing controls behavior of missing values (default last) +type SortField struct { + Field string + Desc bool + Type SortFieldType + Mode SortFieldMode + Missing SortFieldMissing + values [][]byte + tmp [][]byte +} + +// UpdateVisitor notifies this sort field that in this document +// this field has the specified term +func (s *SortField) UpdateVisitor(field string, term []byte) { + if field == s.Field { + s.values = append(s.values, term) + } +} + +// Value returns the sort value of the DocumentMatch +// it also resets the state of this SortField for +// processing the next document +func (s *SortField) Value(i *DocumentMatch) string { + iTerms := s.filterTermsByType(s.values) + iTerm := s.filterTermsByMode(iTerms) + s.values = s.values[:0] + return iTerm +} + +// Descending determines the order of the sort +func (s *SortField) Descending() bool { + return s.Desc +} + +func (s *SortField) filterTermsByMode(terms [][]byte) string { + if len(terms) == 1 || (len(terms) > 1 && s.Mode == SortFieldDefault) { + return string(terms[0]) + } else if len(terms) > 1 { + switch s.Mode { + case SortFieldMin: + sort.Sort(BytesSlice(terms)) + return string(terms[0]) + case SortFieldMax: + sort.Sort(BytesSlice(terms)) + return string(terms[len(terms)-1]) + } + } + + // handle missing terms + if s.Missing == SortFieldMissingLast { + if s.Desc { + return LowTerm + } + return HighTerm + } + if s.Desc { + return HighTerm + } + return LowTerm +} + +// filterTermsByType attempts to make one pass on the terms +// if we are in auto-mode AND all the terms look like prefix-coded numbers +// return only the terms which had shift of 0 +// if we are in explicit number or date mode, return only valid +// prefix coded numbers with shift of 0 +func (s *SortField) filterTermsByType(terms [][]byte) [][]byte { + stype := s.Type + if stype == SortFieldAuto { + allTermsPrefixCoded := true + termsWithShiftZero := s.tmp[:0] + for _, term := range terms { + valid, shift := numeric.ValidPrefixCodedTermBytes(term) + if valid && shift == 0 { + termsWithShiftZero = append(termsWithShiftZero, term) + } else if !valid { + allTermsPrefixCoded = false + } + } + // reset the terms only when valid zero shift terms are found. + if allTermsPrefixCoded && len(termsWithShiftZero) > 0 { + terms = termsWithShiftZero + s.tmp = termsWithShiftZero[:0] + } + } else if stype == SortFieldAsNumber || stype == SortFieldAsDate { + termsWithShiftZero := s.tmp[:0] + for _, term := range terms { + valid, shift := numeric.ValidPrefixCodedTermBytes(term) + if valid && shift == 0 { + termsWithShiftZero = append(termsWithShiftZero, term) + } + } + terms = termsWithShiftZero + s.tmp = termsWithShiftZero[:0] + } + return terms +} + +// RequiresDocID says this SearchSort does not require the DocID be loaded +func (s *SortField) RequiresDocID() bool { return false } + +// RequiresScoring says this SearchStore does not require scoring +func (s *SortField) RequiresScoring() bool { return false } + +// RequiresFields says this SearchStore requires the specified stored field +func (s *SortField) RequiresFields() []string { return []string{s.Field} } + +func (s *SortField) MarshalJSON() ([]byte, error) { + // see if simple format can be used + if s.Missing == SortFieldMissingLast && + s.Mode == SortFieldDefault && + s.Type == SortFieldAuto { + if s.Desc { + return json.Marshal("-" + s.Field) + } + return json.Marshal(s.Field) + } + sfm := map[string]interface{}{ + "by": "field", + "field": s.Field, + } + if s.Desc { + sfm["desc"] = true + } + if s.Missing > SortFieldMissingLast { + switch s.Missing { + case SortFieldMissingFirst: + sfm["missing"] = "first" + } + } + if s.Mode > SortFieldDefault { + switch s.Mode { + case SortFieldMin: + sfm["mode"] = "min" + case SortFieldMax: + sfm["mode"] = "max" + } + } + if s.Type > SortFieldAuto { + switch s.Type { + case SortFieldAsString: + sfm["type"] = "string" + case SortFieldAsNumber: + sfm["type"] = "number" + case SortFieldAsDate: + sfm["type"] = "date" + } + } + + return json.Marshal(sfm) +} + +func (s *SortField) Copy() SearchSort { + rv := *s + return &rv +} + +func (s *SortField) Reverse() { + s.Desc = !s.Desc + if s.Missing == SortFieldMissingFirst { + s.Missing = SortFieldMissingLast + } else { + s.Missing = SortFieldMissingFirst + } +} + +// SortDocID will sort results by the document identifier +type SortDocID struct { + Desc bool +} + +// UpdateVisitor is a no-op for SortDocID as it's value +// is not dependent on any field terms +func (s *SortDocID) UpdateVisitor(field string, term []byte) { +} + +// Value returns the sort value of the DocumentMatch +func (s *SortDocID) Value(i *DocumentMatch) string { + return i.ID +} + +// Descending determines the order of the sort +func (s *SortDocID) Descending() bool { + return s.Desc +} + +// RequiresDocID says this SearchSort does require the DocID be loaded +func (s *SortDocID) RequiresDocID() bool { return true } + +// RequiresScoring says this SearchStore does not require scoring +func (s *SortDocID) RequiresScoring() bool { return false } + +// RequiresFields says this SearchStore does not require any stored fields +func (s *SortDocID) RequiresFields() []string { return nil } + +func (s *SortDocID) MarshalJSON() ([]byte, error) { + if s.Desc { + return json.Marshal("-_id") + } + return json.Marshal("_id") +} + +func (s *SortDocID) Copy() SearchSort { + rv := *s + return &rv +} + +func (s *SortDocID) Reverse() { + s.Desc = !s.Desc +} + +// SortScore will sort results by the document match score +type SortScore struct { + Desc bool +} + +// UpdateVisitor is a no-op for SortScore as it's value +// is not dependent on any field terms +func (s *SortScore) UpdateVisitor(field string, term []byte) { +} + +// Value returns the sort value of the DocumentMatch +func (s *SortScore) Value(i *DocumentMatch) string { + return "_score" +} + +// Descending determines the order of the sort +func (s *SortScore) Descending() bool { + return s.Desc +} + +// RequiresDocID says this SearchSort does not require the DocID be loaded +func (s *SortScore) RequiresDocID() bool { return false } + +// RequiresScoring says this SearchStore does require scoring +func (s *SortScore) RequiresScoring() bool { return true } + +// RequiresFields says this SearchStore does not require any store fields +func (s *SortScore) RequiresFields() []string { return nil } + +func (s *SortScore) MarshalJSON() ([]byte, error) { + if s.Desc { + return json.Marshal("-_score") + } + return json.Marshal("_score") +} + +func (s *SortScore) Copy() SearchSort { + rv := *s + return &rv +} + +func (s *SortScore) Reverse() { + s.Desc = !s.Desc +} + +var maxDistance = string(numeric.MustNewPrefixCodedInt64(math.MaxInt64, 0)) + +// NewSortGeoDistance creates SearchSort instance for sorting documents by +// their distance from the specified point. +func NewSortGeoDistance(field, unit string, lon, lat float64, desc bool) ( + *SortGeoDistance, error) { + rv := &SortGeoDistance{ + Field: field, + Desc: desc, + Unit: unit, + Lon: lon, + Lat: lat, + } + var err error + rv.unitMult, err = geo.ParseDistanceUnit(unit) + if err != nil { + return nil, err + } + return rv, nil +} + +// SortGeoDistance will sort results by the distance of an +// indexed geo point, from the provided location. +// +// Field is the name of the field +// Descending reverse the sort order (default false) +type SortGeoDistance struct { + Field string + Desc bool + Unit string + values []string + Lon float64 + Lat float64 + unitMult float64 +} + +// UpdateVisitor notifies this sort field that in this document +// this field has the specified term +func (s *SortGeoDistance) UpdateVisitor(field string, term []byte) { + if field == s.Field { + s.values = append(s.values, string(term)) + } +} + +// Value returns the sort value of the DocumentMatch +// it also resets the state of this SortField for +// processing the next document +func (s *SortGeoDistance) Value(i *DocumentMatch) string { + iTerms := s.filterTermsByType(s.values) + iTerm := s.filterTermsByMode(iTerms) + s.values = s.values[:0] + + if iTerm == "" { + return maxDistance + } + + i64, err := numeric.PrefixCoded(iTerm).Int64() + if err != nil { + return maxDistance + } + docLon := geo.MortonUnhashLon(uint64(i64)) + docLat := geo.MortonUnhashLat(uint64(i64)) + + dist := geo.Haversin(s.Lon, s.Lat, docLon, docLat) + // dist is returned in km, so convert to m + dist *= 1000 + if s.unitMult != 0 { + dist /= s.unitMult + } + distInt64 := numeric.Float64ToInt64(dist) + return string(numeric.MustNewPrefixCodedInt64(distInt64, 0)) +} + +// Descending determines the order of the sort +func (s *SortGeoDistance) Descending() bool { + return s.Desc +} + +func (s *SortGeoDistance) filterTermsByMode(terms []string) string { + if len(terms) >= 1 { + return terms[0] + } + + return "" +} + +// filterTermsByType attempts to make one pass on the terms +// return only valid prefix coded numbers with shift of 0 +func (s *SortGeoDistance) filterTermsByType(terms []string) []string { + var termsWithShiftZero []string + for _, term := range terms { + valid, shift := numeric.ValidPrefixCodedTerm(term) + if valid && shift == 0 { + termsWithShiftZero = append(termsWithShiftZero, term) + } + } + return termsWithShiftZero +} + +// RequiresDocID says this SearchSort does not require the DocID be loaded +func (s *SortGeoDistance) RequiresDocID() bool { return false } + +// RequiresScoring says this SearchStore does not require scoring +func (s *SortGeoDistance) RequiresScoring() bool { return false } + +// RequiresFields says this SearchStore requires the specified stored field +func (s *SortGeoDistance) RequiresFields() []string { return []string{s.Field} } + +func (s *SortGeoDistance) MarshalJSON() ([]byte, error) { + sfm := map[string]interface{}{ + "by": "geo_distance", + "field": s.Field, + "location": map[string]interface{}{ + "lon": s.Lon, + "lat": s.Lat, + }, + } + if s.Unit != "" { + sfm["unit"] = s.Unit + } + if s.Desc { + sfm["desc"] = true + } + + return json.Marshal(sfm) +} + +func (s *SortGeoDistance) Copy() SearchSort { + rv := *s + return &rv +} + +func (s *SortGeoDistance) Reverse() { + s.Desc = !s.Desc +} + +type BytesSlice [][]byte + +func (p BytesSlice) Len() int { return len(p) } +func (p BytesSlice) Less(i, j int) bool { return bytes.Compare(p[i], p[j]) < 0 } +func (p BytesSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } diff --git a/vendor/github.com/blevesearch/bleve/v2/search/util.go b/vendor/github.com/blevesearch/bleve/v2/search/util.go new file mode 100644 index 00000000..b2cb62a2 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/search/util.go @@ -0,0 +1,135 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package search + +import ( + "context" + + "github.com/blevesearch/geo/s2" +) + +func MergeLocations(locations []FieldTermLocationMap) FieldTermLocationMap { + rv := locations[0] + + for i := 1; i < len(locations); i++ { + nextLocations := locations[i] + for field, termLocationMap := range nextLocations { + rvTermLocationMap, rvHasField := rv[field] + if rvHasField { + rv[field] = MergeTermLocationMaps(rvTermLocationMap, termLocationMap) + } else { + rv[field] = termLocationMap + } + } + } + + return rv +} + +func MergeTermLocationMaps(rv, other TermLocationMap) TermLocationMap { + for term, locationMap := range other { + // for a given term/document there cannot be different locations + // if they came back from different clauses, overwrite is ok + rv[term] = locationMap + } + return rv +} + +func MergeFieldTermLocations(dest []FieldTermLocation, matches []*DocumentMatch) []FieldTermLocation { + n := len(dest) + for _, dm := range matches { + n += len(dm.FieldTermLocations) + } + if cap(dest) < n { + dest = append(make([]FieldTermLocation, 0, n), dest...) + } + + for _, dm := range matches { + for _, ftl := range dm.FieldTermLocations { + dest = append(dest, FieldTermLocation{ + Field: ftl.Field, + Term: ftl.Term, + Location: Location{ + Pos: ftl.Location.Pos, + Start: ftl.Location.Start, + End: ftl.Location.End, + ArrayPositions: append(ArrayPositions(nil), ftl.Location.ArrayPositions...), + }, + }) + } + } + + return dest +} + +const SearchIOStatsCallbackKey = "_search_io_stats_callback_key" + +type SearchIOStatsCallbackFunc func(uint64) + +// Implementation of SearchIncrementalCostCallbackFn should handle the following messages +// - add: increment the cost of a search operation +// (which can be specific to a query type as well) +// - abort: query was aborted due to a cancel of search's context (for eg), +// which can be handled differently as well +// - done: indicates that a search was complete and the tracked cost can be +// handled safely by the implementation. +type SearchIncrementalCostCallbackFn func(SearchIncrementalCostCallbackMsg, + SearchQueryType, uint64) +type SearchIncrementalCostCallbackMsg uint +type SearchQueryType uint + +const ( + Term = SearchQueryType(1 << iota) + Geo + Numeric + GenericCost +) + +const ( + AddM = SearchIncrementalCostCallbackMsg(1 << iota) + AbortM + DoneM +) + +const SearchIncrementalCostKey = "_search_incremental_cost_key" +const QueryTypeKey = "_query_type_key" +const FuzzyMatchPhraseKey = "_fuzzy_match_phrase_key" + +func RecordSearchCost(ctx context.Context, + msg SearchIncrementalCostCallbackMsg, bytes uint64) { + if ctx != nil { + queryType, ok := ctx.Value(QueryTypeKey).(SearchQueryType) + if !ok { + // for the cost of the non query type specific factors such as + // doc values and stored fields section. + queryType = GenericCost + } + + aggCallbackFn := ctx.Value(SearchIncrementalCostKey) + if aggCallbackFn != nil { + aggCallbackFn.(SearchIncrementalCostCallbackFn)(msg, queryType, bytes) + } + } +} + +const GeoBufferPoolCallbackKey = "_geo_buffer_pool_callback_key" + +// Assigning the size of the largest buffer in the pool to 24KB and +// the smallest buffer to 24 bytes. The pools are used to read a +// sequence of vertices which are always 24 bytes each. +const MaxGeoBufPoolSize = 24 * 1024 +const MinGeoBufPoolSize = 24 + +type GeoBufferPoolCallbackFunc func() *s2.GeoBufferPool diff --git a/vendor/github.com/blevesearch/bleve/size/sizes.go b/vendor/github.com/blevesearch/bleve/v2/size/sizes.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/size/sizes.go rename to vendor/github.com/blevesearch/bleve/v2/size/sizes.go diff --git a/vendor/github.com/blevesearch/bleve/v2/util/json.go b/vendor/github.com/blevesearch/bleve/v2/util/json.go new file mode 100644 index 00000000..8745dd8a --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/util/json.go @@ -0,0 +1,25 @@ +// Copyright (c) 2023 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "encoding/json" +) + +// Should only be overwritten during process init()'ialization. +var ( + MarshalJSON = json.Marshal + UnmarshalJSON = json.Unmarshal +) diff --git a/vendor/github.com/blevesearch/bleve_index_api/.golangci.yml b/vendor/github.com/blevesearch/bleve_index_api/.golangci.yml new file mode 100644 index 00000000..a00f6c57 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve_index_api/.golangci.yml @@ -0,0 +1,37 @@ +linters: + # please, do not use `enable-all`: it's deprecated and will be removed soon. + # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint + disable-all: true + enable: + - bodyclose + - deadcode + - depguard + - dogsled + - dupl + - errcheck + - goconst + - gocritic + - gocyclo + - gofmt + - goimports + - gomnd + - goprintffuncname + - gosimple + - govet + - ineffassign + - interfacer + - lll + - misspell + - nakedret + - nolintlint + - rowserrcheck + - scopelint + - staticcheck + - structcheck + - stylecheck + - typecheck + - unconvert + - unparam + - unused + - varcheck + - whitespace diff --git a/vendor/github.com/blevesearch/zap/v11/LICENSE b/vendor/github.com/blevesearch/bleve_index_api/LICENSE similarity index 100% rename from vendor/github.com/blevesearch/zap/v11/LICENSE rename to vendor/github.com/blevesearch/bleve_index_api/LICENSE diff --git a/vendor/github.com/blevesearch/bleve_index_api/README.md b/vendor/github.com/blevesearch/bleve_index_api/README.md new file mode 100644 index 00000000..46daa683 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve_index_api/README.md @@ -0,0 +1,11 @@ +# Bleve Index API + +[![PkgGoDev](https://pkg.go.dev/badge/github.com/blevesearch/bleve_index_api)](https://pkg.go.dev/github.com/blevesearch/bleve_index_api) +[![Tests](https://github.com/blevesearch/bleve_index_api/workflows/Tests/badge.svg?branch=master&event=push)](https://github.com/blevesearch/bleve_index_api/actions?query=workflow%3ATests+event%3Apush+branch%3Amaster) +[![Lint](https://github.com/blevesearch/bleve_index_api/workflows/Lint/badge.svg?branch=master&event=push)](https://github.com/blevesearch/bleve_index_api/actions?query=workflow%3ALint+event%3Apush+branch%3Amaster) + +Bleve supports a pluggable Index interface. + +By placing these interfaces in their own, *hopefully* slowly evolving module, it frees up Bleve and the underlying index to each introduce new major versions without interfering with one another. + +With that in mind, we anticipate introducing non-breaking changes only to this module, and keeping the major version at 1.x for some time. \ No newline at end of file diff --git a/vendor/github.com/blevesearch/bleve_index_api/analysis.go b/vendor/github.com/blevesearch/bleve_index_api/analysis.go new file mode 100644 index 00000000..5ab616c1 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve_index_api/analysis.go @@ -0,0 +1,53 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package index + +type AnalysisWork func() + +type AnalysisQueue struct { + queue chan AnalysisWork + done chan struct{} +} + +func (q *AnalysisQueue) Queue(work AnalysisWork) { + q.queue <- work +} + +func (q *AnalysisQueue) Close() { + close(q.done) +} + +func NewAnalysisQueue(numWorkers int) *AnalysisQueue { + rv := AnalysisQueue{ + queue: make(chan AnalysisWork), + done: make(chan struct{}), + } + for i := 0; i < numWorkers; i++ { + go AnalysisWorker(rv) + } + return &rv +} + +func AnalysisWorker(q AnalysisQueue) { + // read work off the queue + for { + select { + case <-q.done: + return + case w := <-q.queue: + w() + } + } +} diff --git a/vendor/github.com/blevesearch/bleve_index_api/batch.go b/vendor/github.com/blevesearch/bleve_index_api/batch.go new file mode 100644 index 00000000..ff1eaf6c --- /dev/null +++ b/vendor/github.com/blevesearch/bleve_index_api/batch.go @@ -0,0 +1,101 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package index + +import "fmt" + +type BatchCallback func(error) + +type Batch struct { + IndexOps map[string]Document + InternalOps map[string][]byte + persistedCallback BatchCallback +} + +func NewBatch() *Batch { + return &Batch{ + IndexOps: make(map[string]Document), + InternalOps: make(map[string][]byte), + } +} + +func (b *Batch) Update(doc Document) { + b.IndexOps[doc.ID()] = doc +} + +func (b *Batch) Delete(id string) { + b.IndexOps[id] = nil +} + +func (b *Batch) SetInternal(key, val []byte) { + b.InternalOps[string(key)] = val +} + +func (b *Batch) DeleteInternal(key []byte) { + b.InternalOps[string(key)] = nil +} + +func (b *Batch) SetPersistedCallback(f BatchCallback) { + b.persistedCallback = f +} + +func (b *Batch) PersistedCallback() BatchCallback { + return b.persistedCallback +} + +func (b *Batch) String() string { + rv := fmt.Sprintf("Batch (%d ops, %d internal ops)\n", len(b.IndexOps), len(b.InternalOps)) + for k, v := range b.IndexOps { + if v != nil { + rv += fmt.Sprintf("\tINDEX - '%s'\n", k) + } else { + rv += fmt.Sprintf("\tDELETE - '%s'\n", k) + } + } + for k, v := range b.InternalOps { + if v != nil { + rv += fmt.Sprintf("\tSET INTERNAL - '%s'\n", k) + } else { + rv += fmt.Sprintf("\tDELETE INTERNAL - '%s'\n", k) + } + } + return rv +} + +func (b *Batch) Reset() { + b.IndexOps = make(map[string]Document) + b.InternalOps = make(map[string][]byte) + b.persistedCallback = nil +} + +func (b *Batch) Merge(o *Batch) { + for k, v := range o.IndexOps { + b.IndexOps[k] = v + } + for k, v := range o.InternalOps { + b.InternalOps[k] = v + } +} + +func (b *Batch) TotalDocSize() int { + var s int + for k, v := range b.IndexOps { + if v != nil { + s += v.Size() + sizeOfString + } + s += len(k) + } + return s +} diff --git a/vendor/github.com/blevesearch/bleve_index_api/directory.go b/vendor/github.com/blevesearch/bleve_index_api/directory.go new file mode 100644 index 00000000..709a3845 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve_index_api/directory.go @@ -0,0 +1,23 @@ +// Copyright (c) 2021 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package index + +import ( + "io" +) + +type Directory interface { + GetWriter(filePath string) (io.WriteCloser, error) +} diff --git a/vendor/github.com/blevesearch/bleve_index_api/document.go b/vendor/github.com/blevesearch/bleve_index_api/document.go new file mode 100644 index 00000000..0f9012fd --- /dev/null +++ b/vendor/github.com/blevesearch/bleve_index_api/document.go @@ -0,0 +1,93 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package index + +import "time" + +type Document interface { + ID() string + Size() int + + VisitFields(visitor FieldVisitor) + VisitComposite(visitor CompositeFieldVisitor) + HasComposite() bool + + NumPlainTextBytes() uint64 + + AddIDField() + + StoredFieldsBytes() uint64 +} + +type FieldVisitor func(Field) + +type Field interface { + Name() string + Value() []byte + ArrayPositions() []uint64 + + EncodedFieldType() byte + + Analyze() + + Options() FieldIndexingOptions + + AnalyzedLength() int + AnalyzedTokenFrequencies() TokenFrequencies + + NumPlainTextBytes() uint64 +} + +type CompositeFieldVisitor func(field CompositeField) + +type CompositeField interface { + Field + + Compose(field string, length int, freq TokenFrequencies) +} + +type TextField interface { + Text() string +} + +type NumericField interface { + Number() (float64, error) +} + +type DateTimeField interface { + DateTime() (time.Time, string, error) +} + +type BooleanField interface { + Boolean() (bool, error) +} + +type GeoPointField interface { + Lon() (float64, error) + Lat() (float64, error) +} + +type GeoShapeField interface { + GeoShape() (GeoJSON, error) +} + +// TokenizableSpatialField is an optional interface for fields that +// supports pluggable custom hierarchial spatial token generation. +type TokenizableSpatialField interface { + // SetSpatialAnalyzerPlugin lets the index implementations to + // initialise relevant spatial analyzer plugins for the field + // to override the spatial token generations during the analysis phase. + SetSpatialAnalyzerPlugin(SpatialAnalyzerPlugin) +} diff --git a/vendor/github.com/blevesearch/bleve_index_api/freq.go b/vendor/github.com/blevesearch/bleve_index_api/freq.go new file mode 100644 index 00000000..5b6c7e1d --- /dev/null +++ b/vendor/github.com/blevesearch/bleve_index_api/freq.go @@ -0,0 +1,106 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package index + +import "reflect" + +var reflectStaticSizeTokenLocation int +var reflectStaticSizeTokenFreq int + +func init() { + var tl TokenLocation + reflectStaticSizeTokenLocation = int(reflect.TypeOf(tl).Size()) + var tf TokenFreq + reflectStaticSizeTokenFreq = int(reflect.TypeOf(tf).Size()) +} + +// TokenLocation represents one occurrence of a term at a particular location in +// a field. Start, End and Position have the same meaning as in analysis.Token. +// Field and ArrayPositions identify the field value in the source document. +// See document.Field for details. +type TokenLocation struct { + Field string + ArrayPositions []uint64 + Start int + End int + Position int +} + +func (tl *TokenLocation) Size() int { + rv := reflectStaticSizeTokenLocation + rv += len(tl.ArrayPositions) * sizeOfUint64 + return rv +} + +// TokenFreq represents all the occurrences of a term in all fields of a +// document. +type TokenFreq struct { + Term []byte + Locations []*TokenLocation + frequency int +} + +func (tf *TokenFreq) Size() int { + rv := reflectStaticSizeTokenFreq + rv += len(tf.Term) + for _, loc := range tf.Locations { + rv += loc.Size() + } + return rv +} + +func (tf *TokenFreq) Frequency() int { + return tf.frequency +} + +func (tf *TokenFreq) SetFrequency(frequency int) { + tf.frequency = frequency +} + +// TokenFrequencies maps document terms to their combined frequencies from all +// fields. +type TokenFrequencies map[string]*TokenFreq + +func (tfs TokenFrequencies) Size() int { + rv := sizeOfMap + rv += len(tfs) * (sizeOfString + sizeOfPtr) + for k, v := range tfs { + rv += len(k) + rv += v.Size() + } + return rv +} + +func (tfs TokenFrequencies) MergeAll(remoteField string, other TokenFrequencies) { + // walk the new token frequencies + for tfk, tf := range other { + // set the remoteField value in incoming token freqs + for _, l := range tf.Locations { + l.Field = remoteField + } + existingTf, exists := tfs[tfk] + if exists { + existingTf.Locations = append(existingTf.Locations, tf.Locations...) + existingTf.frequency += tf.frequency + } else { + tfs[tfk] = &TokenFreq{ + Term: tf.Term, + frequency: tf.frequency, + Locations: make([]*TokenLocation, len(tf.Locations)), + } + copy(tfs[tfk].Locations, tf.Locations) + } + } +} diff --git a/vendor/github.com/blevesearch/bleve_index_api/index.go b/vendor/github.com/blevesearch/bleve_index_api/index.go new file mode 100644 index 00000000..4c916d5c --- /dev/null +++ b/vendor/github.com/blevesearch/bleve_index_api/index.go @@ -0,0 +1,227 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package index + +import ( + "bytes" + "context" + "reflect" +) + +var reflectStaticSizeTermFieldDoc int +var reflectStaticSizeTermFieldVector int + +func init() { + var tfd TermFieldDoc + reflectStaticSizeTermFieldDoc = int(reflect.TypeOf(tfd).Size()) + var tfv TermFieldVector + reflectStaticSizeTermFieldVector = int(reflect.TypeOf(tfv).Size()) +} + +type Index interface { + Open() error + Close() error + + Update(doc Document) error + Delete(id string) error + Batch(batch *Batch) error + + SetInternal(key, val []byte) error + DeleteInternal(key []byte) error + + // Reader returns a low-level accessor on the index data. Close it to + // release associated resources. + Reader() (IndexReader, error) + + StatsMap() map[string]interface{} +} + +type IndexReader interface { + TermFieldReader(ctx context.Context, term []byte, field string, includeFreq, includeNorm, includeTermVectors bool) (TermFieldReader, error) + + // DocIDReader returns an iterator over all doc ids + // The caller must close returned instance to release associated resources. + DocIDReaderAll() (DocIDReader, error) + + DocIDReaderOnly(ids []string) (DocIDReader, error) + + FieldDict(field string) (FieldDict, error) + + // FieldDictRange is currently defined to include the start and end terms + FieldDictRange(field string, startTerm []byte, endTerm []byte) (FieldDict, error) + FieldDictPrefix(field string, termPrefix []byte) (FieldDict, error) + + Document(id string) (Document, error) + + DocValueReader(fields []string) (DocValueReader, error) + + Fields() ([]string, error) + + GetInternal(key []byte) ([]byte, error) + + DocCount() (uint64, error) + + ExternalID(id IndexInternalID) (string, error) + InternalID(id string) (IndexInternalID, error) + + Close() error +} + +type IndexReaderRegexp interface { + FieldDictRegexp(field string, regex string) (FieldDict, error) +} + +type IndexReaderFuzzy interface { + FieldDictFuzzy(field string, term string, fuzziness int, prefix string) (FieldDict, error) +} + +type IndexReaderContains interface { + FieldDictContains(field string) (FieldDictContains, error) +} + +// SpatialIndexPlugin is an optional interface for exposing the +// support for any custom analyzer plugins that are capable of +// generating hierarchial spatial tokens for both indexing and +// query purposes from the geo location data. +type SpatialIndexPlugin interface { + GetSpatialAnalyzerPlugin(typ string) (SpatialAnalyzerPlugin, error) +} + +type TermFieldVector struct { + Field string + ArrayPositions []uint64 + Pos uint64 + Start uint64 + End uint64 +} + +func (tfv *TermFieldVector) Size() int { + return reflectStaticSizeTermFieldVector + sizeOfPtr + + len(tfv.Field) + len(tfv.ArrayPositions)*sizeOfUint64 +} + +// IndexInternalID is an opaque document identifier interal to the index impl +type IndexInternalID []byte + +func (id IndexInternalID) Equals(other IndexInternalID) bool { + return id.Compare(other) == 0 +} + +func (id IndexInternalID) Compare(other IndexInternalID) int { + return bytes.Compare(id, other) +} + +type TermFieldDoc struct { + Term string + ID IndexInternalID + Freq uint64 + Norm float64 + Vectors []*TermFieldVector +} + +func (tfd *TermFieldDoc) Size() int { + sizeInBytes := reflectStaticSizeTermFieldDoc + sizeOfPtr + + len(tfd.Term) + len(tfd.ID) + + for _, entry := range tfd.Vectors { + sizeInBytes += entry.Size() + } + + return sizeInBytes +} + +// Reset allows an already allocated TermFieldDoc to be reused +func (tfd *TermFieldDoc) Reset() *TermFieldDoc { + // remember the []byte used for the ID + id := tfd.ID + vectors := tfd.Vectors + // idiom to copy over from empty TermFieldDoc (0 allocations) + *tfd = TermFieldDoc{} + // reuse the []byte already allocated (and reset len to 0) + tfd.ID = id[:0] + tfd.Vectors = vectors[:0] + return tfd +} + +// TermFieldReader is the interface exposing the enumeration of documents +// containing a given term in a given field. Documents are returned in byte +// lexicographic order over their identifiers. +type TermFieldReader interface { + // Next returns the next document containing the term in this field, or nil + // when it reaches the end of the enumeration. The preAlloced TermFieldDoc + // is optional, and when non-nil, will be used instead of allocating memory. + Next(preAlloced *TermFieldDoc) (*TermFieldDoc, error) + + // Advance resets the enumeration at specified document or its immediate + // follower. + Advance(ID IndexInternalID, preAlloced *TermFieldDoc) (*TermFieldDoc, error) + + // Count returns the number of documents contains the term in this field. + Count() uint64 + Close() error + + Size() int +} + +type DictEntry struct { + Term string + Count uint64 +} + +type FieldDict interface { + Next() (*DictEntry, error) + Close() error + + BytesRead() uint64 +} + +type FieldDictContains interface { + Contains(key []byte) (bool, error) + + BytesRead() uint64 +} + +// DocIDReader is the interface exposing enumeration of documents identifiers. +// Close the reader to release associated resources. +type DocIDReader interface { + // Next returns the next document internal identifier in the natural + // index order, nil when the end of the sequence is reached. + Next() (IndexInternalID, error) + + // Advance resets the iteration to the first internal identifier greater than + // or equal to ID. If ID is smaller than the start of the range, the iteration + // will start there instead. If ID is greater than or equal to the end of + // the range, Next() call will return io.EOF. + Advance(ID IndexInternalID) (IndexInternalID, error) + + Size() int + + Close() error +} + +type DocValueVisitor func(field string, term []byte) + +type DocValueReader interface { + VisitDocValues(id IndexInternalID, visitor DocValueVisitor) error + + BytesRead() uint64 +} + +// IndexBuilder is an interface supported by some index schemes +// to allow direct write-only index building +type IndexBuilder interface { + Index(doc Document) error + Close() error +} diff --git a/vendor/github.com/blevesearch/bleve_index_api/indexing_options.go b/vendor/github.com/blevesearch/bleve_index_api/indexing_options.go new file mode 100644 index 00000000..9724ccae --- /dev/null +++ b/vendor/github.com/blevesearch/bleve_index_api/indexing_options.go @@ -0,0 +1,77 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package index + +type FieldIndexingOptions int + +const ( + IndexField FieldIndexingOptions = 1 << iota + StoreField + IncludeTermVectors + DocValues + SkipFreqNorm +) + +func (o FieldIndexingOptions) IsIndexed() bool { + return o&IndexField != 0 +} + +func (o FieldIndexingOptions) IsStored() bool { + return o&StoreField != 0 +} + +func (o FieldIndexingOptions) IncludeTermVectors() bool { + return o&IncludeTermVectors != 0 +} + +func (o FieldIndexingOptions) IncludeDocValues() bool { + return o&DocValues != 0 +} + +func (o FieldIndexingOptions) SkipFreqNorm() bool { + return o&SkipFreqNorm != 0 +} + +func (o FieldIndexingOptions) String() string { + rv := "" + if o.IsIndexed() { + rv += "INDEXED" + } + if o.IsStored() { + if rv != "" { + rv += ", " + } + rv += "STORE" + } + if o.IncludeTermVectors() { + if rv != "" { + rv += ", " + } + rv += "TV" + } + if o.IncludeDocValues() { + if rv != "" { + rv += ", " + } + rv += "DV" + } + if !o.SkipFreqNorm() { + if rv != "" { + rv += ", " + } + rv += "FN" + } + return rv +} diff --git a/vendor/github.com/blevesearch/bleve_index_api/optimize.go b/vendor/github.com/blevesearch/bleve_index_api/optimize.go new file mode 100644 index 00000000..2b4e1244 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve_index_api/optimize.go @@ -0,0 +1,39 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package index + +// Optimizable represents an optional interface that implementable by +// optimizable resources (e.g., TermFieldReaders, Searchers). These +// optimizable resources are provided the same OptimizableContext +// instance, so that they can coordinate via dynamic interface +// casting. +type Optimizable interface { + Optimize(kind string, octx OptimizableContext) (OptimizableContext, error) +} + +// Represents a result of optimization -- see the Finish() method. +type Optimized interface{} + +type OptimizableContext interface { + // Once all the optimzable resources have been provided the same + // OptimizableContext instance, the optimization preparations are + // finished or completed via the Finish() method. + // + // Depending on the optimization being performed, the Finish() + // method might return a non-nil Optimized instance. For example, + // the Optimized instance might represent an optimized + // TermFieldReader instance. + Finish() (Optimized, error) +} diff --git a/vendor/github.com/blevesearch/bleve_index_api/sizes.go b/vendor/github.com/blevesearch/bleve_index_api/sizes.go new file mode 100644 index 00000000..65c2dd00 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve_index_api/sizes.go @@ -0,0 +1,35 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package index + +import ( + "reflect" +) + +func init() { + var m map[int]int + sizeOfMap = int(reflect.TypeOf(m).Size()) + var ptr *int + sizeOfPtr = int(reflect.TypeOf(ptr).Size()) + var str string + sizeOfString = int(reflect.TypeOf(str).Size()) + var u64 uint64 + sizeOfUint64 = int(reflect.TypeOf(u64).Size()) +} + +var sizeOfMap int +var sizeOfPtr int +var sizeOfString int +var sizeOfUint64 int diff --git a/vendor/github.com/blevesearch/bleve_index_api/spatial_plugin.go b/vendor/github.com/blevesearch/bleve_index_api/spatial_plugin.go new file mode 100644 index 00000000..bee04769 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve_index_api/spatial_plugin.go @@ -0,0 +1,47 @@ +// Copyright (c) 2022 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package index + +// SpatialAnalyzerPlugin is an interface for the custom spatial +// tokenizer implementations that supports the generation of spatial +// hierarchial tokens for both indexing and querying of geoJSON data. +type SpatialAnalyzerPlugin interface { + // Type returns the plugin type. eg: "s2". + Type() string + + // GetIndexTokens returns the tokens to be indexed for the + // given GeoJSON type data in the document. + GetIndexTokens(GeoJSON) []string + + // GetQueryTokens returns the tokens to be queried for the + // given GeoJSON type data in the document. + GetQueryTokens(GeoJSON) []string +} + +// GeoJSON is generic interface for any geoJSON shapes like +// points, polygon etc. +type GeoJSON interface { + // Returns the type of geoJSON shape. + Type() string + + // Checks whether the given shape intersects with current shape. + Intersects(GeoJSON) (bool, error) + + // Checks whether the given shape resides within the current shape. + Contains(GeoJSON) (bool, error) + + // Value returns the byte value for the shape. + Value() ([]byte, error) +} diff --git a/vendor/github.com/blevesearch/geo/LICENSE b/vendor/github.com/blevesearch/geo/LICENSE new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/vendor/github.com/blevesearch/geo/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/blevesearch/geo/geojson/geojson_s2_util.go b/vendor/github.com/blevesearch/geo/geojson/geojson_s2_util.go new file mode 100644 index 00000000..60d9aa89 --- /dev/null +++ b/vendor/github.com/blevesearch/geo/geojson/geojson_s2_util.go @@ -0,0 +1,319 @@ +// Copyright (c) 2022 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package geojson + +import ( + "strconv" + "strings" + + index "github.com/blevesearch/bleve_index_api" + "github.com/blevesearch/geo/s2" + "github.com/golang/geo/s1" +) + +// ------------------------------------------------------------------------ + +func polylineIntersectsPoint(pls []*s2.Polyline, + point *s2.Point) bool { + s2cell := s2.CellFromPoint(*point) + + for _, pl := range pls { + if pl.IntersectsCell(s2cell) { + return true + } + } + + return false +} + +func polylineIntersectsPolygons(pls []*s2.Polyline, + s2pgns []*s2.Polygon) bool { + // Early exit if the polygon contains any of the line's vertices. + for _, pl := range pls { + for i := 0; i < pl.NumEdges(); i++ { + edge := pl.Edge(i) + for _, s2pgn := range s2pgns { + if s2pgn.IntersectsCell(s2.CellFromPoint(edge.V0)) || + s2pgn.IntersectsCell(s2.CellFromPoint(edge.V1)) { + return true + } + } + } + } + + for _, pl := range pls { + for _, s2pgn := range s2pgns { + for i := 0; i < pl.NumEdges(); i++ { + for i := 0; i < s2pgn.NumEdges(); i++ { + edgeB := s2pgn.Edge(i) + latLng1 := s2.LatLngFromPoint(edgeB.V0) + latLng2 := s2.LatLngFromPoint(edgeB.V1) + pl2 := s2.PolylineFromLatLngs([]s2.LatLng{latLng1, latLng2}) + + if pl.Intersects(pl2) { + return true + } + } + } + } + } + + return false +} + +func geometryCollectionIntersectsShape(gc *GeometryCollection, + shapeIn index.GeoJSON) bool { + for _, shape := range gc.Members() { + intersects, err := shapeIn.Intersects(shape) + if err == nil && intersects { + return true + } + } + return false +} + +func polygonsContainsLineStrings(s2pgns []*s2.Polygon, + pls []*s2.Polyline) bool { + linesWithIn := make(map[int]struct{}) + checker := s2.NewCrossingEdgeQuery(s2.NewShapeIndex()) +nextLine: + for lineIndex, pl := range pls { + for i := 0; i < len(*pl)-1; i++ { + start := (*pl)[i] + end := (*pl)[i+1] + + for _, s2pgn := range s2pgns { + containsStart := s2pgn.ContainsPoint(start) + containsEnd := s2pgn.ContainsPoint(end) + if containsStart && containsEnd { + crossings := checker.Crossings(start, end, s2pgn, s2.CrossingTypeInterior) + if len(crossings) > 0 { + continue nextLine + } + linesWithIn[lineIndex] = struct{}{} + continue nextLine + } else { + for _, loop := range s2pgn.Loops() { + for i := 0; i < loop.NumVertices(); i++ { + if !containsStart && start.ApproxEqual(loop.Vertex(i)) { + containsStart = true + } else if !containsEnd && end.ApproxEqual(loop.Vertex(i)) { + containsEnd = true + } + if containsStart && containsEnd { + linesWithIn[lineIndex] = struct{}{} + continue nextLine + } + } + } + } + } + } + } + + return len(pls) == len(linesWithIn) +} + +func rectangleIntersectsWithPolygons(s2rect *s2.Rect, + s2pgns []*s2.Polygon) bool { + s2pgnFromRect := s2PolygonFromS2Rectangle(s2rect) + for _, s2pgn := range s2pgns { + if s2pgn.Intersects(s2pgnFromRect) { + return true + } + } + + return false +} + +func rectangleIntersectsWithLineStrings(s2rect *s2.Rect, + polylines []*s2.Polyline) bool { + // Early exit path if the envelope contains any of the linestring's vertices. + for _, pl := range polylines { + for i := 0; i < pl.NumEdges(); i++ { + edge := pl.Edge(i) + if s2rect.IntersectsCell(s2.CellFromPoint(edge.V0)) || + s2rect.IntersectsCell(s2.CellFromPoint(edge.V1)) { + return true + } + } + } + + for _, pl := range polylines { + for i := 0; i < pl.NumEdges(); i++ { + for j := 0; j < 4; j++ { + pl2 := s2.PolylineFromLatLngs([]s2.LatLng{s2rect.Vertex(j), + s2rect.Vertex((j + 1) % 4)}) + + if pl.Intersects(pl2) { + return true + } + } + } + } + + return false +} + +func s2PolygonFromCoordinates(coordinates [][][]float64) *s2.Polygon { + loops := make([]*s2.Loop, 0, len(coordinates)) + for _, loop := range coordinates { + var points []s2.Point + if loop[0][0] == loop[len(loop)-1][0] && loop[0][1] == loop[len(loop)-1][1] { + loop = loop[:len(loop)-1] + } + for _, point := range loop { + p := s2.PointFromLatLng(s2.LatLngFromDegrees(point[1], point[0])) + points = append(points, p) + } + s2loop := s2.LoopFromPoints(points) + loops = append(loops, s2loop) + } + + rv := s2.PolygonFromOrientedLoops(loops) + return rv +} + +func s2PolygonFromS2Rectangle(s2rect *s2.Rect) *s2.Polygon { + loops := make([]*s2.Loop, 0, 1) + var points []s2.Point + for j := 0; j < 4; j++ { + points = append(points, s2.PointFromLatLng(s2rect.Vertex(j%4))) + } + + loops = append(loops, s2.LoopFromPoints(points)) + return s2.PolygonFromLoops(loops) +} + +func DeduplicateTerms(terms []string) []string { + var rv []string + hash := make(map[string]struct{}, len(terms)) + for _, term := range terms { + if _, exists := hash[term]; !exists { + rv = append(rv, term) + hash[term] = struct{}{} + } + } + + return rv +} + +//---------------------------------------------------------------------- + +var earthRadiusInMeter = 6378137.0 + +func radiusInMetersToS1Angle(radius float64) s1.Angle { + return s1.Angle(radius / earthRadiusInMeter) +} + +func s2PolylinesFromCoordinates(coordinates [][][]float64) []*s2.Polyline { + var polylines []*s2.Polyline + for _, lines := range coordinates { + var latlngs []s2.LatLng + for _, line := range lines { + v := s2.LatLngFromDegrees(line[1], line[0]) + latlngs = append(latlngs, v) + } + polylines = append(polylines, s2.PolylineFromLatLngs(latlngs)) + } + return polylines +} + +func s2RectFromBounds(topLeft, bottomRight []float64) *s2.Rect { + rect := s2.EmptyRect() + rect = rect.AddPoint(s2.LatLngFromDegrees(topLeft[1], topLeft[0])) + rect = rect.AddPoint(s2.LatLngFromDegrees(bottomRight[1], bottomRight[0])) + return &rect +} + +func s2Cap(vertices []float64, radiusInMeter float64) *s2.Cap { + cp := s2.PointFromLatLng(s2.LatLngFromDegrees(vertices[1], vertices[0])) + angle := radiusInMetersToS1Angle(float64(radiusInMeter)) + cap := s2.CapFromCenterAngle(cp, angle) + return &cap +} + +func max(a, b float64) float64 { + if a >= b { + return a + } + return b +} + +func min(a, b float64) float64 { + if a >= b { + return b + } + return a +} + +func StripCoveringTerms(terms []string) []string { + rv := make([]string, 0, len(terms)) + for _, term := range terms { + if strings.HasPrefix(term, "$") { + rv = append(rv, term[1:]) + continue + } + rv = append(rv, term) + } + return DeduplicateTerms(rv) +} + +type distanceUnit struct { + conv float64 + suffixes []string +} + +var inch = distanceUnit{0.0254, []string{"in", "inch"}} +var yard = distanceUnit{0.9144, []string{"yd", "yards"}} +var feet = distanceUnit{0.3048, []string{"ft", "feet"}} +var kilom = distanceUnit{1000, []string{"km", "kilometers"}} +var nauticalm = distanceUnit{1852.0, []string{"nm", "nauticalmiles"}} +var millim = distanceUnit{0.001, []string{"mm", "millimeters"}} +var centim = distanceUnit{0.01, []string{"cm", "centimeters"}} +var miles = distanceUnit{1609.344, []string{"mi", "miles"}} +var meters = distanceUnit{1, []string{"m", "meters"}} + +var distanceUnits = []*distanceUnit{ + &inch, &yard, &feet, &kilom, &nauticalm, &millim, ¢im, &miles, &meters, +} + +// ParseDistance attempts to parse a distance string and return distance in +// meters. Example formats supported: +// "5in" "5inch" "7yd" "7yards" "9ft" "9feet" "11km" "11kilometers" +// "3nm" "3nauticalmiles" "13mm" "13millimeters" "15cm" "15centimeters" +// "17mi" "17miles" "19m" "19meters" +// If the unit cannot be determined, the entire string is parsed and the +// unit of meters is assumed. +// If the number portion cannot be parsed, 0 and the parse error are returned. +func ParseDistance(d string) (float64, error) { + for _, unit := range distanceUnits { + for _, unitSuffix := range unit.suffixes { + if strings.HasSuffix(d, unitSuffix) { + parsedNum, err := strconv.ParseFloat(d[0:len(d)-len(unitSuffix)], 64) + if err != nil { + return 0, err + } + return parsedNum * unit.conv, nil + } + } + } + // no unit matched, try assuming meters? + parsedNum, err := strconv.ParseFloat(d, 64) + if err != nil { + return 0, err + } + return parsedNum, nil +} diff --git a/vendor/github.com/blevesearch/geo/geojson/geojson_shapes_impl.go b/vendor/github.com/blevesearch/geo/geojson/geojson_shapes_impl.go new file mode 100644 index 00000000..39734589 --- /dev/null +++ b/vendor/github.com/blevesearch/geo/geojson/geojson_shapes_impl.go @@ -0,0 +1,1861 @@ +// Copyright (c) 2022 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package geojson + +import ( + "bufio" + "bytes" + "encoding/binary" + "encoding/json" + "fmt" + "strings" + + index "github.com/blevesearch/bleve_index_api" + + "github.com/blevesearch/geo/s2" +) + +// s2Serializable is an optional interface for implementations +// supporting custom serialisation of data based out of s2's +// encode method. +type s2Serializable interface { + // Marshal implementation should encode the shape using the + // s2's encode methods with appropriate prefix bytes to + // identify the type of the contents. + Marshal() ([]byte, error) +} + +const ( + PointType = "point" + MultiPointType = "multipoint" + LineStringType = "linestring" + MultiLineStringType = "multilinestring" + PolygonType = "polygon" + MultiPolygonType = "multipolygon" + GeometryCollectionType = "geometrycollection" + CircleType = "circle" + EnvelopeType = "envelope" +) + +// These are the byte prefixes for identifying the +// shape contained within the doc values byte slice +// while decoding the contents during the query +// filtering phase. +const ( + PointTypePrefix = byte(1) + MultiPointTypePrefix = byte(2) + LineStringTypePrefix = byte(3) + MultiLineStringTypePrefix = byte(4) + PolygonTypePrefix = byte(5) + MultiPolygonTypePrefix = byte(6) + GeometryCollectionTypePrefix = byte(7) + CircleTypePrefix = byte(8) + EnvelopeTypePrefix = byte(9) +) + +// compositeShape is an optional interface for the +// composite geoJSON shapes which is composed of +// multiple spatial shapes within it. Composite shapes +// like multipoint, multilinestring, multipolygon and +// geometrycollection shapes are supposed to implement +// this interface. +type compositeShape interface { + // Members implementation returns the + // geoJSON shapes composed within the shape. + Members() []index.GeoJSON +} + +// -------------------------------------------------------- +// Point represents the geoJSON point type and it +// implements the index.GeoJSON interface. +type Point struct { + Typ string `json:"type"` + Vertices []float64 `json:"coordinates"` + s2point *s2.Point +} + +func (p *Point) Type() string { + return strings.ToLower(p.Typ) +} + +func (p *Point) Value() ([]byte, error) { + return jsoniter.Marshal(p) +} + +func NewGeoJsonPoint(v []float64) index.GeoJSON { + rv := &Point{Typ: PointType, Vertices: v} + rv.init() + return rv +} + +func (p *Point) init() { + if p.s2point == nil { + s2point := s2.PointFromLatLng(s2.LatLngFromDegrees( + p.Vertices[1], p.Vertices[0])) + p.s2point = &s2point + } +} + +func (p *Point) Marshal() ([]byte, error) { + p.init() + + var b bytes.Buffer + b.Grow(32) + w := bufio.NewWriter(&b) + err := p.s2point.Encode(w) + if err != nil { + return nil, err + } + + w.Flush() + return append([]byte{PointTypePrefix}, b.Bytes()...), nil +} + +func (p *Point) Intersects(other index.GeoJSON) (bool, error) { + p.init() + s2cell := s2.CellFromPoint(*p.s2point) + + return checkCellIntersectsShape(&s2cell, p, other) +} + +func (p *Point) Contains(other index.GeoJSON) (bool, error) { + p.init() + s2cell := s2.CellFromPoint(*p.s2point) + + return checkCellContainsShape([]*s2.Cell{&s2cell}, other) +} + +func (p *Point) Coordinates() []float64 { + return p.Vertices +} + +// -------------------------------------------------------- +// MultiPoint represents the geoJSON multipoint type and it +// implements the index.GeoJSON interface as well as the +// compositeShap interface. +type MultiPoint struct { + Typ string `json:"type"` + Vertices [][]float64 `json:"coordinates"` + s2points []*s2.Point +} + +func NewGeoJsonMultiPoint(v [][]float64) index.GeoJSON { + rv := &MultiPoint{Typ: MultiPointType, Vertices: v} + rv.init() + return rv +} + +func (mp *MultiPoint) init() { + if mp.s2points == nil { + mp.s2points = make([]*s2.Point, len(mp.Vertices)) + for i, point := range mp.Vertices { + s2point := s2.PointFromLatLng(s2.LatLngFromDegrees( + point[1], point[0])) + mp.s2points[i] = &s2point + } + } +} + +func (p *MultiPoint) Marshal() ([]byte, error) { + p.init() + + var b bytes.Buffer + b.Grow(64) + w := bufio.NewWriter(&b) + + // first write the number of points. + count := int32(len(p.s2points)) + err := binary.Write(w, binary.BigEndian, count) + if err != nil { + return nil, err + } + // write the points. + for _, s2point := range p.s2points { + err := s2point.Encode(w) + if err != nil { + return nil, err + } + } + + w.Flush() + return append([]byte{MultiPointTypePrefix}, b.Bytes()...), nil +} + +func (p *MultiPoint) Type() string { + return strings.ToLower(p.Typ) +} + +func (mp *MultiPoint) Value() ([]byte, error) { + return jsoniter.Marshal(mp) +} + +func (p *MultiPoint) Intersects(other index.GeoJSON) (bool, error) { + p.init() + + for _, s2point := range p.s2points { + cell := s2.CellFromPoint(*s2point) + rv, err := checkCellIntersectsShape(&cell, p, other) + if rv && err == nil { + return rv, nil + } + } + + return false, nil +} + +func (p *MultiPoint) Contains(other index.GeoJSON) (bool, error) { + p.init() + s2cells := make([]*s2.Cell, 0, len(p.s2points)) + + for _, s2point := range p.s2points { + cell := s2.CellFromPoint(*s2point) + s2cells = append(s2cells, &cell) + } + + return checkCellContainsShape(s2cells, other) +} + +func (p *MultiPoint) Coordinates() [][]float64 { + return p.Vertices +} + +func (p *MultiPoint) Members() []index.GeoJSON { + if len(p.Vertices) > 0 && len(p.s2points) == 0 { + points := make([]index.GeoJSON, len(p.Vertices)) + for pos, vertices := range p.Vertices { + points[pos] = NewGeoJsonPoint(vertices) + } + return points + } + + points := make([]index.GeoJSON, len(p.s2points)) + for pos, point := range p.s2points { + points[pos] = &Point{s2point: point} + } + return points +} + +// -------------------------------------------------------- +// LineString represents the geoJSON linestring type and it +// implements the index.GeoJSON interface. +type LineString struct { + Typ string `json:"type"` + Vertices [][]float64 `json:"coordinates"` + pl *s2.Polyline +} + +func NewGeoJsonLinestring(points [][]float64) index.GeoJSON { + rv := &LineString{Typ: LineStringType, Vertices: points} + rv.init() + return rv +} + +func (ls *LineString) init() { + if ls.pl == nil { + latlngs := make([]s2.LatLng, len(ls.Vertices)) + for i, vertex := range ls.Vertices { + latlngs[i] = s2.LatLngFromDegrees(vertex[1], vertex[0]) + } + ls.pl = s2.PolylineFromLatLngs(latlngs) + } +} + +func (ls *LineString) Type() string { + return strings.ToLower(ls.Typ) +} + +func (ls *LineString) Value() ([]byte, error) { + return jsoniter.Marshal(ls) +} + +func (ls *LineString) Marshal() ([]byte, error) { + ls.init() + + var b bytes.Buffer + b.Grow(50) + w := bufio.NewWriter(&b) + err := ls.pl.Encode(w) + if err != nil { + return nil, err + } + + w.Flush() + return append([]byte{LineStringTypePrefix}, b.Bytes()...), nil +} + +func (ls *LineString) Intersects(other index.GeoJSON) (bool, error) { + ls.init() + + return checkLineStringsIntersectsShape([]*s2.Polyline{ls.pl}, ls, other) +} + +func (ls *LineString) Contains(other index.GeoJSON) (bool, error) { + return checkLineStringsContainsShape([]*s2.Polyline{ls.pl}, other) +} + +func (ls *LineString) Coordinates() [][]float64 { + return ls.Vertices +} + +// -------------------------------------------------------- +// MultiLineString represents the geoJSON multilinestring type +// and it implements the index.GeoJSON interface as well as the +// compositeShap interface. +type MultiLineString struct { + Typ string `json:"type"` + Vertices [][][]float64 `json:"coordinates"` + pls []*s2.Polyline +} + +func NewGeoJsonMultilinestring(points [][][]float64) index.GeoJSON { + rv := &MultiLineString{Typ: MultiLineStringType, Vertices: points} + rv.init() + return rv +} + +func (mls *MultiLineString) init() { + if mls.pls == nil { + mls.pls = s2PolylinesFromCoordinates(mls.Vertices) + } +} + +func (mls *MultiLineString) Type() string { + return strings.ToLower(mls.Typ) +} + +func (mls *MultiLineString) Value() ([]byte, error) { + return jsoniter.Marshal(mls) +} + +func (mls *MultiLineString) Marshal() ([]byte, error) { + mls.init() + + var b bytes.Buffer + b.Grow(256) + w := bufio.NewWriter(&b) + + // first write the number of linestrings. + count := int32(len(mls.pls)) + err := binary.Write(w, binary.BigEndian, count) + if err != nil { + return nil, err + } + // write the lines. + for _, ls := range mls.pls { + err := ls.Encode(w) + if err != nil { + return nil, err + } + } + + w.Flush() + return append([]byte{MultiLineStringTypePrefix}, b.Bytes()...), nil +} + +func (p *MultiLineString) Intersects(other index.GeoJSON) (bool, error) { + p.init() + return checkLineStringsIntersectsShape(p.pls, p, other) +} + +func (p *MultiLineString) Contains(other index.GeoJSON) (bool, error) { + return checkLineStringsContainsShape(p.pls, other) +} + +func (p *MultiLineString) Coordinates() [][][]float64 { + return p.Vertices +} + +func (p *MultiLineString) Members() []index.GeoJSON { + if len(p.Vertices) > 0 && len(p.pls) == 0 { + lines := make([]index.GeoJSON, len(p.Vertices)) + for pos, vertices := range p.Vertices { + lines[pos] = NewGeoJsonLinestring(vertices) + } + return lines + } + + lines := make([]index.GeoJSON, len(p.pls)) + for pos, pl := range p.pls { + lines[pos] = &LineString{pl: pl} + } + return lines +} + +// -------------------------------------------------------- +// Polygon represents the geoJSON polygon type +// and it implements the index.GeoJSON interface. +type Polygon struct { + Typ string `json:"type"` + Vertices [][][]float64 `json:"coordinates"` + s2pgn *s2.Polygon +} + +func NewGeoJsonPolygon(points [][][]float64) index.GeoJSON { + rv := &Polygon{Typ: PolygonType, Vertices: points} + rv.init() + return rv +} + +func (p *Polygon) init() { + if p.s2pgn == nil { + p.s2pgn = s2PolygonFromCoordinates(p.Vertices) + } +} + +func (p *Polygon) Type() string { + return strings.ToLower(p.Typ) +} + +func (p *Polygon) Value() ([]byte, error) { + return jsoniter.Marshal(p) +} + +func (p *Polygon) Marshal() ([]byte, error) { + p.init() + + var b bytes.Buffer + b.Grow(128) + w := bufio.NewWriter(&b) + err := p.s2pgn.Encode(w) + if err != nil { + return nil, err + } + + w.Flush() + return append([]byte{PolygonTypePrefix}, b.Bytes()...), nil +} + +func (p *Polygon) Intersects(other index.GeoJSON) (bool, error) { + // make an s2polygon for reuse. + p.init() + + return checkPolygonIntersectsShape(p.s2pgn, p, other) +} + +func (p *Polygon) Contains(other index.GeoJSON) (bool, error) { + // make an s2polygon for reuse. + p.init() + + return checkMultiPolygonContainsShape([]*s2.Polygon{p.s2pgn}, p, other) +} + +func (p *Polygon) Coordinates() [][][]float64 { + return p.Vertices +} + +// -------------------------------------------------------- +// MultiPolygon represents the geoJSON multipolygon type +// and it implements the index.GeoJSON interface as well as the +// compositeShap interface. +type MultiPolygon struct { + Typ string `json:"type"` + Vertices [][][][]float64 `json:"coordinates"` + s2pgns []*s2.Polygon +} + +func NewGeoJsonMultiPolygon(points [][][][]float64) index.GeoJSON { + rv := &MultiPolygon{Typ: MultiPolygonType, Vertices: points} + rv.init() + return rv +} + +func (p *MultiPolygon) init() { + if p.s2pgns == nil { + p.s2pgns = make([]*s2.Polygon, len(p.Vertices)) + for i, vertices := range p.Vertices { + pgn := s2PolygonFromCoordinates(vertices) + p.s2pgns[i] = pgn + } + } +} + +func (p *MultiPolygon) Type() string { + return strings.ToLower(p.Typ) +} + +func (p *MultiPolygon) Value() ([]byte, error) { + return jsoniter.Marshal(p) +} + +func (p *MultiPolygon) Marshal() ([]byte, error) { + p.init() + + var b bytes.Buffer + b.Grow(512) + w := bufio.NewWriter(&b) + + // first write the number of polygons. + count := int32(len(p.s2pgns)) + err := binary.Write(w, binary.BigEndian, count) + if err != nil { + return nil, err + } + // write the polygons. + for _, pgn := range p.s2pgns { + err := pgn.Encode(w) + if err != nil { + return nil, err + } + } + + w.Flush() + return append([]byte{MultiPolygonTypePrefix}, b.Bytes()...), nil +} + +func (p *MultiPolygon) Intersects(other index.GeoJSON) (bool, error) { + p.init() + + for _, pgn := range p.s2pgns { + rv, err := checkPolygonIntersectsShape(pgn, p, other) + if rv && err == nil { + return true, nil + } + } + + return false, nil +} + +func (p *MultiPolygon) Contains(other index.GeoJSON) (bool, error) { + p.init() + + return checkMultiPolygonContainsShape(p.s2pgns, p, other) +} + +func (p *MultiPolygon) Coordinates() [][][][]float64 { + return p.Vertices +} + +func (p *MultiPolygon) Members() []index.GeoJSON { + if len(p.Vertices) > 0 && len(p.s2pgns) == 0 { + polygons := make([]index.GeoJSON, len(p.Vertices)) + for pos, vertices := range p.Vertices { + polygons[pos] = NewGeoJsonPolygon(vertices) + } + return polygons + } + + polygons := make([]index.GeoJSON, len(p.s2pgns)) + for pos, pgn := range p.s2pgns { + polygons[pos] = &Polygon{s2pgn: pgn} + } + return polygons +} + +// -------------------------------------------------------- +// GeometryCollection represents the geoJSON geometryCollection type +// and it implements the index.GeoJSON interface as well as the +// compositeShap interface. +type GeometryCollection struct { + Typ string `json:"type"` + Shapes []index.GeoJSON `json:"geometries"` +} + +func (gc *GeometryCollection) Type() string { + return strings.ToLower(gc.Typ) +} + +func (gc *GeometryCollection) Value() ([]byte, error) { + return jsoniter.Marshal(gc) +} + +func (gc *GeometryCollection) Members() []index.GeoJSON { + shapes := make([]index.GeoJSON, 0, len(gc.Shapes)) + for _, shape := range gc.Shapes { + if cs, ok := shape.(compositeShape); ok { + shapes = append(shapes, cs.Members()...) + } else { + shapes = append(shapes, shape) + } + } + return shapes +} + +func (gc *GeometryCollection) Marshal() ([]byte, error) { + var b bytes.Buffer + b.Grow(512) + w := bufio.NewWriter(&b) + + // first write the number of shapes. + count := int32(len(gc.Shapes)) + err := binary.Write(w, binary.BigEndian, count) + if err != nil { + return nil, err + } + + var res []byte + for _, shape := range gc.Shapes { + if s, ok := shape.(s2Serializable); ok { + sb, err := s.Marshal() + if err != nil { + return nil, err + } + // write the length of each shape. + err = binary.Write(w, binary.BigEndian, int32(len(sb))) + if err != nil { + return nil, err + } + // track the shape contents. + res = append(res, sb...) + } + } + w.Flush() + + return append([]byte{GeometryCollectionTypePrefix}, append(b.Bytes(), res...)...), nil +} + +func (gc *GeometryCollection) Intersects(other index.GeoJSON) (bool, error) { + for _, shape := range gc.Members() { + + intersects, err := shape.Intersects(other) + if intersects && err == nil { + return true, nil + } + } + return false, nil +} + +func (gc *GeometryCollection) Contains(other index.GeoJSON) (bool, error) { + // handle composite target shapes explicitly + if cs, ok := other.(compositeShape); ok { + otherShapes := cs.Members() + shapesFoundWithIn := make(map[int]struct{}) + + nextShape: + for pos, shapeInDoc := range otherShapes { + for _, shape := range gc.Members() { + within, err := shape.Contains(shapeInDoc) + if within && err == nil { + shapesFoundWithIn[pos] = struct{}{} + continue nextShape + } + } + } + + return len(shapesFoundWithIn) == len(otherShapes), nil + } + + for _, shape := range gc.Members() { + within, err := shape.Contains(other) + if within && err == nil { + return true, nil + } + } + + return false, nil +} + +func (gc *GeometryCollection) UnmarshalJSON(data []byte) error { + tmp := struct { + Typ string `json:"type"` + Shapes []json.RawMessage `json:"geometries"` + }{} + + err := jsoniter.Unmarshal(data, &tmp) + if err != nil { + return err + } + gc.Typ = tmp.Typ + + for _, shape := range tmp.Shapes { + var t map[string]interface{} + err := jsoniter.Unmarshal(shape, &t) + if err != nil { + return err + } + + var typ string + + if val, ok := t["type"]; ok { + typ = strings.ToLower(val.(string)) + } else { + continue + } + + switch typ { + case PointType: + var p Point + err := jsoniter.Unmarshal(shape, &p) + if err != nil { + return err + } + p.init() + gc.Shapes = append(gc.Shapes, &p) + + case MultiPointType: + var mp MultiPoint + err := jsoniter.Unmarshal(shape, &mp) + if err != nil { + return err + } + mp.init() + gc.Shapes = append(gc.Shapes, &mp) + + case LineStringType: + var ls LineString + err := jsoniter.Unmarshal(shape, &ls) + if err != nil { + return err + } + ls.init() + gc.Shapes = append(gc.Shapes, &ls) + + case MultiLineStringType: + var mls MultiLineString + err := jsoniter.Unmarshal(shape, &mls) + if err != nil { + return err + } + mls.init() + gc.Shapes = append(gc.Shapes, &mls) + + case PolygonType: + var pgn Polygon + err := jsoniter.Unmarshal(shape, &pgn) + if err != nil { + return err + } + pgn.init() + gc.Shapes = append(gc.Shapes, &pgn) + + case MultiPolygonType: + var pgn MultiPolygon + err := jsoniter.Unmarshal(shape, &pgn) + if err != nil { + return err + } + pgn.init() + gc.Shapes = append(gc.Shapes, &pgn) + } + } + + return nil +} + +// -------------------------------------------------------- +// Circle represents a custom circle type and it +// implements the index.GeoJSON interface. +type Circle struct { + Typ string `json:"type"` + Vertices []float64 `json:"coordinates"` + Radius string `json:"radius"` + radiusInMeters float64 + s2cap *s2.Cap +} + +func NewGeoCircle(points []float64, + radius string) index.GeoJSON { + r, err := ParseDistance(radius) + if err != nil { + return nil + } + + return &Circle{Typ: CircleType, + Vertices: points, + Radius: radius, + radiusInMeters: r} +} + +func (c *Circle) Type() string { + return strings.ToLower(c.Typ) +} + +func (c *Circle) Value() ([]byte, error) { + return jsoniter.Marshal(c) +} + +func (c *Circle) init() { + if c.s2cap == nil { + c.s2cap = s2Cap(c.Vertices, c.radiusInMeters) + } +} + +func (c *Circle) Marshal() ([]byte, error) { + c.init() + + var b bytes.Buffer + b.Grow(40) + w := bufio.NewWriter(&b) + err := c.s2cap.Encode(w) + if err != nil { + return nil, err + } + + w.Flush() + return append([]byte{CircleTypePrefix}, b.Bytes()...), nil +} + +func (c *Circle) Intersects(other index.GeoJSON) (bool, error) { + c.init() + + return checkCircleIntersectsShape(c.s2cap, c, other) +} + +func (c *Circle) Contains(other index.GeoJSON) (bool, error) { + c.init() + return checkCircleContainsShape(c.s2cap, c, other) +} + +func (c *Circle) UnmarshalJSON(data []byte) error { + tmp := struct { + Typ string `json:"type"` + Vertices []float64 `json:"coordinates"` + Radius string `json:"radius"` + }{} + + err := jsoniter.Unmarshal(data, &tmp) + if err != nil { + return err + } + c.Typ = tmp.Typ + c.Vertices = tmp.Vertices + c.Radius = tmp.Radius + if tmp.Radius != "" { + c.radiusInMeters, err = ParseDistance(tmp.Radius) + } + + return err +} + +// -------------------------------------------------------- +// Envelope represents the envelope/bounding box type and it +// implements the index.GeoJSON interface. +type Envelope struct { + Typ string `json:"type"` + Vertices [][]float64 `json:"coordinates"` + r *s2.Rect +} + +func NewGeoEnvelope(points [][]float64) index.GeoJSON { + return &Envelope{Vertices: points, Typ: EnvelopeType} +} + +func (e *Envelope) Type() string { + return strings.ToLower(e.Typ) +} + +func (e *Envelope) Value() ([]byte, error) { + return jsoniter.Marshal(e) +} + +func (e *Envelope) init() { + if e.r == nil { + e.r = s2RectFromBounds(e.Vertices[0], e.Vertices[1]) + } +} + +func (e *Envelope) Marshal() ([]byte, error) { + e.init() + + var b bytes.Buffer + b.Grow(50) + w := bufio.NewWriter(&b) + err := e.r.Encode(w) + if err != nil { + return nil, err + } + + w.Flush() + return append([]byte{EnvelopeTypePrefix}, b.Bytes()...), nil +} + +func (e *Envelope) Intersects(other index.GeoJSON) (bool, error) { + e.init() + + return checkEnvelopeIntersectsShape(e.r, e, other) +} + +func (e *Envelope) Contains(other index.GeoJSON) (bool, error) { + e.init() + + return checkEnvelopeContainsShape(e.r, e, other) +} + +//-------------------------------------------------------- + +// checkCellIntersectsShape checks for intersection between +// the s2cell and the shape in the document. +func checkCellIntersectsShape(cell *s2.Cell, shapeIn, + other index.GeoJSON) (bool, error) { + // check if the other shape is a point. + if p2, ok := other.(*Point); ok { + s2cell := s2.CellFromPoint(*p2.s2point) + + if cell.IntersectsCell(s2cell) { + return true, nil + } + + return false, nil + } + + // check if the other shape is a multipoint. + if p2, ok := other.(*MultiPoint); ok { + // check the intersection for any point in the array. + for _, point := range p2.s2points { + s2cell := s2.CellFromPoint(*point) + + if cell.IntersectsCell(s2cell) { + return true, nil + } + } + + return false, nil + } + + // check if the other shape is a polygon. + if p2, ok := other.(*Polygon); ok { + + if p2.s2pgn.IntersectsCell(*cell) { + return true, nil + } + + return false, nil + } + + // check if the other shape is a multipolygon. + if p2, ok := other.(*MultiPolygon); ok { + // check the intersection for any polygon in the collection. + for _, s2pgn := range p2.s2pgns { + + if s2pgn.IntersectsCell(*cell) { + return true, nil + } + } + + return false, nil + } + + // check if the other shape is a linestring. + if p2, ok := other.(*LineString); ok { + for i := 0; i < p2.pl.NumEdges(); i++ { + edge := p2.pl.Edge(i) + start := s2.CellFromPoint(edge.V0) + end := s2.CellFromPoint(edge.V1) + if cell.IntersectsCell(start) || cell.IntersectsCell(end) { + return true, nil + } + } + + return false, nil + } + + // check if the other shape is a multilinestring. + if p2, ok := other.(*MultiLineString); ok { + // check the intersection for any linestring in the array. + for _, pl := range p2.pls { + for i := 0; i < pl.NumEdges(); i++ { + edge := pl.Edge(i) + start := s2.CellFromPoint(edge.V0) + end := s2.CellFromPoint(edge.V1) + if cell.IntersectsCell(start) || cell.IntersectsCell(end) { + return true, nil + } + } + } + + return false, nil + } + + // check if the other shape is a geometrycollection. + if gc, ok := other.(*GeometryCollection); ok { + // check for intersection across every member shape. + if geometryCollectionIntersectsShape(gc, shapeIn) { + return true, nil + } + + return false, nil + } + + // check if the other shape is a circle. + if c, ok := other.(*Circle); ok { + + if c.s2cap.IntersectsCell(*cell) { + return true, nil + } + + return false, nil + } + + // check if the other shape is an envelope. + if e, ok := other.(*Envelope); ok { + + if e.r.IntersectsCell(*cell) { + return true, nil + } + + return false, nil + } + + return false, fmt.Errorf("unknown geojson type: %s "+ + " found in document", other.Type()) +} + +// checkCellContainsShape checks whether the given shape in +// in the document is contained with the s2cell. +func checkCellContainsShape(cells []*s2.Cell, + other index.GeoJSON) (bool, error) { + // check if the other shape is a point. + if p2, ok := other.(*Point); ok { + for _, cell := range cells { + + if cell.ContainsPoint(*p2.s2point) { + return true, nil + } + } + + return false, nil + } + + // check if the other shape is a multipoint, if so containment is + // checked for every point in the multipoint with every given cells. + if p2, ok := other.(*MultiPoint); ok { + // check the containment for every point in the collection. + lookup := make(map[int]struct{}) + for _, cell := range cells { + for pos, point := range p2.s2points { + if _, done := lookup[pos]; done { + continue + } + // already processed all the points in the multipoint. + if len(lookup) == len(p2.s2points) { + return true, nil + } + + if cell.ContainsPoint(*point) { + lookup[pos] = struct{}{} + } + } + } + + return len(lookup) == len(p2.s2points), nil + } + + // as point is a non closed shape, containment isn't feasible + // for other higher dimensions. + return false, nil +} + +// ------------------------------------------------------------------------ + +// checkLineStringsIntersectsShape checks whether the given linestrings +// intersects with the shape in the document. +func checkLineStringsIntersectsShape(pls []*s2.Polyline, shapeIn, + other index.GeoJSON) (bool, error) { + // check if the other shape is a point. + if p2, ok := other.(*Point); ok { + if polylineIntersectsPoint(pls, p2.s2point) { + return true, nil + } + + return false, nil + } + + // check if the other shape is a multipoint. + if p2, ok := other.(*MultiPoint); ok { + // check the intersection for any point in the collection. + for _, point := range p2.s2points { + + if polylineIntersectsPoint(pls, point) { + return true, nil + } + } + + return false, nil + } + + // check if the other shape is a polygon. + if p2, ok := other.(*Polygon); ok { + if polylineIntersectsPolygons(pls, []*s2.Polygon{p2.s2pgn}) { + return true, nil + } + + return false, nil + } + + // check if the other shape is a multipolygon. + if p2, ok := other.(*MultiPolygon); ok { + // check the intersection for any polygon in the collection. + if polylineIntersectsPolygons(pls, p2.s2pgns) { + return true, nil + } + + return false, nil + } + + // check if the other shape is a linestring. + if ls, ok := other.(*LineString); ok { + for _, pl := range pls { + if ls.pl.Intersects(pl) { + return true, nil + } + } + + return false, nil + } + + // check if the other shape is a multilinestring. + if mls, ok := other.(*MultiLineString); ok { + for _, ls := range pls { + for _, docLineString := range mls.pls { + if ls.Intersects(docLineString) { + return true, nil + } + } + } + + return false, nil + } + + if gc, ok := other.(*GeometryCollection); ok { + // check whether the linestring intersects with any of the + // shapes Contains a geometrycollection. + if geometryCollectionIntersectsShape(gc, shapeIn) { + return true, nil + } + + return false, nil + } + + // check if the other shape is a circle. + if c, ok := other.(*Circle); ok { + centre := c.s2cap.Center() + for _, pl := range pls { + for i := 0; i < pl.NumEdges(); i++ { + edge := pl.Edge(i) + distance := s2.DistanceFromSegment(centre, edge.V0, edge.V1) + return distance <= c.s2cap.Radius(), nil + } + } + + return false, nil + } + + // check if the other shape is a envelope. + if e, ok := other.(*Envelope); ok { + res := rectangleIntersectsWithLineStrings(e.r, pls) + + return res, nil + } + + return false, fmt.Errorf("unknown geojson type: %s "+ + "found in document", other.Type()) +} + +// checkLineStringsContainsShape checks the containment for +// points and multipoints for the linestring vertices. +func checkLineStringsContainsShape(pls []*s2.Polyline, + other index.GeoJSON) (bool, error) { + return false, nil +} + +// ------------------------------------------------------------------------ + +// checkPolygonIntersectsShape checks the intersection between the +// s2 polygon and the other shapes in the documents. +func checkPolygonIntersectsShape(s2pgn *s2.Polygon, shapeIn, + other index.GeoJSON) (bool, error) { + // check if the other shape is a point. + if p2, ok := other.(*Point); ok { + + s2cell := s2.CellFromPoint(*p2.s2point) + if s2pgn.IntersectsCell(s2cell) { + return true, nil + } + + return false, nil + } + + // check if the other shape is a multipoint. + if p2, ok := other.(*MultiPoint); ok { + + for _, s2point := range p2.s2points { + s2cell := s2.CellFromPoint(*s2point) + if s2pgn.IntersectsCell(s2cell) { + return true, nil + } + } + + return false, nil + } + + // check if the other shape is a polygon. + if p2, ok := other.(*Polygon); ok { + + if s2pgn.Intersects(p2.s2pgn) { + return true, nil + } + + return false, nil + } + + // check if the other shape is a multipolygon. + if p2, ok := other.(*MultiPolygon); ok { + // check the intersection for any polygon in the collection. + for _, s2pgn1 := range p2.s2pgns { + + if s2pgn.Intersects(s2pgn1) { + return true, nil + } + } + + return false, nil + } + + // check if the other shape is a linestring. + if ls, ok := other.(*LineString); ok { + + if polylineIntersectsPolygons([]*s2.Polyline{ls.pl}, + []*s2.Polygon{s2pgn}) { + return true, nil + } + + return false, nil + } + + // check if the other shape is a multilinestring. + if mls, ok := other.(*MultiLineString); ok { + + if polylineIntersectsPolygons(mls.pls, []*s2.Polygon{s2pgn}) { + return true, nil + } + + return false, nil + } + + if gc, ok := other.(*GeometryCollection); ok { + // check whether the polygon intersects with any of the + // member shapes of the geometry collection. + if geometryCollectionIntersectsShape(gc, shapeIn) { + return true, nil + } + + return false, nil + } + + // check if the other shape is a circle. + if c, ok := other.(*Circle); ok { + cp := c.s2cap.Center() + radius := c.s2cap.Radius() + + projected := s2pgn.Project(&cp) + distance := projected.Distance(cp) + + return distance <= radius, nil + } + + // check if the other shape is a envelope. + if e, ok := other.(*Envelope); ok { + + s2pgnInDoc := s2PolygonFromS2Rectangle(e.r) + if s2pgn.Intersects(s2pgnInDoc) { + return true, nil + } + return false, nil + } + + return false, fmt.Errorf("unknown geojson type: %s "+ + " found in document", other.Type()) +} + +// checkMultiPolygonContainsShape checks whether the given polygons +// collectively contains the shape in the document. +func checkMultiPolygonContainsShape(s2pgns []*s2.Polygon, + shapeIn, other index.GeoJSON) (bool, error) { + // check if the other shape is a point. + if p2, ok := other.(*Point); ok { + + for _, s2pgn := range s2pgns { + if s2pgn.ContainsPoint(*p2.s2point) { + return true, nil + } + } + + return false, nil + } + + // check if the other shape is a multipoint. + if p2, ok := other.(*MultiPoint); ok { + // check the containment for every point in the collection. + pointsWithIn := make(map[int]struct{}) + nextPoint: + for pointIndex, point := range p2.s2points { + + for _, s2pgn := range s2pgns { + if s2pgn.ContainsPoint(*point) { + pointsWithIn[pointIndex] = struct{}{} + continue nextPoint + } else { + // double check for containment with the vertices. + for _, loop := range s2pgn.Loops() { + for i := 0; i < loop.NumVertices(); i++ { + if point.ApproxEqual(loop.Vertex(i)) { + pointsWithIn[pointIndex] = struct{}{} + continue nextPoint + } + } + } + } + } + } + + return len(p2.s2points) == len(pointsWithIn), nil + } + + // check if the other shape is a polygon. + if p2, ok := other.(*Polygon); ok { + + for _, s2pgn := range s2pgns { + if s2pgn.Contains(p2.s2pgn) { + return true, nil + } + } + + return false, nil + } + + // check if the other shape is a multipolygon. + if p2, ok := other.(*MultiPolygon); ok { + // check the intersection for every polygon in the collection. + polygonsWithIn := make(map[int]struct{}) + + nextPolygon: + for pgnIndex, pgn := range p2.s2pgns { + for _, s2pgn := range s2pgns { + if s2pgn.Contains(pgn) { + polygonsWithIn[pgnIndex] = struct{}{} + continue nextPolygon + } + } + } + + return len(p2.s2pgns) == len(polygonsWithIn), nil + } + + // check if the other shape is a linestring. + if ls, ok := other.(*LineString); ok { + + if polygonsContainsLineStrings(s2pgns, + []*s2.Polyline{ls.pl}) { + return true, nil + } + + return false, nil + } + + // check if the other shape is a multilinestring. + if mls, ok := other.(*MultiLineString); ok { + // check whether any of the linestring is inside the polygon. + if polygonsContainsLineStrings(s2pgns, mls.pls) { + return true, nil + } + + return false, nil + } + + if gc, ok := other.(*GeometryCollection); ok { + shapesWithIn := make(map[int]struct{}) + nextShape: + for pos, shape := range gc.Members() { + for _, s2pgn := range s2pgns { + contains, err := checkMultiPolygonContainsShape( + []*s2.Polygon{s2pgn}, shapeIn, shape) + if err == nil && contains { + shapesWithIn[pos] = struct{}{} + continue nextShape + } + } + } + return len(shapesWithIn) == len(gc.Members()), nil + } + + // check if the other shape is a circle. + if c, ok := other.(*Circle); ok { + cp := c.s2cap.Center() + radius := c.s2cap.Radius() + + for _, s2pgn := range s2pgns { + + if s2pgn.ContainsPoint(cp) { + projected := s2pgn.ProjectToBoundary(&cp) + distance := projected.Distance(cp) + if distance >= radius { + return true, nil + } + } + } + + return false, nil + } + + // check if the other shape is a envelope. + if e, ok := other.(*Envelope); ok { + // create a polygon from the rectangle and checks the containment. + s2pgnInDoc := s2PolygonFromS2Rectangle(e.r) + for _, s2pgn := range s2pgns { + if s2pgn.Contains(s2pgnInDoc) { + return true, nil + } + } + + return false, nil + } + + return false, fmt.Errorf("unknown geojson type: %s"+ + " found in document", other.Type()) +} + +// ------------------------------------------------------------------------ + +// checkCircleIntersectsShape checks for intersection of the +// shape in the document with the circle. +func checkCircleIntersectsShape(s2cap *s2.Cap, shapeIn, + other index.GeoJSON) (bool, error) { + // check if the other shape is a point. + if p2, ok := other.(*Point); ok { + s2cell := s2.CellFromPoint(*p2.s2point) + + if s2cap.IntersectsCell(s2cell) { + return true, nil + } + + return false, nil + } + + // check if the other shape is a multipoint. + if p2, ok := other.(*MultiPoint); ok { + // check the intersection for any point in the collection. + for _, point := range p2.s2points { + s2cell := s2.CellFromPoint(*point) + + if s2cap.IntersectsCell(s2cell) { + return true, nil + } + } + + return false, nil + } + + // check if the other shape is a polygon. + if p2, ok := other.(*Polygon); ok { + centerPoint := s2cap.Center() + projected := p2.s2pgn.Project(¢erPoint) + distance := projected.Distance(centerPoint) + return distance <= s2cap.Radius(), nil + } + + // check if the other shape is a multipolygon. + if p2, ok := other.(*MultiPolygon); ok { + // check the intersection for any polygon in the collection. + for _, s2pgn := range p2.s2pgns { + centerPoint := s2cap.Center() + projected := s2pgn.Project(¢erPoint) + distance := projected.Distance(centerPoint) + return distance <= s2cap.Radius(), nil + } + + return false, nil + } + + // check if the other shape is a linestring. + if p2, ok := other.(*LineString); ok { + projected, _ := p2.pl.Project(s2cap.Center()) + distance := projected.Distance(s2cap.Center()) + return distance <= s2cap.Radius(), nil + } + + // check if the other shape is a multilinestring. + if p2, ok := other.(*MultiLineString); ok { + for _, pl := range p2.pls { + projected, _ := pl.Project(s2cap.Center()) + distance := projected.Distance(s2cap.Center()) + if distance <= s2cap.Radius() { + return true, nil + } + } + + return false, nil + } + + if gc, ok := other.(*GeometryCollection); ok { + // check whether the circle intersects with any of the + // member shapes Contains the geometrycollection. + if geometryCollectionIntersectsShape(gc, shapeIn) { + return true, nil + } + return false, nil + } + + // check if the other shape is a circle. + if c, ok := other.(*Circle); ok { + if s2cap.Intersects(*c.s2cap) { + return true, nil + } + return false, nil + } + + // check if the other shape is a envelope. + if e, ok := other.(*Envelope); ok { + + if e.r.ContainsPoint(s2cap.Center()) { + return true, nil + } + + latlngs := []s2.LatLng{e.r.Vertex(0), e.r.Vertex(1), + e.r.Vertex(2), e.r.Vertex(3), e.r.Vertex(0)} + pl := s2.PolylineFromLatLngs(latlngs) + projected, _ := pl.Project(s2cap.Center()) + distance := projected.Distance(s2cap.Center()) + if distance <= s2cap.Radius() { + return true, nil + } + + return false, nil + } + + return false, fmt.Errorf("unknown geojson type: %s"+ + " found in document", other.Type()) +} + +// checkCircleContainsShape checks for containment of the +// shape in the document with the circle. +func checkCircleContainsShape(s2cap *s2.Cap, + shapeIn, other index.GeoJSON) (bool, error) { + // check if the other shape is a point. + if p2, ok := other.(*Point); ok { + + if s2cap.ContainsPoint(*p2.s2point) { + return true, nil + } + + return false, nil + } + + // check if the other shape is a multipoint. + if p2, ok := other.(*MultiPoint); ok { + // check the intersection for every point in the collection. + for _, point := range p2.s2points { + if !s2cap.ContainsPoint(*point) { + return false, nil + } + } + + return true, nil + } + + // check if the other shape is a polygon. + if p2, ok := other.(*Polygon); ok { + for i := 0; i < p2.s2pgn.NumEdges(); i++ { + edge := p2.s2pgn.Edge(i) + if !s2cap.ContainsPoint(edge.V0) || + !s2cap.ContainsPoint(edge.V1) { + return false, nil + } + } + return true, nil + } + + // check if the other shape is a multipolygon. + if p2, ok := other.(*MultiPolygon); ok { + // check the containment for every polygon in the collection. + for _, s2pgn := range p2.s2pgns { + for i := 0; i < s2pgn.NumEdges(); i++ { + edge := s2pgn.Edge(i) + if !s2cap.ContainsPoint(edge.V0) || + !s2cap.ContainsPoint(edge.V1) { + return false, nil + } + } + } + + return true, nil + } + + // check if the other shape is a linestring. + if p2, ok := other.(*LineString); ok { + for i := 0; i < p2.pl.NumEdges(); i++ { + edge := p2.pl.Edge(i) + // check whether both the end vertices are inside the circle. + if s2cap.ContainsPoint(edge.V0) && + s2cap.ContainsPoint(edge.V1) { + return true, nil + } + } + return false, nil + } + + // check if the other shape is a multilinestring. + if p2, ok := other.(*MultiLineString); ok { + for _, pl := range p2.pls { + for i := 0; i < pl.NumEdges(); i++ { + edge := pl.Edge(i) + // check whether both the end vertices are inside the circle. + if !(s2cap.ContainsPoint(edge.V0) && s2cap.ContainsPoint(edge.V1)) { + return false, nil + } + } + } + return true, nil + } + + if gc, ok := other.(*GeometryCollection); ok { + for _, shape := range gc.Members() { + contains, err := shapeIn.Contains(shape) + if err == nil && !contains { + return false, nil + } + } + return true, nil + } + + // check if the other shape is a circle. + if c, ok := other.(*Circle); ok { + + if s2cap.Contains(*c.s2cap) { + return true, nil + } + + return false, nil + } + + // check if the other shape is a envelope. + if e, ok := other.(*Envelope); ok { + + for i := 0; i < 4; i++ { + if !s2cap.ContainsPoint( + s2.PointFromLatLng(e.r.Vertex(i))) { + return false, nil + } + } + + return true, nil + } + + return false, fmt.Errorf("unknown geojson type: %s"+ + " found in document", other.Type()) +} + +// ------------------------------------------------------------------------ + +// checkEnvelopeIntersectsShape checks whether the given shape in +// the document is intersecting Contains the envelope/rectangle. +func checkEnvelopeIntersectsShape(s2rect *s2.Rect, shapeIn, + other index.GeoJSON) (bool, error) { + // check if the other shape is a point. + if p2, ok := other.(*Point); ok { + s2cell := s2.CellFromPoint(*p2.s2point) + + if s2rect.IntersectsCell(s2cell) { + return true, nil + } + + return false, nil + } + + // check if the other shape is a multipoint. + if p2, ok := other.(*MultiPoint); ok { + // check the intersection for any point in the collection. + for _, point := range p2.s2points { + s2cell := s2.CellFromPoint(*point) + + if s2rect.IntersectsCell(s2cell) { + return true, nil + } + } + + return false, nil + } + + // check if the other shape is a polygon. + if pgn, ok := other.(*Polygon); ok { + + if rectangleIntersectsWithPolygons(s2rect, + []*s2.Polygon{pgn.s2pgn}) { + return true, nil + } + + return false, nil + } + + // check if the other shape is a multipolygon. + if mpgn, ok := other.(*MultiPolygon); ok { + // check the intersection for any polygon in the collection. + if rectangleIntersectsWithPolygons(s2rect, mpgn.s2pgns) { + return true, nil + } + + return false, nil + } + + // check if the other shape is a linestring. + if ls, ok := other.(*LineString); ok { + + if rectangleIntersectsWithLineStrings(s2rect, + []*s2.Polyline{ls.pl}) { + return true, nil + } + + return false, nil + } + + // check if the other shape is a multilinestring. + if mls, ok := other.(*MultiLineString); ok { + + if rectangleIntersectsWithLineStrings(s2rect, mls.pls) { + return true, nil + } + + return false, nil + } + + if gc, ok := other.(*GeometryCollection); ok { + // check for the intersection of every member shape + // within the geometrycollection. + if geometryCollectionIntersectsShape(gc, shapeIn) { + return true, nil + } + return false, nil + } + + // check if the other shape is a circle. + if c, ok := other.(*Circle); ok { + s2pgn := s2PolygonFromS2Rectangle(s2rect) + cp := c.s2cap.Center() + projected := s2pgn.Project(&cp) + distance := projected.Distance(cp) + return distance <= c.s2cap.Radius(), nil + } + + // check if the other shape is a envelope. + if e, ok := other.(*Envelope); ok { + + if s2rect.Intersects(*e.r) { + return true, nil + } + + return false, nil + } + + return false, fmt.Errorf("unknown geojson type: %s"+ + " found in document", other.Type()) +} + +// checkEnvelopeContainsShape checks whether the given shape in +// the document is contained Contains the envelope/rectangle. +func checkEnvelopeContainsShape(s2rect *s2.Rect, shapeIn, + other index.GeoJSON) (bool, error) { + // check if the other shape is a point. + if p2, ok := other.(*Point); ok { + s2LatLng := s2.LatLngFromPoint(*p2.s2point) + + if s2rect.ContainsLatLng(s2LatLng) { + return true, nil + } + + return false, nil + } + + // check if the other shape is a multipoint. + if p2, ok := other.(*MultiPoint); ok { + // check the intersection for any point in the collection. + for _, point := range p2.s2points { + s2LatLng := s2.LatLngFromPoint(*point) + + if !s2rect.ContainsLatLng(s2LatLng) { + return false, nil + } + } + + return true, nil + } + + // check if the other shape is a polygon. + if p2, ok := other.(*Polygon); ok { + s2pgnRect := s2PolygonFromS2Rectangle(s2rect) + return s2pgnRect.Contains(p2.s2pgn), nil + } + + // check if the other shape is a multipolygon. + if p2, ok := other.(*MultiPolygon); ok { + s2pgnRect := s2PolygonFromS2Rectangle(s2rect) + + // check the containment for every polygon in the collection. + for _, s2pgn := range p2.s2pgns { + if !s2pgnRect.Contains(s2pgn) { + return false, nil + } + } + + return true, nil + } + + // check if the other shape is a linestring. + if p2, ok := other.(*LineString); ok { + for i := 0; i < p2.pl.NumEdges(); i++ { + edge := p2.pl.Edge(i) + if !s2rect.ContainsPoint(edge.V0) || + !s2rect.ContainsPoint(edge.V1) { + return false, nil + } + } + + return true, nil + } + + // check if the other shape is a multilinestring. + if p2, ok := other.(*MultiLineString); ok { + for _, pl := range p2.pls { + for i := 0; i < pl.NumEdges(); i++ { + edge := pl.Edge(i) + if !s2rect.ContainsPoint(edge.V0) || + !s2rect.ContainsPoint(edge.V1) { + return false, nil + } + } + } + return true, nil + } + + if gc, ok := other.(*GeometryCollection); ok { + for _, shape := range gc.Members() { + contains, err := shapeIn.Contains(shape) + if err == nil && !contains { + return false, nil + } + } + return true, nil + } + + // check if the other shape is a circle. + if c, ok := other.(*Circle); ok { + + if s2rect.Contains(c.s2cap.RectBound()) { + return true, nil + } + + return false, nil + } + + // check if the other shape is a envelope. + if e, ok := other.(*Envelope); ok { + + if s2rect.Contains(*e.r) { + return true, nil + } + + return false, nil + } + + return false, fmt.Errorf("unknown geojson type: %s"+ + " found in document", other.Type()) +} diff --git a/vendor/github.com/blevesearch/geo/geojson/geojson_shapes_util.go b/vendor/github.com/blevesearch/geo/geojson/geojson_shapes_util.go new file mode 100644 index 00000000..c61912e0 --- /dev/null +++ b/vendor/github.com/blevesearch/geo/geojson/geojson_shapes_util.go @@ -0,0 +1,586 @@ +// Copyright (c) 2022 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package geojson + +import ( + "bytes" + "encoding/binary" + "fmt" + "strings" + + index "github.com/blevesearch/bleve_index_api" + "github.com/blevesearch/geo/s2" + jsoniterator "github.com/json-iterator/go" +) + +var jsoniter = jsoniterator.ConfigCompatibleWithStandardLibrary + +// FilterGeoShapesOnRelation extracts the shapes in the document, apply +// the `relation` filter and confirms whether the shape in the document +// satisfies the given relation. +func FilterGeoShapesOnRelation(shape index.GeoJSON, targetShapeBytes []byte, + relation string, reader **bytes.Reader, bufPool *s2.GeoBufferPool) (bool, error) { + + shapeInDoc, err := extractShapesFromBytes(targetShapeBytes, reader, bufPool) + if err != nil { + return false, err + } + + return filterShapes(shape, shapeInDoc, relation) +} + +// extractShapesFromBytes unmarshal the bytes to retrieve the +// embedded geojson shape. +func extractShapesFromBytes(targetShapeBytes []byte, r **bytes.Reader, bufPool *s2.GeoBufferPool) ( + index.GeoJSON, error) { + if (*r) == nil { + *r = bytes.NewReader(targetShapeBytes[1:]) + } else { + (*r).Reset(targetShapeBytes[1:]) + } + + switch targetShapeBytes[0] { + case PointTypePrefix: + point := &Point{s2point: &s2.Point{}} + err := point.s2point.Decode(*r) + if err != nil { + return nil, err + } + return point, nil + + case MultiPointTypePrefix: + var numPoints int32 + err := binary.Read(*r, binary.BigEndian, &numPoints) + if err != nil { + return nil, err + } + multipoint := &MultiPoint{ + s2points: make([]*s2.Point, 0, numPoints), + } + for i := 0; i < int(numPoints); i++ { + s2point := s2.Point{} + err := s2point.Decode((*r)) + if err != nil { + return nil, err + } + multipoint.s2points = append(multipoint.s2points, &s2point) + } + + return multipoint, nil + + case LineStringTypePrefix: + ls := &LineString{pl: &s2.Polyline{}} + err := ls.pl.Decode(*r) + if err != nil { + return nil, err + } + return ls, nil + + case MultiLineStringTypePrefix: + var numLineStrings int32 + err := binary.Read(*r, binary.BigEndian, &numLineStrings) + if err != nil { + return nil, err + } + + mls := &MultiLineString{pls: make([]*s2.Polyline, 0, numLineStrings)} + + for i := 0; i < int(numLineStrings); i++ { + pl := &s2.Polyline{} + err := pl.Decode(*r) + if err != nil { + return nil, err + } + mls.pls = append(mls.pls, pl) + } + + return mls, nil + + case PolygonTypePrefix: + pgn := &Polygon{s2pgn: &s2.Polygon{BufPool: bufPool}} + err := pgn.s2pgn.Decode(*r) + if err != nil { + return nil, err + } + + return pgn, nil + + case MultiPolygonTypePrefix: + var numPolygons int32 + err := binary.Read(*r, binary.BigEndian, &numPolygons) + if err != nil { + return nil, err + } + mpgns := &MultiPolygon{s2pgns: make([]*s2.Polygon, 0, numPolygons)} + for i := 0; i < int(numPolygons); i++ { + pgn := &s2.Polygon{} + err := pgn.Decode(*r) + if err != nil { + return nil, err + } + mpgns.s2pgns = append(mpgns.s2pgns, pgn) + } + + return mpgns, nil + + case GeometryCollectionTypePrefix: + var numShapes int32 + err := binary.Read(*r, binary.BigEndian, &numShapes) + if err != nil { + return nil, err + } + + lengths := make([]int32, numShapes) + for i := int32(0); i < numShapes; i++ { + var length int32 + err := binary.Read(*r, binary.BigEndian, &length) + if err != nil { + return nil, err + } + lengths[i] = length + } + + inputBytes := targetShapeBytes[len(targetShapeBytes)-(*r).Len():] + gc := &GeometryCollection{Shapes: make([]index.GeoJSON, numShapes)} + + for i := int32(0); i < numShapes; i++ { + shape, err := extractShapesFromBytes(inputBytes[:lengths[i]], r, nil) + if err != nil { + return nil, err + } + + gc.Shapes[i] = shape + inputBytes = inputBytes[lengths[i]:] + } + + return gc, nil + + case CircleTypePrefix: + c := &Circle{s2cap: &s2.Cap{}} + err := c.s2cap.Decode(*r) + if err != nil { + return nil, err + } + + return c, nil + + case EnvelopeTypePrefix: + e := &Envelope{r: &s2.Rect{}} + err := e.r.Decode(*r) + if err != nil { + return nil, err + } + + return e, nil + } + + return nil, fmt.Errorf("unknown geo shape type: %v", targetShapeBytes[0]) +} + +// filterShapes applies the given relation between the query shape +// and the shape in the document. +func filterShapes(shape index.GeoJSON, + shapeInDoc index.GeoJSON, relation string) (bool, error) { + + if relation == "intersects" { + return shape.Intersects(shapeInDoc) + } + + if relation == "contains" { + return shapeInDoc.Contains(shape) + } + + if relation == "within" { + return shape.Contains(shapeInDoc) + } + + if relation == "disjoint" { + intersects, err := shape.Intersects(shapeInDoc) + return !intersects, err + } + + return false, fmt.Errorf("unknown relation: %s", relation) +} + +// ParseGeoJSONShape unmarshals the geojson/circle/envelope shape +// embedded in the given bytes. +func ParseGeoJSONShape(input []byte) (index.GeoJSON, error) { + var sType string + var tmp struct { + Typ string `json:"type"` + } + err := jsoniter.Unmarshal(input, &tmp) + if err != nil { + return nil, err + } + + sType = strings.ToLower(tmp.Typ) + + switch sType { + case PolygonType: + var rv Polygon + err := jsoniter.Unmarshal(input, &rv) + if err != nil { + return nil, err + } + rv.init() + return &rv, nil + + case MultiPolygonType: + var rv MultiPolygon + err := jsoniter.Unmarshal(input, &rv) + if err != nil { + return nil, err + } + rv.init() + return &rv, nil + + case PointType: + var rv Point + err := jsoniter.Unmarshal(input, &rv) + if err != nil { + return nil, err + } + rv.init() + return &rv, nil + + case MultiPointType: + var rv MultiPoint + err := jsoniter.Unmarshal(input, &rv) + if err != nil { + return nil, err + } + rv.init() + return &rv, nil + + case LineStringType: + var rv LineString + err := jsoniter.Unmarshal(input, &rv) + if err != nil { + return nil, err + } + rv.init() + return &rv, nil + + case MultiLineStringType: + var rv MultiLineString + err := jsoniter.Unmarshal(input, &rv) + if err != nil { + return nil, err + } + rv.init() + return &rv, nil + + case GeometryCollectionType: + var rv GeometryCollection + err := jsoniter.Unmarshal(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + + case CircleType: + var rv Circle + err := jsoniter.Unmarshal(input, &rv) + if err != nil { + return nil, err + } + rv.init() + return &rv, nil + + case EnvelopeType: + var rv Envelope + err := jsoniter.Unmarshal(input, &rv) + if err != nil { + return nil, err + } + rv.init() + return &rv, nil + + default: + return nil, fmt.Errorf("unknown shape type: %s", sType) + } + + return nil, err +} + +// NewGeoJsonShape instantiate a geojson shape/circle or +// an envelope from the given coordinates and type. +func NewGeoJsonShape(coordinates [][][][]float64, typ string) ( + index.GeoJSON, []byte, error) { + if len(coordinates) == 0 { + return nil, nil, fmt.Errorf("missing coordinates") + } + + typ = strings.ToLower(typ) + + switch typ { + case PointType: + point := NewGeoJsonPoint(coordinates[0][0][0]) + value, err := point.(s2Serializable).Marshal() + if err != nil { + return nil, nil, err + } + return point, value, nil + + case MultiPointType: + multipoint := NewGeoJsonMultiPoint(coordinates[0][0]) + value, err := multipoint.(s2Serializable).Marshal() + if err != nil { + return nil, nil, err + } + return multipoint, value, nil + + case LineStringType: + linestring := NewGeoJsonLinestring(coordinates[0][0]) + value, err := linestring.(s2Serializable).Marshal() + if err != nil { + return nil, nil, err + } + return linestring, value, nil + + case MultiLineStringType: + multilinestring := NewGeoJsonMultilinestring(coordinates[0]) + value, err := multilinestring.(s2Serializable).Marshal() + if err != nil { + return nil, nil, err + } + return multilinestring, value, nil + + case PolygonType: + polygon := NewGeoJsonPolygon(coordinates[0]) + value, err := polygon.(s2Serializable).Marshal() + if err != nil { + return nil, nil, err + } + return polygon, value, nil + + case MultiPolygonType: + multipolygon := NewGeoJsonMultiPolygon(coordinates) + value, err := multipolygon.(s2Serializable).Marshal() + if err != nil { + return nil, nil, err + } + return multipolygon, value, nil + + case EnvelopeType: + envelope := NewGeoEnvelope(coordinates[0][0]) + value, err := envelope.(s2Serializable).Marshal() + if err != nil { + return nil, nil, err + } + return envelope, value, nil + } + + return nil, nil, fmt.Errorf("unknown shape type: %s", typ) +} + +// GlueBytes primarily for quicker filtering of docvalues +// during the filtering phase. +var GlueBytes = []byte("##") + +// NewGeometryCollection instantiate a geometrycollection +// and prefix the byte contents with certain glue bytes that +// can be used later while filering the doc values. +func NewGeometryCollection(coordinates [][][][][]float64, + typs []string) (index.GeoJSON, []byte, error) { + if typs == nil { + return nil, nil, fmt.Errorf("nil type information") + } + if len(typs) < len(coordinates) { + return nil, nil, fmt.Errorf("missing type information for some shapes") + } + shapes := make([]index.GeoJSON, 0, len(coordinates)) + for i, vertices := range coordinates { + s, _, err := NewGeoJsonShape(vertices, typs[i]) + if err != nil { + continue + } + shapes = append(shapes, s) + } + + var gc GeometryCollection + gc.Typ = GeometryCollectionType + gc.Shapes = shapes + vbytes, err := gc.Marshal() + if err != nil { + return nil, nil, err + } + + return &gc, vbytes, nil +} + +// NewGeoCircleShape instantiate a circle shape and +// prefix the byte contents with certain glue bytes that +// can be used later while filering the doc values. +func NewGeoCircleShape(cp []float64, + radius string) (*Circle, []byte, error) { + r, err := ParseDistance(radius) + if err != nil { + return nil, nil, err + } + rv := &Circle{Typ: CircleType, Vertices: cp, + Radius: radius, + radiusInMeters: r} + + vbytes, err := rv.Marshal() + if err != nil { + return nil, nil, err + } + + return rv, vbytes, nil +} + +// ------------------------------------------------------------------------ + +func (p *Point) IndexTokens(s *s2.RegionTermIndexer) []string { + p.init() + terms := s.GetIndexTermsForPoint(*p.s2point, "") + return StripCoveringTerms(terms) +} + +func (p *Point) QueryTokens(s *s2.RegionTermIndexer) []string { + p.init() + terms := s.GetQueryTermsForPoint(*p.s2point, "") + return StripCoveringTerms(terms) +} + +// ------------------------------------------------------------------------ + +func (mp *MultiPoint) IndexTokens(s *s2.RegionTermIndexer) []string { + mp.init() + var rv []string + for _, s2point := range mp.s2points { + terms := s.GetIndexTermsForPoint(*s2point, "") + rv = append(rv, terms...) + } + return StripCoveringTerms(rv) +} + +func (mp *MultiPoint) QueryTokens(s *s2.RegionTermIndexer) []string { + mp.init() + var rv []string + for _, s2point := range mp.s2points { + terms := s.GetQueryTermsForPoint(*s2point, "") + rv = append(rv, terms...) + } + + return StripCoveringTerms(rv) +} + +// ------------------------------------------------------------------------ + +func (ls *LineString) IndexTokens(s *s2.RegionTermIndexer) []string { + ls.init() + terms := s.GetIndexTermsForRegion(ls.pl.CapBound(), "") + return StripCoveringTerms(terms) +} + +func (ls *LineString) QueryTokens(s *s2.RegionTermIndexer) []string { + ls.init() + terms := s.GetQueryTermsForRegion(ls.pl.CapBound(), "") + return StripCoveringTerms(terms) +} + +// ------------------------------------------------------------------------ + +func (mls *MultiLineString) IndexTokens(s *s2.RegionTermIndexer) []string { + mls.init() + var rv []string + for _, ls := range mls.pls { + terms := s.GetIndexTermsForRegion(ls.CapBound(), "") + rv = append(rv, terms...) + } + + return StripCoveringTerms(rv) +} + +func (mls *MultiLineString) QueryTokens(s *s2.RegionTermIndexer) []string { + mls.init() + + var rv []string + for _, ls := range mls.pls { + terms := s.GetQueryTermsForRegion(ls.CapBound(), "") + rv = append(rv, terms...) + } + + return StripCoveringTerms(rv) +} + +// ------------------------------------------------------------------------ + +func (mp *MultiPolygon) IndexTokens(s *s2.RegionTermIndexer) []string { + mp.init() + + var rv []string + for _, s2pgn := range mp.s2pgns { + terms := s.GetIndexTermsForRegion(s2pgn.CapBound(), "") + rv = append(rv, terms...) + } + + return StripCoveringTerms(rv) +} + +func (mp *MultiPolygon) QueryTokens(s *s2.RegionTermIndexer) []string { + mp.init() + + var rv []string + for _, s2pgn := range mp.s2pgns { + terms := s.GetQueryTermsForRegion(s2pgn.CapBound(), "") + rv = append(rv, terms...) + } + + return StripCoveringTerms(rv) +} + +// ------------------------------------------------------------------------ + +func (pgn *Polygon) IndexTokens(s *s2.RegionTermIndexer) []string { + pgn.init() + terms := s.GetIndexTermsForRegion( + pgn.s2pgn.CapBound(), "") + return StripCoveringTerms(terms) +} + +func (pgn *Polygon) QueryTokens(s *s2.RegionTermIndexer) []string { + pgn.init() + terms := s.GetQueryTermsForRegion( + pgn.s2pgn.CapBound(), "") + return StripCoveringTerms(terms) +} + +// ------------------------------------------------------------------------ + +func (c *Circle) IndexTokens(s *s2.RegionTermIndexer) []string { + c.init() + return StripCoveringTerms(s.GetIndexTermsForRegion(c.s2cap.CapBound(), "")) +} + +func (c *Circle) QueryTokens(s *s2.RegionTermIndexer) []string { + c.init() + return StripCoveringTerms(s.GetQueryTermsForRegion(c.s2cap.CapBound(), "")) +} + +// ------------------------------------------------------------------------ + +func (e *Envelope) IndexTokens(s *s2.RegionTermIndexer) []string { + e.init() + return StripCoveringTerms(s.GetIndexTermsForRegion(e.r.CapBound(), "")) +} + +func (e *Envelope) QueryTokens(s *s2.RegionTermIndexer) []string { + e.init() + return StripCoveringTerms(s.GetQueryTermsForRegion(e.r.CapBound(), "")) +} diff --git a/vendor/github.com/blevesearch/geo/s2/bits_go18.go b/vendor/github.com/blevesearch/geo/s2/bits_go18.go new file mode 100644 index 00000000..4b8bfef9 --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/bits_go18.go @@ -0,0 +1,54 @@ +// Copyright 2018 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !go1.9 +// +build !go1.9 + +package s2 + +// This file is for the bit manipulation code pre-Go 1.9. + +// findMSBSetNonZero64 returns the index (between 0 and 63) of the most +// significant set bit. Passing zero to this function returns zero. +func findMSBSetNonZero64(x uint64) int { + val := []uint64{0x2, 0xC, 0xF0, 0xFF00, 0xFFFF0000, 0xFFFFFFFF00000000} + shift := []uint64{1, 2, 4, 8, 16, 32} + var msbPos uint64 + for i := 5; i >= 0; i-- { + if x&val[i] != 0 { + x >>= shift[i] + msbPos |= shift[i] + } + } + return int(msbPos) +} + +const deBruijn64 = 0x03f79d71b4ca8b09 +const digitMask = uint64(1<<64 - 1) + +var deBruijn64Lookup = []byte{ + 0, 1, 56, 2, 57, 49, 28, 3, 61, 58, 42, 50, 38, 29, 17, 4, + 62, 47, 59, 36, 45, 43, 51, 22, 53, 39, 33, 30, 24, 18, 12, 5, + 63, 55, 48, 27, 60, 41, 37, 16, 46, 35, 44, 21, 52, 32, 23, 11, + 54, 26, 40, 15, 34, 20, 31, 10, 25, 14, 19, 9, 13, 8, 7, 6, +} + +// findLSBSetNonZero64 returns the index (between 0 and 63) of the least +// significant set bit. Passing zero to this function returns zero. +// +// This code comes from trailingZeroBits in https://golang.org/src/math/big/nat.go +// which references (Knuth, volume 4, section 7.3.1). +func findLSBSetNonZero64(x uint64) int { + return int(deBruijn64Lookup[((x&-x)*(deBruijn64&digitMask))>>58]) +} diff --git a/vendor/github.com/blevesearch/geo/s2/bits_go19.go b/vendor/github.com/blevesearch/geo/s2/bits_go19.go new file mode 100644 index 00000000..0de1ac69 --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/bits_go19.go @@ -0,0 +1,40 @@ +// Copyright 2018 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build go1.9 +// +build go1.9 + +package s2 + +// This file is for the bit manipulation code post-Go 1.9. + +import "math/bits" + +// findMSBSetNonZero64 returns the index (between 0 and 63) of the most +// significant set bit. Passing zero to this function return zero. +func findMSBSetNonZero64(x uint64) int { + if x == 0 { + return 0 + } + return 63 - bits.LeadingZeros64(x) +} + +// findLSBSetNonZero64 returns the index (between 0 and 63) of the least +// significant set bit. Passing zero to this function return zero. +func findLSBSetNonZero64(x uint64) int { + if x == 0 { + return 0 + } + return bits.TrailingZeros64(x) +} diff --git a/vendor/github.com/blevesearch/geo/s2/buffer_pool.go b/vendor/github.com/blevesearch/geo/s2/buffer_pool.go new file mode 100644 index 00000000..90fb20c3 --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/buffer_pool.go @@ -0,0 +1,69 @@ +// Copyright (c) 2023 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +// GeoBufferPool represents a pool of buffers ranging from a given +// max size to a min size in steps of 2. It uses a lazy approach only allocating +// the buffers when it is needed. + +type GeoBufferPool struct { + buffers [][]byte + maxSize int + minSize int +} + +func NewGeoBufferPool(maxSize int, minSize int) *GeoBufferPool { + // Calculating the number of buffers required. Assuming that + // the value of minSize is correct, the buffers will be of size + // minSize, 2 * minSize, 4 * minSize and so on till it is less + // than or equal to the maxSize. If it is not equal to maxSize, + // then a suitable value less than maxSize will be set as maxSize + length := 0 + temp := minSize + for temp <= maxSize { + length = length + 1 + temp = temp * 2 + } + maxSize = temp / 2 + + return &GeoBufferPool{ + buffers: make([][]byte, length), + maxSize: maxSize, + minSize: minSize, + } +} + +func (b *GeoBufferPool) Get(size int) ([]byte) { + if b == nil { + // GeoBufferPool not setup + return make([]byte, size) + } + + bufSize := b.minSize + + for i := range b.buffers { + if size <= bufSize || i == len(b.buffers) - 1 { + if b.buffers[i] == nil { + b.buffers[i] = make([]byte, bufSize) + } + + return b.buffers[i] + } + + bufSize = bufSize * 2 + } + + return nil +} diff --git a/vendor/github.com/blevesearch/geo/s2/cap.go b/vendor/github.com/blevesearch/geo/s2/cap.go new file mode 100644 index 00000000..c4fb2e1e --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/cap.go @@ -0,0 +1,519 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +import ( + "fmt" + "io" + "math" + + "github.com/golang/geo/r1" + "github.com/golang/geo/s1" +) + +var ( + // centerPoint is the default center for Caps + centerPoint = PointFromCoords(1.0, 0, 0) +) + +// Cap represents a disc-shaped region defined by a center and radius. +// Technically this shape is called a "spherical cap" (rather than disc) +// because it is not planar; the cap represents a portion of the sphere that +// has been cut off by a plane. The boundary of the cap is the circle defined +// by the intersection of the sphere and the plane. For containment purposes, +// the cap is a closed set, i.e. it contains its boundary. +// +// For the most part, you can use a spherical cap wherever you would use a +// disc in planar geometry. The radius of the cap is measured along the +// surface of the sphere (rather than the straight-line distance through the +// interior). Thus a cap of radius π/2 is a hemisphere, and a cap of radius +// π covers the entire sphere. +// +// The center is a point on the surface of the unit sphere. (Hence the need for +// it to be of unit length.) +// +// A cap can also be defined by its center point and height. The height is the +// distance from the center point to the cutoff plane. There is also support for +// "empty" and "full" caps, which contain no points and all points respectively. +// +// Here are some useful relationships between the cap height (h), the cap +// radius (r), the maximum chord length from the cap's center (d), and the +// radius of cap's base (a). +// +// h = 1 - cos(r) +// = 2 * sin^2(r/2) +// d^2 = 2 * h +// = a^2 + h^2 +// +// The zero value of Cap is an invalid cap. Use EmptyCap to get a valid empty cap. +type Cap struct { + center Point + radius s1.ChordAngle +} + +// CapFromPoint constructs a cap containing a single point. +func CapFromPoint(p Point) Cap { + return CapFromCenterChordAngle(p, 0) +} + +// CapFromCenterAngle constructs a cap with the given center and angle. +func CapFromCenterAngle(center Point, angle s1.Angle) Cap { + return CapFromCenterChordAngle(center, s1.ChordAngleFromAngle(angle)) +} + +// CapFromCenterChordAngle constructs a cap where the angle is expressed as an +// s1.ChordAngle. This constructor is more efficient than using an s1.Angle. +func CapFromCenterChordAngle(center Point, radius s1.ChordAngle) Cap { + return Cap{ + center: center, + radius: radius, + } +} + +// CapFromCenterHeight constructs a cap with the given center and height. A +// negative height yields an empty cap; a height of 2 or more yields a full cap. +// The center should be unit length. +func CapFromCenterHeight(center Point, height float64) Cap { + return CapFromCenterChordAngle(center, s1.ChordAngleFromSquaredLength(2*height)) +} + +// CapFromCenterArea constructs a cap with the given center and surface area. +// Note that the area can also be interpreted as the solid angle subtended by the +// cap (because the sphere has unit radius). A negative area yields an empty cap; +// an area of 4*π or more yields a full cap. +func CapFromCenterArea(center Point, area float64) Cap { + return CapFromCenterChordAngle(center, s1.ChordAngleFromSquaredLength(area/math.Pi)) +} + +// EmptyCap returns a cap that contains no points. +func EmptyCap() Cap { + return CapFromCenterChordAngle(centerPoint, s1.NegativeChordAngle) +} + +// FullCap returns a cap that contains all points. +func FullCap() Cap { + return CapFromCenterChordAngle(centerPoint, s1.StraightChordAngle) +} + +// IsValid reports whether the Cap is considered valid. +func (c Cap) IsValid() bool { + return c.center.Vector.IsUnit() && c.radius <= s1.StraightChordAngle +} + +// IsEmpty reports whether the cap is empty, i.e. it contains no points. +func (c Cap) IsEmpty() bool { + return c.radius < 0 +} + +// IsFull reports whether the cap is full, i.e. it contains all points. +func (c Cap) IsFull() bool { + return c.radius == s1.StraightChordAngle +} + +// Center returns the cap's center point. +func (c Cap) Center() Point { + return c.center +} + +// Height returns the height of the cap. This is the distance from the center +// point to the cutoff plane. +func (c Cap) Height() float64 { + return float64(0.5 * c.radius) +} + +// Radius returns the cap radius as an s1.Angle. (Note that the cap angle +// is stored internally as a ChordAngle, so this method requires a trigonometric +// operation and may yield a slightly different result than the value passed +// to CapFromCenterAngle). +func (c Cap) Radius() s1.Angle { + return c.radius.Angle() +} + +// Area returns the surface area of the Cap on the unit sphere. +func (c Cap) Area() float64 { + return 2.0 * math.Pi * math.Max(0, c.Height()) +} + +// Contains reports whether this cap contains the other. +func (c Cap) Contains(other Cap) bool { + // In a set containment sense, every cap contains the empty cap. + if c.IsFull() || other.IsEmpty() { + return true + } + return c.radius >= ChordAngleBetweenPoints(c.center, other.center).Add(other.radius) +} + +// Intersects reports whether this cap intersects the other cap. +// i.e. whether they have any points in common. +func (c Cap) Intersects(other Cap) bool { + if c.IsEmpty() || other.IsEmpty() { + return false + } + + return c.radius.Add(other.radius) >= ChordAngleBetweenPoints(c.center, other.center) +} + +// InteriorIntersects reports whether this caps interior intersects the other cap. +func (c Cap) InteriorIntersects(other Cap) bool { + // Make sure this cap has an interior and the other cap is non-empty. + if c.radius <= 0 || other.IsEmpty() { + return false + } + + return c.radius.Add(other.radius) > ChordAngleBetweenPoints(c.center, other.center) +} + +// ContainsPoint reports whether this cap contains the point. +func (c Cap) ContainsPoint(p Point) bool { + return ChordAngleBetweenPoints(c.center, p) <= c.radius +} + +// InteriorContainsPoint reports whether the point is within the interior of this cap. +func (c Cap) InteriorContainsPoint(p Point) bool { + return c.IsFull() || ChordAngleBetweenPoints(c.center, p) < c.radius +} + +// Complement returns the complement of the interior of the cap. A cap and its +// complement have the same boundary but do not share any interior points. +// The complement operator is not a bijection because the complement of a +// singleton cap (containing a single point) is the same as the complement +// of an empty cap. +func (c Cap) Complement() Cap { + if c.IsFull() { + return EmptyCap() + } + if c.IsEmpty() { + return FullCap() + } + + return CapFromCenterChordAngle(Point{c.center.Mul(-1)}, s1.StraightChordAngle.Sub(c.radius)) +} + +// CapBound returns a bounding spherical cap. This is not guaranteed to be exact. +func (c Cap) CapBound() Cap { + return c +} + +// RectBound returns a bounding latitude-longitude rectangle. +// The bounds are not guaranteed to be tight. +func (c Cap) RectBound() Rect { + if c.IsEmpty() { + return EmptyRect() + } + + capAngle := c.Radius().Radians() + allLongitudes := false + lat := r1.Interval{ + Lo: latitude(c.center).Radians() - capAngle, + Hi: latitude(c.center).Radians() + capAngle, + } + lng := s1.FullInterval() + + // Check whether cap includes the south pole. + if lat.Lo <= -math.Pi/2 { + lat.Lo = -math.Pi / 2 + allLongitudes = true + } + + // Check whether cap includes the north pole. + if lat.Hi >= math.Pi/2 { + lat.Hi = math.Pi / 2 + allLongitudes = true + } + + if !allLongitudes { + // Compute the range of longitudes covered by the cap. We use the law + // of sines for spherical triangles. Consider the triangle ABC where + // A is the north pole, B is the center of the cap, and C is the point + // of tangency between the cap boundary and a line of longitude. Then + // C is a right angle, and letting a,b,c denote the sides opposite A,B,C, + // we have sin(a)/sin(A) = sin(c)/sin(C), or sin(A) = sin(a)/sin(c). + // Here "a" is the cap angle, and "c" is the colatitude (90 degrees + // minus the latitude). This formula also works for negative latitudes. + // + // The formula for sin(a) follows from the relationship h = 1 - cos(a). + sinA := c.radius.Sin() + sinC := math.Cos(latitude(c.center).Radians()) + if sinA <= sinC { + angleA := math.Asin(sinA / sinC) + lng.Lo = math.Remainder(longitude(c.center).Radians()-angleA, math.Pi*2) + lng.Hi = math.Remainder(longitude(c.center).Radians()+angleA, math.Pi*2) + } + } + return Rect{lat, lng} +} + +// Equal reports whether this cap is equal to the other cap. +func (c Cap) Equal(other Cap) bool { + return (c.radius == other.radius && c.center == other.center) || + (c.IsEmpty() && other.IsEmpty()) || + (c.IsFull() && other.IsFull()) +} + +// ApproxEqual reports whether this cap is equal to the other cap within the given tolerance. +func (c Cap) ApproxEqual(other Cap) bool { + const epsilon = 1e-14 + r2 := float64(c.radius) + otherR2 := float64(other.radius) + return c.center.ApproxEqual(other.center) && + math.Abs(r2-otherR2) <= epsilon || + c.IsEmpty() && otherR2 <= epsilon || + other.IsEmpty() && r2 <= epsilon || + c.IsFull() && otherR2 >= 2-epsilon || + other.IsFull() && r2 >= 2-epsilon +} + +// AddPoint increases the cap if necessary to include the given point. If this cap is empty, +// then the center is set to the point with a zero height. p must be unit-length. +func (c Cap) AddPoint(p Point) Cap { + if c.IsEmpty() { + c.center = p + c.radius = 0 + return c + } + + // After calling cap.AddPoint(p), cap.Contains(p) must be true. However + // we don't need to do anything special to achieve this because Contains() + // does exactly the same distance calculation that we do here. + if newRad := ChordAngleBetweenPoints(c.center, p); newRad > c.radius { + c.radius = newRad + } + return c +} + +// AddCap increases the cap height if necessary to include the other cap. If this cap is empty, +// it is set to the other cap. +func (c Cap) AddCap(other Cap) Cap { + if c.IsEmpty() { + return other + } + if other.IsEmpty() { + return c + } + + // We round up the distance to ensure that the cap is actually contained. + // TODO(roberts): Do some error analysis in order to guarantee this. + dist := ChordAngleBetweenPoints(c.center, other.center).Add(other.radius) + if newRad := dist.Expanded(dblEpsilon * float64(dist)); newRad > c.radius { + c.radius = newRad + } + return c +} + +// Expanded returns a new cap expanded by the given angle. If the cap is empty, +// it returns an empty cap. +func (c Cap) Expanded(distance s1.Angle) Cap { + if c.IsEmpty() { + return EmptyCap() + } + return CapFromCenterChordAngle(c.center, c.radius.Add(s1.ChordAngleFromAngle(distance))) +} + +func (c Cap) String() string { + return fmt.Sprintf("[Center=%v, Radius=%f]", c.center.Vector, c.Radius().Degrees()) +} + +// radiusToHeight converts an s1.Angle into the height of the cap. +func radiusToHeight(r s1.Angle) float64 { + if r.Radians() < 0 { + return float64(s1.NegativeChordAngle) + } + if r.Radians() >= math.Pi { + return float64(s1.RightChordAngle) + } + return float64(0.5 * s1.ChordAngleFromAngle(r)) + +} + +// ContainsCell reports whether the cap contains the given cell. +func (c Cap) ContainsCell(cell Cell) bool { + // If the cap does not contain all cell vertices, return false. + var vertices [4]Point + for k := 0; k < 4; k++ { + vertices[k] = cell.Vertex(k) + if !c.ContainsPoint(vertices[k]) { + return false + } + } + // Otherwise, return true if the complement of the cap does not intersect the cell. + return !c.Complement().intersects(cell, vertices) +} + +// IntersectsCell reports whether the cap intersects the cell. +func (c Cap) IntersectsCell(cell Cell) bool { + // If the cap contains any cell vertex, return true. + var vertices [4]Point + for k := 0; k < 4; k++ { + vertices[k] = cell.Vertex(k) + if c.ContainsPoint(vertices[k]) { + return true + } + } + return c.intersects(cell, vertices) +} + +// intersects reports whether the cap intersects any point of the cell excluding +// its vertices (which are assumed to already have been checked). +func (c Cap) intersects(cell Cell, vertices [4]Point) bool { + // If the cap is a hemisphere or larger, the cell and the complement of the cap + // are both convex. Therefore since no vertex of the cell is contained, no other + // interior point of the cell is contained either. + if c.radius >= s1.RightChordAngle { + return false + } + + // We need to check for empty caps due to the center check just below. + if c.IsEmpty() { + return false + } + + // Optimization: return true if the cell contains the cap center. This allows half + // of the edge checks below to be skipped. + if cell.ContainsPoint(c.center) { + return true + } + + // At this point we know that the cell does not contain the cap center, and the cap + // does not contain any cell vertex. The only way that they can intersect is if the + // cap intersects the interior of some edge. + sin2Angle := c.radius.Sin2() + for k := 0; k < 4; k++ { + edge := cell.Edge(k).Vector + dot := c.center.Vector.Dot(edge) + if dot > 0 { + // The center is in the interior half-space defined by the edge. We do not need + // to consider these edges, since if the cap intersects this edge then it also + // intersects the edge on the opposite side of the cell, because the center is + // not contained with the cell. + continue + } + + // The Norm2() factor is necessary because "edge" is not normalized. + if dot*dot > sin2Angle*edge.Norm2() { + return false + } + + // Otherwise, the great circle containing this edge intersects the interior of the cap. We just + // need to check whether the point of closest approach occurs between the two edge endpoints. + dir := edge.Cross(c.center.Vector) + if dir.Dot(vertices[k].Vector) < 0 && dir.Dot(vertices[(k+1)&3].Vector) > 0 { + return true + } + } + return false +} + +// CellUnionBound computes a covering of the Cap. In general the covering +// consists of at most 4 cells except for very large caps, which may need +// up to 6 cells. The output is not sorted. +func (c Cap) CellUnionBound() []CellID { + // TODO(roberts): The covering could be made quite a bit tighter by mapping + // the cap to a rectangle in (i,j)-space and finding a covering for that. + + // Find the maximum level such that the cap contains at most one cell vertex + // and such that CellID.AppendVertexNeighbors() can be called. + level := MinWidthMetric.MaxLevel(c.Radius().Radians()) - 1 + + // If level < 0, more than three face cells are required. + if level < 0 { + cellIDs := make([]CellID, 6) + for face := 0; face < 6; face++ { + cellIDs[face] = CellIDFromFace(face) + } + return cellIDs + } + // The covering consists of the 4 cells at the given level that share the + // cell vertex that is closest to the cap center. + return cellIDFromPoint(c.center).VertexNeighbors(level) +} + +// Centroid returns the true centroid of the cap multiplied by its surface area +// The result lies on the ray from the origin through the cap's center, but it +// is not unit length. Note that if you just want the "surface centroid", i.e. +// the normalized result, then it is simpler to call Center. +// +// The reason for multiplying the result by the cap area is to make it +// easier to compute the centroid of more complicated shapes. The centroid +// of a union of disjoint regions can be computed simply by adding their +// Centroid() results. Caveat: for caps that contain a single point +// (i.e., zero radius), this method always returns the origin (0, 0, 0). +// This is because shapes with no area don't affect the centroid of a +// union whose total area is positive. +func (c Cap) Centroid() Point { + // From symmetry, the centroid of the cap must be somewhere on the line + // from the origin to the center of the cap on the surface of the sphere. + // When a sphere is divided into slices of constant thickness by a set of + // parallel planes, all slices have the same surface area. This implies + // that the radial component of the centroid is simply the midpoint of the + // range of radial distances spanned by the cap. That is easily computed + // from the cap height. + if c.IsEmpty() { + return Point{} + } + r := 1 - 0.5*c.Height() + return Point{c.center.Mul(r * c.Area())} +} + +// Union returns the smallest cap which encloses this cap and other. +func (c Cap) Union(other Cap) Cap { + // If the other cap is larger, swap c and other for the rest of the computations. + if c.radius < other.radius { + c, other = other, c + } + + if c.IsFull() || other.IsEmpty() { + return c + } + + // TODO: This calculation would be more efficient using s1.ChordAngles. + cRadius := c.Radius() + otherRadius := other.Radius() + distance := c.center.Distance(other.center) + if cRadius >= distance+otherRadius { + return c + } + + resRadius := 0.5 * (distance + cRadius + otherRadius) + resCenter := InterpolateAtDistance(0.5*(distance-cRadius+otherRadius), c.center, other.center) + return CapFromCenterAngle(resCenter, resRadius) +} + +// Encode encodes the Cap. +func (c Cap) Encode(w io.Writer) error { + e := &encoder{w: w} + c.encode(e) + return e.err +} + +func (c Cap) encode(e *encoder) { + e.writeFloat64(c.center.X) + e.writeFloat64(c.center.Y) + e.writeFloat64(c.center.Z) + e.writeFloat64(float64(c.radius)) +} + +// Decode decodes the Cap. +func (c *Cap) Decode(r io.Reader) error { + d := &decoder{r: asByteReader(r)} + c.decode(d) + return d.err +} + +func (c *Cap) decode(d *decoder) { + c.center.X = d.readFloat64() + c.center.Y = d.readFloat64() + c.center.Z = d.readFloat64() + c.radius = s1.ChordAngle(d.readFloat64()) +} diff --git a/vendor/github.com/blevesearch/geo/s2/cell.go b/vendor/github.com/blevesearch/geo/s2/cell.go new file mode 100644 index 00000000..0a01a4f1 --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/cell.go @@ -0,0 +1,698 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +import ( + "io" + "math" + + "github.com/golang/geo/r1" + "github.com/golang/geo/r2" + "github.com/golang/geo/r3" + "github.com/golang/geo/s1" +) + +// Cell is an S2 region object that represents a cell. Unlike CellIDs, +// it supports efficient containment and intersection tests. However, it is +// also a more expensive representation. +type Cell struct { + face int8 + level int8 + orientation int8 + id CellID + uv r2.Rect +} + +// CellFromCellID constructs a Cell corresponding to the given CellID. +func CellFromCellID(id CellID) Cell { + c := Cell{} + c.id = id + f, i, j, o := c.id.faceIJOrientation() + c.face = int8(f) + c.level = int8(c.id.Level()) + c.orientation = int8(o) + c.uv = ijLevelToBoundUV(i, j, int(c.level)) + return c +} + +// CellFromPoint constructs a cell for the given Point. +func CellFromPoint(p Point) Cell { + return CellFromCellID(cellIDFromPoint(p)) +} + +// CellFromLatLng constructs a cell for the given LatLng. +func CellFromLatLng(ll LatLng) Cell { + return CellFromCellID(CellIDFromLatLng(ll)) +} + +// Face returns the face this cell is on. +func (c Cell) Face() int { + return int(c.face) +} + +// oppositeFace returns the face opposite the given face. +func oppositeFace(face int) int { + return (face + 3) % 6 +} + +// Level returns the level of this cell. +func (c Cell) Level() int { + return int(c.level) +} + +// ID returns the CellID this cell represents. +func (c Cell) ID() CellID { + return c.id +} + +// IsLeaf returns whether this Cell is a leaf or not. +func (c Cell) IsLeaf() bool { + return c.level == maxLevel +} + +// SizeIJ returns the edge length of this cell in (i,j)-space. +func (c Cell) SizeIJ() int { + return sizeIJ(int(c.level)) +} + +// SizeST returns the edge length of this cell in (s,t)-space. +func (c Cell) SizeST() float64 { + return c.id.sizeST(int(c.level)) +} + +// Vertex returns the k-th vertex of the cell (k = 0,1,2,3) in CCW order +// (lower left, lower right, upper right, upper left in the UV plane). +func (c Cell) Vertex(k int) Point { + return Point{faceUVToXYZ(int(c.face), c.uv.Vertices()[k].X, c.uv.Vertices()[k].Y).Normalize()} +} + +// Edge returns the inward-facing normal of the great circle passing through +// the CCW ordered edge from vertex k to vertex k+1 (mod 4) (for k = 0,1,2,3). +func (c Cell) Edge(k int) Point { + switch k { + case 0: + return Point{vNorm(int(c.face), c.uv.Y.Lo).Normalize()} // Bottom + case 1: + return Point{uNorm(int(c.face), c.uv.X.Hi).Normalize()} // Right + case 2: + return Point{vNorm(int(c.face), c.uv.Y.Hi).Mul(-1.0).Normalize()} // Top + default: + return Point{uNorm(int(c.face), c.uv.X.Lo).Mul(-1.0).Normalize()} // Left + } +} + +// BoundUV returns the bounds of this cell in (u,v)-space. +func (c Cell) BoundUV() r2.Rect { + return c.uv +} + +// Center returns the direction vector corresponding to the center in +// (s,t)-space of the given cell. This is the point at which the cell is +// divided into four subcells; it is not necessarily the centroid of the +// cell in (u,v)-space or (x,y,z)-space +func (c Cell) Center() Point { + return Point{c.id.rawPoint().Normalize()} +} + +// Children returns the four direct children of this cell in traversal order +// and returns true. If this is a leaf cell, or the children could not be created, +// false is returned. +// The C++ method is called Subdivide. +func (c Cell) Children() ([4]Cell, bool) { + var children [4]Cell + + if c.id.IsLeaf() { + return children, false + } + + // Compute the cell midpoint in uv-space. + uvMid := c.id.centerUV() + + // Create four children with the appropriate bounds. + cid := c.id.ChildBegin() + for pos := 0; pos < 4; pos++ { + children[pos] = Cell{ + face: c.face, + level: c.level + 1, + orientation: c.orientation ^ int8(posToOrientation[pos]), + id: cid, + } + + // We want to split the cell in half in u and v. To decide which + // side to set equal to the midpoint value, we look at cell's (i,j) + // position within its parent. The index for i is in bit 1 of ij. + ij := posToIJ[c.orientation][pos] + i := ij >> 1 + j := ij & 1 + if i == 1 { + children[pos].uv.X.Hi = c.uv.X.Hi + children[pos].uv.X.Lo = uvMid.X + } else { + children[pos].uv.X.Lo = c.uv.X.Lo + children[pos].uv.X.Hi = uvMid.X + } + if j == 1 { + children[pos].uv.Y.Hi = c.uv.Y.Hi + children[pos].uv.Y.Lo = uvMid.Y + } else { + children[pos].uv.Y.Lo = c.uv.Y.Lo + children[pos].uv.Y.Hi = uvMid.Y + } + cid = cid.Next() + } + return children, true +} + +// ExactArea returns the area of this cell as accurately as possible. +func (c Cell) ExactArea() float64 { + v0, v1, v2, v3 := c.Vertex(0), c.Vertex(1), c.Vertex(2), c.Vertex(3) + return PointArea(v0, v1, v2) + PointArea(v0, v2, v3) +} + +// ApproxArea returns the approximate area of this cell. This method is accurate +// to within 3% percent for all cell sizes and accurate to within 0.1% for cells +// at level 5 or higher (i.e. squares 350km to a side or smaller on the Earth's +// surface). It is moderately cheap to compute. +func (c Cell) ApproxArea() float64 { + // All cells at the first two levels have the same area. + if c.level < 2 { + return c.AverageArea() + } + + // First, compute the approximate area of the cell when projected + // perpendicular to its normal. The cross product of its diagonals gives + // the normal, and the length of the normal is twice the projected area. + flatArea := 0.5 * (c.Vertex(2).Sub(c.Vertex(0).Vector). + Cross(c.Vertex(3).Sub(c.Vertex(1).Vector)).Norm()) + + // Now, compensate for the curvature of the cell surface by pretending + // that the cell is shaped like a spherical cap. The ratio of the + // area of a spherical cap to the area of its projected disc turns out + // to be 2 / (1 + sqrt(1 - r*r)) where r is the radius of the disc. + // For example, when r=0 the ratio is 1, and when r=1 the ratio is 2. + // Here we set Pi*r*r == flatArea to find the equivalent disc. + return flatArea * 2 / (1 + math.Sqrt(1-math.Min(1/math.Pi*flatArea, 1))) +} + +// AverageArea returns the average area of cells at the level of this cell. +// This is accurate to within a factor of 1.7. +func (c Cell) AverageArea() float64 { + return AvgAreaMetric.Value(int(c.level)) +} + +// IntersectsCell reports whether the intersection of this cell and the other cell is not nil. +func (c Cell) IntersectsCell(oc Cell) bool { + return c.id.Intersects(oc.id) +} + +// ContainsCell reports whether this cell contains the other cell. +func (c Cell) ContainsCell(oc Cell) bool { + return c.id.Contains(oc.id) +} + +// CellUnionBound computes a covering of the Cell. +func (c Cell) CellUnionBound() []CellID { + return c.CapBound().CellUnionBound() +} + +// latitude returns the latitude of the cell vertex in radians given by (i,j), +// where i and j indicate the Hi (1) or Lo (0) corner. +func (c Cell) latitude(i, j int) float64 { + var u, v float64 + switch { + case i == 0 && j == 0: + u = c.uv.X.Lo + v = c.uv.Y.Lo + case i == 0 && j == 1: + u = c.uv.X.Lo + v = c.uv.Y.Hi + case i == 1 && j == 0: + u = c.uv.X.Hi + v = c.uv.Y.Lo + case i == 1 && j == 1: + u = c.uv.X.Hi + v = c.uv.Y.Hi + default: + panic("i and/or j is out of bounds") + } + return latitude(Point{faceUVToXYZ(int(c.face), u, v)}).Radians() +} + +// longitude returns the longitude of the cell vertex in radians given by (i,j), +// where i and j indicate the Hi (1) or Lo (0) corner. +func (c Cell) longitude(i, j int) float64 { + var u, v float64 + switch { + case i == 0 && j == 0: + u = c.uv.X.Lo + v = c.uv.Y.Lo + case i == 0 && j == 1: + u = c.uv.X.Lo + v = c.uv.Y.Hi + case i == 1 && j == 0: + u = c.uv.X.Hi + v = c.uv.Y.Lo + case i == 1 && j == 1: + u = c.uv.X.Hi + v = c.uv.Y.Hi + default: + panic("i and/or j is out of bounds") + } + return longitude(Point{faceUVToXYZ(int(c.face), u, v)}).Radians() +} + +var ( + poleMinLat = math.Asin(math.Sqrt(1.0/3)) - 0.5*dblEpsilon +) + +// RectBound returns the bounding rectangle of this cell. +func (c Cell) RectBound() Rect { + if c.level > 0 { + // Except for cells at level 0, the latitude and longitude extremes are + // attained at the vertices. Furthermore, the latitude range is + // determined by one pair of diagonally opposite vertices and the + // longitude range is determined by the other pair. + // + // We first determine which corner (i,j) of the cell has the largest + // absolute latitude. To maximize latitude, we want to find the point in + // the cell that has the largest absolute z-coordinate and the smallest + // absolute x- and y-coordinates. To do this we look at each coordinate + // (u and v), and determine whether we want to minimize or maximize that + // coordinate based on the axis direction and the cell's (u,v) quadrant. + u := c.uv.X.Lo + c.uv.X.Hi + v := c.uv.Y.Lo + c.uv.Y.Hi + var i, j int + if uAxis(int(c.face)).Z == 0 { + if u < 0 { + i = 1 + } + } else if u > 0 { + i = 1 + } + if vAxis(int(c.face)).Z == 0 { + if v < 0 { + j = 1 + } + } else if v > 0 { + j = 1 + } + lat := r1.IntervalFromPoint(c.latitude(i, j)).AddPoint(c.latitude(1-i, 1-j)) + lng := s1.EmptyInterval().AddPoint(c.longitude(i, 1-j)).AddPoint(c.longitude(1-i, j)) + + // We grow the bounds slightly to make sure that the bounding rectangle + // contains LatLngFromPoint(P) for any point P inside the loop L defined by the + // four *normalized* vertices. Note that normalization of a vector can + // change its direction by up to 0.5 * dblEpsilon radians, and it is not + // enough just to add Normalize calls to the code above because the + // latitude/longitude ranges are not necessarily determined by diagonally + // opposite vertex pairs after normalization. + // + // We would like to bound the amount by which the latitude/longitude of a + // contained point P can exceed the bounds computed above. In the case of + // longitude, the normalization error can change the direction of rounding + // leading to a maximum difference in longitude of 2 * dblEpsilon. In + // the case of latitude, the normalization error can shift the latitude by + // up to 0.5 * dblEpsilon and the other sources of error can cause the + // two latitudes to differ by up to another 1.5 * dblEpsilon, which also + // leads to a maximum difference of 2 * dblEpsilon. + return Rect{lat, lng}.expanded(LatLng{s1.Angle(2 * dblEpsilon), s1.Angle(2 * dblEpsilon)}).PolarClosure() + } + + // The 4 cells around the equator extend to +/-45 degrees latitude at the + // midpoints of their top and bottom edges. The two cells covering the + // poles extend down to +/-35.26 degrees at their vertices. The maximum + // error in this calculation is 0.5 * dblEpsilon. + var bound Rect + switch c.face { + case 0: + bound = Rect{r1.Interval{-math.Pi / 4, math.Pi / 4}, s1.Interval{-math.Pi / 4, math.Pi / 4}} + case 1: + bound = Rect{r1.Interval{-math.Pi / 4, math.Pi / 4}, s1.Interval{math.Pi / 4, 3 * math.Pi / 4}} + case 2: + bound = Rect{r1.Interval{poleMinLat, math.Pi / 2}, s1.FullInterval()} + case 3: + bound = Rect{r1.Interval{-math.Pi / 4, math.Pi / 4}, s1.Interval{3 * math.Pi / 4, -3 * math.Pi / 4}} + case 4: + bound = Rect{r1.Interval{-math.Pi / 4, math.Pi / 4}, s1.Interval{-3 * math.Pi / 4, -math.Pi / 4}} + default: + bound = Rect{r1.Interval{-math.Pi / 2, -poleMinLat}, s1.FullInterval()} + } + + // Finally, we expand the bound to account for the error when a point P is + // converted to an LatLng to test for containment. (The bound should be + // large enough so that it contains the computed LatLng of any contained + // point, not just the infinite-precision version.) We don't need to expand + // longitude because longitude is calculated via a single call to math.Atan2, + // which is guaranteed to be semi-monotonic. + return bound.expanded(LatLng{s1.Angle(dblEpsilon), s1.Angle(0)}) +} + +// CapBound returns the bounding cap of this cell. +func (c Cell) CapBound() Cap { + // We use the cell center in (u,v)-space as the cap axis. This vector is very close + // to GetCenter() and faster to compute. Neither one of these vectors yields the + // bounding cap with minimal surface area, but they are both pretty close. + cap := CapFromPoint(Point{faceUVToXYZ(int(c.face), c.uv.Center().X, c.uv.Center().Y).Normalize()}) + for k := 0; k < 4; k++ { + cap = cap.AddPoint(c.Vertex(k)) + } + return cap +} + +// ContainsPoint reports whether this cell contains the given point. Note that +// unlike Loop/Polygon, a Cell is considered to be a closed set. This means +// that a point on a Cell's edge or vertex belong to the Cell and the relevant +// adjacent Cells too. +// +// If you want every point to be contained by exactly one Cell, +// you will need to convert the Cell to a Loop. +func (c Cell) ContainsPoint(p Point) bool { + var uv r2.Point + var ok bool + if uv.X, uv.Y, ok = faceXYZToUV(int(c.face), p); !ok { + return false + } + + // Expand the (u,v) bound to ensure that + // + // CellFromPoint(p).ContainsPoint(p) + // + // is always true. To do this, we need to account for the error when + // converting from (u,v) coordinates to (s,t) coordinates. In the + // normal case the total error is at most dblEpsilon. + return c.uv.ExpandedByMargin(dblEpsilon).ContainsPoint(uv) +} + +// Encode encodes the Cell. +func (c Cell) Encode(w io.Writer) error { + e := &encoder{w: w} + c.encode(e) + return e.err +} + +func (c Cell) encode(e *encoder) { + c.id.encode(e) +} + +// Decode decodes the Cell. +func (c *Cell) Decode(r io.Reader) error { + d := &decoder{r: asByteReader(r)} + c.decode(d) + return d.err +} + +func (c *Cell) decode(d *decoder) { + c.id.decode(d) + *c = CellFromCellID(c.id) +} + +// vertexChordDist2 returns the squared chord distance from point P to the +// given corner vertex specified by the Hi or Lo values of each. +func (c Cell) vertexChordDist2(p Point, xHi, yHi bool) s1.ChordAngle { + x := c.uv.X.Lo + y := c.uv.Y.Lo + if xHi { + x = c.uv.X.Hi + } + if yHi { + y = c.uv.Y.Hi + } + + return ChordAngleBetweenPoints(p, PointFromCoords(x, y, 1)) +} + +// uEdgeIsClosest reports whether a point P is closer to the interior of the specified +// Cell edge (either the lower or upper edge of the Cell) or to the endpoints. +func (c Cell) uEdgeIsClosest(p Point, vHi bool) bool { + u0 := c.uv.X.Lo + u1 := c.uv.X.Hi + v := c.uv.Y.Lo + if vHi { + v = c.uv.Y.Hi + } + // These are the normals to the planes that are perpendicular to the edge + // and pass through one of its two endpoints. + dir0 := r3.Vector{v*v + 1, -u0 * v, -u0} + dir1 := r3.Vector{v*v + 1, -u1 * v, -u1} + return p.Dot(dir0) > 0 && p.Dot(dir1) < 0 +} + +// vEdgeIsClosest reports whether a point P is closer to the interior of the specified +// Cell edge (either the right or left edge of the Cell) or to the endpoints. +func (c Cell) vEdgeIsClosest(p Point, uHi bool) bool { + v0 := c.uv.Y.Lo + v1 := c.uv.Y.Hi + u := c.uv.X.Lo + if uHi { + u = c.uv.X.Hi + } + dir0 := r3.Vector{-u * v0, u*u + 1, -v0} + dir1 := r3.Vector{-u * v1, u*u + 1, -v1} + return p.Dot(dir0) > 0 && p.Dot(dir1) < 0 +} + +// edgeDistance reports the distance from a Point P to a given Cell edge. The point +// P is given by its dot product, and the uv edge by its normal in the +// given coordinate value. +func edgeDistance(ij, uv float64) s1.ChordAngle { + // Let P by the target point and let R be the closest point on the given + // edge AB. The desired distance PR can be expressed as PR^2 = PQ^2 + QR^2 + // where Q is the point P projected onto the plane through the great circle + // through AB. We can compute the distance PQ^2 perpendicular to the plane + // from "dirIJ" (the dot product of the target point P with the edge + // normal) and the squared length the edge normal (1 + uv**2). + pq2 := (ij * ij) / (1 + uv*uv) + + // We can compute the distance QR as (1 - OQ) where O is the sphere origin, + // and we can compute OQ^2 = 1 - PQ^2 using the Pythagorean theorem. + // (This calculation loses accuracy as angle POQ approaches Pi/2.) + qr := 1 - math.Sqrt(1-pq2) + return s1.ChordAngleFromSquaredLength(pq2 + qr*qr) +} + +// distanceInternal reports the distance from the given point to the interior of +// the cell if toInterior is true or to the boundary of the cell otherwise. +func (c Cell) distanceInternal(targetXYZ Point, toInterior bool) s1.ChordAngle { + // All calculations are done in the (u,v,w) coordinates of this cell's face. + target := faceXYZtoUVW(int(c.face), targetXYZ) + + // Compute dot products with all four upward or rightward-facing edge + // normals. dirIJ is the dot product for the edge corresponding to axis + // I, endpoint J. For example, dir01 is the right edge of the Cell + // (corresponding to the upper endpoint of the u-axis). + dir00 := target.X - target.Z*c.uv.X.Lo + dir01 := target.X - target.Z*c.uv.X.Hi + dir10 := target.Y - target.Z*c.uv.Y.Lo + dir11 := target.Y - target.Z*c.uv.Y.Hi + inside := true + if dir00 < 0 { + inside = false // Target is to the left of the cell + if c.vEdgeIsClosest(target, false) { + return edgeDistance(-dir00, c.uv.X.Lo) + } + } + if dir01 > 0 { + inside = false // Target is to the right of the cell + if c.vEdgeIsClosest(target, true) { + return edgeDistance(dir01, c.uv.X.Hi) + } + } + if dir10 < 0 { + inside = false // Target is below the cell + if c.uEdgeIsClosest(target, false) { + return edgeDistance(-dir10, c.uv.Y.Lo) + } + } + if dir11 > 0 { + inside = false // Target is above the cell + if c.uEdgeIsClosest(target, true) { + return edgeDistance(dir11, c.uv.Y.Hi) + } + } + if inside { + if toInterior { + return s1.ChordAngle(0) + } + // Although you might think of Cells as rectangles, they are actually + // arbitrary quadrilaterals after they are projected onto the sphere. + // Therefore the simplest approach is just to find the minimum distance to + // any of the four edges. + return minChordAngle(edgeDistance(-dir00, c.uv.X.Lo), + edgeDistance(dir01, c.uv.X.Hi), + edgeDistance(-dir10, c.uv.Y.Lo), + edgeDistance(dir11, c.uv.Y.Hi)) + } + + // Otherwise, the closest point is one of the four cell vertices. Note that + // it is *not* trivial to narrow down the candidates based on the edge sign + // tests above, because (1) the edges don't meet at right angles and (2) + // there are points on the far side of the sphere that are both above *and* + // below the cell, etc. + return minChordAngle(c.vertexChordDist2(target, false, false), + c.vertexChordDist2(target, true, false), + c.vertexChordDist2(target, false, true), + c.vertexChordDist2(target, true, true)) +} + +// Distance reports the distance from the cell to the given point. Returns zero if +// the point is inside the cell. +func (c Cell) Distance(target Point) s1.ChordAngle { + return c.distanceInternal(target, true) +} + +// MaxDistance reports the maximum distance from the cell (including its interior) to the +// given point. +func (c Cell) MaxDistance(target Point) s1.ChordAngle { + // First check the 4 cell vertices. If all are within the hemisphere + // centered around target, the max distance will be to one of these vertices. + targetUVW := faceXYZtoUVW(int(c.face), target) + maxDist := maxChordAngle(c.vertexChordDist2(targetUVW, false, false), + c.vertexChordDist2(targetUVW, true, false), + c.vertexChordDist2(targetUVW, false, true), + c.vertexChordDist2(targetUVW, true, true)) + + if maxDist <= s1.RightChordAngle { + return maxDist + } + + // Otherwise, find the minimum distance dMin to the antipodal point and the + // maximum distance will be pi - dMin. + return s1.StraightChordAngle - c.BoundaryDistance(Point{target.Mul(-1)}) +} + +// BoundaryDistance reports the distance from the cell boundary to the given point. +func (c Cell) BoundaryDistance(target Point) s1.ChordAngle { + return c.distanceInternal(target, false) +} + +// DistanceToEdge returns the minimum distance from the cell to the given edge AB. Returns +// zero if the edge intersects the cell interior. +func (c Cell) DistanceToEdge(a, b Point) s1.ChordAngle { + // Possible optimizations: + // - Currently the (cell vertex, edge endpoint) distances are computed + // twice each, and the length of AB is computed 4 times. + // - To fix this, refactor GetDistance(target) so that it skips calculating + // the distance to each cell vertex. Instead, compute the cell vertices + // and distances in this function, and add a low-level UpdateMinDistance + // that allows the XA, XB, and AB distances to be passed in. + // - It might also be more efficient to do all calculations in UVW-space, + // since this would involve transforming 2 points rather than 4. + + // First, check the minimum distance to the edge endpoints A and B. + // (This also detects whether either endpoint is inside the cell.) + minDist := minChordAngle(c.Distance(a), c.Distance(b)) + if minDist == 0 { + return minDist + } + + // Otherwise, check whether the edge crosses the cell boundary. + crosser := NewChainEdgeCrosser(a, b, c.Vertex(3)) + for i := 0; i < 4; i++ { + if crosser.ChainCrossingSign(c.Vertex(i)) != DoNotCross { + return 0 + } + } + + // Finally, check whether the minimum distance occurs between a cell vertex + // and the interior of the edge AB. (Some of this work is redundant, since + // it also checks the distance to the endpoints A and B again.) + // + // Note that we don't need to check the distance from the interior of AB to + // the interior of a cell edge, because the only way that this distance can + // be minimal is if the two edges cross (already checked above). + for i := 0; i < 4; i++ { + minDist, _ = UpdateMinDistance(c.Vertex(i), a, b, minDist) + } + return minDist +} + +// MaxDistanceToEdge returns the maximum distance from the cell (including its interior) +// to the given edge AB. +func (c Cell) MaxDistanceToEdge(a, b Point) s1.ChordAngle { + // If the maximum distance from both endpoints to the cell is less than π/2 + // then the maximum distance from the edge to the cell is the maximum of the + // two endpoint distances. + maxDist := maxChordAngle(c.MaxDistance(a), c.MaxDistance(b)) + if maxDist <= s1.RightChordAngle { + return maxDist + } + + return s1.StraightChordAngle - c.DistanceToEdge(Point{a.Mul(-1)}, Point{b.Mul(-1)}) +} + +// DistanceToCell returns the minimum distance from this cell to the given cell. +// It returns zero if one cell contains the other. +func (c Cell) DistanceToCell(target Cell) s1.ChordAngle { + // If the cells intersect, the distance is zero. We use the (u,v) ranges + // rather than CellID intersects so that cells that share a partial edge or + // corner are considered to intersect. + if c.face == target.face && c.uv.Intersects(target.uv) { + return 0 + } + + // Otherwise, the minimum distance always occurs between a vertex of one + // cell and an edge of the other cell (including the edge endpoints). This + // represents a total of 32 possible (vertex, edge) pairs. + // + // TODO(roberts): This could be optimized to be at least 5x faster by pruning + // the set of possible closest vertex/edge pairs using the faces and (u,v) + // ranges of both cells. + var va, vb [4]Point + for i := 0; i < 4; i++ { + va[i] = c.Vertex(i) + vb[i] = target.Vertex(i) + } + minDist := s1.InfChordAngle() + for i := 0; i < 4; i++ { + for j := 0; j < 4; j++ { + minDist, _ = UpdateMinDistance(va[i], vb[j], vb[(j+1)&3], minDist) + minDist, _ = UpdateMinDistance(vb[i], va[j], va[(j+1)&3], minDist) + } + } + return minDist +} + +// MaxDistanceToCell returns the maximum distance from the cell (including its +// interior) to the given target cell. +func (c Cell) MaxDistanceToCell(target Cell) s1.ChordAngle { + // Need to check the antipodal target for intersection with the cell. If it + // intersects, the distance is the straight ChordAngle. + // antipodalUV is the transpose of the original UV, interpreted within the opposite face. + antipodalUV := r2.Rect{target.uv.Y, target.uv.X} + if int(c.face) == oppositeFace(int(target.face)) && c.uv.Intersects(antipodalUV) { + return s1.StraightChordAngle + } + + // Otherwise, the maximum distance always occurs between a vertex of one + // cell and an edge of the other cell (including the edge endpoints). This + // represents a total of 32 possible (vertex, edge) pairs. + // + // TODO(roberts): When the maximum distance is at most π/2, the maximum is + // always attained between a pair of vertices, and this could be made much + // faster by testing each vertex pair once rather than the current 4 times. + var va, vb [4]Point + for i := 0; i < 4; i++ { + va[i] = c.Vertex(i) + vb[i] = target.Vertex(i) + } + maxDist := s1.NegativeChordAngle + for i := 0; i < 4; i++ { + for j := 0; j < 4; j++ { + maxDist, _ = UpdateMaxDistance(va[i], vb[j], vb[(j+1)&3], maxDist) + maxDist, _ = UpdateMaxDistance(vb[i], va[j], va[(j+1)&3], maxDist) + } + } + return maxDist +} diff --git a/vendor/github.com/blevesearch/geo/s2/cell_index.go b/vendor/github.com/blevesearch/geo/s2/cell_index.go new file mode 100644 index 00000000..879df48a --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/cell_index.go @@ -0,0 +1,584 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +import ( + "sort" +) + +const ( + // A special label indicating that the ContentsIterator done is true. + cellIndexDoneContents = -1 +) + +// cellIndexNode represents a node in the CellIndex. Cells are organized in a +// tree such that the ancestors of a given node contain that node. +type cellIndexNode struct { + cellID CellID + label int32 + parent int32 +} + +// newCellIndexNode returns a node with the appropriate default values. +func newCellIndexNode() cellIndexNode { + return cellIndexNode{ + cellID: 0, + label: cellIndexDoneContents, + parent: -1, + } +} + +// A rangeNode represents a range of leaf CellIDs. The range starts at +// startID (a leaf cell) and ends at the startID field of the next +// rangeNode. contents points to the node of the CellIndex cellTree +// representing the cells that overlap this range. +type rangeNode struct { + startID CellID // First leaf cell contained by this range. + contents int32 // Contents of this node (an index within the cell tree). +} + +// CellIndexIterator is an iterator that visits the entire set of indexed +// (CellID, label) pairs in an unspecified order. +type CellIndexIterator struct { + // TODO(roberts): Implement + cellTree []cellIndexNode + pos int +} + +// NewCellIndexIterator creates an iterator for the given CellIndex. +func NewCellIndexIterator(index *CellIndex) *CellIndexIterator { + return &CellIndexIterator{ + cellTree: index.cellTree, + } +} + +// CellID returns the current CellID. +func (c *CellIndexIterator) CellID() CellID { + return c.cellTree[c.pos].cellID +} + +// Label returns the current Label. +func (c *CellIndexIterator) Label() int32 { + return c.cellTree[c.pos].label +} + +func (c *CellIndexIterator) Done() bool { + return c.pos == len(c.cellTree)-1 +} + +func (c *CellIndexIterator) Next() { + c.pos++ +} + +// CellIndexRangeIterator is an iterator that seeks and iterates over a set of +// non-overlapping leaf cell ranges that cover the entire sphere. The indexed +// (CellID, label) pairs that intersect the current leaf cell range can be +// visited using CellIndexContentsIterator (see below). +type CellIndexRangeIterator struct { + rangeNodes []rangeNode + pos int + nonEmpty bool +} + +// NewCellIndexRangeIterator creates an iterator for the given CellIndex. +// The iterator is initially *unpositioned*; you must call a positioning method +// such as Begin() or Seek() before accessing its contents. +func NewCellIndexRangeIterator(index *CellIndex) *CellIndexRangeIterator { + return &CellIndexRangeIterator{ + rangeNodes: index.rangeNodes, + } +} + +// NewCellIndexNonEmptyRangeIterator creates an iterator for the given CellIndex. +// The iterator is initially *unpositioned*; you must call a positioning method such as +// Begin() or Seek() before accessing its contents. +func NewCellIndexNonEmptyRangeIterator(index *CellIndex) *CellIndexRangeIterator { + return &CellIndexRangeIterator{ + rangeNodes: index.rangeNodes, + nonEmpty: true, + } +} + +// StartID reports the CellID of the start of the current range of leaf CellIDs. +// +// If done is true, this returns the last possible CellID. This property means +// that most loops do not need to test done explicitly. +func (c *CellIndexRangeIterator) StartID() CellID { + return c.rangeNodes[c.pos].startID +} + +// LimitID reports the non-inclusive end of the current range of leaf CellIDs. +// +// This assumes the iterator is not done. +func (c *CellIndexRangeIterator) LimitID() CellID { + return c.rangeNodes[c.pos+1].startID +} + +// IsEmpty reports if no (CellID, label) pairs intersect this range. +// Also returns true if done() is true. +func (c *CellIndexRangeIterator) IsEmpty() bool { + return c.rangeNodes[c.pos].contents == cellIndexDoneContents +} + +// Begin positions the iterator at the first range of leaf cells (if any). +func (c *CellIndexRangeIterator) Begin() { + c.pos = 0 + for c.nonEmpty && c.IsEmpty() && !c.Done() { + c.pos++ + } +} + +// Prev positions the iterator at the previous entry and reports whether it was not +// already positioned at the beginning. +func (c *CellIndexRangeIterator) Prev() bool { + if c.nonEmpty { + return c.nonEmptyPrev() + } + return c.prev() +} + +// prev is used to position the iterator at the previous entry without checking +// if nonEmpty is true to prevent unwanted recursion. +func (c *CellIndexRangeIterator) prev() bool { + if c.pos == 0 { + return false + } + + c.pos-- + return true +} + +// Prev positions the iterator at the previous entry, and reports whether it was +// already positioned at the beginning. +func (c *CellIndexRangeIterator) nonEmptyPrev() bool { + for c.prev() { + if !c.IsEmpty() { + return true + } + } + + // Return the iterator to its original position. + if c.IsEmpty() && !c.Done() { + c.Next() + } + return false +} + +// Next advances the iterator to the next range of leaf cells. +// +// This assumes the iterator is not done. +func (c *CellIndexRangeIterator) Next() { + c.pos++ + for c.nonEmpty && c.IsEmpty() && !c.Done() { + c.pos++ + } +} + +// Advance reports if advancing would leave it positioned on a valid range. If +// the value would not be valid, the positioning is not changed. +func (c *CellIndexRangeIterator) Advance(n int) bool { + // Note that the last element of rangeNodes is a sentinel value. + if n >= len(c.rangeNodes)-1-c.pos { + return false + } + c.pos += n + return true +} + +// Finish positions the iterator so that done is true. +func (c *CellIndexRangeIterator) Finish() { + // Note that the last element of rangeNodes is a sentinel value. + c.pos = len(c.rangeNodes) - 1 +} + +// Done reports if the iterator is positioned beyond the last valid range. +func (c *CellIndexRangeIterator) Done() bool { + return c.pos >= len(c.rangeNodes)-1 +} + +// Seek positions the iterator at the first range with startID >= target. +// Such an entry always exists as long as "target" is a valid leaf cell. +// +// Note that it is valid to access startID even when done is true. +func (c *CellIndexRangeIterator) Seek(target CellID) { + c.pos = sort.Search(len(c.rangeNodes), func(i int) bool { + return c.rangeNodes[i].startID > target + }) - 1 + + // Ensure we don't go beyond the beginning. + if c.pos < 0 { + c.pos = 0 + } + + // Nonempty needs to find the next non-empty entry. + for c.nonEmpty && c.IsEmpty() && !c.Done() { + // c.Next() + c.pos++ + } +} + +// CellIndexContentsIterator is an iterator that visits the (CellID, label) pairs +// that cover a set of leaf cell ranges (see CellIndexRangeIterator). Note that +// when multiple leaf cell ranges are visited, this iterator only guarantees that +// each result will be reported at least once, i.e. duplicate values may be +// suppressed. If you want duplicate values to be reported again, be sure to call +// Clear first. +// +// In particular, the implementation guarantees that when multiple leaf +// cell ranges are visited in monotonically increasing order, then each +// (CellID, label) pair is reported exactly once. +type CellIndexContentsIterator struct { + // The maximum index within the cellTree slice visited during the + // previous call to StartUnion. This is used to eliminate duplicate + // values when StartUnion is called multiple times. + nodeCutoff int32 + + // The maximum index within the cellTree visited during the + // current call to StartUnion. This is used to update nodeCutoff. + nextNodeCutoff int32 + + // The value of startID from the previous call to StartUnion. + // This is used to check whether these values are monotonically + // increasing. + prevStartID CellID + + // The cell tree from CellIndex + cellTree []cellIndexNode + + // A copy of the current node in the cell tree. + node cellIndexNode +} + +// NewCellIndexContentsIterator returns a new contents iterator. +// +// Note that the iterator needs to be positioned using StartUnion before +// it can be safely used. +func NewCellIndexContentsIterator(index *CellIndex) *CellIndexContentsIterator { + it := &CellIndexContentsIterator{ + cellTree: index.cellTree, + prevStartID: 0, + nodeCutoff: -1, + nextNodeCutoff: -1, + node: cellIndexNode{label: cellIndexDoneContents}, + } + return it +} + +// Clear clears all state with respect to which range(s) have been visited. +func (c *CellIndexContentsIterator) Clear() { + c.prevStartID = 0 + c.nodeCutoff = -1 + c.nextNodeCutoff = -1 + c.node.label = cellIndexDoneContents +} + +// CellID returns the current CellID. +func (c *CellIndexContentsIterator) CellID() CellID { + return c.node.cellID +} + +// Label returns the current Label. +func (c *CellIndexContentsIterator) Label() int32 { + return c.node.label +} + +// Next advances the iterator to the next (CellID, label) pair covered by the +// current leaf cell range. +// +// This requires the iterator to not be done. +func (c *CellIndexContentsIterator) Next() { + if c.node.parent <= c.nodeCutoff { + // We have already processed this node and its ancestors. + c.nodeCutoff = c.nextNodeCutoff + c.node.label = cellIndexDoneContents + } else { + c.node = c.cellTree[c.node.parent] + } +} + +// Done reports if all (CellID, label) pairs have been visited. +func (c *CellIndexContentsIterator) Done() bool { + return c.node.label == cellIndexDoneContents +} + +// StartUnion positions the ContentsIterator at the first (cell_id, label) pair +// that covers the given leaf cell range. Note that when multiple leaf cell +// ranges are visited using the same ContentsIterator, duplicate values +// may be suppressed. If you don't want this behavior, call Reset() first. +func (c *CellIndexContentsIterator) StartUnion(r *CellIndexRangeIterator) { + if r.StartID() < c.prevStartID { + c.nodeCutoff = -1 // Can't automatically eliminate duplicates. + } + c.prevStartID = r.StartID() + + contents := r.rangeNodes[r.pos].contents + if contents <= c.nodeCutoff { + c.node.label = cellIndexDoneContents + } else { + c.node = c.cellTree[contents] + } + + // When visiting ancestors, we can stop as soon as the node index is smaller + // than any previously visited node index. Because indexes are assigned + // using a preorder traversal, such nodes are guaranteed to have already + // been reported. + c.nextNodeCutoff = contents +} + +// CellIndex stores a collection of (CellID, label) pairs. +// +// The CellIDs may be overlapping or contain duplicate values. For example, a +// CellIndex could store a collection of CellUnions, where each CellUnion +// gets its own non-negative int32 label. +// +// Similar to ShapeIndex and PointIndex which map each stored element to an +// identifier, CellIndex stores a label that is typically used to map the +// results of queries back to client's specific data. +// +// The zero value for a CellIndex is sufficient when constructing a CellIndex. +// +// To build a CellIndex where each Cell has a distinct label, call Add for each +// (CellID, label) pair, and then Build the index. For example: +// +// // contents is a mapping of an identifier in my system (restaurantID, +// // vehicleID, etc) to a CellID +// var contents = map[int32]CellID{...} +// +// for key, val := range contents { +// index.Add(val, key) +// } +// +// index.Build() +// +// There is also a helper method that adds all elements of CellUnion with the +// same label: +// +// index.AddCellUnion(cellUnion, label) +// +// Note that the index is not dynamic; the contents of the index cannot be +// changed once it has been built. Adding more after calling Build results in +// undefined behavior of the index. +// +// There are several options for retrieving data from the index. The simplest +// is to use a built-in method such as IntersectingLabels (which returns +// the labels of all cells that intersect a given target CellUnion): +// +// labels := index.IntersectingLabels(targetUnion); +// +// Alternatively, you can use a ClosestCellQuery which computes the cell(s) +// that are closest to a given target geometry. +// +// For example, here is how to find all cells that are closer than +// distanceLimit to a given target point: +// +// query := NewClosestCellQuery(cellIndex, opts) +// target := NewMinDistanceToPointTarget(targetPoint); +// for result := range query.FindCells(target) { +// // result.Distance() is the distance to the target. +// // result.CellID() is the indexed CellID. +// // result.Label() is the label associated with the CellID. +// DoSomething(targetPoint, result); +// } +// +// Internally, the index consists of a set of non-overlapping leaf cell ranges +// that subdivide the sphere and such that each range intersects a particular +// set of (cellID, label) pairs. +// +// Most clients should use either the methods such as VisitIntersectingCells +// and IntersectingLabels, or a helper such as ClosestCellQuery. +type CellIndex struct { + // A tree of (cellID, label) pairs such that if X is an ancestor of Y, then + // X.cellID contains Y.cellID. The contents of a given range of leaf + // cells can be represented by pointing to a node of this tree. + cellTree []cellIndexNode + + // The last element of rangeNodes is a sentinel value, which is necessary + // in order to represent the range covered by the previous element. + rangeNodes []rangeNode +} + +// Add adds the given CellID and Label to the index. +func (c *CellIndex) Add(id CellID, label int32) { + if label < 0 { + panic("labels must be non-negative") + } + c.cellTree = append(c.cellTree, cellIndexNode{cellID: id, label: label, parent: -1}) +} + +// AddCellUnion adds all of the elements of the given CellUnion to the index with the same label. +func (c *CellIndex) AddCellUnion(cu CellUnion, label int32) { + if label < 0 { + panic("labels must be non-negative") + } + for _, cell := range cu { + c.Add(cell, label) + } +} + +// Build builds the index for use. This method should only be called once. +func (c *CellIndex) Build() { + // To build the cell tree and leaf cell ranges, we maintain a stack of + // (CellID, label) pairs that contain the current leaf cell. This struct + // represents an instruction to push or pop a (cellID, label) pair. + // + // If label >= 0, the (cellID, label) pair is pushed on the stack. + // If CellID == SentinelCellID, a pair is popped from the stack. + // Otherwise the stack is unchanged but a rangeNode is still emitted. + + // delta represents an entry in a stack of (CellID, label) pairs used in the + // construction of the CellIndex structure. + type delta struct { + startID CellID + cellID CellID + label int32 + } + + deltas := make([]delta, 0, 2*len(c.cellTree)+2) + + // Create two deltas for each (cellID, label) pair: one to add the pair to + // the stack (at the start of its leaf cell range), and one to remove it from + // the stack (at the end of its leaf cell range). + for _, node := range c.cellTree { + deltas = append(deltas, delta{ + startID: node.cellID.RangeMin(), + cellID: node.cellID, + label: node.label, + }) + deltas = append(deltas, delta{ + startID: node.cellID.RangeMax().Next(), + cellID: SentinelCellID, + label: -1, + }) + } + + // We also create two special deltas to ensure that a RangeNode is emitted at + // the beginning and end of the CellID range. + deltas = append(deltas, delta{ + startID: CellIDFromFace(0).ChildBeginAtLevel(maxLevel), + cellID: CellID(0), + label: -1, + }) + deltas = append(deltas, delta{ + startID: CellIDFromFace(5).ChildEndAtLevel(maxLevel), + cellID: CellID(0), + label: -1, + }) + + sort.Slice(deltas, func(i, j int) bool { + // deltas are sorted first by startID, then in reverse order by cellID, + // and then by label. This is necessary to ensure that (1) larger cells + // are pushed on the stack before smaller cells, and (2) cells are popped + // off the stack before any new cells are added. + + if si, sj := deltas[i].startID, deltas[j].startID; si != sj { + return si < sj + } + if si, sj := deltas[i].cellID, deltas[j].cellID; si != sj { + return si > sj + } + return deltas[i].label < deltas[j].label + }) + + // Now walk through the deltas to build the leaf cell ranges and cell tree + // (which is essentially a permanent form of the "stack" described above). + c.cellTree = nil + c.rangeNodes = nil + contents := int32(-1) + for i := 0; i < len(deltas); { + startID := deltas[i].startID + // Process all the deltas associated with the current startID. + for ; i < len(deltas) && deltas[i].startID == startID; i++ { + if deltas[i].label >= 0 { + c.cellTree = append(c.cellTree, cellIndexNode{ + cellID: deltas[i].cellID, + label: deltas[i].label, + parent: contents}) + contents = int32(len(c.cellTree) - 1) + } else if deltas[i].cellID == SentinelCellID { + contents = c.cellTree[contents].parent + } + } + c.rangeNodes = append(c.rangeNodes, rangeNode{startID, contents}) + } +} + +type CellVisitor func(CellID, int32) bool + +func (c *CellIndex) GetIntersectingLabels(target CellUnion) []int32 { + var rv []int32 + c.IntersectingLabels(target, &rv) + return rv +} + +func (c *CellIndex) IntersectingLabels(target CellUnion, labels *[]int32) { + c.VisitIntersectingCells(target, func(cellID CellID, label int32) bool { + *labels = append(*labels, label) + return true + }) + dedupe(labels) + sort.Slice(*labels, func(i, j int) bool { return (*labels)[i] < (*labels)[j] }) +} + +func dedupe(labels *[]int32) { + encountered := make(map[int32]struct{}) + + for v := range *labels { + encountered[(*labels)[v]] = struct{}{} + } + + (*labels) = (*labels)[:0] + for key, _ := range encountered { + *labels = append(*labels, key) + } +} + +func (c *CellIndex) VisitIntersectingCells(target CellUnion, + visitor CellVisitor) bool { + if len(target) == 0 { + return true + } + + var pos int + cItr := NewCellIndexContentsIterator(c) + rItr := NewCellIndexNonEmptyRangeIterator(c) + rItr.Begin() + for pos < len(target) { + if rItr.LimitID() <= target[pos].RangeMin() { + rItr.Seek(target[pos].RangeMin()) + } + + for rItr.StartID() <= target[pos].RangeMax() { + for cItr.StartUnion(rItr); cItr.Done(); cItr.Next() { + if !visitor(cItr.CellID(), cItr.Label()) { + return false + } + } + } + + pos++ + if pos < len(target) && target[pos].RangeMax() < rItr.StartID() { + pos = target.lowerBound(pos, len(target), rItr.StartID()) + if target[pos-1].RangeMax() >= rItr.StartID() { + pos-- + } + } + } + return true +} + +// TODO(roberts): Differences from C++ +// IntersectingLabels +// VisitIntersectingCells +// CellIndexIterator diff --git a/vendor/github.com/blevesearch/geo/s2/cellid.go b/vendor/github.com/blevesearch/geo/s2/cellid.go new file mode 100644 index 00000000..c6cbaf2d --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/cellid.go @@ -0,0 +1,944 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +import ( + "bytes" + "fmt" + "io" + "math" + "sort" + "strconv" + "strings" + + "github.com/golang/geo/r1" + "github.com/golang/geo/r2" + "github.com/golang/geo/r3" + "github.com/golang/geo/s1" +) + +// CellID uniquely identifies a cell in the S2 cell decomposition. +// The most significant 3 bits encode the face number (0-5). The +// remaining 61 bits encode the position of the center of this cell +// along the Hilbert curve on that face. The zero value and the value +// (1<<64)-1 are invalid cell IDs. The first compares less than any +// valid cell ID, the second as greater than any valid cell ID. +// +// Sequentially increasing cell IDs follow a continuous space-filling curve +// over the entire sphere. They have the following properties: +// +// - The ID of a cell at level k consists of a 3-bit face number followed +// by k bit pairs that recursively select one of the four children of +// each cell. The next bit is always 1, and all other bits are 0. +// Therefore, the level of a cell is determined by the position of its +// lowest-numbered bit that is turned on (for a cell at level k, this +// position is 2 * (maxLevel - k)). +// +// - The ID of a parent cell is at the midpoint of the range of IDs spanned +// by its children (or by its descendants at any level). +// +// Leaf cells are often used to represent points on the unit sphere, and +// this type provides methods for converting directly between these two +// representations. For cells that represent 2D regions rather than +// discrete point, it is better to use Cells. +type CellID uint64 + +// SentinelCellID is an invalid cell ID guaranteed to be larger than any +// valid cell ID. It is used primarily by ShapeIndex. The value is also used +// by some S2 types when encoding data. +// Note that the sentinel's RangeMin == RangeMax == itself. +const SentinelCellID = CellID(^uint64(0)) + +// sortCellIDs sorts the slice of CellIDs in place. +func sortCellIDs(ci []CellID) { + sort.Sort(cellIDs(ci)) +} + +// cellIDs implements the Sort interface for slices of CellIDs. +type cellIDs []CellID + +func (c cellIDs) Len() int { return len(c) } +func (c cellIDs) Swap(i, j int) { c[i], c[j] = c[j], c[i] } +func (c cellIDs) Less(i, j int) bool { return c[i] < c[j] } + +// TODO(dsymonds): Some of these constants should probably be exported. +const ( + faceBits = 3 + numFaces = 6 + + // This is the number of levels needed to specify a leaf cell. + maxLevel = 30 + + // The extra position bit (61 rather than 60) lets us encode each cell as its + // Hilbert curve position at the cell center (which is halfway along the + // portion of the Hilbert curve that fills that cell). + posBits = 2*maxLevel + 1 + + // The maximum index of a valid leaf cell plus one. The range of valid leaf + // cell indices is [0..maxSize-1]. + maxSize = 1 << maxLevel + + wrapOffset = uint64(numFaces) << posBits +) + +// CellIDFromFacePosLevel returns a cell given its face in the range +// [0,5], the 61-bit Hilbert curve position pos within that face, and +// the level in the range [0,maxLevel]. The position in the cell ID +// will be truncated to correspond to the Hilbert curve position at +// the center of the returned cell. +func CellIDFromFacePosLevel(face int, pos uint64, level int) CellID { + return CellID(uint64(face)< 16 { + return CellID(0) + } + n, err := strconv.ParseUint(s, 16, 64) + if err != nil { + return CellID(0) + } + // Equivalent to right-padding string with zeros to 16 characters. + if len(s) < 16 { + n = n << (4 * uint(16-len(s))) + } + return CellID(n) +} + +// ToToken returns a hex-encoded string of the uint64 cell id, with leading +// zeros included but trailing zeros stripped. +func (ci CellID) ToToken() string { + s := strings.TrimRight(fmt.Sprintf("%016x", uint64(ci)), "0") + if len(s) == 0 { + return "X" + } + return s +} + +// IsValid reports whether ci represents a valid cell. +func (ci CellID) IsValid() bool { + return ci.Face() < numFaces && (ci.lsb()&0x1555555555555555 != 0) +} + +// Face returns the cube face for this cell ID, in the range [0,5]. +func (ci CellID) Face() int { return int(uint64(ci) >> posBits) } + +// Pos returns the position along the Hilbert curve of this cell ID, in the range [0,2^posBits-1]. +func (ci CellID) Pos() uint64 { return uint64(ci) & (^uint64(0) >> faceBits) } + +// Level returns the subdivision level of this cell ID, in the range [0, maxLevel]. +func (ci CellID) Level() int { + return maxLevel - findLSBSetNonZero64(uint64(ci))>>1 +} + +// IsLeaf returns whether this cell ID is at the deepest level; +// that is, the level at which the cells are smallest. +func (ci CellID) IsLeaf() bool { return uint64(ci)&1 != 0 } + +// ChildPosition returns the child position (0..3) of this cell's +// ancestor at the given level, relative to its parent. The argument +// should be in the range 1..kMaxLevel. For example, +// ChildPosition(1) returns the position of this cell's level-1 +// ancestor within its top-level face cell. +func (ci CellID) ChildPosition(level int) int { + return int(uint64(ci)>>uint64(2*(maxLevel-level)+1)) & 3 +} + +// lsbForLevel returns the lowest-numbered bit that is on for cells at the given level. +func lsbForLevel(level int) uint64 { return 1 << uint64(2*(maxLevel-level)) } + +// Parent returns the cell at the given level, which must be no greater than the current level. +func (ci CellID) Parent(level int) CellID { + lsb := lsbForLevel(level) + return CellID((uint64(ci) & -lsb) | lsb) +} + +// immediateParent is cheaper than Parent, but assumes !ci.isFace(). +func (ci CellID) immediateParent() CellID { + nlsb := CellID(ci.lsb() << 2) + return (ci & -nlsb) | nlsb +} + +// isFace returns whether this is a top-level (face) cell. +func (ci CellID) isFace() bool { return uint64(ci)&(lsbForLevel(0)-1) == 0 } + +// lsb returns the least significant bit that is set. +func (ci CellID) lsb() uint64 { return uint64(ci) & -uint64(ci) } + +// Children returns the four immediate children of this cell. +// If ci is a leaf cell, it returns four identical cells that are not the children. +func (ci CellID) Children() [4]CellID { + var ch [4]CellID + lsb := CellID(ci.lsb()) + ch[0] = ci - lsb + lsb>>2 + lsb >>= 1 + ch[1] = ch[0] + lsb + ch[2] = ch[1] + lsb + ch[3] = ch[2] + lsb + return ch +} + +func sizeIJ(level int) int { + return 1 << uint(maxLevel-level) +} + +// EdgeNeighbors returns the four cells that are adjacent across the cell's four edges. +// Edges 0, 1, 2, 3 are in the down, right, up, left directions in the face space. +// All neighbors are guaranteed to be distinct. +func (ci CellID) EdgeNeighbors() [4]CellID { + level := ci.Level() + size := sizeIJ(level) + f, i, j, _ := ci.faceIJOrientation() + return [4]CellID{ + cellIDFromFaceIJWrap(f, i, j-size).Parent(level), + cellIDFromFaceIJWrap(f, i+size, j).Parent(level), + cellIDFromFaceIJWrap(f, i, j+size).Parent(level), + cellIDFromFaceIJWrap(f, i-size, j).Parent(level), + } +} + +// VertexNeighbors returns the neighboring cellIDs with vertex closest to this cell at the given level. +// (Normally there are four neighbors, but the closest vertex may only have three neighbors if it is one of +// the 8 cube vertices.) +func (ci CellID) VertexNeighbors(level int) []CellID { + halfSize := sizeIJ(level + 1) + size := halfSize << 1 + f, i, j, _ := ci.faceIJOrientation() + + var isame, jsame bool + var ioffset, joffset int + if i&halfSize != 0 { + ioffset = size + isame = (i + size) < maxSize + } else { + ioffset = -size + isame = (i - size) >= 0 + } + if j&halfSize != 0 { + joffset = size + jsame = (j + size) < maxSize + } else { + joffset = -size + jsame = (j - size) >= 0 + } + + results := []CellID{ + ci.Parent(level), + cellIDFromFaceIJSame(f, i+ioffset, j, isame).Parent(level), + cellIDFromFaceIJSame(f, i, j+joffset, jsame).Parent(level), + } + + if isame || jsame { + results = append(results, cellIDFromFaceIJSame(f, i+ioffset, j+joffset, isame && jsame).Parent(level)) + } + + return results +} + +// AllNeighbors returns all neighbors of this cell at the given level. Two +// cells X and Y are neighbors if their boundaries intersect but their +// interiors do not. In particular, two cells that intersect at a single +// point are neighbors. Note that for cells adjacent to a face vertex, the +// same neighbor may be returned more than once. There could be up to eight +// neighbors including the diagonal ones that share the vertex. +// +// This requires level >= ci.Level(). +func (ci CellID) AllNeighbors(level int) []CellID { + var neighbors []CellID + + face, i, j, _ := ci.faceIJOrientation() + + // Find the coordinates of the lower left-hand leaf cell. We need to + // normalize (i,j) to a known position within the cell because level + // may be larger than this cell's level. + size := sizeIJ(ci.Level()) + i &= -size + j &= -size + + nbrSize := sizeIJ(level) + + // We compute the top-bottom, left-right, and diagonal neighbors in one + // pass. The loop test is at the end of the loop to avoid 32-bit overflow. + for k := -nbrSize; ; k += nbrSize { + var sameFace bool + if k < 0 { + sameFace = (j+k >= 0) + } else if k >= size { + sameFace = (j+k < maxSize) + } else { + sameFace = true + // Top and bottom neighbors. + neighbors = append(neighbors, cellIDFromFaceIJSame(face, i+k, j-nbrSize, + j-size >= 0).Parent(level)) + neighbors = append(neighbors, cellIDFromFaceIJSame(face, i+k, j+size, + j+size < maxSize).Parent(level)) + } + + // Left, right, and diagonal neighbors. + neighbors = append(neighbors, cellIDFromFaceIJSame(face, i-nbrSize, j+k, + sameFace && i-size >= 0).Parent(level)) + neighbors = append(neighbors, cellIDFromFaceIJSame(face, i+size, j+k, + sameFace && i+size < maxSize).Parent(level)) + + if k >= size { + break + } + } + + return neighbors +} + +// RangeMin returns the minimum CellID that is contained within this cell. +func (ci CellID) RangeMin() CellID { return CellID(uint64(ci) - (ci.lsb() - 1)) } + +// RangeMax returns the maximum CellID that is contained within this cell. +func (ci CellID) RangeMax() CellID { return CellID(uint64(ci) + (ci.lsb() - 1)) } + +// Contains returns true iff the CellID contains oci. +func (ci CellID) Contains(oci CellID) bool { + return uint64(ci.RangeMin()) <= uint64(oci) && uint64(oci) <= uint64(ci.RangeMax()) +} + +// Intersects returns true iff the CellID intersects oci. +func (ci CellID) Intersects(oci CellID) bool { + return uint64(oci.RangeMin()) <= uint64(ci.RangeMax()) && uint64(oci.RangeMax()) >= uint64(ci.RangeMin()) +} + +// String returns the string representation of the cell ID in the form "1/3210". +func (ci CellID) String() string { + if !ci.IsValid() { + return "Invalid: " + strconv.FormatInt(int64(ci), 16) + } + var b bytes.Buffer + b.WriteByte("012345"[ci.Face()]) // values > 5 will have been picked off by !IsValid above + b.WriteByte('/') + for level := 1; level <= ci.Level(); level++ { + b.WriteByte("0123"[ci.ChildPosition(level)]) + } + return b.String() +} + +// cellIDFromString returns a CellID from a string in the form "1/3210". +func cellIDFromString(s string) CellID { + level := len(s) - 2 + if level < 0 || level > maxLevel { + return CellID(0) + } + face := int(s[0] - '0') + if face < 0 || face > 5 || s[1] != '/' { + return CellID(0) + } + id := CellIDFromFace(face) + for i := 2; i < len(s); i++ { + childPos := s[i] - '0' + if childPos < 0 || childPos > 3 { + return CellID(0) + } + id = id.Children()[childPos] + } + return id +} + +// Point returns the center of the s2 cell on the sphere as a Point. +// The maximum directional error in Point (compared to the exact +// mathematical result) is 1.5 * dblEpsilon radians, and the maximum length +// error is 2 * dblEpsilon (the same as Normalize). +func (ci CellID) Point() Point { return Point{ci.rawPoint().Normalize()} } + +// LatLng returns the center of the s2 cell on the sphere as a LatLng. +func (ci CellID) LatLng() LatLng { return LatLngFromPoint(Point{ci.rawPoint()}) } + +// ChildBegin returns the first child in a traversal of the children of this cell, in Hilbert curve order. +// +// for ci := c.ChildBegin(); ci != c.ChildEnd(); ci = ci.Next() { +// ... +// } +func (ci CellID) ChildBegin() CellID { + ol := ci.lsb() + return CellID(uint64(ci) - ol + ol>>2) +} + +// ChildBeginAtLevel returns the first cell in a traversal of children a given level deeper than this cell, in +// Hilbert curve order. The given level must be no smaller than the cell's level. +// See ChildBegin for example use. +func (ci CellID) ChildBeginAtLevel(level int) CellID { + return CellID(uint64(ci) - ci.lsb() + lsbForLevel(level)) +} + +// ChildEnd returns the first cell after a traversal of the children of this cell in Hilbert curve order. +// The returned cell may be invalid. +func (ci CellID) ChildEnd() CellID { + ol := ci.lsb() + return CellID(uint64(ci) + ol + ol>>2) +} + +// ChildEndAtLevel returns the first cell after the last child in a traversal of children a given level deeper +// than this cell, in Hilbert curve order. +// The given level must be no smaller than the cell's level. +// The returned cell may be invalid. +func (ci CellID) ChildEndAtLevel(level int) CellID { + return CellID(uint64(ci) + ci.lsb() + lsbForLevel(level)) +} + +// Next returns the next cell along the Hilbert curve. +// This is expected to be used with ChildBegin and ChildEnd, +// or ChildBeginAtLevel and ChildEndAtLevel. +func (ci CellID) Next() CellID { + return CellID(uint64(ci) + ci.lsb()<<1) +} + +// Prev returns the previous cell along the Hilbert curve. +func (ci CellID) Prev() CellID { + return CellID(uint64(ci) - ci.lsb()<<1) +} + +// NextWrap returns the next cell along the Hilbert curve, wrapping from last to +// first as necessary. This should not be used with ChildBegin and ChildEnd. +func (ci CellID) NextWrap() CellID { + n := ci.Next() + if uint64(n) < wrapOffset { + return n + } + return CellID(uint64(n) - wrapOffset) +} + +// PrevWrap returns the previous cell along the Hilbert curve, wrapping around from +// first to last as necessary. This should not be used with ChildBegin and ChildEnd. +func (ci CellID) PrevWrap() CellID { + p := ci.Prev() + if uint64(p) < wrapOffset { + return p + } + return CellID(uint64(p) + wrapOffset) +} + +// AdvanceWrap advances or retreats the indicated number of steps along the +// Hilbert curve at the current level and returns the new position. The +// position wraps between the first and last faces as necessary. +func (ci CellID) AdvanceWrap(steps int64) CellID { + if steps == 0 { + return ci + } + + // We clamp the number of steps if necessary to ensure that we do not + // advance past the End() or before the Begin() of this level. + shift := uint(2*(maxLevel-ci.Level()) + 1) + if steps < 0 { + if min := -int64(uint64(ci) >> shift); steps < min { + wrap := int64(wrapOffset >> shift) + steps %= wrap + if steps < min { + steps += wrap + } + } + } else { + // Unlike Advance(), we don't want to return End(level). + if max := int64((wrapOffset - uint64(ci)) >> shift); steps > max { + wrap := int64(wrapOffset >> shift) + steps %= wrap + if steps > max { + steps -= wrap + } + } + } + + // If steps is negative, then shifting it left has undefined behavior. + // Cast to uint64 for a 2's complement answer. + return CellID(uint64(ci) + (uint64(steps) << shift)) +} + +// Encode encodes the CellID. +func (ci CellID) Encode(w io.Writer) error { + e := &encoder{w: w} + ci.encode(e) + return e.err +} + +func (ci CellID) encode(e *encoder) { + e.writeUint64(uint64(ci)) +} + +// Decode decodes the CellID. +func (ci *CellID) Decode(r io.Reader) error { + d := &decoder{r: asByteReader(r)} + ci.decode(d) + return d.err +} + +func (ci *CellID) decode(d *decoder) { + *ci = CellID(d.readUint64()) +} + +// TODO: the methods below are not exported yet. Settle on the entire API design +// before doing this. Do we want to mirror the C++ one as closely as possible? + +// distanceFromBegin returns the number of steps along the Hilbert curve that +// this cell is from the first node in the S2 hierarchy at our level. (i.e., +// FromFace(0).ChildBeginAtLevel(ci.Level())). This is analogous to Pos(), but +// for this cell's level. +// The return value is always non-negative. +func (ci CellID) distanceFromBegin() int64 { + return int64(ci >> uint64(2*(maxLevel-ci.Level())+1)) +} + +// rawPoint returns an unnormalized r3 vector from the origin through the center +// of the s2 cell on the sphere. +func (ci CellID) rawPoint() r3.Vector { + face, si, ti := ci.faceSiTi() + return faceUVToXYZ(face, stToUV((0.5/maxSize)*float64(si)), stToUV((0.5/maxSize)*float64(ti))) +} + +// faceSiTi returns the Face/Si/Ti coordinates of the center of the cell. +func (ci CellID) faceSiTi() (face int, si, ti uint32) { + face, i, j, _ := ci.faceIJOrientation() + delta := 0 + if ci.IsLeaf() { + delta = 1 + } else { + if (i^(int(ci)>>2))&1 != 0 { + delta = 2 + } + } + return face, uint32(2*i + delta), uint32(2*j + delta) +} + +// faceIJOrientation uses the global lookupIJ table to unfiddle the bits of ci. +func (ci CellID) faceIJOrientation() (f, i, j, orientation int) { + f = ci.Face() + orientation = f & swapMask + nbits := maxLevel - 7*lookupBits // first iteration + + // Each iteration maps 8 bits of the Hilbert curve position into + // 4 bits of "i" and "j". The lookup table transforms a key of the + // form "ppppppppoo" to a value of the form "iiiijjjjoo", where the + // letters [ijpo] represents bits of "i", "j", the Hilbert curve + // position, and the Hilbert curve orientation respectively. + // + // On the first iteration we need to be careful to clear out the bits + // representing the cube face. + for k := 7; k >= 0; k-- { + orientation += (int(uint64(ci)>>uint64(k*2*lookupBits+1)) & ((1 << uint(2*nbits)) - 1)) << 2 + orientation = lookupIJ[orientation] + i += (orientation >> (lookupBits + 2)) << uint(k*lookupBits) + j += ((orientation >> 2) & ((1 << lookupBits) - 1)) << uint(k*lookupBits) + orientation &= (swapMask | invertMask) + nbits = lookupBits // following iterations + } + + // The position of a non-leaf cell at level "n" consists of a prefix of + // 2*n bits that identifies the cell, followed by a suffix of + // 2*(maxLevel-n)+1 bits of the form 10*. If n==maxLevel, the suffix is + // just "1" and has no effect. Otherwise, it consists of "10", followed + // by (maxLevel-n-1) repetitions of "00", followed by "0". The "10" has + // no effect, while each occurrence of "00" has the effect of reversing + // the swapMask bit. + if ci.lsb()&0x1111111111111110 != 0 { + orientation ^= swapMask + } + + return +} + +// cellIDFromFaceIJ returns a leaf cell given its cube face (range 0..5) and IJ coordinates. +func cellIDFromFaceIJ(f, i, j int) CellID { + // Note that this value gets shifted one bit to the left at the end + // of the function. + n := uint64(f) << (posBits - 1) + // Alternating faces have opposite Hilbert curve orientations; this + // is necessary in order for all faces to have a right-handed + // coordinate system. + bits := f & swapMask + // Each iteration maps 4 bits of "i" and "j" into 8 bits of the Hilbert + // curve position. The lookup table transforms a 10-bit key of the form + // "iiiijjjjoo" to a 10-bit value of the form "ppppppppoo", where the + // letters [ijpo] denote bits of "i", "j", Hilbert curve position, and + // Hilbert curve orientation respectively. + for k := 7; k >= 0; k-- { + mask := (1 << lookupBits) - 1 + bits += ((i >> uint(k*lookupBits)) & mask) << (lookupBits + 2) + bits += ((j >> uint(k*lookupBits)) & mask) << 2 + bits = lookupPos[bits] + n |= uint64(bits>>2) << (uint(k) * 2 * lookupBits) + bits &= (swapMask | invertMask) + } + return CellID(n*2 + 1) +} + +func cellIDFromFaceIJWrap(f, i, j int) CellID { + // Convert i and j to the coordinates of a leaf cell just beyond the + // boundary of this face. This prevents 32-bit overflow in the case + // of finding the neighbors of a face cell. + i = clampInt(i, -1, maxSize) + j = clampInt(j, -1, maxSize) + + // We want to wrap these coordinates onto the appropriate adjacent face. + // The easiest way to do this is to convert the (i,j) coordinates to (x,y,z) + // (which yields a point outside the normal face boundary), and then call + // xyzToFaceUV to project back onto the correct face. + // + // The code below converts (i,j) to (si,ti), and then (si,ti) to (u,v) using + // the linear projection (u=2*s-1 and v=2*t-1). (The code further below + // converts back using the inverse projection, s=0.5*(u+1) and t=0.5*(v+1). + // Any projection would work here, so we use the simplest.) We also clamp + // the (u,v) coordinates so that the point is barely outside the + // [-1,1]x[-1,1] face rectangle, since otherwise the reprojection step + // (which divides by the new z coordinate) might change the other + // coordinates enough so that we end up in the wrong leaf cell. + const scale = 1.0 / maxSize + limit := math.Nextafter(1, 2) + u := math.Max(-limit, math.Min(limit, scale*float64((i<<1)+1-maxSize))) + v := math.Max(-limit, math.Min(limit, scale*float64((j<<1)+1-maxSize))) + + // Find the leaf cell coordinates on the adjacent face, and convert + // them to a cell id at the appropriate level. + f, u, v = xyzToFaceUV(faceUVToXYZ(f, u, v)) + return cellIDFromFaceIJ(f, stToIJ(0.5*(u+1)), stToIJ(0.5*(v+1))) +} + +func cellIDFromFaceIJSame(f, i, j int, sameFace bool) CellID { + if sameFace { + return cellIDFromFaceIJ(f, i, j) + } + return cellIDFromFaceIJWrap(f, i, j) +} + +// ijToSTMin converts the i- or j-index of a leaf cell to the minimum corresponding +// s- or t-value contained by that cell. The argument must be in the range +// [0..2**30], i.e. up to one position beyond the normal range of valid leaf +// cell indices. +func ijToSTMin(i int) float64 { + return float64(i) / float64(maxSize) +} + +// stToIJ converts value in ST coordinates to a value in IJ coordinates. +func stToIJ(s float64) int { + return clampInt(int(math.Floor(maxSize*s)), 0, maxSize-1) +} + +// cellIDFromPoint returns a leaf cell containing point p. Usually there is +// exactly one such cell, but for points along the edge of a cell, any +// adjacent cell may be (deterministically) chosen. This is because +// s2.CellIDs are considered to be closed sets. The returned cell will +// always contain the given point, i.e. +// +// CellFromPoint(p).ContainsPoint(p) +// +// is always true. +func cellIDFromPoint(p Point) CellID { + f, u, v := xyzToFaceUV(r3.Vector{p.X, p.Y, p.Z}) + i := stToIJ(uvToST(u)) + j := stToIJ(uvToST(v)) + return cellIDFromFaceIJ(f, i, j) +} + +// ijLevelToBoundUV returns the bounds in (u,v)-space for the cell at the given +// level containing the leaf cell with the given (i,j)-coordinates. +func ijLevelToBoundUV(i, j, level int) r2.Rect { + cellSize := sizeIJ(level) + xLo := i & -cellSize + yLo := j & -cellSize + + return r2.Rect{ + X: r1.Interval{ + Lo: stToUV(ijToSTMin(xLo)), + Hi: stToUV(ijToSTMin(xLo + cellSize)), + }, + Y: r1.Interval{ + Lo: stToUV(ijToSTMin(yLo)), + Hi: stToUV(ijToSTMin(yLo + cellSize)), + }, + } +} + +// Constants related to the bit mangling in the Cell ID. +const ( + lookupBits = 4 + swapMask = 0x01 + invertMask = 0x02 +) + +// The following lookup tables are used to convert efficiently between an +// (i,j) cell index and the corresponding position along the Hilbert curve. +// +// lookupPos maps 4 bits of "i", 4 bits of "j", and 2 bits representing the +// orientation of the current cell into 8 bits representing the order in which +// that subcell is visited by the Hilbert curve, plus 2 bits indicating the +// new orientation of the Hilbert curve within that subcell. (Cell +// orientations are represented as combination of swapMask and invertMask.) +// +// lookupIJ is an inverted table used for mapping in the opposite +// direction. +// +// We also experimented with looking up 16 bits at a time (14 bits of position +// plus 2 of orientation) but found that smaller lookup tables gave better +// performance. (2KB fits easily in the primary cache.) +var ( + ijToPos = [4][4]int{ + {0, 1, 3, 2}, // canonical order + {0, 3, 1, 2}, // axes swapped + {2, 3, 1, 0}, // bits inverted + {2, 1, 3, 0}, // swapped & inverted + } + posToIJ = [4][4]int{ + {0, 1, 3, 2}, // canonical order: (0,0), (0,1), (1,1), (1,0) + {0, 2, 3, 1}, // axes swapped: (0,0), (1,0), (1,1), (0,1) + {3, 2, 0, 1}, // bits inverted: (1,1), (1,0), (0,0), (0,1) + {3, 1, 0, 2}, // swapped & inverted: (1,1), (0,1), (0,0), (1,0) + } + posToOrientation = [4]int{swapMask, 0, 0, invertMask | swapMask} + lookupIJ [1 << (2*lookupBits + 2)]int + lookupPos [1 << (2*lookupBits + 2)]int +) + +func init() { + initLookupCell(0, 0, 0, 0, 0, 0) + initLookupCell(0, 0, 0, swapMask, 0, swapMask) + initLookupCell(0, 0, 0, invertMask, 0, invertMask) + initLookupCell(0, 0, 0, swapMask|invertMask, 0, swapMask|invertMask) +} + +// initLookupCell initializes the lookupIJ table at init time. +func initLookupCell(level, i, j, origOrientation, pos, orientation int) { + if level == lookupBits { + ij := (i << lookupBits) + j + lookupPos[(ij<<2)+origOrientation] = (pos << 2) + orientation + lookupIJ[(pos<<2)+origOrientation] = (ij << 2) + orientation + return + } + + level++ + i <<= 1 + j <<= 1 + pos <<= 2 + r := posToIJ[orientation] + initLookupCell(level, i+(r[0]>>1), j+(r[0]&1), origOrientation, pos, orientation^posToOrientation[0]) + initLookupCell(level, i+(r[1]>>1), j+(r[1]&1), origOrientation, pos+1, orientation^posToOrientation[1]) + initLookupCell(level, i+(r[2]>>1), j+(r[2]&1), origOrientation, pos+2, orientation^posToOrientation[2]) + initLookupCell(level, i+(r[3]>>1), j+(r[3]&1), origOrientation, pos+3, orientation^posToOrientation[3]) +} + +// CommonAncestorLevel returns the level of the common ancestor of the two S2 CellIDs. +func (ci CellID) CommonAncestorLevel(other CellID) (level int, ok bool) { + bits := uint64(ci ^ other) + if bits < ci.lsb() { + bits = ci.lsb() + } + if bits < other.lsb() { + bits = other.lsb() + } + + msbPos := findMSBSetNonZero64(bits) + if msbPos > 60 { + return 0, false + } + return (60 - msbPos) >> 1, true +} + +// Advance advances or retreats the indicated number of steps along the +// Hilbert curve at the current level, and returns the new position. The +// position is never advanced past End() or before Begin(). +func (ci CellID) Advance(steps int64) CellID { + if steps == 0 { + return ci + } + + // We clamp the number of steps if necessary to ensure that we do not + // advance past the End() or before the Begin() of this level. Note that + // minSteps and maxSteps always fit in a signed 64-bit integer. + stepShift := uint(2*(maxLevel-ci.Level()) + 1) + if steps < 0 { + minSteps := -int64(uint64(ci) >> stepShift) + if steps < minSteps { + steps = minSteps + } + } else { + maxSteps := int64((wrapOffset + ci.lsb() - uint64(ci)) >> stepShift) + if steps > maxSteps { + steps = maxSteps + } + } + return ci + CellID(steps)<= limit.RangeMin() { + return limit + } + + if ci.RangeMax() >= limit { + // The cell is too large, shrink it. Note that when generating coverings + // of CellID ranges, this loop usually executes only once. Also because + // ci.RangeMin() < limit.RangeMin(), we will always exit the loop by the + // time we reach a leaf cell. + for { + ci = ci.Children()[0] + if ci.RangeMax() < limit { + break + } + } + return ci + } + + // The cell may be too small. Grow it if necessary. Note that generally + // this loop only iterates once. + for !ci.isFace() { + parent := ci.immediateParent() + if parent.RangeMin() != start || parent.RangeMax() >= limit { + break + } + ci = parent + } + return ci +} + +// centerFaceSiTi returns the (face, si, ti) coordinates of the center of the cell. +// Note that although (si,ti) coordinates span the range [0,2**31] in general, +// the cell center coordinates are always in the range [1,2**31-1] and +// therefore can be represented using a signed 32-bit integer. +func (ci CellID) centerFaceSiTi() (face, si, ti int) { + // First we compute the discrete (i,j) coordinates of a leaf cell contained + // within the given cell. Given that cells are represented by the Hilbert + // curve position corresponding at their center, it turns out that the cell + // returned by faceIJOrientation is always one of two leaf cells closest + // to the center of the cell (unless the given cell is a leaf cell itself, + // in which case there is only one possibility). + // + // Given a cell of size s >= 2 (i.e. not a leaf cell), and letting (imin, + // jmin) be the coordinates of its lower left-hand corner, the leaf cell + // returned by faceIJOrientation is either (imin + s/2, jmin + s/2) + // (imin + s/2 - 1, jmin + s/2 - 1). The first case is the one we want. + // We can distinguish these two cases by looking at the low bit of i or + // j. In the second case the low bit is one, unless s == 2 (i.e. the + // level just above leaf cells) in which case the low bit is zero. + // + // In the code below, the expression ((i ^ (int(id) >> 2)) & 1) is true + // if we are in the second case described above. + face, i, j, _ := ci.faceIJOrientation() + delta := 0 + if ci.IsLeaf() { + delta = 1 + } else if (int64(i)^(int64(ci)>>2))&1 == 1 { + delta = 2 + } + + // Note that (2 * {i,j} + delta) will never overflow a 32-bit integer. + return face, 2*i + delta, 2*j + delta +} diff --git a/vendor/github.com/blevesearch/geo/s2/cellunion.go b/vendor/github.com/blevesearch/geo/s2/cellunion.go new file mode 100644 index 00000000..0654de97 --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/cellunion.go @@ -0,0 +1,590 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +import ( + "fmt" + "io" + "sort" + + "github.com/golang/geo/s1" +) + +// A CellUnion is a collection of CellIDs. +// +// It is normalized if it is sorted, and does not contain redundancy. +// Specifically, it may not contain the same CellID twice, nor a CellID that +// is contained by another, nor the four sibling CellIDs that are children of +// a single higher level CellID. +// +// CellUnions are not required to be normalized, but certain operations will +// return different results if they are not (e.g. Contains). +type CellUnion []CellID + +// CellUnionFromRange creates a CellUnion that covers the half-open range +// of leaf cells [begin, end). If begin == end the resulting union is empty. +// This requires that begin and end are both leaves, and begin <= end. +// To create a closed-ended range, pass in end.Next(). +func CellUnionFromRange(begin, end CellID) CellUnion { + // We repeatedly add the largest cell we can. + var cu CellUnion + for id := begin.MaxTile(end); id != end; id = id.Next().MaxTile(end) { + cu = append(cu, id) + } + // The output is normalized because the cells are added in order by the iteration. + return cu +} + +// CellUnionFromUnion creates a CellUnion from the union of the given CellUnions. +func CellUnionFromUnion(cellUnions ...CellUnion) CellUnion { + var cu CellUnion + for _, cellUnion := range cellUnions { + cu = append(cu, cellUnion...) + } + cu.Normalize() + return cu +} + +// CellUnionFromIntersection creates a CellUnion from the intersection of the given CellUnions. +func CellUnionFromIntersection(x, y CellUnion) CellUnion { + var cu CellUnion + + // This is a fairly efficient calculation that uses binary search to skip + // over sections of both input vectors. It takes constant time if all the + // cells of x come before or after all the cells of y in CellID order. + var i, j int + for i < len(x) && j < len(y) { + iMin := x[i].RangeMin() + jMin := y[j].RangeMin() + if iMin > jMin { + // Either j.Contains(i) or the two cells are disjoint. + if x[i] <= y[j].RangeMax() { + cu = append(cu, x[i]) + i++ + } else { + // Advance j to the first cell possibly contained by x[i]. + j = y.lowerBound(j+1, len(y), iMin) + // The previous cell y[j-1] may now contain x[i]. + if x[i] <= y[j-1].RangeMax() { + j-- + } + } + } else if jMin > iMin { + // Identical to the code above with i and j reversed. + if y[j] <= x[i].RangeMax() { + cu = append(cu, y[j]) + j++ + } else { + i = x.lowerBound(i+1, len(x), jMin) + if y[j] <= x[i-1].RangeMax() { + i-- + } + } + } else { + // i and j have the same RangeMin(), so one contains the other. + if x[i] < y[j] { + cu = append(cu, x[i]) + i++ + } else { + cu = append(cu, y[j]) + j++ + } + } + } + + // The output is generated in sorted order. + cu.Normalize() + return cu +} + +// CellUnionFromIntersectionWithCellID creates a CellUnion from the intersection +// of a CellUnion with the given CellID. This can be useful for splitting a +// CellUnion into chunks. +func CellUnionFromIntersectionWithCellID(x CellUnion, id CellID) CellUnion { + var cu CellUnion + if x.ContainsCellID(id) { + cu = append(cu, id) + cu.Normalize() + return cu + } + + idmax := id.RangeMax() + for i := x.lowerBound(0, len(x), id.RangeMin()); i < len(x) && x[i] <= idmax; i++ { + cu = append(cu, x[i]) + } + + cu.Normalize() + return cu +} + +// CellUnionFromDifference creates a CellUnion from the difference (x - y) +// of the given CellUnions. +func CellUnionFromDifference(x, y CellUnion) CellUnion { + // TODO(roberts): This is approximately O(N*log(N)), but could probably + // use similar techniques as CellUnionFromIntersectionWithCellID to be more efficient. + + var cu CellUnion + for _, xid := range x { + cu.cellUnionDifferenceInternal(xid, &y) + } + + // The output is generated in sorted order, and there should not be any + // cells that can be merged (provided that both inputs were normalized). + return cu +} + +// The C++ constructor methods FromNormalized and FromVerbatim are not necessary +// since they don't call Normalize, and just set the CellIDs directly on the object, +// so straight casting is sufficient in Go to replicate this behavior. + +// IsValid reports whether the cell union is valid, meaning that the CellIDs are +// valid, non-overlapping, and sorted in increasing order. +func (cu *CellUnion) IsValid() bool { + for i, cid := range *cu { + if !cid.IsValid() { + return false + } + if i == 0 { + continue + } + if (*cu)[i-1].RangeMax() >= cid.RangeMin() { + return false + } + } + return true +} + +// IsNormalized reports whether the cell union is normalized, meaning that it is +// satisfies IsValid and that no four cells have a common parent. +// Certain operations such as Contains will return a different +// result if the cell union is not normalized. +func (cu *CellUnion) IsNormalized() bool { + for i, cid := range *cu { + if !cid.IsValid() { + return false + } + if i == 0 { + continue + } + if (*cu)[i-1].RangeMax() >= cid.RangeMin() { + return false + } + if i < 3 { + continue + } + if areSiblings((*cu)[i-3], (*cu)[i-2], (*cu)[i-1], cid) { + return false + } + } + return true +} + +// Normalize normalizes the CellUnion. +func (cu *CellUnion) Normalize() { + sortCellIDs(*cu) + + output := make([]CellID, 0, len(*cu)) // the list of accepted cells + // Loop invariant: output is a sorted list of cells with no redundancy. + for _, ci := range *cu { + // The first two passes here either ignore this new candidate, + // or remove previously accepted cells that are covered by this candidate. + + // Ignore this cell if it is contained by the previous one. + // We only need to check the last accepted cell. The ordering of the + // cells implies containment (but not the converse), and output has no redundancy, + // so if this candidate is not contained by the last accepted cell + // then it cannot be contained by any previously accepted cell. + if len(output) > 0 && output[len(output)-1].Contains(ci) { + continue + } + + // Discard any previously accepted cells contained by this one. + // This could be any contiguous trailing subsequence, but it can't be + // a discontiguous subsequence because of the containment property of + // sorted S2 cells mentioned above. + j := len(output) - 1 // last index to keep + for j >= 0 { + if !ci.Contains(output[j]) { + break + } + j-- + } + output = output[:j+1] + + // See if the last three cells plus this one can be collapsed. + // We loop because collapsing three accepted cells and adding a higher level cell + // could cascade into previously accepted cells. + for len(output) >= 3 && areSiblings(output[len(output)-3], output[len(output)-2], output[len(output)-1], ci) { + // Replace four children by their parent cell. + output = output[:len(output)-3] + ci = ci.immediateParent() // checked !ci.isFace above + } + output = append(output, ci) + } + *cu = output +} + +// IntersectsCellID reports whether this CellUnion intersects the given cell ID. +func (cu *CellUnion) IntersectsCellID(id CellID) bool { + // Find index of array item that occurs directly after our probe cell: + i := sort.Search(len(*cu), func(i int) bool { return id < (*cu)[i] }) + + if i != len(*cu) && (*cu)[i].RangeMin() <= id.RangeMax() { + return true + } + return i != 0 && (*cu)[i-1].RangeMax() >= id.RangeMin() +} + +// ContainsCellID reports whether the CellUnion contains the given cell ID. +// Containment is defined with respect to regions, e.g. a cell contains its 4 children. +// +// CAVEAT: If you have constructed a non-normalized CellUnion, note that groups +// of 4 child cells are *not* considered to contain their parent cell. To get +// this behavior you must use one of the call Normalize() explicitly. +func (cu *CellUnion) ContainsCellID(id CellID) bool { + // Find index of array item that occurs directly after our probe cell: + i := sort.Search(len(*cu), func(i int) bool { return id < (*cu)[i] }) + + if i != len(*cu) && (*cu)[i].RangeMin() <= id { + return true + } + return i != 0 && (*cu)[i-1].RangeMax() >= id +} + +// Denormalize replaces this CellUnion with an expanded version of the +// CellUnion where any cell whose level is less than minLevel or where +// (level - minLevel) is not a multiple of levelMod is replaced by its +// children, until either both of these conditions are satisfied or the +// maximum level is reached. +func (cu *CellUnion) Denormalize(minLevel, levelMod int) { + var denorm CellUnion + for _, id := range *cu { + level := id.Level() + newLevel := level + if newLevel < minLevel { + newLevel = minLevel + } + if levelMod > 1 { + newLevel += (maxLevel - (newLevel - minLevel)) % levelMod + if newLevel > maxLevel { + newLevel = maxLevel + } + } + if newLevel == level { + denorm = append(denorm, id) + } else { + end := id.ChildEndAtLevel(newLevel) + for ci := id.ChildBeginAtLevel(newLevel); ci != end; ci = ci.Next() { + denorm = append(denorm, ci) + } + } + } + *cu = denorm +} + +// RectBound returns a Rect that bounds this entity. +func (cu *CellUnion) RectBound() Rect { + bound := EmptyRect() + for _, c := range *cu { + bound = bound.Union(CellFromCellID(c).RectBound()) + } + return bound +} + +// CapBound returns a Cap that bounds this entity. +func (cu *CellUnion) CapBound() Cap { + if len(*cu) == 0 { + return EmptyCap() + } + + // Compute the approximate centroid of the region. This won't produce the + // bounding cap of minimal area, but it should be close enough. + var centroid Point + + for _, ci := range *cu { + area := AvgAreaMetric.Value(ci.Level()) + centroid = Point{centroid.Add(ci.Point().Mul(area))} + } + + if zero := (Point{}); centroid == zero { + centroid = PointFromCoords(1, 0, 0) + } else { + centroid = Point{centroid.Normalize()} + } + + // Use the centroid as the cap axis, and expand the cap angle so that it + // contains the bounding caps of all the individual cells. Note that it is + // *not* sufficient to just bound all the cell vertices because the bounding + // cap may be concave (i.e. cover more than one hemisphere). + c := CapFromPoint(centroid) + for _, ci := range *cu { + c = c.AddCap(CellFromCellID(ci).CapBound()) + } + + return c +} + +// ContainsCell reports whether this cell union contains the given cell. +func (cu *CellUnion) ContainsCell(c Cell) bool { + return cu.ContainsCellID(c.id) +} + +// IntersectsCell reports whether this cell union intersects the given cell. +func (cu *CellUnion) IntersectsCell(c Cell) bool { + return cu.IntersectsCellID(c.id) +} + +// ContainsPoint reports whether this cell union contains the given point. +func (cu *CellUnion) ContainsPoint(p Point) bool { + return cu.ContainsCell(CellFromPoint(p)) +} + +// CellUnionBound computes a covering of the CellUnion. +func (cu *CellUnion) CellUnionBound() []CellID { + return cu.CapBound().CellUnionBound() +} + +// LeafCellsCovered reports the number of leaf cells covered by this cell union. +// This will be no more than 6*2^60 for the whole sphere. +func (cu *CellUnion) LeafCellsCovered() int64 { + var numLeaves int64 + for _, c := range *cu { + numLeaves += 1 << uint64((maxLevel-int64(c.Level()))<<1) + } + return numLeaves +} + +// Returns true if the given four cells have a common parent. +// This requires that the four CellIDs are distinct. +func areSiblings(a, b, c, d CellID) bool { + // A necessary (but not sufficient) condition is that the XOR of the + // four cell IDs must be zero. This is also very fast to test. + if (a ^ b ^ c) != d { + return false + } + + // Now we do a slightly more expensive but exact test. First, compute a + // mask that blocks out the two bits that encode the child position of + // "id" with respect to its parent, then check that the other three + // children all agree with "mask". + mask := d.lsb() << 1 + mask = ^(mask + (mask << 1)) + idMasked := (uint64(d) & mask) + return ((uint64(a)&mask) == idMasked && + (uint64(b)&mask) == idMasked && + (uint64(c)&mask) == idMasked && + !d.isFace()) +} + +// Contains reports whether this CellUnion contains all of the CellIDs of the given CellUnion. +func (cu *CellUnion) Contains(o CellUnion) bool { + // TODO(roberts): Investigate alternatives such as divide-and-conquer + // or alternating-skip-search that may be significantly faster in both + // the average and worst case. This applies to Intersects as well. + for _, id := range o { + if !cu.ContainsCellID(id) { + return false + } + } + + return true +} + +// Intersects reports whether this CellUnion intersects any of the CellIDs of the given CellUnion. +func (cu *CellUnion) Intersects(o CellUnion) bool { + for _, c := range *cu { + if o.IntersectsCellID(c) { + return true + } + } + + return false +} + +// lowerBound returns the index in this CellUnion to the first element whose value +// is not considered to go before the given cell id. (i.e., either it is equivalent +// or comes after the given id.) If there is no match, then end is returned. +func (cu *CellUnion) lowerBound(begin, end int, id CellID) int { + for i := begin; i < end; i++ { + if (*cu)[i] >= id { + return i + } + } + + return end +} + +// cellUnionDifferenceInternal adds the difference between the CellID and the union to +// the result CellUnion. If they intersect but the difference is non-empty, it divides +// and conquers. +func (cu *CellUnion) cellUnionDifferenceInternal(id CellID, other *CellUnion) { + if !other.IntersectsCellID(id) { + (*cu) = append((*cu), id) + return + } + + if !other.ContainsCellID(id) { + for _, child := range id.Children() { + cu.cellUnionDifferenceInternal(child, other) + } + } +} + +// ExpandAtLevel expands this CellUnion by adding a rim of cells at expandLevel +// around the unions boundary. +// +// For each cell c in the union, we add all cells at level +// expandLevel that abut c. There are typically eight of those +// (four edge-abutting and four sharing a vertex). However, if c is +// finer than expandLevel, we add all cells abutting +// c.Parent(expandLevel) as well as c.Parent(expandLevel) itself, +// as an expandLevel cell rarely abuts a smaller cell. +// +// Note that the size of the output is exponential in +// expandLevel. For example, if expandLevel == 20 and the input +// has a cell at level 10, there will be on the order of 4000 +// adjacent cells in the output. For most applications the +// ExpandByRadius method below is easier to use. +func (cu *CellUnion) ExpandAtLevel(level int) { + var output CellUnion + levelLsb := lsbForLevel(level) + for i := len(*cu) - 1; i >= 0; i-- { + id := (*cu)[i] + if id.lsb() < levelLsb { + id = id.Parent(level) + // Optimization: skip over any cells contained by this one. This is + // especially important when very small regions are being expanded. + for i > 0 && id.Contains((*cu)[i-1]) { + i-- + } + } + output = append(output, id) + output = append(output, id.AllNeighbors(level)...) + } + sortCellIDs(output) + + *cu = output + cu.Normalize() +} + +// ExpandByRadius expands this CellUnion such that it contains all points whose +// distance to the CellUnion is at most minRadius, but do not use cells that +// are more than maxLevelDiff levels higher than the largest cell in the input. +// The second parameter controls the tradeoff between accuracy and output size +// when a large region is being expanded by a small amount (e.g. expanding Canada +// by 1km). For example, if maxLevelDiff == 4 the region will always be expanded +// by approximately 1/16 the width of its largest cell. Note that in the worst case, +// the number of cells in the output can be up to 4 * (1 + 2 ** maxLevelDiff) times +// larger than the number of cells in the input. +func (cu *CellUnion) ExpandByRadius(minRadius s1.Angle, maxLevelDiff int) { + minLevel := maxLevel + for _, cid := range *cu { + minLevel = minInt(minLevel, cid.Level()) + } + + // Find the maximum level such that all cells are at least "minRadius" wide. + radiusLevel := MinWidthMetric.MaxLevel(minRadius.Radians()) + if radiusLevel == 0 && minRadius.Radians() > MinWidthMetric.Value(0) { + // The requested expansion is greater than the width of a face cell. + // The easiest way to handle this is to expand twice. + cu.ExpandAtLevel(0) + } + cu.ExpandAtLevel(minInt(minLevel+maxLevelDiff, radiusLevel)) +} + +// Equal reports whether the two CellUnions are equal. +func (cu CellUnion) Equal(o CellUnion) bool { + if len(cu) != len(o) { + return false + } + for i := 0; i < len(cu); i++ { + if cu[i] != o[i] { + return false + } + } + return true +} + +// AverageArea returns the average area of this CellUnion. +// This is accurate to within a factor of 1.7. +func (cu *CellUnion) AverageArea() float64 { + return AvgAreaMetric.Value(maxLevel) * float64(cu.LeafCellsCovered()) +} + +// ApproxArea returns the approximate area of this CellUnion. This method is accurate +// to within 3% percent for all cell sizes and accurate to within 0.1% for cells +// at level 5 or higher within the union. +func (cu *CellUnion) ApproxArea() float64 { + var area float64 + for _, id := range *cu { + area += CellFromCellID(id).ApproxArea() + } + return area +} + +// ExactArea returns the area of this CellUnion as accurately as possible. +func (cu *CellUnion) ExactArea() float64 { + var area float64 + for _, id := range *cu { + area += CellFromCellID(id).ExactArea() + } + return area +} + +// Encode encodes the CellUnion. +func (cu *CellUnion) Encode(w io.Writer) error { + e := &encoder{w: w} + cu.encode(e) + return e.err +} + +func (cu *CellUnion) encode(e *encoder) { + e.writeInt8(encodingVersion) + e.writeInt64(int64(len(*cu))) + for _, ci := range *cu { + ci.encode(e) + } +} + +// Decode decodes the CellUnion. +func (cu *CellUnion) Decode(r io.Reader) error { + d := &decoder{r: asByteReader(r)} + cu.decode(d) + return d.err +} + +func (cu *CellUnion) decode(d *decoder) { + version := d.readInt8() + if d.err != nil { + return + } + if version != encodingVersion { + d.err = fmt.Errorf("only version %d is supported", encodingVersion) + return + } + n := d.readInt64() + if d.err != nil { + return + } + const maxCells = 1000000 + if n > maxCells { + d.err = fmt.Errorf("too many cells (%d; max is %d)", n, maxCells) + return + } + *cu = make([]CellID, n) + for i := range *cu { + (*cu)[i].decode(d) + } +} diff --git a/vendor/github.com/blevesearch/geo/s2/centroids.go b/vendor/github.com/blevesearch/geo/s2/centroids.go new file mode 100644 index 00000000..e8a91c44 --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/centroids.go @@ -0,0 +1,133 @@ +// Copyright 2018 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +import ( + "math" + + "github.com/golang/geo/r3" +) + +// There are several notions of the "centroid" of a triangle. First, there +// is the planar centroid, which is simply the centroid of the ordinary +// (non-spherical) triangle defined by the three vertices. Second, there is +// the surface centroid, which is defined as the intersection of the three +// medians of the spherical triangle. It is possible to show that this +// point is simply the planar centroid projected to the surface of the +// sphere. Finally, there is the true centroid (mass centroid), which is +// defined as the surface integral over the spherical triangle of (x,y,z) +// divided by the triangle area. This is the point that the triangle would +// rotate around if it was spinning in empty space. +// +// The best centroid for most purposes is the true centroid. Unlike the +// planar and surface centroids, the true centroid behaves linearly as +// regions are added or subtracted. That is, if you split a triangle into +// pieces and compute the average of their centroids (weighted by triangle +// area), the result equals the centroid of the original triangle. This is +// not true of the other centroids. +// +// Also note that the surface centroid may be nowhere near the intuitive +// "center" of a spherical triangle. For example, consider the triangle +// with vertices A=(1,eps,0), B=(0,0,1), C=(-1,eps,0) (a quarter-sphere). +// The surface centroid of this triangle is at S=(0, 2*eps, 1), which is +// within a distance of 2*eps of the vertex B. Note that the median from A +// (the segment connecting A to the midpoint of BC) passes through S, since +// this is the shortest path connecting the two endpoints. On the other +// hand, the true centroid is at M=(0, 0.5, 0.5), which when projected onto +// the surface is a much more reasonable interpretation of the "center" of +// this triangle. +// + +// TrueCentroid returns the true centroid of the spherical triangle ABC +// multiplied by the signed area of spherical triangle ABC. The reasons for +// multiplying by the signed area are (1) this is the quantity that needs to be +// summed to compute the centroid of a union or difference of triangles, and +// (2) it's actually easier to calculate this way. All points must have unit length. +// +// Note that the result of this function is defined to be Point(0, 0, 0) if +// the triangle is degenerate. +func TrueCentroid(a, b, c Point) Point { + // Use Distance to get accurate results for small triangles. + ra := float64(1) + if sa := float64(b.Distance(c)); sa != 0 { + ra = sa / math.Sin(sa) + } + rb := float64(1) + if sb := float64(c.Distance(a)); sb != 0 { + rb = sb / math.Sin(sb) + } + rc := float64(1) + if sc := float64(a.Distance(b)); sc != 0 { + rc = sc / math.Sin(sc) + } + + // Now compute a point M such that: + // + // [Ax Ay Az] [Mx] [ra] + // [Bx By Bz] [My] = 0.5 * det(A,B,C) * [rb] + // [Cx Cy Cz] [Mz] [rc] + // + // To improve the numerical stability we subtract the first row (A) from the + // other two rows; this reduces the cancellation error when A, B, and C are + // very close together. Then we solve it using Cramer's rule. + // + // The result is the true centroid of the triangle multiplied by the + // triangle's area. + // + // This code still isn't as numerically stable as it could be. + // The biggest potential improvement is to compute B-A and C-A more + // accurately so that (B-A)x(C-A) is always inside triangle ABC. + x := r3.Vector{a.X, b.X - a.X, c.X - a.X} + y := r3.Vector{a.Y, b.Y - a.Y, c.Y - a.Y} + z := r3.Vector{a.Z, b.Z - a.Z, c.Z - a.Z} + r := r3.Vector{ra, rb - ra, rc - ra} + + return Point{r3.Vector{y.Cross(z).Dot(r), z.Cross(x).Dot(r), x.Cross(y).Dot(r)}.Mul(0.5)} +} + +// EdgeTrueCentroid returns the true centroid of the spherical geodesic edge AB +// multiplied by the length of the edge AB. As with triangles, the true centroid +// of a collection of line segments may be computed simply by summing the result +// of this method for each segment. +// +// Note that the planar centroid of a line segment is simply 0.5 * (a + b), +// while the surface centroid is (a + b).Normalize(). However neither of +// these values is appropriate for computing the centroid of a collection of +// edges (such as a polyline). +// +// Also note that the result of this function is defined to be Point(0, 0, 0) +// if the edge is degenerate. +func EdgeTrueCentroid(a, b Point) Point { + // The centroid (multiplied by length) is a vector toward the midpoint + // of the edge, whose length is twice the sine of half the angle between + // the two vertices. Defining theta to be this angle, we have: + vDiff := a.Sub(b.Vector) // Length == 2*sin(theta) + vSum := a.Add(b.Vector) // Length == 2*cos(theta) + sin2 := vDiff.Norm2() + cos2 := vSum.Norm2() + if cos2 == 0 { + return Point{} // Ignore antipodal edges. + } + return Point{vSum.Mul(math.Sqrt(sin2 / cos2))} // Length == 2*sin(theta) +} + +// PlanarCentroid returns the centroid of the planar triangle ABC. This can be +// normalized to unit length to obtain the "surface centroid" of the corresponding +// spherical triangle, i.e. the intersection of the three medians. However, note +// that for large spherical triangles the surface centroid may be nowhere near +// the intuitive "center". +func PlanarCentroid(a, b, c Point) Point { + return Point{a.Add(b.Vector).Add(c.Vector).Mul(1. / 3)} +} diff --git a/vendor/github.com/blevesearch/geo/s2/contains_point_query.go b/vendor/github.com/blevesearch/geo/s2/contains_point_query.go new file mode 100644 index 00000000..3026f360 --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/contains_point_query.go @@ -0,0 +1,190 @@ +// Copyright 2018 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +// VertexModel defines whether shapes are considered to contain their vertices. +// Note that these definitions differ from the ones used by BooleanOperation. +// +// Note that points other than vertices are never contained by polylines. +// If you want need this behavior, use ClosestEdgeQuery's IsDistanceLess +// with a suitable distance threshold instead. +type VertexModel int + +const ( + // VertexModelOpen means no shapes contain their vertices (not even + // points). Therefore Contains(Point) returns true if and only if the + // point is in the interior of some polygon. + VertexModelOpen VertexModel = iota + + // VertexModelSemiOpen means that polygon point containment is defined + // such that if several polygons tile the region around a vertex, then + // exactly one of those polygons contains that vertex. Points and + // polylines still do not contain any vertices. + VertexModelSemiOpen + + // VertexModelClosed means all shapes contain their vertices (including + // points and polylines). + VertexModelClosed +) + +// ContainsPointQuery determines whether one or more shapes in a ShapeIndex +// contain a given Point. The ShapeIndex may contain any number of points, +// polylines, and/or polygons (possibly overlapping). Shape boundaries may be +// modeled as Open, SemiOpen, or Closed (this affects whether or not shapes are +// considered to contain their vertices). +// +// This type is not safe for concurrent use. +// +// However, note that if you need to do a large number of point containment +// tests, it is more efficient to re-use the query rather than creating a new +// one each time. +type ContainsPointQuery struct { + model VertexModel + index *ShapeIndex + iter *ShapeIndexIterator +} + +// NewContainsPointQuery creates a new instance of the ContainsPointQuery for the index +// and given vertex model choice. +func NewContainsPointQuery(index *ShapeIndex, model VertexModel) *ContainsPointQuery { + return &ContainsPointQuery{ + index: index, + model: model, + iter: index.Iterator(), + } +} + +// Contains reports whether any shape in the queries index contains the point p +// under the queries vertex model (Open, SemiOpen, or Closed). +func (q *ContainsPointQuery) Contains(p Point) bool { + if !q.iter.LocatePoint(p) { + return false + } + + cell := q.iter.IndexCell() + for _, clipped := range cell.shapes { + if q.shapeContains(clipped, q.iter.Center(), p) { + return true + } + } + return false +} + +// shapeContains reports whether the clippedShape from the iterator's center position contains +// the given point. +func (q *ContainsPointQuery) shapeContains(clipped *clippedShape, center, p Point) bool { + inside := clipped.containsCenter + numEdges := clipped.numEdges() + if numEdges <= 0 { + return inside + } + + shape := q.index.Shape(clipped.shapeID) + if shape.Dimension() != 2 { + // Points and polylines can be ignored unless the vertex model is Closed. + if q.model != VertexModelClosed { + return false + } + + // Otherwise, the point is contained if and only if it matches a vertex. + for _, edgeID := range clipped.edges { + edge := shape.Edge(edgeID) + if edge.V0 == p || edge.V1 == p { + return true + } + } + return false + } + + // Test containment by drawing a line segment from the cell center to the + // given point and counting edge crossings. + crosser := NewEdgeCrosser(center, p) + for _, edgeID := range clipped.edges { + edge := shape.Edge(edgeID) + sign := crosser.CrossingSign(edge.V0, edge.V1) + if sign == DoNotCross { + continue + } + if sign == MaybeCross { + // For the Open and Closed models, check whether p is a vertex. + if q.model != VertexModelSemiOpen && (edge.V0 == p || edge.V1 == p) { + return (q.model == VertexModelClosed) + } + // C++ plays fast and loose with the int <-> bool conversions here. + if VertexCrossing(crosser.a, crosser.b, edge.V0, edge.V1) { + sign = Cross + } else { + sign = DoNotCross + } + } + inside = inside != (sign == Cross) + } + + return inside +} + +// ShapeContains reports whether the given shape contains the point under this +// queries vertex model (Open, SemiOpen, or Closed). +// +// This requires the shape belongs to this queries index. +func (q *ContainsPointQuery) ShapeContains(shape Shape, p Point) bool { + if !q.iter.LocatePoint(p) { + return false + } + + clipped := q.iter.IndexCell().findByShapeID(q.index.idForShape(shape)) + if clipped == nil { + return false + } + return q.shapeContains(clipped, q.iter.Center(), p) +} + +// shapeVisitorFunc is a type of function that can be called against shaped in an index. +type shapeVisitorFunc func(shape Shape) bool + +// visitContainingShapes visits all shapes in the given index that contain the +// given point p, terminating early if the given visitor function returns false, +// in which case visitContainingShapes returns false. Each shape is +// visited at most once. +func (q *ContainsPointQuery) visitContainingShapes(p Point, f shapeVisitorFunc) bool { + // This function returns false only if the algorithm terminates early + // because the visitor function returned false. + if !q.iter.LocatePoint(p) { + return true + } + + cell := q.iter.IndexCell() + for _, clipped := range cell.shapes { + if q.shapeContains(clipped, q.iter.Center(), p) && + !f(q.index.Shape(clipped.shapeID)) { + return false + } + } + return true +} + +// ContainingShapes returns a slice of all shapes that contain the given point. +func (q *ContainsPointQuery) ContainingShapes(p Point) []Shape { + var shapes []Shape + q.visitContainingShapes(p, func(shape Shape) bool { + shapes = append(shapes, shape) + return true + }) + return shapes +} + +// TODO(roberts): Remaining methods from C++ +// type edgeVisitorFunc func(shape ShapeEdge) bool +// func (q *ContainsPointQuery) visitIncidentEdges(p Point, v edgeVisitorFunc) bool diff --git a/vendor/github.com/blevesearch/geo/s2/contains_vertex_query.go b/vendor/github.com/blevesearch/geo/s2/contains_vertex_query.go new file mode 100644 index 00000000..8e74f9e5 --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/contains_vertex_query.go @@ -0,0 +1,63 @@ +// Copyright 2017 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +// ContainsVertexQuery is used to track the edges entering and leaving the +// given vertex of a Polygon in order to be able to determine if the point is +// contained by the Polygon. +// +// Point containment is defined according to the semi-open boundary model +// which means that if several polygons tile the region around a vertex, +// then exactly one of those polygons contains that vertex. +type ContainsVertexQuery struct { + target Point + edgeMap map[Point]int +} + +// NewContainsVertexQuery returns a new query for the given vertex whose +// containment will be determined. +func NewContainsVertexQuery(target Point) *ContainsVertexQuery { + return &ContainsVertexQuery{ + target: target, + edgeMap: make(map[Point]int), + } +} + +// AddEdge adds the edge between target and v with the given direction. +// (+1 = outgoing, -1 = incoming, 0 = degenerate). +func (q *ContainsVertexQuery) AddEdge(v Point, direction int) { + q.edgeMap[v] += direction +} + +// ContainsVertex reports a +1 if the target vertex is contained, -1 if it is +// not contained, and 0 if the incident edges consisted of matched sibling pairs. +func (q *ContainsVertexQuery) ContainsVertex() int { + // Find the unmatched edge that is immediately clockwise from Ortho(P). + referenceDir := Point{q.target.Ortho()} + + bestPoint := referenceDir + bestDir := 0 + + for k, v := range q.edgeMap { + if v == 0 { + continue // This is a "matched" edge. + } + if OrderedCCW(referenceDir, bestPoint, k, q.target) { + bestPoint = k + bestDir = v + } + } + return bestDir +} diff --git a/vendor/github.com/blevesearch/geo/s2/convex_hull_query.go b/vendor/github.com/blevesearch/geo/s2/convex_hull_query.go new file mode 100644 index 00000000..68539abb --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/convex_hull_query.go @@ -0,0 +1,258 @@ +// Copyright 2018 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +import ( + "sort" + + "github.com/golang/geo/r3" +) + +// ConvexHullQuery builds the convex hull of any collection of points, +// polylines, loops, and polygons. It returns a single convex loop. +// +// The convex hull is defined as the smallest convex region on the sphere that +// contains all of your input geometry. Recall that a region is "convex" if +// for every pair of points inside the region, the straight edge between them +// is also inside the region. In our case, a "straight" edge is a geodesic, +// i.e. the shortest path on the sphere between two points. +// +// Containment of input geometry is defined as follows: +// +// - Each input loop and polygon is contained by the convex hull exactly +// (i.e., according to Polygon's Contains(Polygon)). +// +// - Each input point is either contained by the convex hull or is a vertex +// of the convex hull. (Recall that S2Loops do not necessarily contain their +// vertices.) +// +// - For each input polyline, the convex hull contains all of its vertices +// according to the rule for points above. (The definition of convexity +// then ensures that the convex hull also contains the polyline edges.) +// +// To use this type, call the various Add... methods to add your input geometry, and +// then call ConvexHull. Note that ConvexHull does *not* reset the +// state; you can continue adding geometry if desired and compute the convex +// hull again. If you want to start from scratch, simply create a new +// ConvexHullQuery value. +// +// This implement Andrew's monotone chain algorithm, which is a variant of the +// Graham scan (see https://en.wikipedia.org/wiki/Graham_scan). The time +// complexity is O(n log n), and the space required is O(n). In fact only the +// call to "sort" takes O(n log n) time; the rest of the algorithm is linear. +// +// Demonstration of the algorithm and code: +// en.wikibooks.org/wiki/Algorithm_Implementation/Geometry/Convex_hull/Monotone_chain +// +// This type is not safe for concurrent use. +type ConvexHullQuery struct { + bound Rect + points []Point +} + +// NewConvexHullQuery creates a new ConvexHullQuery. +func NewConvexHullQuery() *ConvexHullQuery { + return &ConvexHullQuery{ + bound: EmptyRect(), + } +} + +// AddPoint adds the given point to the input geometry. +func (q *ConvexHullQuery) AddPoint(p Point) { + q.bound = q.bound.AddPoint(LatLngFromPoint(p)) + q.points = append(q.points, p) +} + +// AddPolyline adds the given polyline to the input geometry. +func (q *ConvexHullQuery) AddPolyline(p *Polyline) { + q.bound = q.bound.Union(p.RectBound()) + q.points = append(q.points, (*p)...) +} + +// AddLoop adds the given loop to the input geometry. +func (q *ConvexHullQuery) AddLoop(l *Loop) { + q.bound = q.bound.Union(l.RectBound()) + if l.isEmptyOrFull() { + return + } + q.points = append(q.points, l.vertices...) +} + +// AddPolygon adds the given polygon to the input geometry. +func (q *ConvexHullQuery) AddPolygon(p *Polygon) { + q.bound = q.bound.Union(p.RectBound()) + for _, l := range p.loops { + // Only loops at depth 0 can contribute to the convex hull. + if l.depth == 0 { + q.AddLoop(l) + } + } +} + +// CapBound returns a bounding cap for the input geometry provided. +// +// Note that this method does not clear the geometry; you can continue +// adding to it and call this method again if desired. +func (q *ConvexHullQuery) CapBound() Cap { + // We keep track of a rectangular bound rather than a spherical cap because + // it is easy to compute a tight bound for a union of rectangles, whereas it + // is quite difficult to compute a tight bound around a union of caps. + // Also, polygons and polylines implement CapBound() in terms of + // RectBound() for this same reason, so it is much better to keep track + // of a rectangular bound as we go along and convert it at the end. + // + // TODO(roberts): We could compute an optimal bound by implementing Welzl's + // algorithm. However we would still need to have special handling of loops + // and polygons, since if a loop spans more than 180 degrees in any + // direction (i.e., if it contains two antipodal points), then it is not + // enough just to bound its vertices. In this case the only convex bounding + // cap is FullCap(), and the only convex bounding loop is the full loop. + return q.bound.CapBound() +} + +// ConvexHull returns a Loop representing the convex hull of the input geometry provided. +// +// If there is no geometry, this method returns an empty loop containing no +// points. +// +// If the geometry spans more than half of the sphere, this method returns a +// full loop containing the entire sphere. +// +// If the geometry contains 1 or 2 points, or a single edge, this method +// returns a very small loop consisting of three vertices (which are a +// superset of the input vertices). +// +// Note that this method does not clear the geometry; you can continue +// adding to the query and call this method again. +func (q *ConvexHullQuery) ConvexHull() *Loop { + c := q.CapBound() + if c.Height() >= 1 { + // The bounding cap is not convex. The current bounding cap + // implementation is not optimal, but nevertheless it is likely that the + // input geometry itself is not contained by any convex polygon. In any + // case, we need a convex bounding cap to proceed with the algorithm below + // (in order to construct a point "origin" that is definitely outside the + // convex hull). + return FullLoop() + } + + // Remove duplicates. We need to do this before checking whether there are + // fewer than 3 points. + x := make(map[Point]bool) + r, w := 0, 0 // read/write indexes + for ; r < len(q.points); r++ { + if x[q.points[r]] { + continue + } + q.points[w] = q.points[r] + x[q.points[r]] = true + w++ + } + q.points = q.points[:w] + + // This code implements Andrew's monotone chain algorithm, which is a simple + // variant of the Graham scan. Rather than sorting by x-coordinate, instead + // we sort the points in CCW order around an origin O such that all points + // are guaranteed to be on one side of some geodesic through O. This + // ensures that as we scan through the points, each new point can only + // belong at the end of the chain (i.e., the chain is monotone in terms of + // the angle around O from the starting point). + origin := Point{c.Center().Ortho()} + sort.Slice(q.points, func(i, j int) bool { + return RobustSign(origin, q.points[i], q.points[j]) == CounterClockwise + }) + + // Special cases for fewer than 3 points. + switch len(q.points) { + case 0: + return EmptyLoop() + case 1: + return singlePointLoop(q.points[0]) + case 2: + return singleEdgeLoop(q.points[0], q.points[1]) + } + + // Generate the lower and upper halves of the convex hull. Each half + // consists of the maximal subset of vertices such that the edge chain + // makes only left (CCW) turns. + lower := q.monotoneChain() + + // reverse the points + for left, right := 0, len(q.points)-1; left < right; left, right = left+1, right-1 { + q.points[left], q.points[right] = q.points[right], q.points[left] + } + upper := q.monotoneChain() + + // Remove the duplicate vertices and combine the chains. + lower = lower[:len(lower)-1] + upper = upper[:len(upper)-1] + lower = append(lower, upper...) + + return LoopFromPoints(lower) +} + +// monotoneChain iterates through the points, selecting the maximal subset of points +// such that the edge chain makes only left (CCW) turns. +func (q *ConvexHullQuery) monotoneChain() []Point { + var output []Point + for _, p := range q.points { + // Remove any points that would cause the chain to make a clockwise turn. + for len(output) >= 2 && RobustSign(output[len(output)-2], output[len(output)-1], p) != CounterClockwise { + output = output[:len(output)-1] + } + output = append(output, p) + } + return output +} + +// singlePointLoop constructs a 3-vertex polygon consisting of "p" and two nearby +// vertices. Note that ContainsPoint(p) may be false for the resulting loop. +func singlePointLoop(p Point) *Loop { + const offset = 1e-15 + d0 := p.Ortho() + d1 := p.Cross(d0) + vertices := []Point{ + p, + {p.Add(d0.Mul(offset)).Normalize()}, + {p.Add(d1.Mul(offset)).Normalize()}, + } + return LoopFromPoints(vertices) +} + +// singleEdgeLoop constructs a loop consisting of the two vertices and their midpoint. +func singleEdgeLoop(a, b Point) *Loop { + // If the points are exactly antipodal we return the full loop. + // + // Note that we could use the code below even in this case (which would + // return a zero-area loop that follows the edge AB), except that (1) the + // direction of AB is defined using symbolic perturbations and therefore is + // not predictable by ordinary users, and (2) Loop disallows anitpodal + // adjacent vertices and so we would need to use 4 vertices to define the + // degenerate loop. (Note that the Loop antipodal vertex restriction is + // historical and now could easily be removed, however it would still have + // the problem that the edge direction is not easily predictable.) + if a.Add(b.Vector) == (r3.Vector{}) { + return FullLoop() + } + + // Construct a loop consisting of the two vertices and their midpoint. We + // use Interpolate() to ensure that the midpoint is very close to + // the edge even when its endpoints nearly antipodal. + vertices := []Point{a, b, Interpolate(0.5, a, b)} + loop := LoopFromPoints(vertices) + // The resulting loop may be clockwise, so invert it if necessary. + loop.Normalize() + return loop +} diff --git a/vendor/github.com/blevesearch/geo/s2/crossing_edge_query.go b/vendor/github.com/blevesearch/geo/s2/crossing_edge_query.go new file mode 100644 index 00000000..51852dab --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/crossing_edge_query.go @@ -0,0 +1,409 @@ +// Copyright 2017 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +import ( + "sort" + + "github.com/golang/geo/r2" +) + +// CrossingEdgeQuery is used to find the Edge IDs of Shapes that are crossed by +// a given edge(s). +// +// Note that if you need to query many edges, it is more efficient to declare +// a single CrossingEdgeQuery instance and reuse it. +// +// If you want to find *all* the pairs of crossing edges, it is more efficient to +// use the not yet implemented VisitCrossings in shapeutil. +type CrossingEdgeQuery struct { + index *ShapeIndex + + // temporary values used while processing a query. + a, b r2.Point + iter *ShapeIndexIterator + + // candidate cells generated when finding crossings. + cells []*ShapeIndexCell +} + +// NewCrossingEdgeQuery creates a CrossingEdgeQuery for the given index. +func NewCrossingEdgeQuery(index *ShapeIndex) *CrossingEdgeQuery { + c := &CrossingEdgeQuery{ + index: index, + iter: index.Iterator(), + } + return c +} + +// Crossings returns the set of edge of the shape S that intersect the given edge AB. +// If the CrossingType is Interior, then only intersections at a point interior to both +// edges are reported, while if it is CrossingTypeAll then edges that share a vertex +// are also reported. +func (c *CrossingEdgeQuery) Crossings(a, b Point, shape Shape, crossType CrossingType) []int { + edges := c.candidates(a, b, shape) + if len(edges) == 0 { + return nil + } + + crosser := NewEdgeCrosser(a, b) + out := 0 + n := len(edges) + + for in := 0; in < n; in++ { + b := shape.Edge(edges[in]) + sign := crosser.CrossingSign(b.V0, b.V1) + if crossType == CrossingTypeAll && (sign == MaybeCross || sign == Cross) || crossType != CrossingTypeAll && sign == Cross { + edges[out] = edges[in] + out++ + } + } + + if out < n { + edges = edges[0:out] + } + return edges +} + +// EdgeMap stores a sorted set of edge ids for each shape. +type EdgeMap map[Shape][]int + +// CrossingsEdgeMap returns the set of all edges in the index that intersect the given +// edge AB. If crossType is CrossingTypeInterior, then only intersections at a +// point interior to both edges are reported, while if it is CrossingTypeAll +// then edges that share a vertex are also reported. +// +// The edges are returned as a mapping from shape to the edges of that shape +// that intersect AB. Every returned shape has at least one crossing edge. +func (c *CrossingEdgeQuery) CrossingsEdgeMap(a, b Point, crossType CrossingType) EdgeMap { + edgeMap := c.candidatesEdgeMap(a, b) + if len(edgeMap) == 0 { + return nil + } + + crosser := NewEdgeCrosser(a, b) + for shape, edges := range edgeMap { + out := 0 + n := len(edges) + for in := 0; in < n; in++ { + edge := shape.Edge(edges[in]) + sign := crosser.CrossingSign(edge.V0, edge.V1) + if (crossType == CrossingTypeAll && (sign == MaybeCross || sign == Cross)) || (crossType != CrossingTypeAll && sign == Cross) { + edgeMap[shape][out] = edges[in] + out++ + } + } + + if out == 0 { + delete(edgeMap, shape) + } else { + if out < n { + edgeMap[shape] = edgeMap[shape][0:out] + } + } + } + return edgeMap +} + +// candidates returns a superset of the edges of the given shape that intersect +// the edge AB. +func (c *CrossingEdgeQuery) candidates(a, b Point, shape Shape) []int { + var edges []int + + // For small loops it is faster to use brute force. The threshold below was + // determined using benchmarks. + const maxBruteForceEdges = 27 + maxEdges := shape.NumEdges() + if maxEdges <= maxBruteForceEdges { + edges = make([]int, maxEdges) + for i := 0; i < maxEdges; i++ { + edges[i] = i + } + return edges + } + + // Compute the set of index cells intersected by the query edge. + c.getCellsForEdge(a, b) + if len(c.cells) == 0 { + return nil + } + + // Gather all the edges that intersect those cells and sort them. + // TODO(roberts): Shapes don't track their ID, so we need to range over + // the index to find the ID manually. + var shapeID int32 + for k, v := range c.index.shapes { + if v == shape { + shapeID = k + } + } + + for _, cell := range c.cells { + if cell == nil { + continue + } + clipped := cell.findByShapeID(shapeID) + if clipped == nil { + continue + } + edges = append(edges, clipped.edges...) + } + + if len(c.cells) > 1 { + edges = uniqueInts(edges) + } + + return edges +} + +// uniqueInts returns the sorted uniqued values from the given input. +func uniqueInts(in []int) []int { + var edges []int + m := make(map[int]bool) + for _, i := range in { + if m[i] { + continue + } + m[i] = true + edges = append(edges, i) + } + sort.Ints(edges) + return edges +} + +// candidatesEdgeMap returns a map from shapes to the superse of edges for that +// shape that intersect the edge AB. +// +// CAVEAT: This method may return shapes that have an empty set of candidate edges. +// However the return value is non-empty only if at least one shape has a candidate edge. +func (c *CrossingEdgeQuery) candidatesEdgeMap(a, b Point) EdgeMap { + edgeMap := make(EdgeMap) + + // If there are only a few edges then it's faster to use brute force. We + // only bother with this optimization when there is a single shape. + if len(c.index.shapes) == 1 { + // Typically this method is called many times, so it is worth checking + // whether the edge map is empty or already consists of a single entry for + // this shape, and skip clearing edge map in that case. + shape := c.index.Shape(0) + + // Note that we leave the edge map non-empty even if there are no candidates + // (i.e., there is a single entry with an empty set of edges). + edgeMap[shape] = c.candidates(a, b, shape) + return edgeMap + } + + // Compute the set of index cells intersected by the query edge. + c.getCellsForEdge(a, b) + if len(c.cells) == 0 { + return edgeMap + } + + // Gather all the edges that intersect those cells and sort them. + for _, cell := range c.cells { + for _, clipped := range cell.shapes { + s := c.index.Shape(clipped.shapeID) + for j := 0; j < clipped.numEdges(); j++ { + edgeMap[s] = append(edgeMap[s], clipped.edges[j]) + } + } + } + + if len(c.cells) > 1 { + for s, edges := range edgeMap { + edgeMap[s] = uniqueInts(edges) + } + } + + return edgeMap +} + +// getCells returns the set of ShapeIndexCells that might contain edges intersecting +// the edge AB in the given cell root. This method is used primarily by loop and shapeutil. +func (c *CrossingEdgeQuery) getCells(a, b Point, root *PaddedCell) []*ShapeIndexCell { + aUV, bUV, ok := ClipToFace(a, b, root.id.Face()) + if ok { + c.a = aUV + c.b = bUV + edgeBound := r2.RectFromPoints(c.a, c.b) + if root.Bound().Intersects(edgeBound) { + c.computeCellsIntersected(root, edgeBound) + } + } + + if len(c.cells) == 0 { + return nil + } + + return c.cells +} + +// getCellsForEdge populates the cells field to the set of index cells intersected by an edge AB. +func (c *CrossingEdgeQuery) getCellsForEdge(a, b Point) { + c.cells = nil + + segments := FaceSegments(a, b) + for _, segment := range segments { + c.a = segment.a + c.b = segment.b + + // Optimization: rather than always starting the recursive subdivision at + // the top level face cell, instead we start at the smallest S2CellId that + // contains the edge (the edge root cell). This typically lets us skip + // quite a few levels of recursion since most edges are short. + edgeBound := r2.RectFromPoints(c.a, c.b) + pcell := PaddedCellFromCellID(CellIDFromFace(segment.face), 0) + edgeRoot := pcell.ShrinkToFit(edgeBound) + + // Now we need to determine how the edge root cell is related to the cells + // in the spatial index (cellMap). There are three cases: + // + // 1. edgeRoot is an index cell or is contained within an index cell. + // In this case we only need to look at the contents of that cell. + // 2. edgeRoot is subdivided into one or more index cells. In this case + // we recursively subdivide to find the cells intersected by AB. + // 3. edgeRoot does not intersect any index cells. In this case there + // is nothing to do. + relation := c.iter.LocateCellID(edgeRoot) + if relation == Indexed { + // edgeRoot is an index cell or is contained by an index cell (case 1). + c.cells = append(c.cells, c.iter.IndexCell()) + } else if relation == Subdivided { + // edgeRoot is subdivided into one or more index cells (case 2). We + // find the cells intersected by AB using recursive subdivision. + if !edgeRoot.isFace() { + pcell = PaddedCellFromCellID(edgeRoot, 0) + } + c.computeCellsIntersected(pcell, edgeBound) + } + } +} + +// computeCellsIntersected computes the index cells intersected by the current +// edge that are descendants of pcell and adds them to this queries set of cells. +func (c *CrossingEdgeQuery) computeCellsIntersected(pcell *PaddedCell, edgeBound r2.Rect) { + + c.iter.seek(pcell.id.RangeMin()) + if c.iter.Done() || c.iter.CellID() > pcell.id.RangeMax() { + // The index does not contain pcell or any of its descendants. + return + } + if c.iter.CellID() == pcell.id { + // The index contains this cell exactly. + c.cells = append(c.cells, c.iter.IndexCell()) + return + } + + // Otherwise, split the edge among the four children of pcell. + center := pcell.Middle().Lo() + + if edgeBound.X.Hi < center.X { + // Edge is entirely contained in the two left children. + c.clipVAxis(edgeBound, center.Y, 0, pcell) + return + } else if edgeBound.X.Lo >= center.X { + // Edge is entirely contained in the two right children. + c.clipVAxis(edgeBound, center.Y, 1, pcell) + return + } + + childBounds := c.splitUBound(edgeBound, center.X) + if edgeBound.Y.Hi < center.Y { + // Edge is entirely contained in the two lower children. + c.computeCellsIntersected(PaddedCellFromParentIJ(pcell, 0, 0), childBounds[0]) + c.computeCellsIntersected(PaddedCellFromParentIJ(pcell, 1, 0), childBounds[1]) + } else if edgeBound.Y.Lo >= center.Y { + // Edge is entirely contained in the two upper children. + c.computeCellsIntersected(PaddedCellFromParentIJ(pcell, 0, 1), childBounds[0]) + c.computeCellsIntersected(PaddedCellFromParentIJ(pcell, 1, 1), childBounds[1]) + } else { + // The edge bound spans all four children. The edge itself intersects + // at most three children (since no padding is being used). + c.clipVAxis(childBounds[0], center.Y, 0, pcell) + c.clipVAxis(childBounds[1], center.Y, 1, pcell) + } +} + +// clipVAxis computes the intersected cells recursively for a given padded cell. +// Given either the left (i=0) or right (i=1) side of a padded cell pcell, +// determine whether the current edge intersects the lower child, upper child, +// or both children, and call c.computeCellsIntersected recursively on those children. +// The center is the v-coordinate at the center of pcell. +func (c *CrossingEdgeQuery) clipVAxis(edgeBound r2.Rect, center float64, i int, pcell *PaddedCell) { + if edgeBound.Y.Hi < center { + // Edge is entirely contained in the lower child. + c.computeCellsIntersected(PaddedCellFromParentIJ(pcell, i, 0), edgeBound) + } else if edgeBound.Y.Lo >= center { + // Edge is entirely contained in the upper child. + c.computeCellsIntersected(PaddedCellFromParentIJ(pcell, i, 1), edgeBound) + } else { + // The edge intersects both children. + childBounds := c.splitVBound(edgeBound, center) + c.computeCellsIntersected(PaddedCellFromParentIJ(pcell, i, 0), childBounds[0]) + c.computeCellsIntersected(PaddedCellFromParentIJ(pcell, i, 1), childBounds[1]) + } +} + +// splitUBound returns the bound for two children as a result of spliting the +// current edge at the given value U. +func (c *CrossingEdgeQuery) splitUBound(edgeBound r2.Rect, u float64) [2]r2.Rect { + v := edgeBound.Y.ClampPoint(interpolateFloat64(u, c.a.X, c.b.X, c.a.Y, c.b.Y)) + // diag indicates which diagonal of the bounding box is spanned by AB: + // it is 0 if AB has positive slope, and 1 if AB has negative slope. + var diag int + if (c.a.X > c.b.X) != (c.a.Y > c.b.Y) { + diag = 1 + } + return splitBound(edgeBound, 0, diag, u, v) +} + +// splitVBound returns the bound for two children as a result of spliting the +// current edge into two child edges at the given value V. +func (c *CrossingEdgeQuery) splitVBound(edgeBound r2.Rect, v float64) [2]r2.Rect { + u := edgeBound.X.ClampPoint(interpolateFloat64(v, c.a.Y, c.b.Y, c.a.X, c.b.X)) + var diag int + if (c.a.X > c.b.X) != (c.a.Y > c.b.Y) { + diag = 1 + } + return splitBound(edgeBound, diag, 0, u, v) +} + +// splitBound returns the bounds for the two childrenn as a result of spliting +// the current edge into two child edges at the given point (u,v). uEnd and vEnd +// indicate which bound endpoints of the first child will be updated. +func splitBound(edgeBound r2.Rect, uEnd, vEnd int, u, v float64) [2]r2.Rect { + var childBounds = [2]r2.Rect{ + edgeBound, + edgeBound, + } + + if uEnd == 1 { + childBounds[0].X.Lo = u + childBounds[1].X.Hi = u + } else { + childBounds[0].X.Hi = u + childBounds[1].X.Lo = u + } + + if vEnd == 1 { + childBounds[0].Y.Lo = v + childBounds[1].Y.Hi = v + } else { + childBounds[0].Y.Hi = v + childBounds[1].Y.Lo = v + } + + return childBounds +} diff --git a/vendor/github.com/blevesearch/geo/s2/distance_target.go b/vendor/github.com/blevesearch/geo/s2/distance_target.go new file mode 100644 index 00000000..066bbacf --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/distance_target.go @@ -0,0 +1,149 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +import ( + "github.com/golang/geo/s1" +) + +// The distance interface represents a set of common methods used by algorithms +// that compute distances between various S2 types. +type distance interface { + // chordAngle returns this type as a ChordAngle. + chordAngle() s1.ChordAngle + + // fromChordAngle is used to type convert a ChordAngle to this type. + // This is to work around needing to be clever in parts of the code + // where a distanceTarget interface method expects distances, but the + // user only supplies a ChordAngle, and we need to dynamically cast it + // to an appropriate distance interface types. + fromChordAngle(o s1.ChordAngle) distance + + // zero returns a zero distance. + zero() distance + // negative returns a value smaller than any valid value. + negative() distance + // infinity returns a value larger than any valid value. + infinity() distance + + // less is similar to the Less method in Sort. To get minimum values, + // this would be a less than type operation. For maximum, this would + // be a greater than type operation. + less(other distance) bool + + // sub subtracts the other value from this one and returns the new value. + // This is done as a method and not simple mathematical operation to + // allow closest and furthest to implement this in opposite ways. + sub(other distance) distance + + // chordAngleBound reports the upper bound on a ChordAngle corresponding + // to this distance. For example, if distance measures WGS84 ellipsoid + // distance then the corresponding angle needs to be 0.56% larger. + chordAngleBound() s1.ChordAngle + + // updateDistance may update the value this distance represents + // based on the given input. The updated value and a boolean reporting + // if the value was changed are returned. + updateDistance(other distance) (distance, bool) +} + +// distanceTarget is an interface that represents a geometric type to which distances +// are measured. +// +// For example, there are implementations that measure distances to a Point, +// an Edge, a Cell, a CellUnion, and even to an arbitrary collection of geometry +// stored in ShapeIndex. +// +// The distanceTarget types are provided for the benefit of types that measure +// distances and/or find nearby geometry, such as ClosestEdgeQuery, FurthestEdgeQuery, +// ClosestPointQuery, and ClosestCellQuery, etc. +type distanceTarget interface { + // capBound returns a Cap that bounds the set of points whose distance to the + // target is distance.zero(). + capBound() Cap + + // updateDistanceToPoint updates the distance if the distance to + // the point P is within than the given dist. + // The boolean reports if the value was updated. + updateDistanceToPoint(p Point, dist distance) (distance, bool) + + // updateDistanceToEdge updates the distance if the distance to + // the edge E is within than the given dist. + // The boolean reports if the value was updated. + updateDistanceToEdge(e Edge, dist distance) (distance, bool) + + // updateDistanceToCell updates the distance if the distance to the cell C + // (including its interior) is within than the given dist. + // The boolean reports if the value was updated. + updateDistanceToCell(c Cell, dist distance) (distance, bool) + + // setMaxError potentially updates the value of MaxError, and reports if + // the specific type supports altering it. Whenever one of the + // updateDistanceTo... methods above returns true, the returned distance + // is allowed to be up to maxError larger than the true minimum distance. + // In other words, it gives this target object permission to terminate its + // distance calculation as soon as it has determined that (1) the minimum + // distance is less than minDist and (2) the best possible further + // improvement is less than maxError. + // + // If the target takes advantage of maxError to optimize its distance + // calculation, this method must return true. (Most target types will + // default to return false.) + setMaxError(maxErr s1.ChordAngle) bool + + // maxBruteForceIndexSize reports the maximum number of indexed objects for + // which it is faster to compute the distance by brute force (e.g., by testing + // every edge) rather than by using an index. + // + // The following method is provided as a convenience for types that compute + // distances to a collection of indexed geometry, such as ClosestEdgeQuery + // and ClosestPointQuery. + // + // Types that do not support this should return a -1. + maxBruteForceIndexSize() int + + // distance returns an instance of the underlying distance type this + // target uses. This is to work around the use of Templates in the C++. + distance() distance + + // visitContainingShapes finds all polygons in the given index that + // completely contain a connected component of the target geometry. (For + // example, if the target consists of 10 points, this method finds + // polygons that contain any of those 10 points.) For each such polygon, + // the visit function is called with the Shape of the polygon along with + // a point of the target geometry that is contained by that polygon. + // + // Optionally, any polygon that intersects the target geometry may also be + // returned. In other words, this method returns all polygons that + // contain any connected component of the target, along with an arbitrary + // subset of the polygons that intersect the target. + // + // For example, suppose that the index contains two abutting polygons + // A and B. If the target consists of two points "a" contained by A and + // "b" contained by B, then both A and B are returned. But if the target + // consists of the edge "ab", then any subset of {A, B} could be returned + // (because both polygons intersect the target but neither one contains + // the edge "ab"). + // + // If the visit function returns false, this method terminates early and + // returns false as well. Otherwise returns true. + // + // NOTE(roberts): This method exists only for the purpose of implementing + // edgeQuery IncludeInteriors efficiently. + visitContainingShapes(index *ShapeIndex, v shapePointVisitorFunc) bool +} + +// shapePointVisitorFunc defines a type of function the visitContainingShapes can call. +type shapePointVisitorFunc func(containingShape Shape, targetPoint Point) bool diff --git a/vendor/github.com/blevesearch/geo/s2/doc.go b/vendor/github.com/blevesearch/geo/s2/doc.go new file mode 100644 index 00000000..43e7a634 --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/doc.go @@ -0,0 +1,29 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* +Package s2 is a library for working with geometry in S² (spherical geometry). + +Its related packages, parallel to this one, are s1 (operates on S¹), r1 (operates on ℝ¹), +r2 (operates on ℝ²) and r3 (operates on ℝ³). + +This package provides types and functions for the S2 cell hierarchy and coordinate systems. +The S2 cell hierarchy is a hierarchical decomposition of the surface of a unit sphere (S²) +into ``cells''; it is highly efficient, scales from continental size to under 1 cm² +and preserves spatial locality (nearby cells have close IDs). + +More information including an in-depth introduction to S2 can be found on the +S2 website https://s2geometry.io/ +*/ +package s2 diff --git a/vendor/github.com/blevesearch/geo/s2/edge_clipping.go b/vendor/github.com/blevesearch/geo/s2/edge_clipping.go new file mode 100644 index 00000000..57a53bf0 --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/edge_clipping.go @@ -0,0 +1,672 @@ +// Copyright 2017 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +// This file contains a collection of methods for: +// +// (1) Robustly clipping geodesic edges to the faces of the S2 biunit cube +// (see s2stuv), and +// +// (2) Robustly clipping 2D edges against 2D rectangles. +// +// These functions can be used to efficiently find the set of CellIDs that +// are intersected by a geodesic edge (e.g., see CrossingEdgeQuery). + +import ( + "math" + + "github.com/golang/geo/r1" + "github.com/golang/geo/r2" + "github.com/golang/geo/r3" +) + +const ( + // edgeClipErrorUVCoord is the maximum error in a u- or v-coordinate + // compared to the exact result, assuming that the points A and B are in + // the rectangle [-1,1]x[1,1] or slightly outside it (by 1e-10 or less). + edgeClipErrorUVCoord = 2.25 * dblEpsilon + + // edgeClipErrorUVDist is the maximum distance from a clipped point to + // the corresponding exact result. It is equal to the error in a single + // coordinate because at most one coordinate is subject to error. + edgeClipErrorUVDist = 2.25 * dblEpsilon + + // faceClipErrorRadians is the maximum angle between a returned vertex + // and the nearest point on the exact edge AB. It is equal to the + // maximum directional error in PointCross, plus the error when + // projecting points onto a cube face. + faceClipErrorRadians = 3 * dblEpsilon + + // faceClipErrorDist is the same angle expressed as a maximum distance + // in (u,v)-space. In other words, a returned vertex is at most this far + // from the exact edge AB projected into (u,v)-space. + faceClipErrorUVDist = 9 * dblEpsilon + + // faceClipErrorUVCoord is the maximum angle between a returned vertex + // and the nearest point on the exact edge AB expressed as the maximum error + // in an individual u- or v-coordinate. In other words, for each + // returned vertex there is a point on the exact edge AB whose u- and + // v-coordinates differ from the vertex by at most this amount. + faceClipErrorUVCoord = 9.0 * (1.0 / math.Sqrt2) * dblEpsilon + + // intersectsRectErrorUVDist is the maximum error when computing if a point + // intersects with a given Rect. If some point of AB is inside the + // rectangle by at least this distance, the result is guaranteed to be true; + // if all points of AB are outside the rectangle by at least this distance, + // the result is guaranteed to be false. This bound assumes that rect is + // a subset of the rectangle [-1,1]x[-1,1] or extends slightly outside it + // (e.g., by 1e-10 or less). + intersectsRectErrorUVDist = 3 * math.Sqrt2 * dblEpsilon +) + +// ClipToFace returns the (u,v) coordinates for the portion of the edge AB that +// intersects the given face, or false if the edge AB does not intersect. +// This method guarantees that the clipped vertices lie within the [-1,1]x[-1,1] +// cube face rectangle and are within faceClipErrorUVDist of the line AB, but +// the results may differ from those produced by FaceSegments. +func ClipToFace(a, b Point, face int) (aUV, bUV r2.Point, intersects bool) { + return ClipToPaddedFace(a, b, face, 0.0) +} + +// ClipToPaddedFace returns the (u,v) coordinates for the portion of the edge AB that +// intersects the given face, but rather than clipping to the square [-1,1]x[-1,1] +// in (u,v) space, this method clips to [-R,R]x[-R,R] where R=(1+padding). +// Padding must be non-negative. +func ClipToPaddedFace(a, b Point, f int, padding float64) (aUV, bUV r2.Point, intersects bool) { + // Fast path: both endpoints are on the given face. + if face(a.Vector) == f && face(b.Vector) == f { + au, av := validFaceXYZToUV(f, a.Vector) + bu, bv := validFaceXYZToUV(f, b.Vector) + return r2.Point{au, av}, r2.Point{bu, bv}, true + } + + // Convert everything into the (u,v,w) coordinates of the given face. Note + // that the cross product *must* be computed in the original (x,y,z) + // coordinate system because PointCross (unlike the mathematical cross + // product) can produce different results in different coordinate systems + // when one argument is a linear multiple of the other, due to the use of + // symbolic perturbations. + normUVW := pointUVW(faceXYZtoUVW(f, a.PointCross(b))) + aUVW := pointUVW(faceXYZtoUVW(f, a)) + bUVW := pointUVW(faceXYZtoUVW(f, b)) + + // Padding is handled by scaling the u- and v-components of the normal. + // Letting R=1+padding, this means that when we compute the dot product of + // the normal with a cube face vertex (such as (-1,-1,1)), we will actually + // compute the dot product with the scaled vertex (-R,-R,1). This allows + // methods such as intersectsFace, exitAxis, etc, to handle padding + // with no further modifications. + scaleUV := 1 + padding + scaledN := pointUVW{r3.Vector{X: scaleUV * normUVW.X, Y: scaleUV * normUVW.Y, Z: normUVW.Z}} + if !scaledN.intersectsFace() { + return aUV, bUV, false + } + + // TODO(roberts): This is a workaround for extremely small vectors where some + // loss of precision can occur in Normalize causing underflow. When PointCross + // is updated to work around this, this can be removed. + if math.Max(math.Abs(normUVW.X), math.Max(math.Abs(normUVW.Y), math.Abs(normUVW.Z))) < math.Ldexp(1, -511) { + normUVW = pointUVW{normUVW.Mul(math.Ldexp(1, 563))} + } + + normUVW = pointUVW{normUVW.Normalize()} + + aTan := pointUVW{normUVW.Cross(aUVW.Vector)} + bTan := pointUVW{bUVW.Cross(normUVW.Vector)} + + // As described in clipDestination, if the sum of the scores from clipping the two + // endpoints is 3 or more, then the segment does not intersect this face. + aUV, aScore := clipDestination(bUVW, aUVW, pointUVW{scaledN.Mul(-1)}, bTan, aTan, scaleUV) + bUV, bScore := clipDestination(aUVW, bUVW, scaledN, aTan, bTan, scaleUV) + + return aUV, bUV, aScore+bScore < 3 +} + +// ClipEdge returns the portion of the edge defined by AB that is contained by the +// given rectangle. If there is no intersection, false is returned and aClip and bClip +// are undefined. +func ClipEdge(a, b r2.Point, clip r2.Rect) (aClip, bClip r2.Point, intersects bool) { + // Compute the bounding rectangle of AB, clip it, and then extract the new + // endpoints from the clipped bound. + bound := r2.RectFromPoints(a, b) + if bound, intersects = clipEdgeBound(a, b, clip, bound); !intersects { + return aClip, bClip, false + } + ai := 0 + if a.X > b.X { + ai = 1 + } + aj := 0 + if a.Y > b.Y { + aj = 1 + } + + return bound.VertexIJ(ai, aj), bound.VertexIJ(1-ai, 1-aj), true +} + +// The three functions below (sumEqual, intersectsFace, intersectsOppositeEdges) +// all compare a sum (u + v) to a third value w. They are implemented in such a +// way that they produce an exact result even though all calculations are done +// with ordinary floating-point operations. Here are the principles on which these +// functions are based: +// +// A. If u + v < w in floating-point, then u + v < w in exact arithmetic. +// +// B. If u + v < w in exact arithmetic, then at least one of the following +// expressions is true in floating-point: +// u + v < w +// u < w - v +// v < w - u +// +// Proof: By rearranging terms and substituting ">" for "<", we can assume +// that all values are non-negative. Now clearly "w" is not the smallest +// value, so assume WLOG that "u" is the smallest. We want to show that +// u < w - v in floating-point. If v >= w/2, the calculation of w - v is +// exact since the result is smaller in magnitude than either input value, +// so the result holds. Otherwise we have u <= v < w/2 and w - v >= w/2 +// (even in floating point), so the result also holds. + +// sumEqual reports whether u + v == w exactly. +func sumEqual(u, v, w float64) bool { + return (u+v == w) && (u == w-v) && (v == w-u) +} + +// pointUVW represents a Point in (u,v,w) coordinate space of a cube face. +type pointUVW Point + +// intersectsFace reports whether a given directed line L intersects the cube face F. +// The line L is defined by its normal N in the (u,v,w) coordinates of F. +func (p pointUVW) intersectsFace() bool { + // L intersects the [-1,1]x[-1,1] square in (u,v) if and only if the dot + // products of N with the four corner vertices (-1,-1,1), (1,-1,1), (1,1,1), + // and (-1,1,1) do not all have the same sign. This is true exactly when + // |Nu| + |Nv| >= |Nw|. The code below evaluates this expression exactly. + u := math.Abs(p.X) + v := math.Abs(p.Y) + w := math.Abs(p.Z) + + // We only need to consider the cases where u or v is the smallest value, + // since if w is the smallest then both expressions below will have a + // positive LHS and a negative RHS. + return (v >= w-u) && (u >= w-v) +} + +// intersectsOppositeEdges reports whether a directed line L intersects two +// opposite edges of a cube face F. This includs the case where L passes +// exactly through a corner vertex of F. The directed line L is defined +// by its normal N in the (u,v,w) coordinates of F. +func (p pointUVW) intersectsOppositeEdges() bool { + // The line L intersects opposite edges of the [-1,1]x[-1,1] (u,v) square if + // and only exactly two of the corner vertices lie on each side of L. This + // is true exactly when ||Nu| - |Nv|| >= |Nw|. The code below evaluates this + // expression exactly. + u := math.Abs(p.X) + v := math.Abs(p.Y) + w := math.Abs(p.Z) + + // If w is the smallest, the following line returns an exact result. + if math.Abs(u-v) != w { + return math.Abs(u-v) >= w + } + + // Otherwise u - v = w exactly, or w is not the smallest value. In either + // case the following returns the correct result. + if u >= v { + return u-w >= v + } + return v-w >= u +} + +// axis represents the possible results of exitAxis. +type axis int + +const ( + axisU axis = iota + axisV +) + +// exitAxis reports which axis the directed line L exits the cube face F on. +// The directed line L is represented by its CCW normal N in the (u,v,w) coordinates +// of F. It returns axisU if L exits through the u=-1 or u=+1 edge, and axisV if L exits +// through the v=-1 or v=+1 edge. Either result is acceptable if L exits exactly +// through a corner vertex of the cube face. +func (p pointUVW) exitAxis() axis { + if p.intersectsOppositeEdges() { + // The line passes through through opposite edges of the face. + // It exits through the v=+1 or v=-1 edge if the u-component of N has a + // larger absolute magnitude than the v-component. + if math.Abs(p.X) >= math.Abs(p.Y) { + return axisV + } + return axisU + } + + // The line passes through through two adjacent edges of the face. + // It exits the v=+1 or v=-1 edge if an even number of the components of N + // are negative. We test this using signbit() rather than multiplication + // to avoid the possibility of underflow. + var x, y, z int + if math.Signbit(p.X) { + x = 1 + } + if math.Signbit(p.Y) { + y = 1 + } + if math.Signbit(p.Z) { + z = 1 + } + + if x^y^z == 0 { + return axisV + } + return axisU +} + +// exitPoint returns the UV coordinates of the point where a directed line L (represented +// by the CCW normal of this point), exits the cube face this point is derived from along +// the given axis. +func (p pointUVW) exitPoint(a axis) r2.Point { + if a == axisU { + u := -1.0 + if p.Y > 0 { + u = 1.0 + } + return r2.Point{u, (-u*p.X - p.Z) / p.Y} + } + + v := -1.0 + if p.X < 0 { + v = 1.0 + } + return r2.Point{(-v*p.Y - p.Z) / p.X, v} +} + +// clipDestination returns a score which is used to indicate if the clipped edge AB +// on the given face intersects the face at all. This function returns the score for +// the given endpoint, which is an integer ranging from 0 to 3. If the sum of the scores +// from both of the endpoints is 3 or more, then edge AB does not intersect this face. +// +// First, it clips the line segment AB to find the clipped destination B' on a given +// face. (The face is specified implicitly by expressing *all arguments* in the (u,v,w) +// coordinates of that face.) Second, it partially computes whether the segment AB +// intersects this face at all. The actual condition is fairly complicated, but it +// turns out that it can be expressed as a "score" that can be computed independently +// when clipping the two endpoints A and B. +func clipDestination(a, b, scaledN, aTan, bTan pointUVW, scaleUV float64) (r2.Point, int) { + var uv r2.Point + + // Optimization: if B is within the safe region of the face, use it. + maxSafeUVCoord := 1 - faceClipErrorUVCoord + if b.Z > 0 { + uv = r2.Point{b.X / b.Z, b.Y / b.Z} + if math.Max(math.Abs(uv.X), math.Abs(uv.Y)) <= maxSafeUVCoord { + return uv, 0 + } + } + + // Otherwise find the point B' where the line AB exits the face. + uv = scaledN.exitPoint(scaledN.exitAxis()).Mul(scaleUV) + + p := pointUVW(Point{r3.Vector{uv.X, uv.Y, 1.0}}) + + // Determine if the exit point B' is contained within the segment. We do this + // by computing the dot products with two inward-facing tangent vectors at A + // and B. If either dot product is negative, we say that B' is on the "wrong + // side" of that point. As the point B' moves around the great circle AB past + // the segment endpoint B, it is initially on the wrong side of B only; as it + // moves further it is on the wrong side of both endpoints; and then it is on + // the wrong side of A only. If the exit point B' is on the wrong side of + // either endpoint, we can't use it; instead the segment is clipped at the + // original endpoint B. + // + // We reject the segment if the sum of the scores of the two endpoints is 3 + // or more. Here is what that rule encodes: + // - If B' is on the wrong side of A, then the other clipped endpoint A' + // must be in the interior of AB (otherwise AB' would go the wrong way + // around the circle). There is a similar rule for A'. + // - If B' is on the wrong side of either endpoint (and therefore we must + // use the original endpoint B instead), then it must be possible to + // project B onto this face (i.e., its w-coordinate must be positive). + // This rule is only necessary to handle certain zero-length edges (A=B). + score := 0 + if p.Sub(a.Vector).Dot(aTan.Vector) < 0 { + score = 2 // B' is on wrong side of A. + } else if p.Sub(b.Vector).Dot(bTan.Vector) < 0 { + score = 1 // B' is on wrong side of B. + } + + if score > 0 { // B' is not in the interior of AB. + if b.Z <= 0 { + score = 3 // B cannot be projected onto this face. + } else { + uv = r2.Point{b.X / b.Z, b.Y / b.Z} + } + } + + return uv, score +} + +// updateEndpoint returns the interval with the specified endpoint updated to +// the given value. If the value lies beyond the opposite endpoint, nothing is +// changed and false is returned. +func updateEndpoint(bound r1.Interval, highEndpoint bool, value float64) (r1.Interval, bool) { + if !highEndpoint { + if bound.Hi < value { + return bound, false + } + if bound.Lo < value { + bound.Lo = value + } + return bound, true + } + + if bound.Lo > value { + return bound, false + } + if bound.Hi > value { + bound.Hi = value + } + return bound, true +} + +// clipBoundAxis returns the clipped versions of the bounding intervals for the given +// axes for the line segment from (a0,a1) to (b0,b1) so that neither extends beyond the +// given clip interval. negSlope is a precomputed helper variable that indicates which +// diagonal of the bounding box is spanned by AB; it is false if AB has positive slope, +// and true if AB has negative slope. If the clipping interval doesn't overlap the bounds, +// false is returned. +func clipBoundAxis(a0, b0 float64, bound0 r1.Interval, a1, b1 float64, bound1 r1.Interval, + negSlope bool, clip r1.Interval) (bound0c, bound1c r1.Interval, updated bool) { + + if bound0.Lo < clip.Lo { + // If the upper bound is below the clips lower bound, there is nothing to do. + if bound0.Hi < clip.Lo { + return bound0, bound1, false + } + // narrow the intervals lower bound to the clip bound. + bound0.Lo = clip.Lo + if bound1, updated = updateEndpoint(bound1, negSlope, interpolateFloat64(clip.Lo, a0, b0, a1, b1)); !updated { + return bound0, bound1, false + } + } + + if bound0.Hi > clip.Hi { + // If the lower bound is above the clips upper bound, there is nothing to do. + if bound0.Lo > clip.Hi { + return bound0, bound1, false + } + // narrow the intervals upper bound to the clip bound. + bound0.Hi = clip.Hi + if bound1, updated = updateEndpoint(bound1, !negSlope, interpolateFloat64(clip.Hi, a0, b0, a1, b1)); !updated { + return bound0, bound1, false + } + } + return bound0, bound1, true +} + +// edgeIntersectsRect reports whether the edge defined by AB intersects the +// given closed rectangle to within the error bound. +func edgeIntersectsRect(a, b r2.Point, r r2.Rect) bool { + // First check whether the bounds of a Rect around AB intersects the given rect. + if !r.Intersects(r2.RectFromPoints(a, b)) { + return false + } + + // Otherwise AB intersects the rect if and only if all four vertices of rect + // do not lie on the same side of the extended line AB. We test this by finding + // the two vertices of rect with minimum and maximum projections onto the normal + // of AB, and computing their dot products with the edge normal. + n := b.Sub(a).Ortho() + + i := 0 + if n.X >= 0 { + i = 1 + } + j := 0 + if n.Y >= 0 { + j = 1 + } + + max := n.Dot(r.VertexIJ(i, j).Sub(a)) + min := n.Dot(r.VertexIJ(1-i, 1-j).Sub(a)) + + return (max >= 0) && (min <= 0) +} + +// clippedEdgeBound returns the bounding rectangle of the portion of the edge defined +// by AB intersected by clip. The resulting bound may be empty. This is a convenience +// function built on top of clipEdgeBound. +func clippedEdgeBound(a, b r2.Point, clip r2.Rect) r2.Rect { + bound := r2.RectFromPoints(a, b) + if b1, intersects := clipEdgeBound(a, b, clip, bound); intersects { + return b1 + } + return r2.EmptyRect() +} + +// clipEdgeBound clips an edge AB to sequence of rectangles efficiently. +// It represents the clipped edges by their bounding boxes rather than as a pair of +// endpoints. Specifically, let A'B' be some portion of an edge AB, and let bound be +// a tight bound of A'B'. This function returns the bound that is a tight bound +// of A'B' intersected with a given rectangle. If A'B' does not intersect clip, +// it returns false and the original bound. +func clipEdgeBound(a, b r2.Point, clip, bound r2.Rect) (r2.Rect, bool) { + // negSlope indicates which diagonal of the bounding box is spanned by AB: it + // is false if AB has positive slope, and true if AB has negative slope. This is + // used to determine which interval endpoints need to be updated each time + // the edge is clipped. + negSlope := (a.X > b.X) != (a.Y > b.Y) + + b0x, b0y, up1 := clipBoundAxis(a.X, b.X, bound.X, a.Y, b.Y, bound.Y, negSlope, clip.X) + if !up1 { + return bound, false + } + b1y, b1x, up2 := clipBoundAxis(a.Y, b.Y, b0y, a.X, b.X, b0x, negSlope, clip.Y) + if !up2 { + return r2.Rect{b0x, b0y}, false + } + return r2.Rect{X: b1x, Y: b1y}, true +} + +// interpolateFloat64 returns a value with the same combination of a1 and b1 as the +// given value x is of a and b. This function makes the following guarantees: +// - If x == a, then x1 = a1 (exactly). +// - If x == b, then x1 = b1 (exactly). +// - If a <= x <= b, then a1 <= x1 <= b1 (even if a1 == b1). +// This requires a != b. +func interpolateFloat64(x, a, b, a1, b1 float64) float64 { + // To get results that are accurate near both A and B, we interpolate + // starting from the closer of the two points. + if math.Abs(a-x) <= math.Abs(b-x) { + return a1 + (b1-a1)*(x-a)/(b-a) + } + return b1 + (a1-b1)*(x-b)/(a-b) +} + +// FaceSegment represents an edge AB clipped to an S2 cube face. It is +// represented by a face index and a pair of (u,v) coordinates. +type FaceSegment struct { + face int + a, b r2.Point +} + +// FaceSegments subdivides the given edge AB at every point where it crosses the +// boundary between two S2 cube faces and returns the corresponding FaceSegments. +// The segments are returned in order from A toward B. The input points must be +// unit length. +// +// This function guarantees that the returned segments form a continuous path +// from A to B, and that all vertices are within faceClipErrorUVDist of the +// line AB. All vertices lie within the [-1,1]x[-1,1] cube face rectangles. +// The results are consistent with Sign, i.e. the edge is well-defined even its +// endpoints are antipodal. +// TODO(roberts): Extend the implementation of PointCross so that this is true. +func FaceSegments(a, b Point) []FaceSegment { + var segment FaceSegment + + // Fast path: both endpoints are on the same face. + var aFace, bFace int + aFace, segment.a.X, segment.a.Y = xyzToFaceUV(a.Vector) + bFace, segment.b.X, segment.b.Y = xyzToFaceUV(b.Vector) + if aFace == bFace { + segment.face = aFace + return []FaceSegment{segment} + } + + // Starting at A, we follow AB from face to face until we reach the face + // containing B. The following code is designed to ensure that we always + // reach B, even in the presence of numerical errors. + // + // First we compute the normal to the plane containing A and B. This normal + // becomes the ultimate definition of the line AB; it is used to resolve all + // questions regarding where exactly the line goes. Unfortunately due to + // numerical errors, the line may not quite intersect the faces containing + // the original endpoints. We handle this by moving A and/or B slightly if + // necessary so that they are on faces intersected by the line AB. + ab := a.PointCross(b) + + aFace, segment.a = moveOriginToValidFace(aFace, a, ab, segment.a) + bFace, segment.b = moveOriginToValidFace(bFace, b, Point{ab.Mul(-1)}, segment.b) + + // Now we simply follow AB from face to face until we reach B. + var segments []FaceSegment + segment.face = aFace + bSaved := segment.b + + for face := aFace; face != bFace; { + // Complete the current segment by finding the point where AB + // exits the current face. + z := faceXYZtoUVW(face, ab) + n := pointUVW{z.Vector} + + exitAxis := n.exitAxis() + segment.b = n.exitPoint(exitAxis) + segments = append(segments, segment) + + // Compute the next face intersected by AB, and translate the exit + // point of the current segment into the (u,v) coordinates of the + // next face. This becomes the first point of the next segment. + exitXyz := faceUVToXYZ(face, segment.b.X, segment.b.Y) + face = nextFace(face, segment.b, exitAxis, n, bFace) + exitUvw := faceXYZtoUVW(face, Point{exitXyz}) + segment.face = face + segment.a = r2.Point{exitUvw.X, exitUvw.Y} + } + // Finish the last segment. + segment.b = bSaved + return append(segments, segment) +} + +// moveOriginToValidFace updates the origin point to a valid face if necessary. +// Given a line segment AB whose origin A has been projected onto a given cube +// face, determine whether it is necessary to project A onto a different face +// instead. This can happen because the normal of the line AB is not computed +// exactly, so that the line AB (defined as the set of points perpendicular to +// the normal) may not intersect the cube face containing A. Even if it does +// intersect the face, the exit point of the line from that face may be on +// the wrong side of A (i.e., in the direction away from B). If this happens, +// we reproject A onto the adjacent face where the line AB approaches A most +// closely. This moves the origin by a small amount, but never more than the +// error tolerances. +func moveOriginToValidFace(face int, a, ab Point, aUV r2.Point) (int, r2.Point) { + // Fast path: if the origin is sufficiently far inside the face, it is + // always safe to use it. + const maxSafeUVCoord = 1 - faceClipErrorUVCoord + if math.Max(math.Abs((aUV).X), math.Abs((aUV).Y)) <= maxSafeUVCoord { + return face, aUV + } + + // Otherwise check whether the normal AB even intersects this face. + z := faceXYZtoUVW(face, ab) + n := pointUVW{z.Vector} + if n.intersectsFace() { + // Check whether the point where the line AB exits this face is on the + // wrong side of A (by more than the acceptable error tolerance). + uv := n.exitPoint(n.exitAxis()) + exit := faceUVToXYZ(face, uv.X, uv.Y) + aTangent := ab.Normalize().Cross(a.Vector) + + // We can use the given face. + if exit.Sub(a.Vector).Dot(aTangent) >= -faceClipErrorRadians { + return face, aUV + } + } + + // Otherwise we reproject A to the nearest adjacent face. (If line AB does + // not pass through a given face, it must pass through all adjacent faces.) + var dir int + if math.Abs((aUV).X) >= math.Abs((aUV).Y) { + // U-axis + if aUV.X > 0 { + dir = 1 + } + face = uvwFace(face, 0, dir) + } else { + // V-axis + if aUV.Y > 0 { + dir = 1 + } + face = uvwFace(face, 1, dir) + } + + aUV.X, aUV.Y = validFaceXYZToUV(face, a.Vector) + aUV.X = math.Max(-1.0, math.Min(1.0, aUV.X)) + aUV.Y = math.Max(-1.0, math.Min(1.0, aUV.Y)) + + return face, aUV +} + +// nextFace returns the next face that should be visited by FaceSegments, given that +// we have just visited face and we are following the line AB (represented +// by its normal N in the (u,v,w) coordinates of that face). The other +// arguments include the point where AB exits face, the corresponding +// exit axis, and the target face containing the destination point B. +func nextFace(face int, exit r2.Point, axis axis, n pointUVW, targetFace int) int { + // this bit is to work around C++ cleverly casting bools to ints for you. + exitA := exit.X + exit1MinusA := exit.Y + + if axis == axisV { + exitA = exit.Y + exit1MinusA = exit.X + } + exitAPos := 0 + if exitA > 0 { + exitAPos = 1 + } + exit1MinusAPos := 0 + if exit1MinusA > 0 { + exit1MinusAPos = 1 + } + + // We return the face that is adjacent to the exit point along the given + // axis. If line AB exits *exactly* through a corner of the face, there are + // two possible next faces. If one is the target face containing B, then + // we guarantee that we advance to that face directly. + // + // The three conditions below check that (1) AB exits approximately through + // a corner, (2) the adjacent face along the non-exit axis is the target + // face, and (3) AB exits *exactly* through the corner. (The sumEqual + // code checks whether the dot product of (u,v,1) and n is exactly zero.) + if math.Abs(exit1MinusA) == 1 && + uvwFace(face, int(1-axis), exit1MinusAPos) == targetFace && + sumEqual(exit.X*n.X, exit.Y*n.Y, -n.Z) { + return targetFace + } + + // Otherwise return the face that is adjacent to the exit point in the + // direction of the exit axis. + return uvwFace(face, int(axis), exitAPos) +} diff --git a/vendor/github.com/blevesearch/geo/s2/edge_crosser.go b/vendor/github.com/blevesearch/geo/s2/edge_crosser.go new file mode 100644 index 00000000..69c6da6b --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/edge_crosser.go @@ -0,0 +1,227 @@ +// Copyright 2017 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +import ( + "math" +) + +// EdgeCrosser allows edges to be efficiently tested for intersection with a +// given fixed edge AB. It is especially efficient when testing for +// intersection with an edge chain connecting vertices v0, v1, v2, ... +// +// Example usage: +// +// func CountIntersections(a, b Point, edges []Edge) int { +// count := 0 +// crosser := NewEdgeCrosser(a, b) +// for _, edge := range edges { +// if crosser.CrossingSign(&edge.First, &edge.Second) != DoNotCross { +// count++ +// } +// } +// return count +// } +// +type EdgeCrosser struct { + a Point + b Point + aXb Point + + // To reduce the number of calls to expensiveSign, we compute an + // outward-facing tangent at A and B if necessary. If the plane + // perpendicular to one of these tangents separates AB from CD (i.e., one + // edge on each side) then there is no intersection. + aTangent Point // Outward-facing tangent at A. + bTangent Point // Outward-facing tangent at B. + + // The fields below are updated for each vertex in the chain. + c Point // Previous vertex in the vertex chain. + acb Direction // The orientation of triangle ACB. +} + +// NewEdgeCrosser returns an EdgeCrosser with the fixed edge AB. +func NewEdgeCrosser(a, b Point) *EdgeCrosser { + norm := a.PointCross(b) + return &EdgeCrosser{ + a: a, + b: b, + aXb: Point{a.Cross(b.Vector)}, + aTangent: Point{a.Cross(norm.Vector)}, + bTangent: Point{norm.Cross(b.Vector)}, + } +} + +// CrossingSign reports whether the edge AB intersects the edge CD. If any two +// vertices from different edges are the same, returns MaybeCross. If either edge +// is degenerate (A == B or C == D), returns either DoNotCross or MaybeCross. +// +// Properties of CrossingSign: +// +// (1) CrossingSign(b,a,c,d) == CrossingSign(a,b,c,d) +// (2) CrossingSign(c,d,a,b) == CrossingSign(a,b,c,d) +// (3) CrossingSign(a,b,c,d) == MaybeCross if a==c, a==d, b==c, b==d +// (3) CrossingSign(a,b,c,d) == DoNotCross or MaybeCross if a==b or c==d +// +// Note that if you want to check an edge against a chain of other edges, +// it is slightly more efficient to use the single-argument version +// ChainCrossingSign below. +func (e *EdgeCrosser) CrossingSign(c, d Point) Crossing { + if c != e.c { + e.RestartAt(c) + } + return e.ChainCrossingSign(d) +} + +// EdgeOrVertexCrossing reports whether if CrossingSign(c, d) > 0, or AB and +// CD share a vertex and VertexCrossing(a, b, c, d) is true. +// +// This method extends the concept of a "crossing" to the case where AB +// and CD have a vertex in common. The two edges may or may not cross, +// according to the rules defined in VertexCrossing above. The rules +// are designed so that point containment tests can be implemented simply +// by counting edge crossings. Similarly, determining whether one edge +// chain crosses another edge chain can be implemented by counting. +func (e *EdgeCrosser) EdgeOrVertexCrossing(c, d Point) bool { + if c != e.c { + e.RestartAt(c) + } + return e.EdgeOrVertexChainCrossing(d) +} + +// NewChainEdgeCrosser is a convenience constructor that uses AB as the fixed edge, +// and C as the first vertex of the vertex chain (equivalent to calling RestartAt(c)). +// +// You don't need to use this or any of the chain functions unless you're trying to +// squeeze out every last drop of performance. Essentially all you are saving is a test +// whether the first vertex of the current edge is the same as the second vertex of the +// previous edge. +func NewChainEdgeCrosser(a, b, c Point) *EdgeCrosser { + e := NewEdgeCrosser(a, b) + e.RestartAt(c) + return e +} + +// RestartAt sets the current point of the edge crosser to be c. +// Call this method when your chain 'jumps' to a new place. +// The argument must point to a value that persists until the next call. +func (e *EdgeCrosser) RestartAt(c Point) { + e.c = c + e.acb = -triageSign(e.a, e.b, e.c) +} + +// ChainCrossingSign is like CrossingSign, but uses the last vertex passed to one of +// the crossing methods (or RestartAt) as the first vertex of the current edge. +func (e *EdgeCrosser) ChainCrossingSign(d Point) Crossing { + // For there to be an edge crossing, the triangles ACB, CBD, BDA, DAC must + // all be oriented the same way (CW or CCW). We keep the orientation of ACB + // as part of our state. When each new point D arrives, we compute the + // orientation of BDA and check whether it matches ACB. This checks whether + // the points C and D are on opposite sides of the great circle through AB. + + // Recall that triageSign is invariant with respect to rotating its + // arguments, i.e. ABD has the same orientation as BDA. + bda := triageSign(e.a, e.b, d) + if e.acb == -bda && bda != Indeterminate { + // The most common case -- triangles have opposite orientations. Save the + // current vertex D as the next vertex C, and also save the orientation of + // the new triangle ACB (which is opposite to the current triangle BDA). + e.c = d + e.acb = -bda + return DoNotCross + } + return e.crossingSign(d, bda) +} + +// EdgeOrVertexChainCrossing is like EdgeOrVertexCrossing, but uses the last vertex +// passed to one of the crossing methods (or RestartAt) as the first vertex of the current edge. +func (e *EdgeCrosser) EdgeOrVertexChainCrossing(d Point) bool { + // We need to copy e.c since it is clobbered by ChainCrossingSign. + c := e.c + switch e.ChainCrossingSign(d) { + case DoNotCross: + return false + case Cross: + return true + } + return VertexCrossing(e.a, e.b, c, d) +} + +// crossingSign handle the slow path of CrossingSign. +func (e *EdgeCrosser) crossingSign(d Point, bda Direction) Crossing { + // Compute the actual result, and then save the current vertex D as the next + // vertex C, and save the orientation of the next triangle ACB (which is + // opposite to the current triangle BDA). + defer func() { + e.c = d + e.acb = -bda + }() + + // At this point, a very common situation is that A,B,C,D are four points on + // a line such that AB does not overlap CD. (For example, this happens when + // a line or curve is sampled finely, or when geometry is constructed by + // computing the union of S2CellIds.) Most of the time, we can determine + // that AB and CD do not intersect using the two outward-facing + // tangents at A and B (parallel to AB) and testing whether AB and CD are on + // opposite sides of the plane perpendicular to one of these tangents. This + // is moderately expensive but still much cheaper than expensiveSign. + + // The error in RobustCrossProd is insignificant. The maximum error in + // the call to CrossProd (i.e., the maximum norm of the error vector) is + // (0.5 + 1/sqrt(3)) * dblEpsilon. The maximum error in each call to + // DotProd below is dblEpsilon. (There is also a small relative error + // term that is insignificant because we are comparing the result against a + // constant that is very close to zero.) + maxError := (1.5 + 1/math.Sqrt(3)) * dblEpsilon + if (e.c.Dot(e.aTangent.Vector) > maxError && d.Dot(e.aTangent.Vector) > maxError) || (e.c.Dot(e.bTangent.Vector) > maxError && d.Dot(e.bTangent.Vector) > maxError) { + return DoNotCross + } + + // Otherwise, eliminate the cases where two vertices from different edges are + // equal. (These cases could be handled in the code below, but we would rather + // avoid calling ExpensiveSign if possible.) + if e.a == e.c || e.a == d || e.b == e.c || e.b == d { + return MaybeCross + } + + // Eliminate the cases where an input edge is degenerate. (Note that in + // most cases, if CD is degenerate then this method is not even called + // because acb and bda have different signs.) + if e.a == e.b || e.c == d { + return DoNotCross + } + + // Otherwise it's time to break out the big guns. + if e.acb == Indeterminate { + e.acb = -expensiveSign(e.a, e.b, e.c) + } + if bda == Indeterminate { + bda = expensiveSign(e.a, e.b, d) + } + + if bda != e.acb { + return DoNotCross + } + + cbd := -RobustSign(e.c, d, e.b) + if cbd != e.acb { + return DoNotCross + } + dac := RobustSign(e.c, d, e.a) + if dac != e.acb { + return DoNotCross + } + return Cross +} diff --git a/vendor/github.com/blevesearch/geo/s2/edge_crossings.go b/vendor/github.com/blevesearch/geo/s2/edge_crossings.go new file mode 100644 index 00000000..a98ec76f --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/edge_crossings.go @@ -0,0 +1,396 @@ +// Copyright 2017 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +import ( + "fmt" + "math" + + "github.com/golang/geo/r3" + "github.com/golang/geo/s1" +) + +const ( + // intersectionError can be set somewhat arbitrarily, because the algorithm + // uses more precision if necessary in order to achieve the specified error. + // The only strict requirement is that intersectionError >= dblEpsilon + // radians. However, using a larger error tolerance makes the algorithm more + // efficient because it reduces the number of cases where exact arithmetic is + // needed. + intersectionError = s1.Angle(8 * dblError) + + // intersectionMergeRadius is used to ensure that intersection points that + // are supposed to be coincident are merged back together into a single + // vertex. This is required in order for various polygon operations (union, + // intersection, etc) to work correctly. It is twice the intersection error + // because two coincident intersection points might have errors in + // opposite directions. + intersectionMergeRadius = 2 * intersectionError +) + +// A Crossing indicates how edges cross. +type Crossing int + +const ( + // Cross means the edges cross. + Cross Crossing = iota + // MaybeCross means two vertices from different edges are the same. + MaybeCross + // DoNotCross means the edges do not cross. + DoNotCross +) + +func (c Crossing) String() string { + switch c { + case Cross: + return "Cross" + case MaybeCross: + return "MaybeCross" + case DoNotCross: + return "DoNotCross" + default: + return fmt.Sprintf("(BAD CROSSING %d)", c) + } +} + +// CrossingSign reports whether the edge AB intersects the edge CD. +// If AB crosses CD at a point that is interior to both edges, Cross is returned. +// If any two vertices from different edges are the same it returns MaybeCross. +// Otherwise it returns DoNotCross. +// If either edge is degenerate (A == B or C == D), the return value is MaybeCross +// if two vertices from different edges are the same and DoNotCross otherwise. +// +// Properties of CrossingSign: +// +// (1) CrossingSign(b,a,c,d) == CrossingSign(a,b,c,d) +// (2) CrossingSign(c,d,a,b) == CrossingSign(a,b,c,d) +// (3) CrossingSign(a,b,c,d) == MaybeCross if a==c, a==d, b==c, b==d +// (3) CrossingSign(a,b,c,d) == DoNotCross or MaybeCross if a==b or c==d +// +// This method implements an exact, consistent perturbation model such +// that no three points are ever considered to be collinear. This means +// that even if you have 4 points A, B, C, D that lie exactly in a line +// (say, around the equator), C and D will be treated as being slightly to +// one side or the other of AB. This is done in a way such that the +// results are always consistent (see RobustSign). +func CrossingSign(a, b, c, d Point) Crossing { + crosser := NewChainEdgeCrosser(a, b, c) + return crosser.ChainCrossingSign(d) +} + +// VertexCrossing reports whether two edges "cross" in such a way that point-in-polygon +// containment tests can be implemented by counting the number of edge crossings. +// +// Given two edges AB and CD where at least two vertices are identical +// (i.e. CrossingSign(a,b,c,d) == 0), the basic rule is that a "crossing" +// occurs if AB is encountered after CD during a CCW sweep around the shared +// vertex starting from a fixed reference point. +// +// Note that according to this rule, if AB crosses CD then in general CD +// does not cross AB. However, this leads to the correct result when +// counting polygon edge crossings. For example, suppose that A,B,C are +// three consecutive vertices of a CCW polygon. If we now consider the edge +// crossings of a segment BP as P sweeps around B, the crossing number +// changes parity exactly when BP crosses BA or BC. +// +// Useful properties of VertexCrossing (VC): +// +// (1) VC(a,a,c,d) == VC(a,b,c,c) == false +// (2) VC(a,b,a,b) == VC(a,b,b,a) == true +// (3) VC(a,b,c,d) == VC(a,b,d,c) == VC(b,a,c,d) == VC(b,a,d,c) +// (3) If exactly one of a,b equals one of c,d, then exactly one of +// VC(a,b,c,d) and VC(c,d,a,b) is true +// +// It is an error to call this method with 4 distinct vertices. +func VertexCrossing(a, b, c, d Point) bool { + // If A == B or C == D there is no intersection. We need to check this + // case first in case 3 or more input points are identical. + if a == b || c == d { + return false + } + + // If any other pair of vertices is equal, there is a crossing if and only + // if OrderedCCW indicates that the edge AB is further CCW around the + // shared vertex O (either A or B) than the edge CD, starting from an + // arbitrary fixed reference point. + + // Optimization: if AB=CD or AB=DC, we can avoid most of the calculations. + switch { + case a == c: + return (b == d) || OrderedCCW(Point{a.Ortho()}, d, b, a) + case b == d: + return OrderedCCW(Point{b.Ortho()}, c, a, b) + case a == d: + return (b == c) || OrderedCCW(Point{a.Ortho()}, c, b, a) + case b == c: + return OrderedCCW(Point{b.Ortho()}, d, a, b) + } + + return false +} + +// EdgeOrVertexCrossing is a convenience function that calls CrossingSign to +// handle cases where all four vertices are distinct, and VertexCrossing to +// handle cases where two or more vertices are the same. This defines a crossing +// function such that point-in-polygon containment tests can be implemented +// by simply counting edge crossings. +func EdgeOrVertexCrossing(a, b, c, d Point) bool { + switch CrossingSign(a, b, c, d) { + case DoNotCross: + return false + case Cross: + return true + default: + return VertexCrossing(a, b, c, d) + } +} + +// Intersection returns the intersection point of two edges AB and CD that cross +// (CrossingSign(a,b,c,d) == Crossing). +// +// Useful properties of Intersection: +// +// (1) Intersection(b,a,c,d) == Intersection(a,b,d,c) == Intersection(a,b,c,d) +// (2) Intersection(c,d,a,b) == Intersection(a,b,c,d) +// +// The returned intersection point X is guaranteed to be very close to the +// true intersection point of AB and CD, even if the edges intersect at a +// very small angle. +func Intersection(a0, a1, b0, b1 Point) Point { + // It is difficult to compute the intersection point of two edges accurately + // when the angle between the edges is very small. Previously we handled + // this by only guaranteeing that the returned intersection point is within + // intersectionError of each edge. However, this means that when the edges + // cross at a very small angle, the computed result may be very far from the + // true intersection point. + // + // Instead this function now guarantees that the result is always within + // intersectionError of the true intersection. This requires using more + // sophisticated techniques and in some cases extended precision. + // + // - intersectionStable computes the intersection point using + // projection and interpolation, taking care to minimize cancellation + // error. + // + // - intersectionExact computes the intersection point using precision + // arithmetic and converts the final result back to an Point. + pt, ok := intersectionStable(a0, a1, b0, b1) + if !ok { + pt = intersectionExact(a0, a1, b0, b1) + } + + // Make sure the intersection point is on the correct side of the sphere. + // Since all vertices are unit length, and edges are less than 180 degrees, + // (a0 + a1) and (b0 + b1) both have positive dot product with the + // intersection point. We use the sum of all vertices to make sure that the + // result is unchanged when the edges are swapped or reversed. + if pt.Dot((a0.Add(a1.Vector)).Add(b0.Add(b1.Vector))) < 0 { + pt = Point{pt.Mul(-1)} + } + + return pt +} + +// Computes the cross product of two vectors, normalized to be unit length. +// Also returns the length of the cross +// product before normalization, which is useful for estimating the amount of +// error in the result. For numerical stability, the vectors should both be +// approximately unit length. +func robustNormalWithLength(x, y r3.Vector) (r3.Vector, float64) { + var pt r3.Vector + // This computes 2 * (x.Cross(y)), but has much better numerical + // stability when x and y are unit length. + tmp := x.Sub(y).Cross(x.Add(y)) + length := tmp.Norm() + if length != 0 { + pt = tmp.Mul(1 / length) + } + return pt, 0.5 * length // Since tmp == 2 * (x.Cross(y)) +} + +/* +// intersectionSimple is not used by the C++ so it is skipped here. +*/ + +// projection returns the projection of aNorm onto X (x.Dot(aNorm)), and a bound +// on the error in the result. aNorm is not necessarily unit length. +// +// The remaining parameters (the length of aNorm (aNormLen) and the edge endpoints +// a0 and a1) allow this dot product to be computed more accurately and efficiently. +func projection(x, aNorm r3.Vector, aNormLen float64, a0, a1 Point) (proj, bound float64) { + // The error in the dot product is proportional to the lengths of the input + // vectors, so rather than using x itself (a unit-length vector) we use + // the vectors from x to the closer of the two edge endpoints. This + // typically reduces the error by a huge factor. + x0 := x.Sub(a0.Vector) + x1 := x.Sub(a1.Vector) + x0Dist2 := x0.Norm2() + x1Dist2 := x1.Norm2() + + // If both distances are the same, we need to be careful to choose one + // endpoint deterministically so that the result does not change if the + // order of the endpoints is reversed. + var dist float64 + if x0Dist2 < x1Dist2 || (x0Dist2 == x1Dist2 && x0.Cmp(x1) == -1) { + dist = math.Sqrt(x0Dist2) + proj = x0.Dot(aNorm) + } else { + dist = math.Sqrt(x1Dist2) + proj = x1.Dot(aNorm) + } + + // This calculation bounds the error from all sources: the computation of + // the normal, the subtraction of one endpoint, and the dot product itself. + // dblError appears because the input points are assumed to be + // normalized in double precision. + // + // For reference, the bounds that went into this calculation are: + // ||N'-N|| <= ((1 + 2 * sqrt(3))||N|| + 32 * sqrt(3) * dblError) * epsilon + // |(A.B)'-(A.B)| <= (1.5 * (A.B) + 1.5 * ||A|| * ||B||) * epsilon + // ||(X-Y)'-(X-Y)|| <= ||X-Y|| * epsilon + bound = (((3.5+2*math.Sqrt(3))*aNormLen+32*math.Sqrt(3)*dblError)*dist + 1.5*math.Abs(proj)) * epsilon + return proj, bound +} + +// compareEdges reports whether (a0,a1) is less than (b0,b1) with respect to a total +// ordering on edges that is invariant under edge reversals. +func compareEdges(a0, a1, b0, b1 Point) bool { + if a0.Cmp(a1.Vector) != -1 { + a0, a1 = a1, a0 + } + if b0.Cmp(b1.Vector) != -1 { + b0, b1 = b1, b0 + } + return a0.Cmp(b0.Vector) == -1 || (a0 == b0 && b0.Cmp(b1.Vector) == -1) +} + +// intersectionStable returns the intersection point of the edges (a0,a1) and +// (b0,b1) if it can be computed to within an error of at most intersectionError +// by this function. +// +// The intersection point is not guaranteed to have the correct sign because we +// choose to use the longest of the two edges first. The sign is corrected by +// Intersection. +func intersectionStable(a0, a1, b0, b1 Point) (Point, bool) { + // Sort the two edges so that (a0,a1) is longer, breaking ties in a + // deterministic way that does not depend on the ordering of the endpoints. + // This is desirable for two reasons: + // - So that the result doesn't change when edges are swapped or reversed. + // - It reduces error, since the first edge is used to compute the edge + // normal (where a longer edge means less error), and the second edge + // is used for interpolation (where a shorter edge means less error). + aLen2 := a1.Sub(a0.Vector).Norm2() + bLen2 := b1.Sub(b0.Vector).Norm2() + if aLen2 < bLen2 || (aLen2 == bLen2 && compareEdges(a0, a1, b0, b1)) { + return intersectionStableSorted(b0, b1, a0, a1) + } + return intersectionStableSorted(a0, a1, b0, b1) +} + +// intersectionStableSorted is a helper function for intersectionStable. +// It expects that the edges (a0,a1) and (b0,b1) have been sorted so that +// the first edge passed in is longer. +func intersectionStableSorted(a0, a1, b0, b1 Point) (Point, bool) { + var pt Point + + // Compute the normal of the plane through (a0, a1) in a stable way. + aNorm := a0.Sub(a1.Vector).Cross(a0.Add(a1.Vector)) + aNormLen := aNorm.Norm() + bLen := b1.Sub(b0.Vector).Norm() + + // Compute the projection (i.e., signed distance) of b0 and b1 onto the + // plane through (a0, a1). Distances are scaled by the length of aNorm. + b0Dist, b0Error := projection(b0.Vector, aNorm, aNormLen, a0, a1) + b1Dist, b1Error := projection(b1.Vector, aNorm, aNormLen, a0, a1) + + // The total distance from b0 to b1 measured perpendicularly to (a0,a1) is + // |b0Dist - b1Dist|. Note that b0Dist and b1Dist generally have + // opposite signs because b0 and b1 are on opposite sides of (a0, a1). The + // code below finds the intersection point by interpolating along the edge + // (b0, b1) to a fractional distance of b0Dist / (b0Dist - b1Dist). + // + // It can be shown that the maximum error in the interpolation fraction is + // + // (b0Dist * b1Error - b1Dist * b0Error) / (distSum * (distSum - errorSum)) + // + // We save ourselves some work by scaling the result and the error bound by + // "distSum", since the result is normalized to be unit length anyway. + distSum := math.Abs(b0Dist - b1Dist) + errorSum := b0Error + b1Error + if distSum <= errorSum { + return pt, false // Error is unbounded in this case. + } + + x := b1.Mul(b0Dist).Sub(b0.Mul(b1Dist)) + err := bLen*math.Abs(b0Dist*b1Error-b1Dist*b0Error)/ + (distSum-errorSum) + 2*distSum*epsilon + + // Finally we normalize the result, compute the corresponding error, and + // check whether the total error is acceptable. + xLen := x.Norm() + maxError := intersectionError + if err > (float64(maxError)-epsilon)*xLen { + return pt, false + } + + return Point{x.Mul(1 / xLen)}, true +} + +// intersectionExact returns the intersection point of (a0, a1) and (b0, b1) +// using precise arithmetic. Note that the result is not exact because it is +// rounded down to double precision at the end. Also, the intersection point +// is not guaranteed to have the correct sign (i.e., the return value may need +// to be negated). +func intersectionExact(a0, a1, b0, b1 Point) Point { + // Since we are using presice arithmetic, we don't need to worry about + // numerical stability. + a0P := r3.PreciseVectorFromVector(a0.Vector) + a1P := r3.PreciseVectorFromVector(a1.Vector) + b0P := r3.PreciseVectorFromVector(b0.Vector) + b1P := r3.PreciseVectorFromVector(b1.Vector) + aNormP := a0P.Cross(a1P) + bNormP := b0P.Cross(b1P) + xP := aNormP.Cross(bNormP) + + // The final Normalize() call is done in double precision, which creates a + // directional error of up to 2*dblError. (Precise conversion and Normalize() + // each contribute up to dblError of directional error.) + x := xP.Vector() + + if x == (r3.Vector{}) { + // The two edges are exactly collinear, but we still consider them to be + // "crossing" because of simulation of simplicity. Out of the four + // endpoints, exactly two lie in the interior of the other edge. Of + // those two we return the one that is lexicographically smallest. + x = r3.Vector{10, 10, 10} // Greater than any valid S2Point + + aNorm := Point{aNormP.Vector()} + bNorm := Point{bNormP.Vector()} + if OrderedCCW(b0, a0, b1, bNorm) && a0.Cmp(x) == -1 { + return a0 + } + if OrderedCCW(b0, a1, b1, bNorm) && a1.Cmp(x) == -1 { + return a1 + } + if OrderedCCW(a0, b0, a1, aNorm) && b0.Cmp(x) == -1 { + return b0 + } + if OrderedCCW(a0, b1, a1, aNorm) && b1.Cmp(x) == -1 { + return b1 + } + } + + return Point{x} +} diff --git a/vendor/github.com/blevesearch/geo/s2/edge_distances.go b/vendor/github.com/blevesearch/geo/s2/edge_distances.go new file mode 100644 index 00000000..ca197af1 --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/edge_distances.go @@ -0,0 +1,408 @@ +// Copyright 2017 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +// This file defines a collection of methods for computing the distance to an edge, +// interpolating along an edge, projecting points onto edges, etc. + +import ( + "math" + + "github.com/golang/geo/s1" +) + +// DistanceFromSegment returns the distance of point X from line segment AB. +// The points are expected to be normalized. The result is very accurate for small +// distances but may have some numerical error if the distance is large +// (approximately pi/2 or greater). The case A == B is handled correctly. +func DistanceFromSegment(x, a, b Point) s1.Angle { + var minDist s1.ChordAngle + minDist, _ = updateMinDistance(x, a, b, minDist, true) + return minDist.Angle() +} + +// IsDistanceLess reports whether the distance from X to the edge AB is less +// than limit. (For less than or equal to, specify limit.Successor()). +// This method is faster than DistanceFromSegment(). If you want to +// compare against a fixed s1.Angle, you should convert it to an s1.ChordAngle +// once and save the value, since this conversion is relatively expensive. +func IsDistanceLess(x, a, b Point, limit s1.ChordAngle) bool { + _, less := UpdateMinDistance(x, a, b, limit) + return less +} + +// UpdateMinDistance checks if the distance from X to the edge AB is less +// than minDist, and if so, returns the updated value and true. +// The case A == B is handled correctly. +// +// Use this method when you want to compute many distances and keep track of +// the minimum. It is significantly faster than using DistanceFromSegment +// because (1) using s1.ChordAngle is much faster than s1.Angle, and (2) it +// can save a lot of work by not actually computing the distance when it is +// obviously larger than the current minimum. +func UpdateMinDistance(x, a, b Point, minDist s1.ChordAngle) (s1.ChordAngle, bool) { + return updateMinDistance(x, a, b, minDist, false) +} + +// UpdateMaxDistance checks if the distance from X to the edge AB is greater +// than maxDist, and if so, returns the updated value and true. +// Otherwise it returns false. The case A == B is handled correctly. +func UpdateMaxDistance(x, a, b Point, maxDist s1.ChordAngle) (s1.ChordAngle, bool) { + dist := maxChordAngle(ChordAngleBetweenPoints(x, a), ChordAngleBetweenPoints(x, b)) + if dist > s1.RightChordAngle { + dist, _ = updateMinDistance(Point{x.Mul(-1)}, a, b, dist, true) + dist = s1.StraightChordAngle - dist + } + if maxDist < dist { + return dist, true + } + + return maxDist, false +} + +// IsInteriorDistanceLess reports whether the minimum distance from X to the edge +// AB is attained at an interior point of AB (i.e., not an endpoint), and that +// distance is less than limit. (Specify limit.Successor() for less than or equal to). +func IsInteriorDistanceLess(x, a, b Point, limit s1.ChordAngle) bool { + _, less := UpdateMinInteriorDistance(x, a, b, limit) + return less +} + +// UpdateMinInteriorDistance reports whether the minimum distance from X to AB +// is attained at an interior point of AB (i.e., not an endpoint), and that distance +// is less than minDist. If so, the value of minDist is updated and true is returned. +// Otherwise it is unchanged and returns false. +func UpdateMinInteriorDistance(x, a, b Point, minDist s1.ChordAngle) (s1.ChordAngle, bool) { + return interiorDist(x, a, b, minDist, false) +} + +// Project returns the point along the edge AB that is closest to the point X. +// The fractional distance of this point along the edge AB can be obtained +// using DistanceFraction. +// +// This requires that all points are unit length. +func Project(x, a, b Point) Point { + aXb := a.PointCross(b) + // Find the closest point to X along the great circle through AB. + p := x.Sub(aXb.Mul(x.Dot(aXb.Vector) / aXb.Vector.Norm2())) + + // If this point is on the edge AB, then it's the closest point. + if Sign(aXb, a, Point{p}) && Sign(Point{p}, b, aXb) { + return Point{p.Normalize()} + } + + // Otherwise, the closest point is either A or B. + if x.Sub(a.Vector).Norm2() <= x.Sub(b.Vector).Norm2() { + return a + } + return b +} + +// DistanceFraction returns the distance ratio of the point X along an edge AB. +// If X is on the line segment AB, this is the fraction T such +// that X == Interpolate(T, A, B). +// +// This requires that A and B are distinct. +func DistanceFraction(x, a, b Point) float64 { + d0 := x.Angle(a.Vector) + d1 := x.Angle(b.Vector) + return float64(d0 / (d0 + d1)) +} + +// Interpolate returns the point X along the line segment AB whose distance from A +// is the given fraction "t" of the distance AB. Does NOT require that "t" be +// between 0 and 1. Note that all distances are measured on the surface of +// the sphere, so this is more complicated than just computing (1-t)*a + t*b +// and normalizing the result. +func Interpolate(t float64, a, b Point) Point { + if t == 0 { + return a + } + if t == 1 { + return b + } + ab := a.Angle(b.Vector) + return InterpolateAtDistance(s1.Angle(t)*ab, a, b) +} + +// InterpolateAtDistance returns the point X along the line segment AB whose +// distance from A is the angle ax. +func InterpolateAtDistance(ax s1.Angle, a, b Point) Point { + aRad := ax.Radians() + + // Use PointCross to compute the tangent vector at A towards B. The + // result is always perpendicular to A, even if A=B or A=-B, but it is not + // necessarily unit length. (We effectively normalize it below.) + normal := a.PointCross(b) + tangent := normal.Vector.Cross(a.Vector) + + // Now compute the appropriate linear combination of A and "tangent". With + // infinite precision the result would always be unit length, but we + // normalize it anyway to ensure that the error is within acceptable bounds. + // (Otherwise errors can build up when the result of one interpolation is + // fed into another interpolation.) + return Point{(a.Mul(math.Cos(aRad)).Add(tangent.Mul(math.Sin(aRad) / tangent.Norm()))).Normalize()} +} + +// minUpdateDistanceMaxError returns the maximum error in the result of +// UpdateMinDistance (and the associated functions such as +// UpdateMinInteriorDistance, IsDistanceLess, etc), assuming that all +// input points are normalized to within the bounds guaranteed by r3.Vector's +// Normalize. The error can be added or subtracted from an s1.ChordAngle +// using its Expanded method. +func minUpdateDistanceMaxError(dist s1.ChordAngle) float64 { + // There are two cases for the maximum error in UpdateMinDistance(), + // depending on whether the closest point is interior to the edge. + return math.Max(minUpdateInteriorDistanceMaxError(dist), dist.MaxPointError()) +} + +// minUpdateInteriorDistanceMaxError returns the maximum error in the result of +// UpdateMinInteriorDistance, assuming that all input points are normalized +// to within the bounds guaranteed by Point's Normalize. The error can be added +// or subtracted from an s1.ChordAngle using its Expanded method. +// +// Note that accuracy goes down as the distance approaches 0 degrees or 180 +// degrees (for different reasons). Near 0 degrees the error is acceptable +// for all practical purposes (about 1.2e-15 radians ~= 8 nanometers). For +// exactly antipodal points the maximum error is quite high (0.5 meters), +// but this error drops rapidly as the points move away from antipodality +// (approximately 1 millimeter for points that are 50 meters from antipodal, +// and 1 micrometer for points that are 50km from antipodal). +// +// TODO(roberts): Currently the error bound does not hold for edges whose endpoints +// are antipodal to within about 1e-15 radians (less than 1 micron). This could +// be fixed by extending PointCross to use higher precision when necessary. +func minUpdateInteriorDistanceMaxError(dist s1.ChordAngle) float64 { + // If a point is more than 90 degrees from an edge, then the minimum + // distance is always to one of the endpoints, not to the edge interior. + if dist >= s1.RightChordAngle { + return 0.0 + } + + // This bound includes all source of error, assuming that the input points + // are normalized. a and b are components of chord length that are + // perpendicular and parallel to a plane containing the edge respectively. + b := math.Min(1.0, 0.5*float64(dist)) + a := math.Sqrt(b * (2 - b)) + return ((2.5+2*math.Sqrt(3)+8.5*a)*a + + (2+2*math.Sqrt(3)/3+6.5*(1-b))*b + + (23+16/math.Sqrt(3))*dblEpsilon) * dblEpsilon +} + +// updateMinDistance computes the distance from a point X to a line segment AB, +// and if either the distance was less than the given minDist, or alwaysUpdate is +// true, the value and whether it was updated are returned. +func updateMinDistance(x, a, b Point, minDist s1.ChordAngle, alwaysUpdate bool) (s1.ChordAngle, bool) { + if d, ok := interiorDist(x, a, b, minDist, alwaysUpdate); ok { + // Minimum distance is attained along the edge interior. + return d, true + } + + // Otherwise the minimum distance is to one of the endpoints. + xa2, xb2 := (x.Sub(a.Vector)).Norm2(), x.Sub(b.Vector).Norm2() + dist := s1.ChordAngle(math.Min(xa2, xb2)) + if !alwaysUpdate && dist >= minDist { + return minDist, false + } + return dist, true +} + +// interiorDist returns the shortest distance from point x to edge ab, assuming +// that the closest point to X is interior to AB. If the closest point is not +// interior to AB, interiorDist returns (minDist, false). If alwaysUpdate is set to +// false, the distance is only updated when the value exceeds certain the given minDist. +func interiorDist(x, a, b Point, minDist s1.ChordAngle, alwaysUpdate bool) (s1.ChordAngle, bool) { + // Chord distance of x to both end points a and b. + xa2, xb2 := (x.Sub(a.Vector)).Norm2(), x.Sub(b.Vector).Norm2() + + // The closest point on AB could either be one of the two vertices (the + // vertex case) or in the interior (the interior case). Let C = A x B. + // If X is in the spherical wedge extending from A to B around the axis + // through C, then we are in the interior case. Otherwise we are in the + // vertex case. + // + // Check whether we might be in the interior case. For this to be true, XAB + // and XBA must both be acute angles. Checking this condition exactly is + // expensive, so instead we consider the planar triangle ABX (which passes + // through the sphere's interior). The planar angles XAB and XBA are always + // less than the corresponding spherical angles, so if we are in the + // interior case then both of these angles must be acute. + // + // We check this by computing the squared edge lengths of the planar + // triangle ABX, and testing whether angles XAB and XBA are both acute using + // the law of cosines: + // + // | XA^2 - XB^2 | < AB^2 (*) + // + // This test must be done conservatively (taking numerical errors into + // account) since otherwise we might miss a situation where the true minimum + // distance is achieved by a point on the edge interior. + // + // There are two sources of error in the expression above (*). The first is + // that points are not normalized exactly; they are only guaranteed to be + // within 2 * dblEpsilon of unit length. Under the assumption that the two + // sides of (*) are nearly equal, the total error due to normalization errors + // can be shown to be at most + // + // 2 * dblEpsilon * (XA^2 + XB^2 + AB^2) + 8 * dblEpsilon ^ 2 . + // + // The other source of error is rounding of results in the calculation of (*). + // Each of XA^2, XB^2, AB^2 has a maximum relative error of 2.5 * dblEpsilon, + // plus an additional relative error of 0.5 * dblEpsilon in the final + // subtraction which we further bound as 0.25 * dblEpsilon * (XA^2 + XB^2 + + // AB^2) for convenience. This yields a final error bound of + // + // 4.75 * dblEpsilon * (XA^2 + XB^2 + AB^2) + 8 * dblEpsilon ^ 2 . + ab2 := a.Sub(b.Vector).Norm2() + maxError := (4.75*dblEpsilon*(xa2+xb2+ab2) + 8*dblEpsilon*dblEpsilon) + if math.Abs(xa2-xb2) >= ab2+maxError { + return minDist, false + } + + // The minimum distance might be to a point on the edge interior. Let R + // be closest point to X that lies on the great circle through AB. Rather + // than computing the geodesic distance along the surface of the sphere, + // instead we compute the "chord length" through the sphere's interior. + // + // The squared chord length XR^2 can be expressed as XQ^2 + QR^2, where Q + // is the point X projected onto the plane through the great circle AB. + // The distance XQ^2 can be written as (X.C)^2 / |C|^2 where C = A x B. + // We ignore the QR^2 term and instead use XQ^2 as a lower bound, since it + // is faster and the corresponding distance on the Earth's surface is + // accurate to within 1% for distances up to about 1800km. + c := a.PointCross(b) + c2 := c.Norm2() + xDotC := x.Dot(c.Vector) + xDotC2 := xDotC * xDotC + if !alwaysUpdate && xDotC2 > c2*float64(minDist) { + // The closest point on the great circle AB is too far away. We need to + // test this using ">" rather than ">=" because the actual minimum bound + // on the distance is (xDotC2 / c2), which can be rounded differently + // than the (more efficient) multiplicative test above. + return minDist, false + } + + // Otherwise we do the exact, more expensive test for the interior case. + // This test is very likely to succeed because of the conservative planar + // test we did initially. + // + // TODO(roberts): Ensure that the errors in test are accurately reflected in the + // minUpdateInteriorDistanceMaxError. + cx := c.Cross(x.Vector) + if a.Sub(x.Vector).Dot(cx) >= 0 || b.Sub(x.Vector).Dot(cx) <= 0 { + return minDist, false + } + + // Compute the squared chord length XR^2 = XQ^2 + QR^2 (see above). + // This calculation has good accuracy for all chord lengths since it + // is based on both the dot product and cross product (rather than + // deriving one from the other). However, note that the chord length + // representation itself loses accuracy as the angle approaches π. + qr := 1 - math.Sqrt(cx.Norm2()/c2) + dist := s1.ChordAngle((xDotC2 / c2) + (qr * qr)) + + if !alwaysUpdate && dist >= minDist { + return minDist, false + } + + return dist, true +} + +// updateEdgePairMinDistance computes the minimum distance between the given +// pair of edges. If the two edges cross, the distance is zero. The cases +// a0 == a1 and b0 == b1 are handled correctly. +func updateEdgePairMinDistance(a0, a1, b0, b1 Point, minDist s1.ChordAngle) (s1.ChordAngle, bool) { + if minDist == 0 { + return 0, false + } + if CrossingSign(a0, a1, b0, b1) == Cross { + minDist = 0 + return 0, true + } + + // Otherwise, the minimum distance is achieved at an endpoint of at least + // one of the two edges. We ensure that all four possibilities are always checked. + // + // The calculation below computes each of the six vertex-vertex distances + // twice (this could be optimized). + var ok1, ok2, ok3, ok4 bool + minDist, ok1 = UpdateMinDistance(a0, b0, b1, minDist) + minDist, ok2 = UpdateMinDistance(a1, b0, b1, minDist) + minDist, ok3 = UpdateMinDistance(b0, a0, a1, minDist) + minDist, ok4 = UpdateMinDistance(b1, a0, a1, minDist) + return minDist, ok1 || ok2 || ok3 || ok4 +} + +// updateEdgePairMaxDistance reports the minimum distance between the given pair of edges. +// If one edge crosses the antipodal reflection of the other, the distance is pi. +func updateEdgePairMaxDistance(a0, a1, b0, b1 Point, maxDist s1.ChordAngle) (s1.ChordAngle, bool) { + if maxDist == s1.StraightChordAngle { + return s1.StraightChordAngle, false + } + if CrossingSign(a0, a1, Point{b0.Mul(-1)}, Point{b1.Mul(-1)}) == Cross { + return s1.StraightChordAngle, true + } + + // Otherwise, the maximum distance is achieved at an endpoint of at least + // one of the two edges. We ensure that all four possibilities are always checked. + // + // The calculation below computes each of the six vertex-vertex distances + // twice (this could be optimized). + var ok1, ok2, ok3, ok4 bool + maxDist, ok1 = UpdateMaxDistance(a0, b0, b1, maxDist) + maxDist, ok2 = UpdateMaxDistance(a1, b0, b1, maxDist) + maxDist, ok3 = UpdateMaxDistance(b0, a0, a1, maxDist) + maxDist, ok4 = UpdateMaxDistance(b1, a0, a1, maxDist) + return maxDist, ok1 || ok2 || ok3 || ok4 +} + +// EdgePairClosestPoints returns the pair of points (a, b) that achieves the +// minimum distance between edges a0a1 and b0b1, where a is a point on a0a1 and +// b is a point on b0b1. If the two edges intersect, a and b are both equal to +// the intersection point. Handles a0 == a1 and b0 == b1 correctly. +func EdgePairClosestPoints(a0, a1, b0, b1 Point) (Point, Point) { + if CrossingSign(a0, a1, b0, b1) == Cross { + x := Intersection(a0, a1, b0, b1) + return x, x + } + // We save some work by first determining which vertex/edge pair achieves + // the minimum distance, and then computing the closest point on that edge. + var minDist s1.ChordAngle + var ok bool + + minDist, ok = updateMinDistance(a0, b0, b1, minDist, true) + closestVertex := 0 + if minDist, ok = UpdateMinDistance(a1, b0, b1, minDist); ok { + closestVertex = 1 + } + if minDist, ok = UpdateMinDistance(b0, a0, a1, minDist); ok { + closestVertex = 2 + } + if minDist, ok = UpdateMinDistance(b1, a0, a1, minDist); ok { + closestVertex = 3 + } + switch closestVertex { + case 0: + return a0, Project(a0, b0, b1) + case 1: + return a1, Project(a1, b0, b1) + case 2: + return Project(b0, a0, a1), b0 + case 3: + return Project(b1, a0, a1), b1 + default: + panic("illegal case reached") + } +} diff --git a/vendor/github.com/blevesearch/geo/s2/edge_query.go b/vendor/github.com/blevesearch/geo/s2/edge_query.go new file mode 100644 index 00000000..6c86962b --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/edge_query.go @@ -0,0 +1,816 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +import ( + "sort" + + "github.com/golang/geo/s1" +) + +// EdgeQueryOptions holds the options for controlling how EdgeQuery operates. +// +// Options can be chained together builder-style: +// +// opts = NewClosestEdgeQueryOptions(). +// MaxResults(1). +// DistanceLimit(s1.ChordAngleFromAngle(3 * s1.Degree)). +// MaxError(s1.ChordAngleFromAngle(0.001 * s1.Degree)) +// query = NewClosestEdgeQuery(index, opts) +// +// or set individually: +// +// opts = NewClosestEdgeQueryOptions() +// opts.IncludeInteriors(true) +// +// or just inline: +// +// query = NewClosestEdgeQuery(index, NewClosestEdgeQueryOptions().MaxResults(3)) +// +// If you pass a nil as the options you get the default values for the options. +type EdgeQueryOptions struct { + common *queryOptions +} + +// DistanceLimit specifies that only edges whose distance to the target is +// within, this distance should be returned. Edges whose distance is equal +// are not returned. To include values that are equal, specify the limit with +// the next largest representable distance. i.e. limit.Successor(). +func (e *EdgeQueryOptions) DistanceLimit(limit s1.ChordAngle) *EdgeQueryOptions { + e.common = e.common.DistanceLimit(limit) + return e +} + +// IncludeInteriors specifies whether polygon interiors should be +// included when measuring distances. +func (e *EdgeQueryOptions) IncludeInteriors(x bool) *EdgeQueryOptions { + e.common = e.common.IncludeInteriors(x) + return e +} + +// UseBruteForce sets or disables the use of brute force in a query. +func (e *EdgeQueryOptions) UseBruteForce(x bool) *EdgeQueryOptions { + e.common = e.common.UseBruteForce(x) + return e +} + +// MaxError specifies that edges up to dist away than the true +// matching edges may be substituted in the result set, as long as such +// edges satisfy all the remaining search criteria (such as DistanceLimit). +// This option only has an effect if MaxResults is also specified; +// otherwise all edges closer than MaxDistance will always be returned. +func (e *EdgeQueryOptions) MaxError(dist s1.ChordAngle) *EdgeQueryOptions { + e.common = e.common.MaxError(dist) + return e +} + +// MaxResults specifies that at most MaxResults edges should be returned. +// This must be at least 1. +func (e *EdgeQueryOptions) MaxResults(n int) *EdgeQueryOptions { + e.common = e.common.MaxResults(n) + return e +} + +// NewClosestEdgeQueryOptions returns a set of edge query options suitable +// for performing closest edge queries. +func NewClosestEdgeQueryOptions() *EdgeQueryOptions { + return &EdgeQueryOptions{ + common: newQueryOptions(minDistance(0)), + } +} + +// NewFurthestEdgeQueryOptions returns a set of edge query options suitable +// for performing furthest edge queries. +func NewFurthestEdgeQueryOptions() *EdgeQueryOptions { + return &EdgeQueryOptions{ + common: newQueryOptions(maxDistance(0)), + } +} + +// EdgeQueryResult represents an edge that meets the target criteria for the +// query. Note the following special cases: +// +// - ShapeID >= 0 && EdgeID < 0 represents the interior of a shape. +// Such results may be returned when the option IncludeInteriors is true. +// +// - ShapeID < 0 && EdgeID < 0 is returned to indicate that no edge +// satisfies the requested query options. +type EdgeQueryResult struct { + distance distance + shapeID int32 + edgeID int32 +} + +// Distance reports the distance between the edge in this shape that satisfied +// the query's parameters. +func (e EdgeQueryResult) Distance() s1.ChordAngle { return e.distance.chordAngle() } + +// ShapeID reports the ID of the Shape this result is for. +func (e EdgeQueryResult) ShapeID() int32 { return e.shapeID } + +// EdgeID reports the ID of the edge in the results Shape. +func (e EdgeQueryResult) EdgeID() int32 { return e.edgeID } + +// newEdgeQueryResult returns a result instance with default values. +func newEdgeQueryResult(target distanceTarget) EdgeQueryResult { + return EdgeQueryResult{ + distance: target.distance().infinity(), + shapeID: -1, + edgeID: -1, + } +} + +// IsInterior reports if this result represents the interior of a Shape. +func (e EdgeQueryResult) IsInterior() bool { + return e.shapeID >= 0 && e.edgeID < 0 +} + +// IsEmpty reports if this has no edge that satisfies the given edge query options. +// This result is only returned in one special case, namely when FindEdge() does +// not find any suitable edges. +func (e EdgeQueryResult) IsEmpty() bool { + return e.shapeID < 0 +} + +// Less reports if this results is less that the other first by distance, +// then by (shapeID, edgeID). This is used for sorting. +func (e EdgeQueryResult) Less(other EdgeQueryResult) bool { + if e.distance.chordAngle() != other.distance.chordAngle() { + return e.distance.less(other.distance) + } + if e.shapeID != other.shapeID { + return e.shapeID < other.shapeID + } + return e.edgeID < other.edgeID +} + +// EdgeQuery is used to find the edge(s) between two geometries that match a +// given set of options. It is flexible enough so that it can be adapted to +// compute maximum distances and even potentially Hausdorff distances. +// +// By using the appropriate options, this type can answer questions such as: +// +// - Find the minimum distance between two geometries A and B. +// - Find all edges of geometry A that are within a distance D of geometry B. +// - Find the k edges of geometry A that are closest to a given point P. +// +// You can also specify whether polygons should include their interiors (i.e., +// if a point is contained by a polygon, should the distance be zero or should +// it be measured to the polygon boundary?) +// +// The input geometries may consist of any number of points, polylines, and +// polygons (collectively referred to as "shapes"). Shapes do not need to be +// disjoint; they may overlap or intersect arbitrarily. The implementation is +// designed to be fast for both simple and complex geometries. +type EdgeQuery struct { + index *ShapeIndex + opts *queryOptions + target distanceTarget + + // True if opts.maxError must be subtracted from ShapeIndex cell distances + // in order to ensure that such distances are measured conservatively. This + // is true only if the target takes advantage of maxError in order to + // return faster results, and 0 < maxError < distanceLimit. + useConservativeCellDistance bool + + // The decision about whether to use the brute force algorithm is based on + // counting the total number of edges in the index. However if the index + // contains a large number of shapes, this in itself might take too long. + // So instead we only count edges up to (maxBruteForceIndexSize() + 1) + // for the current target type (stored as indexNumEdgesLimit). + indexNumEdges int + indexNumEdgesLimit int + + // The distance beyond which we can safely ignore further candidate edges. + // (Candidates that are exactly at the limit are ignored; this is more + // efficient for UpdateMinDistance and should not affect clients since + // distance measurements have a small amount of error anyway.) + // + // Initially this is the same as the maximum distance specified by the user, + // but it can also be updated by the algorithm (see maybeAddResult). + distanceLimit distance + + // The current set of results of the query. + results []EdgeQueryResult + + // This field is true when duplicates must be avoided explicitly. This + // is achieved by maintaining a separate set keyed by (shapeID, edgeID) + // only, and checking whether each edge is in that set before computing the + // distance to it. + avoidDuplicates bool + + // testedEdges tracks the set of shape and edges that have already been tested. + testedEdges map[ShapeEdgeID]uint32 + + // For the optimized algorihm we precompute the top-level CellIDs that + // will be added to the priority queue. There can be at most 6 of these + // cells. Essentially this is just a covering of the indexed edges, except + // that we also store pointers to the corresponding ShapeIndexCells to + // reduce the number of index seeks required. + indexCovering []CellID + indexCells []*ShapeIndexCell + + // The algorithm maintains a priority queue of unprocessed CellIDs, sorted + // in increasing order of distance from the target. + queue *queryQueue + + iter *ShapeIndexIterator + maxDistanceCovering []CellID + initialCells []CellID +} + +// NewClosestEdgeQuery returns an EdgeQuery that is used for finding the +// closest edge(s) to a given Point, Edge, Cell, or geometry collection. +// +// You can find either the k closest edges, or all edges within a given +// radius, or both (i.e., the k closest edges up to a given maximum radius). +// E.g. to find all the edges within 5 kilometers, set the DistanceLimit in +// the options. +// +// By default *all* edges are returned, so you should always specify either +// MaxResults or DistanceLimit options or both. +// +// Note that by default, distances are measured to the boundary and interior +// of polygons. For example, if a point is inside a polygon then its distance +// is zero. To change this behavior, set the IncludeInteriors option to false. +// +// If you only need to test whether the distance is above or below a given +// threshold (e.g., 10 km), you can use the IsDistanceLess() method. This is +// much faster than actually calculating the distance with FindEdge, +// since the implementation can stop as soon as it can prove that the minimum +// distance is either above or below the threshold. +func NewClosestEdgeQuery(index *ShapeIndex, opts *EdgeQueryOptions) *EdgeQuery { + if opts == nil { + opts = NewClosestEdgeQueryOptions() + } + e := &EdgeQuery{ + testedEdges: make(map[ShapeEdgeID]uint32), + index: index, + opts: opts.common, + queue: newQueryQueue(), + } + + return e +} + +// NewFurthestEdgeQuery returns an EdgeQuery that is used for finding the +// furthest edge(s) to a given Point, Edge, Cell, or geometry collection. +// +// The furthest edge is defined as the one which maximizes the +// distance from any point on that edge to any point on the target geometry. +// +// Similar to the example in NewClosestEdgeQuery, to find the 5 furthest edges +// from a given Point: +func NewFurthestEdgeQuery(index *ShapeIndex, opts *EdgeQueryOptions) *EdgeQuery { + if opts == nil { + opts = NewFurthestEdgeQueryOptions() + } + e := &EdgeQuery{ + testedEdges: make(map[ShapeEdgeID]uint32), + index: index, + opts: opts.common, + queue: newQueryQueue(), + } + + return e +} + +// Reset resets the state of this EdgeQuery. +func (e *EdgeQuery) Reset() { + e.indexNumEdges = 0 + e.indexNumEdgesLimit = 0 + e.indexCovering = nil + e.indexCells = nil +} + +// FindEdges returns the edges for the given target that satisfy the current options. +// +// Note that if opts.IncludeInteriors is true, the results may include some +// entries with edge_id == -1. This indicates that the target intersects +// the indexed polygon with the given ShapeID. +func (e *EdgeQuery) FindEdges(target distanceTarget) []EdgeQueryResult { + return e.findEdges(target, e.opts) +} + +// Distance reports the distance to the target. If the index or target is empty, +// returns the EdgeQuery's maximal sentinel. +// +// Use IsDistanceLess()/IsDistanceGreater() if you only want to compare the +// distance against a threshold value, since it is often much faster. +func (e *EdgeQuery) Distance(target distanceTarget) s1.ChordAngle { + return e.findEdge(target, e.opts).Distance() +} + +// IsDistanceLess reports if the distance to target is less than the given limit. +// +// This method is usually much faster than Distance(), since it is much +// less work to determine whether the minimum distance is above or below a +// threshold than it is to calculate the actual minimum distance. +// +// If you wish to check if the distance is less than or equal to the limit, use: +// +// query.IsDistanceLess(target, limit.Successor()) +// +func (e *EdgeQuery) IsDistanceLess(target distanceTarget, limit s1.ChordAngle) bool { + opts := e.opts + opts = opts.MaxResults(1). + DistanceLimit(limit). + MaxError(s1.StraightChordAngle) + return !e.findEdge(target, opts).IsEmpty() +} + +// IsDistanceGreater reports if the distance to target is greater than limit. +// +// This method is usually much faster than Distance, since it is much +// less work to determine whether the maximum distance is above or below a +// threshold than it is to calculate the actual maximum distance. +// If you wish to check if the distance is less than or equal to the limit, use: +// +// query.IsDistanceGreater(target, limit.Predecessor()) +// +func (e *EdgeQuery) IsDistanceGreater(target distanceTarget, limit s1.ChordAngle) bool { + return e.IsDistanceLess(target, limit) +} + +// IsConservativeDistanceLessOrEqual reports if the distance to target is less +// or equal to the limit, where the limit has been expanded by the maximum error +// for the distance calculation. +// +// For example, suppose that we want to test whether two geometries might +// intersect each other after they are snapped together using Builder +// (using the IdentitySnapFunction with a given "snap radius"). Since +// Builder uses exact distance predicates (s2predicates), we need to +// measure the distance between the two geometries conservatively. If the +// distance is definitely greater than "snap radius", then the geometries +// are guaranteed to not intersect after snapping. +func (e *EdgeQuery) IsConservativeDistanceLessOrEqual(target distanceTarget, limit s1.ChordAngle) bool { + return e.IsDistanceLess(target, limit.Expanded(minUpdateDistanceMaxError(limit))) +} + +// IsConservativeDistanceGreaterOrEqual reports if the distance to the target is greater +// than or equal to the given limit with some small tolerance. +func (e *EdgeQuery) IsConservativeDistanceGreaterOrEqual(target distanceTarget, limit s1.ChordAngle) bool { + return e.IsDistanceGreater(target, limit.Expanded(-minUpdateDistanceMaxError(limit))) +} + +// findEdges returns the closest edges to the given target that satisfy the given options. +// +// Note that if opts.includeInteriors is true, the results may include some +// entries with edgeID == -1. This indicates that the target intersects the +// indexed polygon with the given shapeID. +func (e *EdgeQuery) findEdges(target distanceTarget, opts *queryOptions) []EdgeQueryResult { + e.findEdgesInternal(target, opts) + // TODO(roberts): Revisit this if there is a heap or other sorted and + // uniquing datastructure we can use instead of just a slice. + e.results = sortAndUniqueResults(e.results) + if len(e.results) > e.opts.maxResults { + e.results = e.results[:e.opts.maxResults] + } + return e.results +} + +func sortAndUniqueResults(results []EdgeQueryResult) []EdgeQueryResult { + if len(results) <= 1 { + return results + } + sort.Slice(results, func(i, j int) bool { return results[i].Less(results[j]) }) + j := 0 + for i := 1; i < len(results); i++ { + if results[j] == results[i] { + continue + } + j++ + results[j] = results[i] + } + return results[:j+1] +} + +// findEdge is a convenience method that returns exactly one edge, and if no +// edges satisfy the given search criteria, then a default Result is returned. +// +// This is primarily to ease the usage of a number of the methods in the DistanceTargets +// and in EdgeQuery. +func (e *EdgeQuery) findEdge(target distanceTarget, opts *queryOptions) EdgeQueryResult { + opts.MaxResults(1) + e.findEdges(target, opts) + if len(e.results) > 0 { + return e.results[0] + } + + return newEdgeQueryResult(target) +} + +// findEdgesInternal does the actual work for find edges that match the given options. +func (e *EdgeQuery) findEdgesInternal(target distanceTarget, opts *queryOptions) { + e.target = target + e.opts = opts + + e.testedEdges = make(map[ShapeEdgeID]uint32) + e.distanceLimit = target.distance().fromChordAngle(opts.distanceLimit) + e.results = make([]EdgeQueryResult, 0) + + if e.distanceLimit == target.distance().zero() { + return + } + + if opts.includeInteriors { + shapeIDs := map[int32]struct{}{} + e.target.visitContainingShapes(e.index, func(containingShape Shape, targetPoint Point) bool { + shapeIDs[e.index.idForShape(containingShape)] = struct{}{} + return len(shapeIDs) < opts.maxResults + }) + for shapeID := range shapeIDs { + e.addResult(EdgeQueryResult{target.distance().zero(), shapeID, -1}) + } + + if e.distanceLimit == target.distance().zero() { + return + } + } + + // If maxError > 0 and the target takes advantage of this, then we may + // need to adjust the distance estimates to ShapeIndex cells to ensure + // that they are always a lower bound on the true distance. For example, + // suppose max_distance == 100, maxError == 30, and we compute the distance + // to the target from some cell C0 as d(C0) == 80. Then because the target + // takes advantage of maxError, the true distance could be as low as 50. + // In order not to miss edges contained by such cells, we need to subtract + // maxError from the distance estimates. This behavior is controlled by + // the useConservativeCellDistance flag. + // + // However there is one important case where this adjustment is not + // necessary, namely when distanceLimit < maxError, This is because + // maxError only affects the algorithm once at least maxEdges edges + // have been found that satisfy the given distance limit. At that point, + // maxError is subtracted from distanceLimit in order to ensure that + // any further matches are closer by at least that amount. But when + // distanceLimit < maxError, this reduces the distance limit to 0, + // i.e. all remaining candidate cells and edges can safely be discarded. + // (This is how IsDistanceLess() and friends are implemented.) + targetUsesMaxError := opts.maxError != target.distance().zero().chordAngle() && + e.target.setMaxError(opts.maxError) + + // Note that we can't compare maxError and distanceLimit directly + // because one is a Delta and one is a Distance. Instead we subtract them. + e.useConservativeCellDistance = targetUsesMaxError && + (e.distanceLimit == target.distance().infinity() || + target.distance().zero().less(e.distanceLimit.sub(target.distance().fromChordAngle(opts.maxError)))) + + // Use the brute force algorithm if the index is small enough. To avoid + // spending too much time counting edges when there are many shapes, we stop + // counting once there are too many edges. We may need to recount the edges + // if we later see a target with a larger brute force edge threshold. + minOptimizedEdges := e.target.maxBruteForceIndexSize() + 1 + if minOptimizedEdges > e.indexNumEdgesLimit && e.indexNumEdges >= e.indexNumEdgesLimit { + e.indexNumEdges = e.index.NumEdgesUpTo(minOptimizedEdges) + e.indexNumEdgesLimit = minOptimizedEdges + } + + if opts.useBruteForce || e.indexNumEdges < minOptimizedEdges { + // The brute force algorithm already considers each edge exactly once. + e.avoidDuplicates = false + e.findEdgesBruteForce() + } else { + // If the target takes advantage of maxError then we need to avoid + // duplicate edges explicitly. (Otherwise it happens automatically.) + e.avoidDuplicates = targetUsesMaxError && opts.maxResults > 1 + e.findEdgesOptimized() + } +} + +func (e *EdgeQuery) addResult(r EdgeQueryResult) { + e.results = append(e.results, r) + if e.opts.maxResults == 1 { + // Optimization for the common case where only the closest edge is wanted. + e.distanceLimit = r.distance.sub(e.target.distance().fromChordAngle(e.opts.maxError)) + } + // TODO(roberts): Add the other if/else cases when a different data structure + // is used for the results. +} + +func (e *EdgeQuery) maybeAddResult(shape Shape, edgeID int32) { + if _, ok := e.testedEdges[ShapeEdgeID{e.index.idForShape(shape), edgeID}]; e.avoidDuplicates && !ok { + return + } + edge := shape.Edge(int(edgeID)) + dist := e.distanceLimit + + if dist, ok := e.target.updateDistanceToEdge(edge, dist); ok { + e.addResult(EdgeQueryResult{dist, e.index.idForShape(shape), edgeID}) + } +} + +func (e *EdgeQuery) findEdgesBruteForce() { + // Range over all shapes in the index. Does order matter here? if so + // switch to for i = 0 .. n? + for _, shape := range e.index.shapes { + // TODO(roberts): can this happen if we are only ranging over current entries? + if shape == nil { + continue + } + for edgeID := int32(0); edgeID < int32(shape.NumEdges()); edgeID++ { + e.maybeAddResult(shape, edgeID) + } + } +} + +func (e *EdgeQuery) findEdgesOptimized() { + e.initQueue() + // Repeatedly find the closest Cell to "target" and either split it into + // its four children or process all of its edges. + for e.queue.size() > 0 { + // We need to copy the top entry before removing it, and we need to + // remove it before adding any new entries to the queue. + entry := e.queue.pop() + + if !entry.distance.less(e.distanceLimit) { + e.queue.reset() // Clear any remaining entries. + break + } + // If this is already known to be an index cell, just process it. + if entry.indexCell != nil { + e.processEdges(entry) + continue + } + // Otherwise split the cell into its four children. Before adding a + // child back to the queue, we first check whether it is empty. We do + // this in two seek operations rather than four by seeking to the key + // between children 0 and 1 and to the key between children 2 and 3. + id := entry.id + ch := id.Children() + e.iter.seek(ch[1].RangeMin()) + + if !e.iter.Done() && e.iter.CellID() <= ch[1].RangeMax() { + e.processOrEnqueueCell(ch[1]) + } + if e.iter.Prev() && e.iter.CellID() >= id.RangeMin() { + e.processOrEnqueueCell(ch[0]) + } + + e.iter.seek(ch[3].RangeMin()) + if !e.iter.Done() && e.iter.CellID() <= id.RangeMax() { + e.processOrEnqueueCell(ch[3]) + } + if e.iter.Prev() && e.iter.CellID() >= ch[2].RangeMin() { + e.processOrEnqueueCell(ch[2]) + } + } +} + +func (e *EdgeQuery) processOrEnqueueCell(id CellID) { + if e.iter.CellID() == id { + e.processOrEnqueue(id, e.iter.IndexCell()) + } else { + e.processOrEnqueue(id, nil) + } +} + +func (e *EdgeQuery) initQueue() { + if len(e.indexCovering) == 0 { + // We delay iterator initialization until now to make queries on very + // small indexes a bit faster (i.e., where brute force is used). + e.iter = NewShapeIndexIterator(e.index) + } + + // Optimization: if the user is searching for just the closest edge, and the + // center of the target's bounding cap happens to intersect an index cell, + // then we try to limit the search region to a small disc by first + // processing the edges in that cell. This sets distance_limit_ based on + // the closest edge in that cell, which we can then use to limit the search + // area. This means that the cell containing "target" will be processed + // twice, but in general this is still faster. + // + // TODO(roberts): Even if the cap center is not contained, we could still + // process one or both of the adjacent index cells in CellID order, + // provided that those cells are closer than distanceLimit. + cb := e.target.capBound() + if cb.IsEmpty() { + return // Empty target. + } + + if e.opts.maxResults == 1 && e.iter.LocatePoint(cb.Center()) { + e.processEdges(&queryQueueEntry{ + distance: e.target.distance().zero(), + id: e.iter.CellID(), + indexCell: e.iter.IndexCell(), + }) + // Skip the rest of the algorithm if we found an intersecting edge. + if e.distanceLimit == e.target.distance().zero() { + return + } + } + if len(e.indexCovering) == 0 { + e.initCovering() + } + if e.distanceLimit == e.target.distance().infinity() { + // Start with the precomputed index covering. + for i := range e.indexCovering { + e.processOrEnqueue(e.indexCovering[i], e.indexCells[i]) + } + } else { + // Compute a covering of the search disc and intersect it with the + // precomputed index covering. + coverer := &RegionCoverer{MaxCells: 4, LevelMod: 1, MaxLevel: maxLevel} + + radius := cb.Radius() + e.distanceLimit.chordAngleBound().Angle() + searchCB := CapFromCenterAngle(cb.Center(), radius) + maxDistCover := coverer.FastCovering(searchCB) + e.initialCells = CellUnionFromIntersection(e.indexCovering, maxDistCover) + + // Now we need to clean up the initial cells to ensure that they all + // contain at least one cell of the ShapeIndex. (Some may not intersect + // the index at all, while other may be descendants of an index cell.) + i, j := 0, 0 + for i < len(e.initialCells) { + idI := e.initialCells[i] + // Find the top-level cell that contains this initial cell. + for e.indexCovering[j].RangeMax() < idI { + j++ + } + + idJ := e.indexCovering[j] + if idI == idJ { + // This initial cell is one of the top-level cells. Use the + // precomputed ShapeIndexCell pointer to avoid an index seek. + e.processOrEnqueue(idJ, e.indexCells[j]) + i++ + j++ + } else { + // This initial cell is a proper descendant of a top-level cell. + // Check how it is related to the cells of the ShapeIndex. + r := e.iter.LocateCellID(idI) + if r == Indexed { + // This cell is a descendant of an index cell. + // Enqueue it and skip any other initial cells + // that are also descendants of this cell. + e.processOrEnqueue(e.iter.CellID(), e.iter.IndexCell()) + lastID := e.iter.CellID().RangeMax() + for i < len(e.initialCells) && e.initialCells[i] <= lastID { + i++ + } + } else { + // Enqueue the cell only if it contains at least one index cell. + if r == Subdivided { + e.processOrEnqueue(idI, nil) + } + i++ + } + } + } + } +} + +func (e *EdgeQuery) initCovering() { + // Find the range of Cells spanned by the index and choose a level such + // that the entire index can be covered with just a few cells. These are + // the "top-level" cells. There are two cases: + // + // - If the index spans more than one face, then there is one top-level cell + // per spanned face, just big enough to cover the index cells on that face. + // + // - If the index spans only one face, then we find the smallest cell "C" + // that covers the index cells on that face (just like the case above). + // Then for each of the 4 children of "C", if the child contains any index + // cells then we create a top-level cell that is big enough to just fit + // those index cells (i.e., shrinking the child as much as possible to fit + // its contents). This essentially replicates what would happen if we + // started with "C" as the top-level cell, since "C" would immediately be + // split, except that we take the time to prune the children further since + // this will save work on every subsequent query. + e.indexCovering = make([]CellID, 0, 6) + + // TODO(roberts): Use a single iterator below and save position + // information using pair {CellID, ShapeIndexCell}. + next := NewShapeIndexIterator(e.index, IteratorBegin) + last := NewShapeIndexIterator(e.index, IteratorEnd) + last.Prev() + if next.CellID() != last.CellID() { + // The index has at least two cells. Choose a level such that the entire + // index can be spanned with at most 6 cells (if the index spans multiple + // faces) or 4 cells (it the index spans a single face). + level, ok := next.CellID().CommonAncestorLevel(last.CellID()) + if !ok { + level = 0 + } else { + level++ + } + + // Visit each potential top-level cell except the last (handled below). + lastID := last.CellID().Parent(level) + for id := next.CellID().Parent(level); id != lastID; id = id.Next() { + // Skip any top-level cells that don't contain any index cells. + if id.RangeMax() < next.CellID() { + continue + } + + // Find the range of index cells contained by this top-level cell and + // then shrink the cell if necessary so that it just covers them. + cellFirst := next.clone() + next.seek(id.RangeMax().Next()) + cellLast := next.clone() + cellLast.Prev() + e.addInitialRange(cellFirst, cellLast) + break + } + + } + e.addInitialRange(next, last) +} + +// addInitialRange adds an entry to the indexCovering and indexCells that covers the given +// inclusive range of cells. +// +// This requires that first and last cells have a common ancestor. +func (e *EdgeQuery) addInitialRange(first, last *ShapeIndexIterator) { + if first.CellID() == last.CellID() { + // The range consists of a single index cell. + e.indexCovering = append(e.indexCovering, first.CellID()) + e.indexCells = append(e.indexCells, first.IndexCell()) + } else { + // Add the lowest common ancestor of the given range. + level, _ := first.CellID().CommonAncestorLevel(last.CellID()) + e.indexCovering = append(e.indexCovering, first.CellID().Parent(level)) + e.indexCells = append(e.indexCells, nil) + } +} + +// processEdges processes all the edges of the given index cell. +func (e *EdgeQuery) processEdges(entry *queryQueueEntry) { + for _, clipped := range entry.indexCell.shapes { + shape := e.index.Shape(clipped.shapeID) + for j := 0; j < clipped.numEdges(); j++ { + e.maybeAddResult(shape, int32(clipped.edges[j])) + } + } +} + +// processOrEnqueue the given cell id and indexCell. +func (e *EdgeQuery) processOrEnqueue(id CellID, indexCell *ShapeIndexCell) { + if indexCell != nil { + // If this index cell has only a few edges, then it is faster to check + // them directly rather than computing the minimum distance to the Cell + // and inserting it into the queue. + const minEdgesToEnqueue = 10 + numEdges := indexCell.numEdges() + if numEdges == 0 { + return + } + if numEdges < minEdgesToEnqueue { + // Set "distance" to zero to avoid the expense of computing it. + e.processEdges(&queryQueueEntry{ + distance: e.target.distance().zero(), + id: id, + indexCell: indexCell, + }) + return + } + } + + // Otherwise compute the minimum distance to any point in the cell and add + // it to the priority queue. + cell := CellFromCellID(id) + dist := e.distanceLimit + var ok bool + if dist, ok = e.target.updateDistanceToCell(cell, dist); !ok { + return + } + if e.useConservativeCellDistance { + // Ensure that "distance" is a lower bound on the true distance to the cell. + dist = dist.sub(e.target.distance().fromChordAngle(e.opts.maxError)) + } + + e.queue.push(&queryQueueEntry{ + distance: dist, + id: id, + indexCell: indexCell, + }) +} + +func (e *EdgeQuery) GetEdge(result EdgeQueryResult) Edge { + return e.index.Shape(result.shapeID).Edge(int(result.edgeID)) +} + +func (e *EdgeQuery) Project(point Point, result EdgeQueryResult) Point { + if result.edgeID < 0 { + return point + } + + edge := e.GetEdge(result) + return Project(point, edge.V0, edge.V1) +} + +// TODO(roberts): Remaining pieces +// GetEdge +// Project diff --git a/vendor/github.com/blevesearch/geo/s2/edge_tessellator.go b/vendor/github.com/blevesearch/geo/s2/edge_tessellator.go new file mode 100644 index 00000000..1d5805c2 --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/edge_tessellator.go @@ -0,0 +1,291 @@ +// Copyright 2018 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +import ( + "github.com/golang/geo/r2" + "github.com/golang/geo/s1" +) + +// Tessellation is implemented by subdividing the edge until the estimated +// maximum error is below the given tolerance. Estimating error is a hard +// problem, especially when the only methods available are point evaluation of +// the projection and its inverse. (These are the only methods that +// Projection provides, which makes it easier and less error-prone to +// implement new projections.) +// +// One technique that significantly increases robustness is to treat the +// geodesic and projected edges as parametric curves rather than geometric ones. +// Given a spherical edge AB and a projection p:S2->R2, let f(t) be the +// normalized arc length parametrization of AB and let g(t) be the normalized +// arc length parameterization of the projected edge p(A)p(B). (In other words, +// f(0)=A, f(1)=B, g(0)=p(A), g(1)=p(B).) We now define the geometric error as +// the maximum distance from the point p^-1(g(t)) to the geodesic edge AB for +// any t in [0,1], where p^-1 denotes the inverse projection. In other words, +// the geometric error is the maximum distance from any point on the projected +// edge (mapped back onto the sphere) to the geodesic edge AB. On the other +// hand we define the parametric error as the maximum distance between the +// points f(t) and p^-1(g(t)) for any t in [0,1], i.e. the maximum distance +// (measured on the sphere) between the geodesic and projected points at the +// same interpolation fraction t. +// +// The easiest way to estimate the parametric error is to simply evaluate both +// edges at their midpoints and measure the distance between them (the "midpoint +// method"). This is very fast and works quite well for most edges, however it +// has one major drawback: it doesn't handle points of inflection (i.e., points +// where the curvature changes sign). For example, edges in the Mercator and +// Plate Carree projections always curve towards the equator relative to the +// corresponding geodesic edge, so in these projections there is a point of +// inflection whenever the projected edge crosses the equator. The worst case +// occurs when the edge endpoints have different longitudes but the same +// absolute latitude, since in that case the error is non-zero but the edges +// have exactly the same midpoint (on the equator). +// +// One solution to this problem is to split the input edges at all inflection +// points (i.e., along the equator in the case of the Mercator and Plate Carree +// projections). However for general projections these inflection points can +// occur anywhere on the sphere (e.g., consider the Transverse Mercator +// projection). This could be addressed by adding methods to the S2Projection +// interface to split edges at inflection points but this would make it harder +// and more error-prone to implement new projections. +// +// Another problem with this approach is that the midpoint method sometimes +// underestimates the true error even when edges do not cross the equator. +// For the Plate Carree and Mercator projections, the midpoint method can +// underestimate the error by up to 3%. +// +// Both of these problems can be solved as follows. We assume that the error +// can be modeled as a convex combination of two worst-case functions, one +// where the error is maximized at the edge midpoint and another where the +// error is *minimized* (i.e., zero) at the edge midpoint. For example, we +// could choose these functions as: +// +// E1(x) = 1 - x^2 +// E2(x) = x * (1 - x^2) +// +// where for convenience we use an interpolation parameter "x" in the range +// [-1, 1] rather than the original "t" in the range [0, 1]. Note that both +// error functions must have roots at x = {-1, 1} since the error must be zero +// at the edge endpoints. E1 is simply a parabola whose maximum value is 1 +// attained at x = 0, while E2 is a cubic with an additional root at x = 0, +// and whose maximum value is 2 * sqrt(3) / 9 attained at x = 1 / sqrt(3). +// +// Next, it is convenient to scale these functions so that the both have a +// maximum value of 1. E1 already satisfies this requirement, and we simply +// redefine E2 as +// +// E2(x) = x * (1 - x^2) / (2 * sqrt(3) / 9) +// +// Now define x0 to be the point where these two functions intersect, i.e. the +// point in the range (-1, 1) where E1(x0) = E2(x0). This value has the very +// convenient property that if we evaluate the actual error E(x0), then the +// maximum error on the entire interval [-1, 1] is bounded by +// +// E(x) <= E(x0) / E1(x0) +// +// since whether the error is modeled using E1 or E2, the resulting function +// has the same maximum value (namely E(x0) / E1(x0)). If it is modeled as +// some other convex combination of E1 and E2, the maximum value can only +// decrease. +// +// Finally, since E2 is not symmetric about the y-axis, we must also allow for +// the possibility that the error is a convex combination of E1 and -E2. This +// can be handled by evaluating the error at E(-x0) as well, and then +// computing the final error bound as +// +// E(x) <= max(E(x0), E(-x0)) / E1(x0) . +// +// Effectively, this method is simply evaluating the error at two points about +// 1/3 and 2/3 of the way along the edges, and then scaling the maximum of +// these two errors by a constant factor. Intuitively, the reason this works +// is that if the two edges cross somewhere in the interior, then at least one +// of these points will be far from the crossing. +// +// The actual algorithm implemented below has some additional refinements. +// First, edges longer than 90 degrees are always subdivided; this avoids +// various unusual situations that can happen with very long edges, and there +// is really no reason to avoid adding vertices to edges that are so long. +// +// Second, the error function E1 above needs to be modified to take into +// account spherical distortions. (It turns out that spherical distortions are +// beneficial in the case of E2, i.e. they only make its error estimates +// slightly more conservative.) To do this, we model E1 as the maximum error +// in a Plate Carree edge of length 90 degrees or less. This turns out to be +// an edge from 45:-90 to 45:90 (in lat:lng format). The corresponding error +// as a function of "x" in the range [-1, 1] can be computed as the distance +// between the Plate Caree edge point (45, 90 * x) and the geodesic +// edge point (90 - 45 * abs(x), 90 * sgn(x)). Using the Haversine formula, +// the corresponding function E1 (normalized to have a maximum value of 1) is: +// +// E1(x) = +// asin(sqrt(sin(Pi / 8 * (1 - x)) ^ 2 + +// sin(Pi / 4 * (1 - x)) ^ 2 * cos(Pi / 4) * sin(Pi / 4 * x))) / +// asin(sqrt((1 - 1 / sqrt(2)) / 2)) +// +// Note that this function does not need to be evaluated at runtime, it +// simply affects the calculation of the value x0 where E1(x0) = E2(x0) +// and the corresponding scaling factor C = 1 / E1(x0). +// +// ------------------------------------------------------------------ +// +// In the case of the Mercator and Plate Carree projections this strategy +// produces a conservative upper bound (verified using 10 million random +// edges). Furthermore the bound is nearly tight; the scaling constant is +// C = 1.19289, whereas the maximum observed value was 1.19254. +// +// Compared to the simpler midpoint evaluation method, this strategy requires +// more function evaluations (currently twice as many, but with a smarter +// tessellation algorithm it will only be 50% more). It also results in a +// small amount of additional tessellation (about 1.5%) compared to the +// midpoint method, but this is due almost entirely to the fact that the +// midpoint method does not yield conservative error estimates. +// +// For random edges with a tolerance of 1 meter, the expected amount of +// overtessellation is as follows: +// +// Midpoint Method Cubic Method +// Plate Carree 1.8% 3.0% +// Mercator 15.8% 17.4% + +const ( + // tessellationInterpolationFraction is the fraction at which the two edges + // are evaluated in order to measure the error between them. (Edges are + // evaluated at two points measured this fraction from either end.) + tessellationInterpolationFraction = 0.31215691082248312 + tessellationScaleFactor = 0.83829992569888509 + + // minTessellationTolerance is the minimum supported tolerance (which + // corresponds to a distance less than 1 micrometer on the Earth's + // surface, but is still much larger than the expected projection and + // interpolation errors). + minTessellationTolerance s1.Angle = 1e-13 +) + +// EdgeTessellator converts an edge in a given projection (e.g., Mercator) into +// a chain of spherical geodesic edges such that the maximum distance between +// the original edge and the geodesic edge chain is at most the requested +// tolerance. Similarly, it can convert a spherical geodesic edge into a chain +// of edges in a given 2D projection such that the maximum distance between the +// geodesic edge and the chain of projected edges is at most the requested tolerance. +// +// Method | Input | Output +// ------------|------------------------|----------------------- +// Projected | S2 geodesics | Planar projected edges +// Unprojected | Planar projected edges | S2 geodesics +type EdgeTessellator struct { + projection Projection + + // The given tolerance scaled by a constant fraction so that it can be + // compared against the result returned by estimateMaxError. + scaledTolerance s1.ChordAngle +} + +// NewEdgeTessellator creates a new edge tessellator for the given projection and tolerance. +func NewEdgeTessellator(p Projection, tolerance s1.Angle) *EdgeTessellator { + return &EdgeTessellator{ + projection: p, + scaledTolerance: s1.ChordAngleFromAngle(maxAngle(tolerance, minTessellationTolerance)), + } +} + +// AppendProjected converts the spherical geodesic edge AB to a chain of planar edges +// in the given projection and returns the corresponding vertices. +// +// If the given projection has one or more coordinate axes that wrap, then +// every vertex's coordinates will be as close as possible to the previous +// vertex's coordinates. Note that this may yield vertices whose +// coordinates are outside the usual range. For example, tessellating the +// edge (0:170, 0:-170) (in lat:lng notation) yields (0:170, 0:190). +func (e *EdgeTessellator) AppendProjected(a, b Point, vertices []r2.Point) []r2.Point { + pa := e.projection.Project(a) + if len(vertices) == 0 { + vertices = []r2.Point{pa} + } else { + pa = e.projection.WrapDestination(vertices[len(vertices)-1], pa) + } + + pb := e.projection.Project(b) + return e.appendProjected(pa, a, pb, b, vertices) +} + +// appendProjected splits a geodesic edge AB as necessary and returns the +// projected vertices appended to the given vertices. +// +// The maximum recursion depth is (math.Pi / minTessellationTolerance) < 45 +func (e *EdgeTessellator) appendProjected(pa r2.Point, a Point, pbIn r2.Point, b Point, vertices []r2.Point) []r2.Point { + pb := e.projection.WrapDestination(pa, pbIn) + if e.estimateMaxError(pa, a, pb, b) <= e.scaledTolerance { + return append(vertices, pb) + } + + mid := Point{a.Add(b.Vector).Normalize()} + pmid := e.projection.WrapDestination(pa, e.projection.Project(mid)) + vertices = e.appendProjected(pa, a, pmid, mid, vertices) + return e.appendProjected(pmid, mid, pb, b, vertices) +} + +// AppendUnprojected converts the planar edge AB in the given projection to a chain of +// spherical geodesic edges and returns the vertices. +// +// Note that to construct a Loop, you must eliminate the duplicate first and last +// vertex. Note also that if the given projection involves coordinate wrapping +// (e.g. across the 180 degree meridian) then the first and last vertices may not +// be exactly the same. +func (e *EdgeTessellator) AppendUnprojected(pa, pb r2.Point, vertices []Point) []Point { + a := e.projection.Unproject(pa) + b := e.projection.Unproject(pb) + + if len(vertices) == 0 { + vertices = []Point{a} + } + + // Note that coordinate wrapping can create a small amount of error. For + // example in the edge chain "0:-175, 0:179, 0:-177", the first edge is + // transformed into "0:-175, 0:-181" while the second is transformed into + // "0:179, 0:183". The two coordinate pairs for the middle vertex + // ("0:-181" and "0:179") may not yield exactly the same S2Point. + return e.appendUnprojected(pa, a, pb, b, vertices) +} + +// appendUnprojected interpolates a projected edge and appends the corresponding +// points on the sphere. +func (e *EdgeTessellator) appendUnprojected(pa r2.Point, a Point, pbIn r2.Point, b Point, vertices []Point) []Point { + pb := e.projection.WrapDestination(pa, pbIn) + if e.estimateMaxError(pa, a, pb, b) <= e.scaledTolerance { + return append(vertices, b) + } + + pmid := e.projection.Interpolate(0.5, pa, pb) + mid := e.projection.Unproject(pmid) + + vertices = e.appendUnprojected(pa, a, pmid, mid, vertices) + return e.appendUnprojected(pmid, mid, pb, b, vertices) +} + +func (e *EdgeTessellator) estimateMaxError(pa r2.Point, a Point, pb r2.Point, b Point) s1.ChordAngle { + // See the algorithm description at the top of this file. + // We always tessellate edges longer than 90 degrees on the sphere, since the + // approximation below is not robust enough to handle such edges. + if a.Dot(b.Vector) < -1e-14 { + return s1.InfChordAngle() + } + t1 := tessellationInterpolationFraction + t2 := 1 - tessellationInterpolationFraction + mid1 := Interpolate(t1, a, b) + mid2 := Interpolate(t2, a, b) + pmid1 := e.projection.Unproject(e.projection.Interpolate(t1, pa, pb)) + pmid2 := e.projection.Unproject(e.projection.Interpolate(t2, pa, pb)) + return maxChordAngle(ChordAngleBetweenPoints(mid1, pmid1), ChordAngleBetweenPoints(mid2, pmid2)) +} diff --git a/vendor/github.com/blevesearch/geo/s2/encode.go b/vendor/github.com/blevesearch/geo/s2/encode.go new file mode 100644 index 00000000..66eaf98a --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/encode.go @@ -0,0 +1,238 @@ +// Copyright 2017 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +import ( + "encoding/binary" + "io" + "math" +) + +const ( + // encodingVersion is the current version of the encoding + // format that is compatible with C++ and other S2 libraries. + encodingVersion = int8(1) + + // encodingCompressedVersion is the current version of the + // compressed format. + encodingCompressedVersion = int8(4) +) + +// encoder handles the specifics of encoding for S2 types. +type encoder struct { + w io.Writer // the real writer passed to Encode + err error +} + +func (e *encoder) writeUvarint(x uint64) { + if e.err != nil { + return + } + var buf [binary.MaxVarintLen64]byte + n := binary.PutUvarint(buf[:], x) + _, e.err = e.w.Write(buf[:n]) +} + +func (e *encoder) writeBool(x bool) { + if e.err != nil { + return + } + var val int8 + if x { + val = 1 + } + e.err = binary.Write(e.w, binary.LittleEndian, val) +} + +func (e *encoder) writeInt8(x int8) { + if e.err != nil { + return + } + e.err = binary.Write(e.w, binary.LittleEndian, x) +} + +func (e *encoder) writeInt16(x int16) { + if e.err != nil { + return + } + e.err = binary.Write(e.w, binary.LittleEndian, x) +} + +func (e *encoder) writeInt32(x int32) { + if e.err != nil { + return + } + e.err = binary.Write(e.w, binary.LittleEndian, x) +} + +func (e *encoder) writeInt64(x int64) { + if e.err != nil { + return + } + e.err = binary.Write(e.w, binary.LittleEndian, x) +} + +func (e *encoder) writeUint8(x uint8) { + if e.err != nil { + return + } + _, e.err = e.w.Write([]byte{x}) +} + +func (e *encoder) writeUint32(x uint32) { + if e.err != nil { + return + } + e.err = binary.Write(e.w, binary.LittleEndian, x) +} + +func (e *encoder) writeUint64(x uint64) { + if e.err != nil { + return + } + e.err = binary.Write(e.w, binary.LittleEndian, x) +} + +func (e *encoder) writeFloat32(x float32) { + if e.err != nil { + return + } + e.err = binary.Write(e.w, binary.LittleEndian, x) +} + +func (e *encoder) writeFloat64(x float64) { + if e.err != nil { + return + } + e.err = binary.Write(e.w, binary.LittleEndian, x) +} + +type byteReader interface { + io.Reader + io.ByteReader +} + +// byteReaderAdapter embellishes an io.Reader with a ReadByte method, +// so that it implements the io.ByteReader interface. +type byteReaderAdapter struct { + io.Reader +} + +func (b byteReaderAdapter) ReadByte() (byte, error) { + buf := []byte{0} + _, err := io.ReadFull(b, buf) + return buf[0], err +} + +func asByteReader(r io.Reader) byteReader { + if br, ok := r.(byteReader); ok { + return br + } + return byteReaderAdapter{r} +} + +type decoder struct { + r byteReader // the real reader passed to Decode + err error + buf []byte +} + +// Get a buffer of size 8, to avoid allocating over and over. +func (d *decoder) buffer() []byte { + if d.buf == nil { + d.buf = make([]byte, 8) + } + return d.buf +} + +func (d *decoder) readBool() (x bool) { + if d.err != nil { + return + } + var val int8 + d.err = binary.Read(d.r, binary.LittleEndian, &val) + return val == 1 +} + +func (d *decoder) readInt8() (x int8) { + if d.err != nil { + return + } + d.err = binary.Read(d.r, binary.LittleEndian, &x) + return +} + +func (d *decoder) readInt64() (x int64) { + if d.err != nil { + return + } + d.err = binary.Read(d.r, binary.LittleEndian, &x) + return +} + +func (d *decoder) readUint8() (x uint8) { + if d.err != nil { + return + } + x, d.err = d.r.ReadByte() + return +} + +func (d *decoder) readUint32() (x uint32) { + if d.err != nil { + return + } + d.err = binary.Read(d.r, binary.LittleEndian, &x) + return +} + +func (d *decoder) readUint64() (x uint64) { + if d.err != nil { + return + } + d.err = binary.Read(d.r, binary.LittleEndian, &x) + return +} + +func (d *decoder) readFloat64() float64 { + if d.err != nil { + return 0 + } + buf := d.buffer() + _, d.err = io.ReadFull(d.r, buf) + return math.Float64frombits(binary.LittleEndian.Uint64(buf)) +} + +func (d *decoder) readUvarint() (x uint64) { + if d.err != nil { + return + } + x, d.err = binary.ReadUvarint(d.r) + return +} + +func (d *decoder) readFloat64Array(size int, buf []byte) int { + if d.err != nil || buf == nil { + return 0 + } + + if size >= len(buf) { + _, d.err = io.ReadFull(d.r, buf) + return len(buf) + } + + _, d.err = io.ReadFull(d.r, buf[0:size]) + return size +} diff --git a/vendor/github.com/blevesearch/geo/s2/interleave.go b/vendor/github.com/blevesearch/geo/s2/interleave.go new file mode 100644 index 00000000..6ac6ef58 --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/interleave.go @@ -0,0 +1,143 @@ +// Copyright 2017 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +/* +The lookup table below can convert a sequence of interleaved 8 bits into +non-interleaved 4 bits. The table can convert both odd and even bits at the +same time, and lut[x & 0x55] converts the even bits (bits 0, 2, 4 and 6), +while lut[x & 0xaa] converts the odd bits (bits 1, 3, 5 and 7). + +The lookup table below was generated using the following python code: + + def deinterleave(bits): + if bits == 0: return 0 + if bits < 4: return 1 + return deinterleave(bits / 4) * 2 + deinterleave(bits & 3) + + for i in range(256): print "0x%x," % deinterleave(i), +*/ +var deinterleaveLookup = [256]uint32{ + 0x0, 0x1, 0x1, 0x1, 0x2, 0x3, 0x3, 0x3, + 0x2, 0x3, 0x3, 0x3, 0x2, 0x3, 0x3, 0x3, + 0x4, 0x5, 0x5, 0x5, 0x6, 0x7, 0x7, 0x7, + 0x6, 0x7, 0x7, 0x7, 0x6, 0x7, 0x7, 0x7, + 0x4, 0x5, 0x5, 0x5, 0x6, 0x7, 0x7, 0x7, + 0x6, 0x7, 0x7, 0x7, 0x6, 0x7, 0x7, 0x7, + 0x4, 0x5, 0x5, 0x5, 0x6, 0x7, 0x7, 0x7, + 0x6, 0x7, 0x7, 0x7, 0x6, 0x7, 0x7, 0x7, + + 0x8, 0x9, 0x9, 0x9, 0xa, 0xb, 0xb, 0xb, + 0xa, 0xb, 0xb, 0xb, 0xa, 0xb, 0xb, 0xb, + 0xc, 0xd, 0xd, 0xd, 0xe, 0xf, 0xf, 0xf, + 0xe, 0xf, 0xf, 0xf, 0xe, 0xf, 0xf, 0xf, + 0xc, 0xd, 0xd, 0xd, 0xe, 0xf, 0xf, 0xf, + 0xe, 0xf, 0xf, 0xf, 0xe, 0xf, 0xf, 0xf, + 0xc, 0xd, 0xd, 0xd, 0xe, 0xf, 0xf, 0xf, + 0xe, 0xf, 0xf, 0xf, 0xe, 0xf, 0xf, 0xf, + + 0x8, 0x9, 0x9, 0x9, 0xa, 0xb, 0xb, 0xb, + 0xa, 0xb, 0xb, 0xb, 0xa, 0xb, 0xb, 0xb, + 0xc, 0xd, 0xd, 0xd, 0xe, 0xf, 0xf, 0xf, + 0xe, 0xf, 0xf, 0xf, 0xe, 0xf, 0xf, 0xf, + 0xc, 0xd, 0xd, 0xd, 0xe, 0xf, 0xf, 0xf, + 0xe, 0xf, 0xf, 0xf, 0xe, 0xf, 0xf, 0xf, + 0xc, 0xd, 0xd, 0xd, 0xe, 0xf, 0xf, 0xf, + 0xe, 0xf, 0xf, 0xf, 0xe, 0xf, 0xf, 0xf, + + 0x8, 0x9, 0x9, 0x9, 0xa, 0xb, 0xb, 0xb, + 0xa, 0xb, 0xb, 0xb, 0xa, 0xb, 0xb, 0xb, + 0xc, 0xd, 0xd, 0xd, 0xe, 0xf, 0xf, 0xf, + 0xe, 0xf, 0xf, 0xf, 0xe, 0xf, 0xf, 0xf, + 0xc, 0xd, 0xd, 0xd, 0xe, 0xf, 0xf, 0xf, + 0xe, 0xf, 0xf, 0xf, 0xe, 0xf, 0xf, 0xf, + 0xc, 0xd, 0xd, 0xd, 0xe, 0xf, 0xf, 0xf, + 0xe, 0xf, 0xf, 0xf, 0xe, 0xf, 0xf, 0xf, +} + +// deinterleaveUint32 decodes the interleaved values. +func deinterleaveUint32(code uint64) (uint32, uint32) { + x := (deinterleaveLookup[code&0x55]) | + (deinterleaveLookup[(code>>8)&0x55] << 4) | + (deinterleaveLookup[(code>>16)&0x55] << 8) | + (deinterleaveLookup[(code>>24)&0x55] << 12) | + (deinterleaveLookup[(code>>32)&0x55] << 16) | + (deinterleaveLookup[(code>>40)&0x55] << 20) | + (deinterleaveLookup[(code>>48)&0x55] << 24) | + (deinterleaveLookup[(code>>56)&0x55] << 28) + y := (deinterleaveLookup[code&0xaa]) | + (deinterleaveLookup[(code>>8)&0xaa] << 4) | + (deinterleaveLookup[(code>>16)&0xaa] << 8) | + (deinterleaveLookup[(code>>24)&0xaa] << 12) | + (deinterleaveLookup[(code>>32)&0xaa] << 16) | + (deinterleaveLookup[(code>>40)&0xaa] << 20) | + (deinterleaveLookup[(code>>48)&0xaa] << 24) | + (deinterleaveLookup[(code>>56)&0xaa] << 28) + return x, y +} + +var interleaveLookup = [256]uint64{ + 0x0000, 0x0001, 0x0004, 0x0005, 0x0010, 0x0011, 0x0014, 0x0015, + 0x0040, 0x0041, 0x0044, 0x0045, 0x0050, 0x0051, 0x0054, 0x0055, + 0x0100, 0x0101, 0x0104, 0x0105, 0x0110, 0x0111, 0x0114, 0x0115, + 0x0140, 0x0141, 0x0144, 0x0145, 0x0150, 0x0151, 0x0154, 0x0155, + 0x0400, 0x0401, 0x0404, 0x0405, 0x0410, 0x0411, 0x0414, 0x0415, + 0x0440, 0x0441, 0x0444, 0x0445, 0x0450, 0x0451, 0x0454, 0x0455, + 0x0500, 0x0501, 0x0504, 0x0505, 0x0510, 0x0511, 0x0514, 0x0515, + 0x0540, 0x0541, 0x0544, 0x0545, 0x0550, 0x0551, 0x0554, 0x0555, + + 0x1000, 0x1001, 0x1004, 0x1005, 0x1010, 0x1011, 0x1014, 0x1015, + 0x1040, 0x1041, 0x1044, 0x1045, 0x1050, 0x1051, 0x1054, 0x1055, + 0x1100, 0x1101, 0x1104, 0x1105, 0x1110, 0x1111, 0x1114, 0x1115, + 0x1140, 0x1141, 0x1144, 0x1145, 0x1150, 0x1151, 0x1154, 0x1155, + 0x1400, 0x1401, 0x1404, 0x1405, 0x1410, 0x1411, 0x1414, 0x1415, + 0x1440, 0x1441, 0x1444, 0x1445, 0x1450, 0x1451, 0x1454, 0x1455, + 0x1500, 0x1501, 0x1504, 0x1505, 0x1510, 0x1511, 0x1514, 0x1515, + 0x1540, 0x1541, 0x1544, 0x1545, 0x1550, 0x1551, 0x1554, 0x1555, + + 0x4000, 0x4001, 0x4004, 0x4005, 0x4010, 0x4011, 0x4014, 0x4015, + 0x4040, 0x4041, 0x4044, 0x4045, 0x4050, 0x4051, 0x4054, 0x4055, + 0x4100, 0x4101, 0x4104, 0x4105, 0x4110, 0x4111, 0x4114, 0x4115, + 0x4140, 0x4141, 0x4144, 0x4145, 0x4150, 0x4151, 0x4154, 0x4155, + 0x4400, 0x4401, 0x4404, 0x4405, 0x4410, 0x4411, 0x4414, 0x4415, + 0x4440, 0x4441, 0x4444, 0x4445, 0x4450, 0x4451, 0x4454, 0x4455, + 0x4500, 0x4501, 0x4504, 0x4505, 0x4510, 0x4511, 0x4514, 0x4515, + 0x4540, 0x4541, 0x4544, 0x4545, 0x4550, 0x4551, 0x4554, 0x4555, + + 0x5000, 0x5001, 0x5004, 0x5005, 0x5010, 0x5011, 0x5014, 0x5015, + 0x5040, 0x5041, 0x5044, 0x5045, 0x5050, 0x5051, 0x5054, 0x5055, + 0x5100, 0x5101, 0x5104, 0x5105, 0x5110, 0x5111, 0x5114, 0x5115, + 0x5140, 0x5141, 0x5144, 0x5145, 0x5150, 0x5151, 0x5154, 0x5155, + 0x5400, 0x5401, 0x5404, 0x5405, 0x5410, 0x5411, 0x5414, 0x5415, + 0x5440, 0x5441, 0x5444, 0x5445, 0x5450, 0x5451, 0x5454, 0x5455, + 0x5500, 0x5501, 0x5504, 0x5505, 0x5510, 0x5511, 0x5514, 0x5515, + 0x5540, 0x5541, 0x5544, 0x5545, 0x5550, 0x5551, 0x5554, 0x5555, +} + +// interleaveUint32 interleaves the given arguments into the return value. +// +// The 0-bit in val0 will be the 0-bit in the return value. +// The 0-bit in val1 will be the 1-bit in the return value. +// The 1-bit of val0 will be the 2-bit in the return value, and so on. +func interleaveUint32(x, y uint32) uint64 { + return (interleaveLookup[x&0xff]) | + (interleaveLookup[(x>>8)&0xff] << 16) | + (interleaveLookup[(x>>16)&0xff] << 32) | + (interleaveLookup[x>>24] << 48) | + (interleaveLookup[y&0xff] << 1) | + (interleaveLookup[(y>>8)&0xff] << 17) | + (interleaveLookup[(y>>16)&0xff] << 33) | + (interleaveLookup[y>>24] << 49) +} diff --git a/vendor/github.com/blevesearch/geo/s2/latlng.go b/vendor/github.com/blevesearch/geo/s2/latlng.go new file mode 100644 index 00000000..a750304a --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/latlng.go @@ -0,0 +1,101 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +import ( + "fmt" + "math" + + "github.com/golang/geo/r3" + "github.com/golang/geo/s1" +) + +const ( + northPoleLat = s1.Angle(math.Pi/2) * s1.Radian + southPoleLat = -northPoleLat +) + +// LatLng represents a point on the unit sphere as a pair of angles. +type LatLng struct { + Lat, Lng s1.Angle +} + +// LatLngFromDegrees returns a LatLng for the coordinates given in degrees. +func LatLngFromDegrees(lat, lng float64) LatLng { + return LatLng{s1.Angle(lat) * s1.Degree, s1.Angle(lng) * s1.Degree} +} + +// IsValid returns true iff the LatLng is normalized, with Lat ∈ [-π/2,π/2] and Lng ∈ [-π,π]. +func (ll LatLng) IsValid() bool { + return math.Abs(ll.Lat.Radians()) <= math.Pi/2 && math.Abs(ll.Lng.Radians()) <= math.Pi +} + +// Normalized returns the normalized version of the LatLng, +// with Lat clamped to [-π/2,π/2] and Lng wrapped in [-π,π]. +func (ll LatLng) Normalized() LatLng { + lat := ll.Lat + if lat > northPoleLat { + lat = northPoleLat + } else if lat < southPoleLat { + lat = southPoleLat + } + lng := s1.Angle(math.Remainder(ll.Lng.Radians(), 2*math.Pi)) * s1.Radian + return LatLng{lat, lng} +} + +func (ll LatLng) String() string { return fmt.Sprintf("[%v, %v]", ll.Lat, ll.Lng) } + +// Distance returns the angle between two LatLngs. +func (ll LatLng) Distance(ll2 LatLng) s1.Angle { + // Haversine formula, as used in C++ S2LatLng::GetDistance. + lat1, lat2 := ll.Lat.Radians(), ll2.Lat.Radians() + lng1, lng2 := ll.Lng.Radians(), ll2.Lng.Radians() + dlat := math.Sin(0.5 * (lat2 - lat1)) + dlng := math.Sin(0.5 * (lng2 - lng1)) + x := dlat*dlat + dlng*dlng*math.Cos(lat1)*math.Cos(lat2) + return s1.Angle(2*math.Atan2(math.Sqrt(x), math.Sqrt(math.Max(0, 1-x)))) * s1.Radian +} + +// NOTE(mikeperrow): The C++ implementation publicly exposes latitude/longitude +// functions. Let's see if that's really necessary before exposing the same functionality. + +func latitude(p Point) s1.Angle { + return s1.Angle(math.Atan2(p.Z, math.Sqrt(p.X*p.X+p.Y*p.Y))) * s1.Radian +} + +func longitude(p Point) s1.Angle { + return s1.Angle(math.Atan2(p.Y, p.X)) * s1.Radian +} + +// PointFromLatLng returns an Point for the given LatLng. +// The maximum error in the result is 1.5 * dblEpsilon. (This does not +// include the error of converting degrees, E5, E6, or E7 into radians.) +func PointFromLatLng(ll LatLng) Point { + phi := ll.Lat.Radians() + theta := ll.Lng.Radians() + cosphi := math.Cos(phi) + return Point{r3.Vector{math.Cos(theta) * cosphi, math.Sin(theta) * cosphi, math.Sin(phi)}} +} + +// LatLngFromPoint returns an LatLng for a given Point. +func LatLngFromPoint(p Point) LatLng { + return LatLng{latitude(p), longitude(p)} +} + +// ApproxEqual reports whether the latitude and longitude of the two LatLngs +// are the same up to a small tolerance. +func (ll LatLng) ApproxEqual(other LatLng) bool { + return ll.Lat.ApproxEqual(other.Lat) && ll.Lng.ApproxEqual(other.Lng) +} diff --git a/vendor/github.com/blevesearch/geo/s2/lexicon.go b/vendor/github.com/blevesearch/geo/s2/lexicon.go new file mode 100644 index 00000000..41cbffdc --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/lexicon.go @@ -0,0 +1,175 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +import ( + "encoding/binary" + "hash/adler32" + "math" + "sort" +) + +// TODO(roberts): If any of these are worth making public, change the +// method signatures and type names. + +// emptySetID represents the last ID that will ever be generated. +// (Non-negative IDs are reserved for singleton sets.) +var emptySetID = int32(math.MinInt32) + +// idSetLexicon compactly represents a set of non-negative +// integers such as array indices ("ID sets"). It is especially suitable when +// either (1) there are many duplicate sets, or (2) there are many singleton +// or empty sets. See also sequenceLexicon. +// +// Each distinct ID set is mapped to a 32-bit integer. Empty and singleton +// sets take up no additional space; the set itself is represented +// by the unique ID assigned to the set. Duplicate sets are automatically +// eliminated. Note also that ID sets are referred to using 32-bit integers +// rather than pointers. +type idSetLexicon struct { + idSets *sequenceLexicon +} + +func newIDSetLexicon() *idSetLexicon { + return &idSetLexicon{ + idSets: newSequenceLexicon(), + } +} + +// add adds the given set of integers to the lexicon if it is not already +// present, and return the unique ID for this set. The values are automatically +// sorted and duplicates are removed. +// +// The primary difference between this and sequenceLexicon are: +// 1. Empty and singleton sets are represented implicitly; they use no space. +// 2. Sets are represented rather than sequences; the ordering of values is +// not important and duplicates are removed. +// 3. The values must be 32-bit non-negative integers only. +func (l *idSetLexicon) add(ids ...int32) int32 { + // Empty sets have a special ID chosen not to conflict with other IDs. + if len(ids) == 0 { + return emptySetID + } + + // Singleton sets are represented by their element. + if len(ids) == 1 { + return ids[0] + } + + // Canonicalize the set by sorting and removing duplicates. + // + // Creates a new slice in order to not alter the supplied values. + set := uniqueInt32s(ids) + + // Non-singleton sets are represented by the bitwise complement of the ID + // returned by the sequenceLexicon + return ^l.idSets.add(set) +} + +// idSet returns the set of integers corresponding to an ID returned by add. +func (l *idSetLexicon) idSet(setID int32) []int32 { + if setID >= 0 { + return []int32{setID} + } + if setID == emptySetID { + return []int32{} + } + + return l.idSets.sequence(^setID) +} + +func (l *idSetLexicon) clear() { + l.idSets.clear() +} + +// sequenceLexicon compactly represents a sequence of values (e.g., tuples). +// It automatically eliminates duplicates slices, and maps the remaining +// sequences to sequentially increasing integer IDs. See also idSetLexicon. +// +// Each distinct sequence is mapped to a 32-bit integer. +type sequenceLexicon struct { + values []int32 + begins []uint32 + + // idSet is a mapping of a sequence hash to sequence index in the lexicon. + idSet map[uint32]int32 +} + +func newSequenceLexicon() *sequenceLexicon { + return &sequenceLexicon{ + begins: []uint32{0}, + idSet: make(map[uint32]int32), + } +} + +// clears all data from the lexicon. +func (l *sequenceLexicon) clear() { + l.values = nil + l.begins = []uint32{0} + l.idSet = make(map[uint32]int32) +} + +// add adds the given value to the lexicon if it is not already present, and +// returns its ID. IDs are assigned sequentially starting from zero. +func (l *sequenceLexicon) add(ids []int32) int32 { + if id, ok := l.idSet[hashSet(ids)]; ok { + return id + } + for _, v := range ids { + l.values = append(l.values, v) + } + l.begins = append(l.begins, uint32(len(l.values))) + + id := int32(len(l.begins)) - 2 + l.idSet[hashSet(ids)] = id + + return id +} + +// sequence returns the original sequence of values for the given ID. +func (l *sequenceLexicon) sequence(id int32) []int32 { + return l.values[l.begins[id]:l.begins[id+1]] +} + +// size reports the number of value sequences in the lexicon. +func (l *sequenceLexicon) size() int { + // Subtract one because the list of begins starts out with the first element set to 0. + return len(l.begins) - 1 +} + +// hash returns a hash of this sequence of int32s. +func hashSet(s []int32) uint32 { + // TODO(roberts): We just need a way to nicely hash all the values down to + // a 32-bit value. To ensure no unnecessary dependencies we use the core + // library types available to do this. Is there a better option? + a := adler32.New() + binary.Write(a, binary.LittleEndian, s) + return a.Sum32() +} + +// uniqueInt32s returns the sorted and uniqued set of int32s from the input. +func uniqueInt32s(in []int32) []int32 { + var vals []int32 + m := make(map[int32]bool) + for _, i := range in { + if m[i] { + continue + } + m[i] = true + vals = append(vals, i) + } + sort.Slice(vals, func(i, j int) bool { return vals[i] < vals[j] }) + return vals +} diff --git a/vendor/github.com/blevesearch/geo/s2/loop.go b/vendor/github.com/blevesearch/geo/s2/loop.go new file mode 100644 index 00000000..95cd17e4 --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/loop.go @@ -0,0 +1,1873 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +import ( + "encoding/binary" + "fmt" + "io" + "math" + "reflect" + + "github.com/golang/geo/r1" + "github.com/golang/geo/r3" + "github.com/golang/geo/s1" +) + +// Loop represents a simple spherical polygon. It consists of a sequence +// of vertices where the first vertex is implicitly connected to the +// last. All loops are defined to have a CCW orientation, i.e. the interior of +// the loop is on the left side of the edges. This implies that a clockwise +// loop enclosing a small area is interpreted to be a CCW loop enclosing a +// very large area. +// +// Loops are not allowed to have any duplicate vertices (whether adjacent or +// not). Non-adjacent edges are not allowed to intersect, and furthermore edges +// of length 180 degrees are not allowed (i.e., adjacent vertices cannot be +// antipodal). Loops must have at least 3 vertices (except for the "empty" and +// "full" loops discussed below). +// +// There are two special loops: the "empty" loop contains no points and the +// "full" loop contains all points. These loops do not have any edges, but to +// preserve the invariant that every loop can be represented as a vertex +// chain, they are defined as having exactly one vertex each (see EmptyLoop +// and FullLoop). +type Loop struct { + vertices []Point + + // originInside keeps a precomputed value whether this loop contains the origin + // versus computing from the set of vertices every time. + originInside bool + + // depth is the nesting depth of this Loop if it is contained by a Polygon + // or other shape and is used to determine if this loop represents a hole + // or a filled in portion. + depth int + + // bound is a conservative bound on all points contained by this loop. + // If l.ContainsPoint(P), then l.bound.ContainsPoint(P). + bound Rect + + // Since bound is not exact, it is possible that a loop A contains + // another loop B whose bounds are slightly larger. subregionBound + // has been expanded sufficiently to account for this error, i.e. + // if A.Contains(B), then A.subregionBound.Contains(B.bound). + subregionBound Rect + + // index is the spatial index for this Loop. + index *ShapeIndex + + // A buffer pool to be used while decoding the polygon + BufPool *GeoBufferPool +} + +// LoopFromPoints constructs a loop from the given points. +func LoopFromPoints(pts []Point) *Loop { + l := &Loop{ + vertices: pts, + index: NewShapeIndex(), + } + + l.initOriginAndBound() + return l +} + +// LoopFromCell constructs a loop corresponding to the given cell. +// +// Note that the loop and cell *do not* contain exactly the same set of +// points, because Loop and Cell have slightly different definitions of +// point containment. For example, a Cell vertex is contained by all +// four neighboring Cells, but it is contained by exactly one of four +// Loops constructed from those cells. As another example, the cell +// coverings of cell and LoopFromCell(cell) will be different, because the +// loop contains points on its boundary that actually belong to other cells +// (i.e., the covering will include a layer of neighboring cells). +func LoopFromCell(c Cell) *Loop { + l := &Loop{ + vertices: []Point{ + c.Vertex(0), + c.Vertex(1), + c.Vertex(2), + c.Vertex(3), + }, + index: NewShapeIndex(), + } + + l.initOriginAndBound() + return l +} + +// These two points are used for the special Empty and Full loops. +var ( + emptyLoopPoint = Point{r3.Vector{X: 0, Y: 0, Z: 1}} + fullLoopPoint = Point{r3.Vector{X: 0, Y: 0, Z: -1}} +) + +// EmptyLoop returns a special "empty" loop. +func EmptyLoop() *Loop { + return LoopFromPoints([]Point{emptyLoopPoint}) +} + +// FullLoop returns a special "full" loop. +func FullLoop() *Loop { + return LoopFromPoints([]Point{fullLoopPoint}) +} + +// initOriginAndBound sets the origin containment for the given point and then calls +// the initialization for the bounds objects and the internal index. +func (l *Loop) initOriginAndBound() { + if len(l.vertices) < 3 { + // Check for the special "empty" and "full" loops (which have one vertex). + if !l.isEmptyOrFull() { + l.originInside = false + return + } + + // This is the special empty or full loop, so the origin depends on if + // the vertex is in the southern hemisphere or not. + l.originInside = l.vertices[0].Z < 0 + } else { + // Point containment testing is done by counting edge crossings starting + // at a fixed point on the sphere (OriginPoint). We need to know whether + // the reference point (OriginPoint) is inside or outside the loop before + // we can construct the ShapeIndex. We do this by first guessing that + // it is outside, and then seeing whether we get the correct containment + // result for vertex 1. If the result is incorrect, the origin must be + // inside the loop. + // + // A loop with consecutive vertices A,B,C contains vertex B if and only if + // the fixed vector R = B.Ortho is contained by the wedge ABC. The + // wedge is closed at A and open at C, i.e. the point B is inside the loop + // if A = R but not if C = R. This convention is required for compatibility + // with VertexCrossing. (Note that we can't use OriginPoint + // as the fixed vector because of the possibility that B == OriginPoint.) + l.originInside = false + v1Inside := OrderedCCW(Point{l.vertices[1].Ortho()}, l.vertices[0], l.vertices[2], l.vertices[1]) + if v1Inside != l.ContainsPoint(l.vertices[1]) { + l.originInside = true + } + } + + // We *must* call initBound before initializing the index, because + // initBound calls ContainsPoint which does a bounds check before using + // the index. + l.initBound() + + // Create a new index and add us to it. + l.index = NewShapeIndex() + l.index.Add(l) +} + +// initBound sets up the approximate bounding Rects for this loop. +func (l *Loop) initBound() { + if len(l.vertices) == 0 { + *l = *EmptyLoop() + return + } + // Check for the special "empty" and "full" loops. + if l.isEmptyOrFull() { + if l.IsEmpty() { + l.bound = EmptyRect() + } else { + l.bound = FullRect() + } + l.subregionBound = l.bound + return + } + + // The bounding rectangle of a loop is not necessarily the same as the + // bounding rectangle of its vertices. First, the maximal latitude may be + // attained along the interior of an edge. Second, the loop may wrap + // entirely around the sphere (e.g. a loop that defines two revolutions of a + // candy-cane stripe). Third, the loop may include one or both poles. + // Note that a small clockwise loop near the equator contains both poles. + bounder := NewRectBounder() + for i := 0; i <= len(l.vertices); i++ { // add vertex 0 twice + bounder.AddPoint(l.Vertex(i)) + } + b := bounder.RectBound() + + if l.ContainsPoint(Point{r3.Vector{0, 0, 1}}) { + b = Rect{r1.Interval{b.Lat.Lo, math.Pi / 2}, s1.FullInterval()} + } + // If a loop contains the south pole, then either it wraps entirely + // around the sphere (full longitude range), or it also contains the + // north pole in which case b.Lng.IsFull() due to the test above. + // Either way, we only need to do the south pole containment test if + // b.Lng.IsFull(). + if b.Lng.IsFull() && l.ContainsPoint(Point{r3.Vector{0, 0, -1}}) { + b.Lat.Lo = -math.Pi / 2 + } + l.bound = b + l.subregionBound = ExpandForSubregions(l.bound) +} + +// Validate checks whether this is a valid loop. +func (l *Loop) Validate() error { + if err := l.findValidationErrorNoIndex(); err != nil { + return err + } + + // Check for intersections between non-adjacent edges (including at vertices) + // TODO(roberts): Once shapeutil gets findAnyCrossing uncomment this. + // return findAnyCrossing(l.index) + + return nil +} + +// findValidationErrorNoIndex reports whether this is not a valid loop, but +// skips checks that would require a ShapeIndex to be built for the loop. This +// is primarily used by Polygon to do validation so it doesn't trigger the +// creation of unneeded ShapeIndices. +func (l *Loop) findValidationErrorNoIndex() error { + // All vertices must be unit length. + for i, v := range l.vertices { + if !v.IsUnit() { + return fmt.Errorf("vertex %d is not unit length", i) + } + } + + // Loops must have at least 3 vertices (except for empty and full). + if len(l.vertices) < 3 { + if l.isEmptyOrFull() { + return nil // Skip remaining tests. + } + return fmt.Errorf("non-empty, non-full loops must have at least 3 vertices") + } + + // Loops are not allowed to have any duplicate vertices or edge crossings. + // We split this check into two parts. First we check that no edge is + // degenerate (identical endpoints). Then we check that there are no + // intersections between non-adjacent edges (including at vertices). The + // second check needs the ShapeIndex, so it does not fall within the scope + // of this method. + for i, v := range l.vertices { + if v == l.Vertex(i+1) { + return fmt.Errorf("edge %d is degenerate (duplicate vertex)", i) + } + + // Antipodal vertices are not allowed. + if other := (Point{l.Vertex(i + 1).Mul(-1)}); v == other { + return fmt.Errorf("vertices %d and %d are antipodal", i, + (i+1)%len(l.vertices)) + } + } + + return nil +} + +// Contains reports whether the region contained by this loop is a superset of the +// region contained by the given other loop. +func (l *Loop) Contains(o *Loop) bool { + // For a loop A to contain the loop B, all of the following must + // be true: + // + // (1) There are no edge crossings between A and B except at vertices. + // + // (2) At every vertex that is shared between A and B, the local edge + // ordering implies that A contains B. + // + // (3) If there are no shared vertices, then A must contain a vertex of B + // and B must not contain a vertex of A. (An arbitrary vertex may be + // chosen in each case.) + // + // The second part of (3) is necessary to detect the case of two loops whose + // union is the entire sphere, i.e. two loops that contains each other's + // boundaries but not each other's interiors. + if !l.subregionBound.Contains(o.bound) { + return false + } + + // Special cases to handle either loop being empty or full. + if l.isEmptyOrFull() || o.isEmptyOrFull() { + return l.IsFull() || o.IsEmpty() + } + + // Check whether there are any edge crossings, and also check the loop + // relationship at any shared vertices. + relation := &containsRelation{} + if hasCrossingRelation(l, o, relation) { + return false + } + + // There are no crossings, and if there are any shared vertices then A + // contains B locally at each shared vertex. + if relation.foundSharedVertex { + return true + } + + // Since there are no edge intersections or shared vertices, we just need to + // test condition (3) above. We can skip this test if we discovered that A + // contains at least one point of B while checking for edge crossings. + if !l.ContainsPoint(o.Vertex(0)) { + return false + } + + // We still need to check whether (A union B) is the entire sphere. + // Normally this check is very cheap due to the bounding box precondition. + if (o.subregionBound.Contains(l.bound) || o.bound.Union(l.bound).IsFull()) && + o.ContainsPoint(l.Vertex(0)) { + return false + } + return true +} + +// Intersects reports whether the region contained by this loop intersects the region +// contained by the other loop. +func (l *Loop) Intersects(o *Loop) bool { + // Given two loops, A and B, A.Intersects(B) if and only if !A.Complement().Contains(B). + // + // This code is similar to Contains, but is optimized for the case + // where both loops enclose less than half of the sphere. + if !l.bound.Intersects(o.bound) { + return false + } + + // Check whether there are any edge crossings, and also check the loop + // relationship at any shared vertices. + relation := &intersectsRelation{} + if hasCrossingRelation(l, o, relation) { + return true + } + if relation.foundSharedVertex { + return false + } + + // Since there are no edge intersections or shared vertices, the loops + // intersect only if A contains B, B contains A, or the two loops contain + // each other's boundaries. These checks are usually cheap because of the + // bounding box preconditions. Note that neither loop is empty (because of + // the bounding box check above), so it is safe to access vertex(0). + + // Check whether A contains B, or A and B contain each other's boundaries. + // (Note that A contains all the vertices of B in either case.) + if l.subregionBound.Contains(o.bound) || l.bound.Union(o.bound).IsFull() { + if l.ContainsPoint(o.Vertex(0)) { + return true + } + } + // Check whether B contains A. + if o.subregionBound.Contains(l.bound) { + if o.ContainsPoint(l.Vertex(0)) { + return true + } + } + return false +} + +// Equal reports whether two loops have the same vertices in the same linear order +// (i.e., cyclic rotations are not allowed). +func (l *Loop) Equal(other *Loop) bool { + if len(l.vertices) != len(other.vertices) { + return false + } + + for i, v := range l.vertices { + if v != other.Vertex(i) { + return false + } + } + return true +} + +// BoundaryEqual reports whether the two loops have the same boundary. This is +// true if and only if the loops have the same vertices in the same cyclic order +// (i.e., the vertices may be cyclically rotated). The empty and full loops are +// considered to have different boundaries. +func (l *Loop) BoundaryEqual(o *Loop) bool { + if len(l.vertices) != len(o.vertices) { + return false + } + + // Special case to handle empty or full loops. Since they have the same + // number of vertices, if one loop is empty/full then so is the other. + if l.isEmptyOrFull() { + return l.IsEmpty() == o.IsEmpty() + } + + // Loop through the vertices to find the first of ours that matches the + // starting vertex of the other loop. Use that offset to then 'align' the + // vertices for comparison. + for offset, vertex := range l.vertices { + if vertex == o.Vertex(0) { + // There is at most one starting offset since loop vertices are unique. + for i := 0; i < len(l.vertices); i++ { + if l.Vertex(i+offset) != o.Vertex(i) { + return false + } + } + return true + } + } + return false +} + +// compareBoundary returns +1 if this loop contains the boundary of the other loop, +// -1 if it excludes the boundary of the other, and 0 if the boundaries of the two +// loops cross. Shared edges are handled as follows: +// +// If XY is a shared edge, define Reversed(XY) to be true if XY +// appears in opposite directions in both loops. +// Then this loop contains XY if and only if Reversed(XY) == the other loop is a hole. +// (Intuitively, this checks whether this loop contains a vanishingly small region +// extending from the boundary of the other toward the interior of the polygon to +// which the other belongs.) +// +// This function is used for testing containment and intersection of +// multi-loop polygons. Note that this method is not symmetric, since the +// result depends on the direction of this loop but not on the direction of +// the other loop (in the absence of shared edges). +// +// This requires that neither loop is empty, and if other loop IsFull, then it must not +// be a hole. +func (l *Loop) compareBoundary(o *Loop) int { + // The bounds must intersect for containment or crossing. + if !l.bound.Intersects(o.bound) { + return -1 + } + + // Full loops are handled as though the loop surrounded the entire sphere. + if l.IsFull() { + return 1 + } + if o.IsFull() { + return -1 + } + + // Check whether there are any edge crossings, and also check the loop + // relationship at any shared vertices. + relation := newCompareBoundaryRelation(o.IsHole()) + if hasCrossingRelation(l, o, relation) { + return 0 + } + if relation.foundSharedVertex { + if relation.containsEdge { + return 1 + } + return -1 + } + + // There are no edge intersections or shared vertices, so we can check + // whether A contains an arbitrary vertex of B. + if l.ContainsPoint(o.Vertex(0)) { + return 1 + } + return -1 +} + +// ContainsOrigin reports true if this loop contains s2.OriginPoint(). +func (l *Loop) ContainsOrigin() bool { + return l.originInside +} + +// ReferencePoint returns the reference point for this loop. +func (l *Loop) ReferencePoint() ReferencePoint { + return OriginReferencePoint(l.originInside) +} + +// NumEdges returns the number of edges in this shape. +func (l *Loop) NumEdges() int { + if l.isEmptyOrFull() { + return 0 + } + return len(l.vertices) +} + +// Edge returns the endpoints for the given edge index. +func (l *Loop) Edge(i int) Edge { + return Edge{l.Vertex(i), l.Vertex(i + 1)} +} + +// NumChains reports the number of contiguous edge chains in the Loop. +func (l *Loop) NumChains() int { + if l.IsEmpty() { + return 0 + } + return 1 +} + +// Chain returns the i-th edge chain in the Shape. +func (l *Loop) Chain(chainID int) Chain { + return Chain{0, l.NumEdges()} +} + +// ChainEdge returns the j-th edge of the i-th edge chain. +func (l *Loop) ChainEdge(chainID, offset int) Edge { + return Edge{l.Vertex(offset), l.Vertex(offset + 1)} +} + +// ChainPosition returns a ChainPosition pair (i, j) such that edgeID is the +// j-th edge of the Loop. +func (l *Loop) ChainPosition(edgeID int) ChainPosition { + return ChainPosition{0, edgeID} +} + +// Dimension returns the dimension of the geometry represented by this Loop. +func (l *Loop) Dimension() int { return 2 } + +func (l *Loop) typeTag() typeTag { return typeTagNone } + +func (l *Loop) privateInterface() {} + +// IsEmpty reports true if this is the special empty loop that contains no points. +func (l *Loop) IsEmpty() bool { + return l.isEmptyOrFull() && !l.ContainsOrigin() +} + +// IsFull reports true if this is the special full loop that contains all points. +func (l *Loop) IsFull() bool { + return l.isEmptyOrFull() && l.ContainsOrigin() +} + +// isEmptyOrFull reports true if this loop is either the "empty" or "full" special loops. +func (l *Loop) isEmptyOrFull() bool { + return len(l.vertices) == 1 +} + +// Vertices returns the vertices in the loop. +func (l *Loop) Vertices() []Point { + return l.vertices +} + +// RectBound returns a tight bounding rectangle. If the loop contains the point, +// the bound also contains it. +func (l *Loop) RectBound() Rect { + return l.bound +} + +// CapBound returns a bounding cap that may have more padding than the corresponding +// RectBound. The bound is conservative such that if the loop contains a point P, +// the bound also contains it. +func (l *Loop) CapBound() Cap { + return l.bound.CapBound() +} + +// Vertex returns the vertex for the given index. For convenience, the vertex indices +// wrap automatically for methods that do index math such as Edge. +// i.e., Vertex(NumEdges() + n) is the same as Vertex(n). +func (l *Loop) Vertex(i int) Point { + return l.vertices[i%len(l.vertices)] +} + +// OrientedVertex returns the vertex in reverse order if the loop represents a polygon +// hole. For example, arguments 0, 1, 2 are mapped to vertices n-1, n-2, n-3, where +// n == len(vertices). This ensures that the interior of the polygon is always to +// the left of the vertex chain. +// +// This requires: 0 <= i < 2 * len(vertices) +func (l *Loop) OrientedVertex(i int) Point { + j := i - len(l.vertices) + if j < 0 { + j = i + } + if l.IsHole() { + j = len(l.vertices) - 1 - j + } + return l.Vertex(j) +} + +// NumVertices returns the number of vertices in this loop. +func (l *Loop) NumVertices() int { + return len(l.vertices) +} + +// bruteForceContainsPoint reports if the given point is contained by this loop. +// This method does not use the ShapeIndex, so it is only preferable below a certain +// size of loop. +func (l *Loop) bruteForceContainsPoint(p Point) bool { + origin := OriginPoint() + inside := l.originInside + crosser := NewChainEdgeCrosser(origin, p, l.Vertex(0)) + for i := 1; i <= len(l.vertices); i++ { // add vertex 0 twice + inside = inside != crosser.EdgeOrVertexChainCrossing(l.Vertex(i)) + } + return inside +} + +// ContainsPoint returns true if the loop contains the point. +func (l *Loop) ContainsPoint(p Point) bool { + if !l.index.IsFresh() && !l.bound.ContainsPoint(p) { + return false + } + + // For small loops it is faster to just check all the crossings. We also + // use this method during loop initialization because InitOriginAndBound() + // calls Contains() before InitIndex(). Otherwise, we keep track of the + // number of calls to Contains() and only build the index when enough calls + // have been made so that we think it is worth the effort. Note that the + // code below is structured so that if many calls are made in parallel only + // one thread builds the index, while the rest continue using brute force + // until the index is actually available. + + const maxBruteForceVertices = 32 + // TODO(roberts): add unindexed contains calls tracking + + if len(l.index.shapes) == 0 || // Index has not been initialized yet. + len(l.vertices) <= maxBruteForceVertices { + return l.bruteForceContainsPoint(p) + } + + // Otherwise, look up the point in the index. + it := l.index.Iterator() + if !it.LocatePoint(p) { + return false + } + return l.iteratorContainsPoint(it, p) +} + +// ContainsCell reports whether the given Cell is contained by this Loop. +func (l *Loop) ContainsCell(target Cell) bool { + it := l.index.Iterator() + relation := it.LocateCellID(target.ID()) + + // If "target" is disjoint from all index cells, it is not contained. + // Similarly, if "target" is subdivided into one or more index cells then it + // is not contained, since index cells are subdivided only if they (nearly) + // intersect a sufficient number of edges. (But note that if "target" itself + // is an index cell then it may be contained, since it could be a cell with + // no edges in the loop interior.) + if relation != Indexed { + return false + } + + // Otherwise check if any edges intersect "target". + if l.boundaryApproxIntersects(it, target) { + return false + } + + // Otherwise check if the loop contains the center of "target". + return l.iteratorContainsPoint(it, target.Center()) +} + +// IntersectsCell reports whether this Loop intersects the given cell. +func (l *Loop) IntersectsCell(target Cell) bool { + it := l.index.Iterator() + relation := it.LocateCellID(target.ID()) + + // If target does not overlap any index cell, there is no intersection. + if relation == Disjoint { + return false + } + // If target is subdivided into one or more index cells, there is an + // intersection to within the ShapeIndex error bound (see Contains). + if relation == Subdivided { + return true + } + // If target is an index cell, there is an intersection because index cells + // are created only if they have at least one edge or they are entirely + // contained by the loop. + if it.CellID() == target.id { + return true + } + // Otherwise check if any edges intersect target. + if l.boundaryApproxIntersects(it, target) { + return true + } + // Otherwise check if the loop contains the center of target. + return l.iteratorContainsPoint(it, target.Center()) +} + +// CellUnionBound computes a covering of the Loop. +func (l *Loop) CellUnionBound() []CellID { + return l.CapBound().CellUnionBound() +} + +// boundaryApproxIntersects reports if the loop's boundary intersects target. +// It may also return true when the loop boundary does not intersect target but +// some edge comes within the worst-case error tolerance. +// +// This requires that it.Locate(target) returned Indexed. +func (l *Loop) boundaryApproxIntersects(it *ShapeIndexIterator, target Cell) bool { + aClipped := it.IndexCell().findByShapeID(0) + + // If there are no edges, there is no intersection. + if len(aClipped.edges) == 0 { + return false + } + + // We can save some work if target is the index cell itself. + if it.CellID() == target.ID() { + return true + } + + // Otherwise check whether any of the edges intersect target. + maxError := (faceClipErrorUVCoord + intersectsRectErrorUVDist) + bound := target.BoundUV().ExpandedByMargin(maxError) + for _, ai := range aClipped.edges { + v0, v1, ok := ClipToPaddedFace(l.Vertex(ai), l.Vertex(ai+1), target.Face(), maxError) + if ok && edgeIntersectsRect(v0, v1, bound) { + return true + } + } + return false +} + +// iteratorContainsPoint reports if the iterator that is positioned at the ShapeIndexCell +// that may contain p, contains the point p. +func (l *Loop) iteratorContainsPoint(it *ShapeIndexIterator, p Point) bool { + // Test containment by drawing a line segment from the cell center to the + // given point and counting edge crossings. + aClipped := it.IndexCell().findByShapeID(0) + inside := aClipped.containsCenter + if len(aClipped.edges) > 0 { + center := it.Center() + crosser := NewEdgeCrosser(center, p) + aiPrev := -2 + for _, ai := range aClipped.edges { + if ai != aiPrev+1 { + crosser.RestartAt(l.Vertex(ai)) + } + aiPrev = ai + inside = inside != crosser.EdgeOrVertexChainCrossing(l.Vertex(ai+1)) + } + } + return inside +} + +// RegularLoop creates a loop with the given number of vertices, all +// located on a circle of the specified radius around the given center. +func RegularLoop(center Point, radius s1.Angle, numVertices int) *Loop { + return RegularLoopForFrame(getFrame(center), radius, numVertices) +} + +// RegularLoopForFrame creates a loop centered around the z-axis of the given +// coordinate frame, with the first vertex in the direction of the positive x-axis. +func RegularLoopForFrame(frame matrix3x3, radius s1.Angle, numVertices int) *Loop { + return LoopFromPoints(regularPointsForFrame(frame, radius, numVertices)) +} + +// CanonicalFirstVertex returns a first index and a direction (either +1 or -1) +// such that the vertex sequence (first, first+dir, ..., first+(n-1)*dir) does +// not change when the loop vertex order is rotated or inverted. This allows the +// loop vertices to be traversed in a canonical order. The return values are +// chosen such that (first, ..., first+n*dir) are in the range [0, 2*n-1] as +// expected by the Vertex method. +func (l *Loop) CanonicalFirstVertex() (firstIdx, direction int) { + firstIdx = 0 + n := len(l.vertices) + for i := 1; i < n; i++ { + if l.Vertex(i).Cmp(l.Vertex(firstIdx).Vector) == -1 { + firstIdx = i + } + } + + // 0 <= firstIdx <= n-1, so (firstIdx+n*dir) <= 2*n-1. + if l.Vertex(firstIdx+1).Cmp(l.Vertex(firstIdx+n-1).Vector) == -1 { + return firstIdx, 1 + } + + // n <= firstIdx <= 2*n-1, so (firstIdx+n*dir) >= 0. + firstIdx += n + return firstIdx, -1 +} + +// TurningAngle returns the sum of the turning angles at each vertex. The return +// value is positive if the loop is counter-clockwise, negative if the loop is +// clockwise, and zero if the loop is a great circle. Degenerate and +// nearly-degenerate loops are handled consistently with Sign. So for example, +// if a loop has zero area (i.e., it is a very small CCW loop) then the turning +// angle will always be negative. +// +// This quantity is also called the "geodesic curvature" of the loop. +func (l *Loop) TurningAngle() float64 { + // For empty and full loops, we return the limit value as the loop area + // approaches 0 or 4*Pi respectively. + if l.isEmptyOrFull() { + if l.ContainsOrigin() { + return -2 * math.Pi + } + return 2 * math.Pi + } + + // Don't crash even if the loop is not well-defined. + if len(l.vertices) < 3 { + return 0 + } + + // To ensure that we get the same result when the vertex order is rotated, + // and that the result is negated when the vertex order is reversed, we need + // to add up the individual turn angles in a consistent order. (In general, + // adding up a set of numbers in a different order can change the sum due to + // rounding errors.) + // + // Furthermore, if we just accumulate an ordinary sum then the worst-case + // error is quadratic in the number of vertices. (This can happen with + // spiral shapes, where the partial sum of the turning angles can be linear + // in the number of vertices.) To avoid this we use the Kahan summation + // algorithm (http://en.wikipedia.org/wiki/Kahan_summation_algorithm). + n := len(l.vertices) + i, dir := l.CanonicalFirstVertex() + sum := TurnAngle(l.Vertex((i+n-dir)%n), l.Vertex(i), l.Vertex((i+dir)%n)) + + compensation := s1.Angle(0) + for n-1 > 0 { + i += dir + angle := TurnAngle(l.Vertex(i-dir), l.Vertex(i), l.Vertex(i+dir)) + oldSum := sum + angle += compensation + sum += angle + compensation = (oldSum - sum) + angle + n-- + } + + const maxCurvature = 2*math.Pi - 4*dblEpsilon + + return math.Max(-maxCurvature, math.Min(maxCurvature, float64(dir)*float64(sum+compensation))) +} + +// turningAngleMaxError return the maximum error in TurningAngle. The value is not +// constant; it depends on the loop. +func (l *Loop) turningAngleMaxError() float64 { + // The maximum error can be bounded as follows: + // 3.00 * dblEpsilon for RobustCrossProd(b, a) + // 3.00 * dblEpsilon for RobustCrossProd(c, b) + // 3.25 * dblEpsilon for Angle() + // 2.00 * dblEpsilon for each addition in the Kahan summation + // ------------------ + // 11.25 * dblEpsilon + maxErrorPerVertex := 11.25 * dblEpsilon + return maxErrorPerVertex * float64(len(l.vertices)) +} + +// IsHole reports whether this loop represents a hole in its containing polygon. +func (l *Loop) IsHole() bool { return l.depth&1 != 0 } + +// Sign returns -1 if this Loop represents a hole in its containing polygon, and +1 otherwise. +func (l *Loop) Sign() int { + if l.IsHole() { + return -1 + } + return 1 +} + +// IsNormalized reports whether the loop area is at most 2*pi. Degenerate loops are +// handled consistently with Sign, i.e., if a loop can be +// expressed as the union of degenerate or nearly-degenerate CCW triangles, +// then it will always be considered normalized. +func (l *Loop) IsNormalized() bool { + // Optimization: if the longitude span is less than 180 degrees, then the + // loop covers less than half the sphere and is therefore normalized. + if l.bound.Lng.Length() < math.Pi { + return true + } + + // We allow some error so that hemispheres are always considered normalized. + // TODO(roberts): This is no longer required by the Polygon implementation, + // so alternatively we could create the invariant that a loop is normalized + // if and only if its complement is not normalized. + return l.TurningAngle() >= -l.turningAngleMaxError() +} + +// Normalize inverts the loop if necessary so that the area enclosed by the loop +// is at most 2*pi. +func (l *Loop) Normalize() { + if !l.IsNormalized() { + l.Invert() + } +} + +// Invert reverses the order of the loop vertices, effectively complementing the +// region represented by the loop. For example, the loop ABCD (with edges +// AB, BC, CD, DA) becomes the loop DCBA (with edges DC, CB, BA, AD). +// Notice that the last edge is the same in both cases except that its +// direction has been reversed. +func (l *Loop) Invert() { + l.index.Reset() + if l.isEmptyOrFull() { + if l.IsFull() { + l.vertices[0] = emptyLoopPoint + } else { + l.vertices[0] = fullLoopPoint + } + } else { + // For non-special loops, reverse the slice of vertices. + for i := len(l.vertices)/2 - 1; i >= 0; i-- { + opp := len(l.vertices) - 1 - i + l.vertices[i], l.vertices[opp] = l.vertices[opp], l.vertices[i] + } + } + + // originInside must be set correctly before building the ShapeIndex. + l.originInside = !l.originInside + if l.bound.Lat.Lo > -math.Pi/2 && l.bound.Lat.Hi < math.Pi/2 { + // The complement of this loop contains both poles. + l.bound = FullRect() + l.subregionBound = l.bound + } else { + l.initBound() + } + l.index.Add(l) +} + +// findVertex returns the index of the vertex at the given Point in the range +// 1..numVertices, and a boolean indicating if a vertex was found. +func (l *Loop) findVertex(p Point) (index int, ok bool) { + const notFound = 0 + if len(l.vertices) < 10 { + // Exhaustive search for loops below a small threshold. + for i := 1; i <= len(l.vertices); i++ { + if l.Vertex(i) == p { + return i, true + } + } + return notFound, false + } + + it := l.index.Iterator() + if !it.LocatePoint(p) { + return notFound, false + } + + aClipped := it.IndexCell().findByShapeID(0) + for i := aClipped.numEdges() - 1; i >= 0; i-- { + ai := aClipped.edges[i] + if l.Vertex(ai) == p { + if ai == 0 { + return len(l.vertices), true + } + return ai, true + } + + if l.Vertex(ai+1) == p { + return ai + 1, true + } + } + return notFound, false +} + +// ContainsNested reports whether the given loops is contained within this loop. +// This function does not test for edge intersections. The two loops must meet +// all of the Polygon requirements; for example this implies that their +// boundaries may not cross or have any shared edges (although they may have +// shared vertices). +func (l *Loop) ContainsNested(other *Loop) bool { + if !l.subregionBound.Contains(other.bound) { + return false + } + + // Special cases to handle either loop being empty or full. Also bail out + // when B has no vertices to avoid heap overflow on the vertex(1) call + // below. (This method is called during polygon initialization before the + // client has an opportunity to call IsValid().) + if l.isEmptyOrFull() || other.NumVertices() < 2 { + return l.IsFull() || other.IsEmpty() + } + + // We are given that A and B do not share any edges, and that either one + // loop contains the other or they do not intersect. + m, ok := l.findVertex(other.Vertex(1)) + if !ok { + // Since other.vertex(1) is not shared, we can check whether A contains it. + return l.ContainsPoint(other.Vertex(1)) + } + + // Check whether the edge order around other.Vertex(1) is compatible with + // A containing B. + return WedgeContains(l.Vertex(m-1), l.Vertex(m), l.Vertex(m+1), other.Vertex(0), other.Vertex(2)) +} + +// surfaceIntegralFloat64 computes the oriented surface integral of some quantity f(x) +// over the loop interior, given a function f(A,B,C) that returns the +// corresponding integral over the spherical triangle ABC. Here "oriented +// surface integral" means: +// +// (1) f(A,B,C) must be the integral of f if ABC is counterclockwise, +// and the integral of -f if ABC is clockwise. +// +// (2) The result of this function is *either* the integral of f over the +// loop interior, or the integral of (-f) over the loop exterior. +// +// Note that there are at least two common situations where it easy to work +// around property (2) above: +// +// - If the integral of f over the entire sphere is zero, then it doesn't +// matter which case is returned because they are always equal. +// +// - If f is non-negative, then it is easy to detect when the integral over +// the loop exterior has been returned, and the integral over the loop +// interior can be obtained by adding the integral of f over the entire +// unit sphere (a constant) to the result. +// +// Any changes to this method may need corresponding changes to surfaceIntegralPoint as well. +func (l *Loop) surfaceIntegralFloat64(f func(a, b, c Point) float64) float64 { + // We sum f over a collection T of oriented triangles, possibly + // overlapping. Let the sign of a triangle be +1 if it is CCW and -1 + // otherwise, and let the sign of a point x be the sum of the signs of the + // triangles containing x. Then the collection of triangles T is chosen + // such that either: + // + // (1) Each point in the loop interior has sign +1, and sign 0 otherwise; or + // (2) Each point in the loop exterior has sign -1, and sign 0 otherwise. + // + // The triangles basically consist of a fan from vertex 0 to every loop + // edge that does not include vertex 0. These triangles will always satisfy + // either (1) or (2). However, what makes this a bit tricky is that + // spherical edges become numerically unstable as their length approaches + // 180 degrees. Of course there is not much we can do if the loop itself + // contains such edges, but we would like to make sure that all the triangle + // edges under our control (i.e., the non-loop edges) are stable. For + // example, consider a loop around the equator consisting of four equally + // spaced points. This is a well-defined loop, but we cannot just split it + // into two triangles by connecting vertex 0 to vertex 2. + // + // We handle this type of situation by moving the origin of the triangle fan + // whenever we are about to create an unstable edge. We choose a new + // location for the origin such that all relevant edges are stable. We also + // create extra triangles with the appropriate orientation so that the sum + // of the triangle signs is still correct at every point. + + // The maximum length of an edge for it to be considered numerically stable. + // The exact value is fairly arbitrary since it depends on the stability of + // the function f. The value below is quite conservative but could be + // reduced further if desired. + const maxLength = math.Pi - 1e-5 + + var sum float64 + origin := l.Vertex(0) + for i := 1; i+1 < len(l.vertices); i++ { + // Let V_i be vertex(i), let O be the current origin, and let length(A,B) + // be the length of edge (A,B). At the start of each loop iteration, the + // "leading edge" of the triangle fan is (O,V_i), and we want to extend + // the triangle fan so that the leading edge is (O,V_i+1). + // + // Invariants: + // 1. length(O,V_i) < maxLength for all (i > 1). + // 2. Either O == V_0, or O is approximately perpendicular to V_0. + // 3. "sum" is the oriented integral of f over the area defined by + // (O, V_0, V_1, ..., V_i). + if l.Vertex(i+1).Angle(origin.Vector) > maxLength { + // We are about to create an unstable edge, so choose a new origin O' + // for the triangle fan. + oldOrigin := origin + if origin == l.Vertex(0) { + // The following point is well-separated from V_i and V_0 (and + // therefore V_i+1 as well). + origin = Point{l.Vertex(0).PointCross(l.Vertex(i)).Normalize()} + } else if l.Vertex(i).Angle(l.Vertex(0).Vector) < maxLength { + // All edges of the triangle (O, V_0, V_i) are stable, so we can + // revert to using V_0 as the origin. + origin = l.Vertex(0) + } else { + // (O, V_i+1) and (V_0, V_i) are antipodal pairs, and O and V_0 are + // perpendicular. Therefore V_0.CrossProd(O) is approximately + // perpendicular to all of {O, V_0, V_i, V_i+1}, and we can choose + // this point O' as the new origin. + origin = Point{l.Vertex(0).Cross(oldOrigin.Vector)} + + // Advance the edge (V_0,O) to (V_0,O'). + sum += f(l.Vertex(0), oldOrigin, origin) + } + // Advance the edge (O,V_i) to (O',V_i). + sum += f(oldOrigin, l.Vertex(i), origin) + } + // Advance the edge (O,V_i) to (O,V_i+1). + sum += f(origin, l.Vertex(i), l.Vertex(i+1)) + } + // If the origin is not V_0, we need to sum one more triangle. + if origin != l.Vertex(0) { + // Advance the edge (O,V_n-1) to (O,V_0). + sum += f(origin, l.Vertex(len(l.vertices)-1), l.Vertex(0)) + } + return sum +} + +// surfaceIntegralPoint mirrors the surfaceIntegralFloat64 method but over Points; +// see that method for commentary. The C++ version uses a templated method. +// Any changes to this method may need corresponding changes to surfaceIntegralFloat64 as well. +func (l *Loop) surfaceIntegralPoint(f func(a, b, c Point) Point) Point { + const maxLength = math.Pi - 1e-5 + var sum r3.Vector + + origin := l.Vertex(0) + for i := 1; i+1 < len(l.vertices); i++ { + if l.Vertex(i+1).Angle(origin.Vector) > maxLength { + oldOrigin := origin + if origin == l.Vertex(0) { + origin = Point{l.Vertex(0).PointCross(l.Vertex(i)).Normalize()} + } else if l.Vertex(i).Angle(l.Vertex(0).Vector) < maxLength { + origin = l.Vertex(0) + } else { + origin = Point{l.Vertex(0).Cross(oldOrigin.Vector)} + sum = sum.Add(f(l.Vertex(0), oldOrigin, origin).Vector) + } + sum = sum.Add(f(oldOrigin, l.Vertex(i), origin).Vector) + } + sum = sum.Add(f(origin, l.Vertex(i), l.Vertex(i+1)).Vector) + } + if origin != l.Vertex(0) { + sum = sum.Add(f(origin, l.Vertex(len(l.vertices)-1), l.Vertex(0)).Vector) + } + return Point{sum} +} + +// Area returns the area of the loop interior, i.e. the region on the left side of +// the loop. The return value is between 0 and 4*pi. (Note that the return +// value is not affected by whether this loop is a "hole" or a "shell".) +func (l *Loop) Area() float64 { + // It is surprisingly difficult to compute the area of a loop robustly. The + // main issues are (1) whether degenerate loops are considered to be CCW or + // not (i.e., whether their area is close to 0 or 4*pi), and (2) computing + // the areas of small loops with good relative accuracy. + // + // With respect to degeneracies, we would like Area to be consistent + // with ContainsPoint in that loops that contain many points + // should have large areas, and loops that contain few points should have + // small areas. For example, if a degenerate triangle is considered CCW + // according to s2predicates Sign, then it will contain very few points and + // its area should be approximately zero. On the other hand if it is + // considered clockwise, then it will contain virtually all points and so + // its area should be approximately 4*pi. + // + // More precisely, let U be the set of Points for which IsUnitLength + // is true, let P(U) be the projection of those points onto the mathematical + // unit sphere, and let V(P(U)) be the Voronoi diagram of the projected + // points. Then for every loop x, we would like Area to approximately + // equal the sum of the areas of the Voronoi regions of the points p for + // which x.ContainsPoint(p) is true. + // + // The second issue is that we want to compute the area of small loops + // accurately. This requires having good relative precision rather than + // good absolute precision. For example, if the area of a loop is 1e-12 and + // the error is 1e-15, then the area only has 3 digits of accuracy. (For + // reference, 1e-12 is about 40 square meters on the surface of the earth.) + // We would like to have good relative accuracy even for small loops. + // + // To achieve these goals, we combine two different methods of computing the + // area. This first method is based on the Gauss-Bonnet theorem, which says + // that the area enclosed by the loop equals 2*pi minus the total geodesic + // curvature of the loop (i.e., the sum of the "turning angles" at all the + // loop vertices). The big advantage of this method is that as long as we + // use Sign to compute the turning angle at each vertex, then + // degeneracies are always handled correctly. In other words, if a + // degenerate loop is CCW according to the symbolic perturbations used by + // Sign, then its turning angle will be approximately 2*pi. + // + // The disadvantage of the Gauss-Bonnet method is that its absolute error is + // about 2e-15 times the number of vertices (see turningAngleMaxError). + // So, it cannot compute the area of small loops accurately. + // + // The second method is based on splitting the loop into triangles and + // summing the area of each triangle. To avoid the difficulty and expense + // of decomposing the loop into a union of non-overlapping triangles, + // instead we compute a signed sum over triangles that may overlap (see the + // comments for surfaceIntegral). The advantage of this method + // is that the area of each triangle can be computed with much better + // relative accuracy (using l'Huilier's theorem). The disadvantage is that + // the result is a signed area: CCW loops may yield a small positive value, + // while CW loops may yield a small negative value (which is converted to a + // positive area by adding 4*pi). This means that small errors in computing + // the signed area may translate into a very large error in the result (if + // the sign of the sum is incorrect). + // + // So, our strategy is to combine these two methods as follows. First we + // compute the area using the "signed sum over triangles" approach (since it + // is generally more accurate). We also estimate the maximum error in this + // result. If the signed area is too close to zero (i.e., zero is within + // the error bounds), then we double-check the sign of the result using the + // Gauss-Bonnet method. (In fact we just call IsNormalized, which is + // based on this method.) If the two methods disagree, we return either 0 + // or 4*pi based on the result of IsNormalized. Otherwise we return the + // area that we computed originally. + if l.isEmptyOrFull() { + if l.ContainsOrigin() { + return 4 * math.Pi + } + return 0 + } + area := l.surfaceIntegralFloat64(SignedArea) + + // TODO(roberts): This error estimate is very approximate. There are two + // issues: (1) SignedArea needs some improvements to ensure that its error + // is actually never higher than GirardArea, and (2) although the number of + // triangles in the sum is typically N-2, in theory it could be as high as + // 2*N for pathological inputs. But in other respects this error bound is + // very conservative since it assumes that the maximum error is achieved on + // every triangle. + maxError := l.turningAngleMaxError() + + // The signed area should be between approximately -4*pi and 4*pi. + if area < 0 { + // We have computed the negative of the area of the loop exterior. + area += 4 * math.Pi + } + + if area > 4*math.Pi { + area = 4 * math.Pi + } + if area < 0 { + area = 0 + } + + // If the area is close enough to zero or 4*pi so that the loop orientation + // is ambiguous, then we compute the loop orientation explicitly. + if area < maxError && !l.IsNormalized() { + return 4 * math.Pi + } else if area > (4*math.Pi-maxError) && l.IsNormalized() { + return 0 + } + + return area +} + +// Centroid returns the true centroid of the loop multiplied by the area of the +// loop. The result is not unit length, so you may want to normalize it. Also +// note that in general, the centroid may not be contained by the loop. +// +// We prescale by the loop area for two reasons: (1) it is cheaper to +// compute this way, and (2) it makes it easier to compute the centroid of +// more complicated shapes (by splitting them into disjoint regions and +// adding their centroids). +// +// Note that the return value is not affected by whether this loop is a +// "hole" or a "shell". +func (l *Loop) Centroid() Point { + // surfaceIntegralPoint() returns either the integral of position over loop + // interior, or the negative of the integral of position over the loop + // exterior. But these two values are the same (!), because the integral of + // position over the entire sphere is (0, 0, 0). + return l.surfaceIntegralPoint(TrueCentroid) +} + +// Encode encodes the Loop. +func (l Loop) Encode(w io.Writer) error { + e := &encoder{w: w} + l.encode(e) + return e.err +} + +func (l Loop) encode(e *encoder) { + e.writeInt8(encodingVersion) + e.writeUint32(uint32(len(l.vertices))) + for _, v := range l.vertices { + e.writeFloat64(v.X) + e.writeFloat64(v.Y) + e.writeFloat64(v.Z) + } + + e.writeBool(l.originInside) + e.writeInt32(int32(l.depth)) + + // Encode the bound. + l.bound.encode(e) +} + +func init() { + var f64 float64 + sizeOfFloat64 = int(reflect.TypeOf(f64).Size()) + sizeOfVertex = 3 * sizeOfFloat64 +} + +var sizeOfFloat64 int +var sizeOfVertex int + +// Decode decodes a loop. +func (l *Loop) Decode(r io.Reader) error { + *l = Loop{} + d := &decoder{r: asByteReader(r)} + l.decode(d) + return d.err +} + +func (l *Loop) decode(d *decoder) { + version := int8(d.readUint8()) + if d.err != nil { + return + } + if version != encodingVersion { + d.err = fmt.Errorf("cannot decode version %d", version) + return + } + + // Empty loops are explicitly allowed here: a newly created loop has zero vertices + // and such loops encode and decode properly. + nvertices := d.readUint32() + if nvertices > maxEncodedVertices { + if d.err == nil { + d.err = fmt.Errorf("too many vertices (%d; max is %d)", nvertices, maxEncodedVertices) + + } + return + } + l.vertices = make([]Point, nvertices) + + // Each vertex requires 24 bytes of storage + numBytesNeeded := int(nvertices) * sizeOfVertex + + i := 0 + + for numBytesNeeded > 0 { + arr := l.BufPool.Get(numBytesNeeded) + numBytesRead := d.readFloat64Array(numBytesNeeded, arr) + + if numBytesRead == 0 { + break + } + + numBytesNeeded -= numBytesRead + + // Parsing one vertex at a time into the vertex array of the loop + // by going through the buffer in steps of sizeOfVertex and converting + // floatSize worth of bytes into the float values + for j := 0; j < int(numBytesRead/sizeOfVertex); j++ { + l.vertices[i+j].X = math.Float64frombits( + binary.LittleEndian.Uint64(arr[sizeOfFloat64*(j*3) : sizeOfFloat64*(j*3+1)])) + l.vertices[i+j].Y = math.Float64frombits( + binary.LittleEndian.Uint64(arr[sizeOfFloat64*(j*3+1) : sizeOfFloat64*(j*3+2)])) + l.vertices[i+j].Z = math.Float64frombits( + binary.LittleEndian.Uint64(arr[sizeOfFloat64*(j*3+2) : sizeOfFloat64*(j*3+3)])) + } + + i += int(numBytesRead/sizeOfVertex) + } + + l.index = NewShapeIndex() + l.originInside = d.readBool() + l.depth = int(d.readUint32()) + l.bound.decode(d) + l.subregionBound = ExpandForSubregions(l.bound) + + l.index.Add(l) +} + +// Bitmasks to read from properties. +const ( + originInside = 1 << iota + boundEncoded +) + +func (l *Loop) xyzFaceSiTiVertices() []xyzFaceSiTi { + ret := make([]xyzFaceSiTi, len(l.vertices)) + for i, v := range l.vertices { + ret[i].xyz = v + ret[i].face, ret[i].si, ret[i].ti, ret[i].level = xyzToFaceSiTi(v) + } + return ret +} + +func (l *Loop) encodeCompressed(e *encoder, snapLevel int, vertices []xyzFaceSiTi) { + if len(l.vertices) != len(vertices) { + panic("encodeCompressed: vertices must be the same length as l.vertices") + } + if len(vertices) > maxEncodedVertices { + if e.err == nil { + e.err = fmt.Errorf("too many vertices (%d; max is %d)", len(vertices), maxEncodedVertices) + } + return + } + e.writeUvarint(uint64(len(vertices))) + encodePointsCompressed(e, vertices, snapLevel) + + props := l.compressedEncodingProperties() + e.writeUvarint(props) + e.writeUvarint(uint64(l.depth)) + if props&boundEncoded != 0 { + l.bound.encode(e) + } +} + +func (l *Loop) compressedEncodingProperties() uint64 { + var properties uint64 + if l.originInside { + properties |= originInside + } + + // Write whether there is a bound so we can change the threshold later. + // Recomputing the bound multiplies the decode time taken per vertex + // by a factor of about 3.5. Without recomputing the bound, decode + // takes approximately 125 ns / vertex. A loop with 63 vertices + // encoded without the bound will take ~30us to decode, which is + // acceptable. At ~3.5 bytes / vertex without the bound, adding + // the bound will increase the size by <15%, which is also acceptable. + const minVerticesForBound = 64 + if len(l.vertices) >= minVerticesForBound { + properties |= boundEncoded + } + + return properties +} + +func (l *Loop) decodeCompressed(d *decoder, snapLevel int) { + nvertices := d.readUvarint() + if d.err != nil { + return + } + if nvertices > maxEncodedVertices { + d.err = fmt.Errorf("too many vertices (%d; max is %d)", nvertices, maxEncodedVertices) + return + } + l.vertices = make([]Point, nvertices) + decodePointsCompressed(d, snapLevel, l.vertices) + properties := d.readUvarint() + + // Make sure values are valid before using. + if d.err != nil { + return + } + + l.index = NewShapeIndex() + l.originInside = (properties & originInside) != 0 + + l.depth = int(d.readUvarint()) + + if (properties & boundEncoded) != 0 { + l.bound.decode(d) + if d.err != nil { + return + } + l.subregionBound = ExpandForSubregions(l.bound) + } else { + l.initBound() + } + + l.index.Add(l) +} + +// crossingTarget is an enum representing the possible crossing target cases for relations. +type crossingTarget int + +const ( + crossingTargetDontCare crossingTarget = iota + crossingTargetDontCross + crossingTargetCross +) + +// loopRelation defines the interface for checking a type of relationship between two loops. +// Some examples of relations are Contains, Intersects, or CompareBoundary. +type loopRelation interface { + // Optionally, aCrossingTarget and bCrossingTarget can specify an early-exit + // condition for the loop relation. If any point P is found such that + // + // A.ContainsPoint(P) == aCrossingTarget() && + // B.ContainsPoint(P) == bCrossingTarget() + // + // then the loop relation is assumed to be the same as if a pair of crossing + // edges were found. For example, the ContainsPoint relation has + // + // aCrossingTarget() == crossingTargetDontCross + // bCrossingTarget() == crossingTargetCross + // + // because if A.ContainsPoint(P) == false and B.ContainsPoint(P) == true + // for any point P, then it is equivalent to finding an edge crossing (i.e., + // since Contains returns false in both cases). + // + // Loop relations that do not have an early-exit condition of this form + // should return crossingTargetDontCare for both crossing targets. + + // aCrossingTarget reports whether loop A crosses the target point with + // the given relation type. + aCrossingTarget() crossingTarget + // bCrossingTarget reports whether loop B crosses the target point with + // the given relation type. + bCrossingTarget() crossingTarget + + // wedgesCross reports if a shared vertex ab1 and the two associated wedges + // (a0, ab1, b2) and (b0, ab1, b2) are equivalent to an edge crossing. + // The loop relation is also allowed to maintain its own internal state, and + // can return true if it observes any sequence of wedges that are equivalent + // to an edge crossing. + wedgesCross(a0, ab1, a2, b0, b2 Point) bool +} + +// loopCrosser is a helper type for determining whether two loops cross. +// It is instantiated twice for each pair of loops to be tested, once for the +// pair (A,B) and once for the pair (B,A), in order to be able to process +// edges in either loop nesting order. +type loopCrosser struct { + a, b *Loop + relation loopRelation + swapped bool + aCrossingTarget crossingTarget + bCrossingTarget crossingTarget + + // state maintained by startEdge and edgeCrossesCell. + crosser *EdgeCrosser + aj, bjPrev int + + // temporary data declared here to avoid repeated memory allocations. + bQuery *CrossingEdgeQuery + bCells []*ShapeIndexCell +} + +// newLoopCrosser creates a loopCrosser from the given values. If swapped is true, +// the loops A and B have been swapped. This affects how arguments are passed to +// the given loop relation, since for example A.Contains(B) is not the same as +// B.Contains(A). +func newLoopCrosser(a, b *Loop, relation loopRelation, swapped bool) *loopCrosser { + l := &loopCrosser{ + a: a, + b: b, + relation: relation, + swapped: swapped, + aCrossingTarget: relation.aCrossingTarget(), + bCrossingTarget: relation.bCrossingTarget(), + bQuery: NewCrossingEdgeQuery(b.index), + } + if swapped { + l.aCrossingTarget, l.bCrossingTarget = l.bCrossingTarget, l.aCrossingTarget + } + + return l +} + +// startEdge sets the crossers state for checking the given edge of loop A. +func (l *loopCrosser) startEdge(aj int) { + l.crosser = NewEdgeCrosser(l.a.Vertex(aj), l.a.Vertex(aj+1)) + l.aj = aj + l.bjPrev = -2 +} + +// edgeCrossesCell reports whether the current edge of loop A has any crossings with +// edges of the index cell of loop B. +func (l *loopCrosser) edgeCrossesCell(bClipped *clippedShape) bool { + // Test the current edge of A against all edges of bClipped + bNumEdges := bClipped.numEdges() + for j := 0; j < bNumEdges; j++ { + bj := bClipped.edges[j] + if bj != l.bjPrev+1 { + l.crosser.RestartAt(l.b.Vertex(bj)) + } + l.bjPrev = bj + if crossing := l.crosser.ChainCrossingSign(l.b.Vertex(bj + 1)); crossing == DoNotCross { + continue + } else if crossing == Cross { + return true + } + + // We only need to check each shared vertex once, so we only + // consider the case where l.aVertex(l.aj+1) == l.b.Vertex(bj+1). + if l.a.Vertex(l.aj+1) == l.b.Vertex(bj+1) { + if l.swapped { + if l.relation.wedgesCross(l.b.Vertex(bj), l.b.Vertex(bj+1), l.b.Vertex(bj+2), l.a.Vertex(l.aj), l.a.Vertex(l.aj+2)) { + return true + } + } else { + if l.relation.wedgesCross(l.a.Vertex(l.aj), l.a.Vertex(l.aj+1), l.a.Vertex(l.aj+2), l.b.Vertex(bj), l.b.Vertex(bj+2)) { + return true + } + } + } + } + + return false +} + +// cellCrossesCell reports whether there are any edge crossings or wedge crossings +// within the two given cells. +func (l *loopCrosser) cellCrossesCell(aClipped, bClipped *clippedShape) bool { + // Test all edges of aClipped against all edges of bClipped. + for _, edge := range aClipped.edges { + l.startEdge(edge) + if l.edgeCrossesCell(bClipped) { + return true + } + } + + return false +} + +// cellCrossesAnySubcell reports whether given an index cell of A, if there are any +// edge or wedge crossings with any index cell of B contained within bID. +func (l *loopCrosser) cellCrossesAnySubcell(aClipped *clippedShape, bID CellID) bool { + // Test all edges of aClipped against all edges of B. The relevant B + // edges are guaranteed to be children of bID, which lets us find the + // correct index cells more efficiently. + bRoot := PaddedCellFromCellID(bID, 0) + for _, aj := range aClipped.edges { + // Use an CrossingEdgeQuery starting at bRoot to find the index cells + // of B that might contain crossing edges. + l.bCells = l.bQuery.getCells(l.a.Vertex(aj), l.a.Vertex(aj+1), bRoot) + if len(l.bCells) == 0 { + continue + } + l.startEdge(aj) + for c := 0; c < len(l.bCells); c++ { + if l.edgeCrossesCell(l.bCells[c].shapes[0]) { + return true + } + } + } + + return false +} + +// hasCrossing reports whether given two iterators positioned such that +// ai.cellID().ContainsCellID(bi.cellID()), there is an edge or wedge crossing +// anywhere within ai.cellID(). This function advances bi only past ai.cellID(). +func (l *loopCrosser) hasCrossing(ai, bi *rangeIterator) bool { + // If ai.CellID() intersects many edges of B, then it is faster to use + // CrossingEdgeQuery to narrow down the candidates. But if it intersects + // only a few edges, it is faster to check all the crossings directly. + // We handle this by advancing bi and keeping track of how many edges we + // would need to test. + const edgeQueryMinEdges = 20 // Tuned from benchmarks. + var totalEdges int + l.bCells = nil + + for { + if n := bi.it.IndexCell().shapes[0].numEdges(); n > 0 { + totalEdges += n + if totalEdges >= edgeQueryMinEdges { + // There are too many edges to test them directly, so use CrossingEdgeQuery. + if l.cellCrossesAnySubcell(ai.it.IndexCell().shapes[0], ai.cellID()) { + return true + } + bi.seekBeyond(ai) + return false + } + l.bCells = append(l.bCells, bi.indexCell()) + } + bi.next() + if bi.cellID() > ai.rangeMax { + break + } + } + + // Test all the edge crossings directly. + for _, c := range l.bCells { + if l.cellCrossesCell(ai.it.IndexCell().shapes[0], c.shapes[0]) { + return true + } + } + + return false +} + +// containsCenterMatches reports if the clippedShapes containsCenter boolean corresponds +// to the crossing target type given. (This is to work around C++ allowing false == 0, +// true == 1 type implicit conversions and comparisons) +func containsCenterMatches(a *clippedShape, target crossingTarget) bool { + return (!a.containsCenter && target == crossingTargetDontCross) || + (a.containsCenter && target == crossingTargetCross) +} + +// hasCrossingRelation reports whether given two iterators positioned such that +// ai.cellID().ContainsCellID(bi.cellID()), there is a crossing relationship +// anywhere within ai.cellID(). Specifically, this method returns true if there +// is an edge crossing, a wedge crossing, or a point P that matches both relations +// crossing targets. This function advances both iterators past ai.cellID. +func (l *loopCrosser) hasCrossingRelation(ai, bi *rangeIterator) bool { + aClipped := ai.it.IndexCell().shapes[0] + if aClipped.numEdges() != 0 { + // The current cell of A has at least one edge, so check for crossings. + if l.hasCrossing(ai, bi) { + return true + } + ai.next() + return false + } + + if !containsCenterMatches(aClipped, l.aCrossingTarget) { + // The crossing target for A is not satisfied, so we skip over these cells of B. + bi.seekBeyond(ai) + ai.next() + return false + } + + // All points within ai.cellID() satisfy the crossing target for A, so it's + // worth iterating through the cells of B to see whether any cell + // centers also satisfy the crossing target for B. + for bi.cellID() <= ai.rangeMax { + bClipped := bi.it.IndexCell().shapes[0] + if containsCenterMatches(bClipped, l.bCrossingTarget) { + return true + } + bi.next() + } + ai.next() + return false +} + +// hasCrossingRelation checks all edges of loop A for intersection against all edges +// of loop B and reports if there are any that satisfy the given relation. If there +// is any shared vertex, the wedges centered at this vertex are sent to the given +// relation to be tested. +// +// If the two loop boundaries cross, this method is guaranteed to return +// true. It also returns true in certain cases if the loop relationship is +// equivalent to crossing. For example, if the relation is Contains and a +// point P is found such that B contains P but A does not contain P, this +// method will return true to indicate that the result is the same as though +// a pair of crossing edges were found (since Contains returns false in +// both cases). +// +// See Contains, Intersects and CompareBoundary for the three uses of this function. +func hasCrossingRelation(a, b *Loop, relation loopRelation) bool { + // We look for CellID ranges where the indexes of A and B overlap, and + // then test those edges for crossings. + ai := newRangeIterator(a.index) + bi := newRangeIterator(b.index) + + ab := newLoopCrosser(a, b, relation, false) // Tests edges of A against B + ba := newLoopCrosser(b, a, relation, true) // Tests edges of B against A + + for !ai.done() || !bi.done() { + if ai.rangeMax < bi.rangeMin { + // The A and B cells don't overlap, and A precedes B. + ai.seekTo(bi) + } else if bi.rangeMax < ai.rangeMin { + // The A and B cells don't overlap, and B precedes A. + bi.seekTo(ai) + } else { + // One cell contains the other. Determine which cell is larger. + abRelation := int64(ai.it.CellID().lsb() - bi.it.CellID().lsb()) + if abRelation > 0 { + // A's index cell is larger. + if ab.hasCrossingRelation(ai, bi) { + return true + } + } else if abRelation < 0 { + // B's index cell is larger. + if ba.hasCrossingRelation(bi, ai) { + return true + } + } else { + // The A and B cells are the same. Since the two cells + // have the same center point P, check whether P satisfies + // the crossing targets. + aClipped := ai.it.IndexCell().shapes[0] + bClipped := bi.it.IndexCell().shapes[0] + if containsCenterMatches(aClipped, ab.aCrossingTarget) && + containsCenterMatches(bClipped, ab.bCrossingTarget) { + return true + } + // Otherwise test all the edge crossings directly. + if aClipped.numEdges() > 0 && bClipped.numEdges() > 0 && ab.cellCrossesCell(aClipped, bClipped) { + return true + } + ai.next() + bi.next() + } + } + } + return false +} + +// containsRelation implements loopRelation for a contains operation. If +// A.ContainsPoint(P) == false && B.ContainsPoint(P) == true, it is equivalent +// to having an edge crossing (i.e., Contains returns false). +type containsRelation struct { + foundSharedVertex bool +} + +func (c *containsRelation) aCrossingTarget() crossingTarget { return crossingTargetDontCross } +func (c *containsRelation) bCrossingTarget() crossingTarget { return crossingTargetCross } +func (c *containsRelation) wedgesCross(a0, ab1, a2, b0, b2 Point) bool { + c.foundSharedVertex = true + return !WedgeContains(a0, ab1, a2, b0, b2) +} + +// intersectsRelation implements loopRelation for an intersects operation. Given +// two loops, A and B, if A.ContainsPoint(P) == true && B.ContainsPoint(P) == true, +// it is equivalent to having an edge crossing (i.e., Intersects returns true). +type intersectsRelation struct { + foundSharedVertex bool +} + +func (i *intersectsRelation) aCrossingTarget() crossingTarget { return crossingTargetCross } +func (i *intersectsRelation) bCrossingTarget() crossingTarget { return crossingTargetCross } +func (i *intersectsRelation) wedgesCross(a0, ab1, a2, b0, b2 Point) bool { + i.foundSharedVertex = true + return WedgeIntersects(a0, ab1, a2, b0, b2) +} + +// compareBoundaryRelation implements loopRelation for comparing boundaries. +// +// The compare boundary relation does not have a useful early-exit condition, +// so we return crossingTargetDontCare for both crossing targets. +// +// Aside: A possible early exit condition could be based on the following. +// If A contains a point of both B and ~B, then A intersects Boundary(B). +// If ~A contains a point of both B and ~B, then ~A intersects Boundary(B). +// So if the intersections of {A, ~A} with {B, ~B} are all non-empty, +// the return value is 0, i.e., Boundary(A) intersects Boundary(B). +// Unfortunately it isn't worth detecting this situation because by the +// time we have seen a point in all four intersection regions, we are also +// guaranteed to have seen at least one pair of crossing edges. +type compareBoundaryRelation struct { + reverse bool // True if the other loop should be reversed. + foundSharedVertex bool // True if any wedge was processed. + containsEdge bool // True if any edge of the other loop is contained by this loop. + excludesEdge bool // True if any edge of the other loop is excluded by this loop. +} + +func newCompareBoundaryRelation(reverse bool) *compareBoundaryRelation { + return &compareBoundaryRelation{reverse: reverse} +} + +func (c *compareBoundaryRelation) aCrossingTarget() crossingTarget { return crossingTargetDontCare } +func (c *compareBoundaryRelation) bCrossingTarget() crossingTarget { return crossingTargetDontCare } +func (c *compareBoundaryRelation) wedgesCross(a0, ab1, a2, b0, b2 Point) bool { + // Because we don't care about the interior of the other, only its boundary, + // it is sufficient to check whether this one contains the semiwedge (ab1, b2). + c.foundSharedVertex = true + if wedgeContainsSemiwedge(a0, ab1, a2, b2, c.reverse) { + c.containsEdge = true + } else { + c.excludesEdge = true + } + return c.containsEdge && c.excludesEdge +} + +// wedgeContainsSemiwedge reports whether the wedge (a0, ab1, a2) contains the +// "semiwedge" defined as any non-empty open set of rays immediately CCW from +// the edge (ab1, b2). If reverse is true, then substitute clockwise for CCW; +// this simulates what would happen if the direction of the other loop was reversed. +func wedgeContainsSemiwedge(a0, ab1, a2, b2 Point, reverse bool) bool { + if b2 == a0 || b2 == a2 { + // We have a shared or reversed edge. + return (b2 == a0) == reverse + } + return OrderedCCW(a0, a2, b2, ab1) +} + +// containsNonCrossingBoundary reports whether given two loops whose boundaries +// do not cross (see compareBoundary), if this loop contains the boundary of the +// other loop. If reverse is true, the boundary of the other loop is reversed +// first (which only affects the result when there are shared edges). This method +// is cheaper than compareBoundary because it does not test for edge intersections. +// +// This function requires that neither loop is empty, and that if the other is full, +// then reverse == false. +func (l *Loop) containsNonCrossingBoundary(other *Loop, reverseOther bool) bool { + // The bounds must intersect for containment. + if !l.bound.Intersects(other.bound) { + return false + } + + // Full loops are handled as though the loop surrounded the entire sphere. + if l.IsFull() { + return true + } + if other.IsFull() { + return false + } + + m, ok := l.findVertex(other.Vertex(0)) + if !ok { + // Since the other loops vertex 0 is not shared, we can check if this contains it. + return l.ContainsPoint(other.Vertex(0)) + } + // Otherwise check whether the edge (b0, b1) is contained by this loop. + return wedgeContainsSemiwedge(l.Vertex(m-1), l.Vertex(m), l.Vertex(m+1), + other.Vertex(1), reverseOther) +} + +// TODO(roberts): Differences from the C++ version: +// DistanceToPoint +// DistanceToBoundary +// Project +// ProjectToBoundary +// BoundaryApproxEqual +// BoundaryNear diff --git a/vendor/github.com/blevesearch/geo/s2/matrix3x3.go b/vendor/github.com/blevesearch/geo/s2/matrix3x3.go new file mode 100644 index 00000000..01696fe8 --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/matrix3x3.go @@ -0,0 +1,127 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +import ( + "fmt" + + "github.com/golang/geo/r3" +) + +// matrix3x3 represents a traditional 3x3 matrix of floating point values. +// This is not a full fledged matrix. It only contains the pieces needed +// to satisfy the computations done within the s2 package. +type matrix3x3 [3][3]float64 + +// col returns the given column as a Point. +func (m *matrix3x3) col(col int) Point { + return Point{r3.Vector{m[0][col], m[1][col], m[2][col]}} +} + +// row returns the given row as a Point. +func (m *matrix3x3) row(row int) Point { + return Point{r3.Vector{m[row][0], m[row][1], m[row][2]}} +} + +// setCol sets the specified column to the value in the given Point. +func (m *matrix3x3) setCol(col int, p Point) *matrix3x3 { + m[0][col] = p.X + m[1][col] = p.Y + m[2][col] = p.Z + + return m +} + +// setRow sets the specified row to the value in the given Point. +func (m *matrix3x3) setRow(row int, p Point) *matrix3x3 { + m[row][0] = p.X + m[row][1] = p.Y + m[row][2] = p.Z + + return m +} + +// scale multiplies the matrix by the given value. +func (m *matrix3x3) scale(f float64) *matrix3x3 { + return &matrix3x3{ + [3]float64{f * m[0][0], f * m[0][1], f * m[0][2]}, + [3]float64{f * m[1][0], f * m[1][1], f * m[1][2]}, + [3]float64{f * m[2][0], f * m[2][1], f * m[2][2]}, + } +} + +// mul returns the multiplication of m by the Point p and converts the +// resulting 1x3 matrix into a Point. +func (m *matrix3x3) mul(p Point) Point { + return Point{r3.Vector{ + m[0][0]*p.X + m[0][1]*p.Y + m[0][2]*p.Z, + m[1][0]*p.X + m[1][1]*p.Y + m[1][2]*p.Z, + m[2][0]*p.X + m[2][1]*p.Y + m[2][2]*p.Z, + }} +} + +// det returns the determinant of this matrix. +func (m *matrix3x3) det() float64 { + // | a b c | + // det | d e f | = aei + bfg + cdh - ceg - bdi - afh + // | g h i | + return m[0][0]*m[1][1]*m[2][2] + m[0][1]*m[1][2]*m[2][0] + m[0][2]*m[1][0]*m[2][1] - + m[0][2]*m[1][1]*m[2][0] - m[0][1]*m[1][0]*m[2][2] - m[0][0]*m[1][2]*m[2][1] +} + +// transpose reflects the matrix along its diagonal and returns the result. +func (m *matrix3x3) transpose() *matrix3x3 { + m[0][1], m[1][0] = m[1][0], m[0][1] + m[0][2], m[2][0] = m[2][0], m[0][2] + m[1][2], m[2][1] = m[2][1], m[1][2] + + return m +} + +// String formats the matrix into an easier to read layout. +func (m *matrix3x3) String() string { + return fmt.Sprintf("[ %0.4f %0.4f %0.4f ] [ %0.4f %0.4f %0.4f ] [ %0.4f %0.4f %0.4f ]", + m[0][0], m[0][1], m[0][2], + m[1][0], m[1][1], m[1][2], + m[2][0], m[2][1], m[2][2], + ) +} + +// getFrame returns the orthonormal frame for the given point on the unit sphere. +func getFrame(p Point) matrix3x3 { + // Given the point p on the unit sphere, extend this into a right-handed + // coordinate frame of unit-length column vectors m = (x,y,z). Note that + // the vectors (x,y) are an orthonormal frame for the tangent space at point p, + // while p itself is an orthonormal frame for the normal space at p. + m := matrix3x3{} + m.setCol(2, p) + m.setCol(1, Point{p.Ortho()}) + m.setCol(0, Point{m.col(1).Cross(p.Vector)}) + return m +} + +// toFrame returns the coordinates of the given point with respect to its orthonormal basis m. +// The resulting point q satisfies the identity (m * q == p). +func toFrame(m matrix3x3, p Point) Point { + // The inverse of an orthonormal matrix is its transpose. + return m.transpose().mul(p) +} + +// fromFrame returns the coordinates of the given point in standard axis-aligned basis +// from its orthonormal basis m. +// The resulting point p satisfies the identity (p == m * q). +func fromFrame(m matrix3x3, q Point) Point { + return m.mul(q) +} diff --git a/vendor/github.com/blevesearch/geo/s2/max_distance_targets.go b/vendor/github.com/blevesearch/geo/s2/max_distance_targets.go new file mode 100644 index 00000000..92e916d9 --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/max_distance_targets.go @@ -0,0 +1,306 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +import ( + "math" + + "github.com/golang/geo/s1" +) + +// maxDistance implements distance as the supplementary distance (Pi - x) to find +// results that are the furthest using the distance related algorithms. +type maxDistance s1.ChordAngle + +func (m maxDistance) chordAngle() s1.ChordAngle { return s1.ChordAngle(m) } +func (m maxDistance) zero() distance { return maxDistance(s1.StraightChordAngle) } +func (m maxDistance) negative() distance { return maxDistance(s1.InfChordAngle()) } +func (m maxDistance) infinity() distance { return maxDistance(s1.NegativeChordAngle) } +func (m maxDistance) less(other distance) bool { return m.chordAngle() > other.chordAngle() } +func (m maxDistance) sub(other distance) distance { + return maxDistance(m.chordAngle() + other.chordAngle()) +} +func (m maxDistance) chordAngleBound() s1.ChordAngle { + return s1.StraightChordAngle - m.chordAngle() +} +func (m maxDistance) updateDistance(dist distance) (distance, bool) { + if dist.less(m) { + m = maxDistance(dist.chordAngle()) + return m, true + } + return m, false +} + +func (m maxDistance) fromChordAngle(o s1.ChordAngle) distance { + return maxDistance(o) +} + +// MaxDistanceToPointTarget is used for computing the maximum distance to a Point. +type MaxDistanceToPointTarget struct { + point Point + dist distance +} + +// NewMaxDistanceToPointTarget returns a new target for the given Point. +func NewMaxDistanceToPointTarget(point Point) *MaxDistanceToPointTarget { + m := maxDistance(0) + return &MaxDistanceToPointTarget{point: point, dist: &m} +} + +func (m *MaxDistanceToPointTarget) capBound() Cap { + return CapFromCenterChordAngle(Point{m.point.Mul(-1)}, (s1.ChordAngle(0))) +} + +func (m *MaxDistanceToPointTarget) updateDistanceToPoint(p Point, dist distance) (distance, bool) { + return dist.updateDistance(maxDistance(ChordAngleBetweenPoints(p, m.point))) +} + +func (m *MaxDistanceToPointTarget) updateDistanceToEdge(edge Edge, dist distance) (distance, bool) { + if d, ok := UpdateMaxDistance(m.point, edge.V0, edge.V1, dist.chordAngle()); ok { + dist, _ = dist.updateDistance(maxDistance(d)) + return dist, true + } + return dist, false +} + +func (m *MaxDistanceToPointTarget) updateDistanceToCell(cell Cell, dist distance) (distance, bool) { + return dist.updateDistance(maxDistance(cell.MaxDistance(m.point))) +} + +func (m *MaxDistanceToPointTarget) visitContainingShapes(index *ShapeIndex, v shapePointVisitorFunc) bool { + // For furthest points, we visit the polygons whose interior contains + // the antipode of the target point. These are the polygons whose + // distance to the target is maxDistance.zero() + q := NewContainsPointQuery(index, VertexModelSemiOpen) + return q.visitContainingShapes(Point{m.point.Mul(-1)}, func(shape Shape) bool { + return v(shape, m.point) + }) +} + +func (m *MaxDistanceToPointTarget) setMaxError(maxErr s1.ChordAngle) bool { return false } +func (m *MaxDistanceToPointTarget) maxBruteForceIndexSize() int { return 30 } +func (m *MaxDistanceToPointTarget) distance() distance { return m.dist } + +// MaxDistanceToEdgeTarget is used for computing the maximum distance to an Edge. +type MaxDistanceToEdgeTarget struct { + e Edge + dist distance +} + +// NewMaxDistanceToEdgeTarget returns a new target for the given Edge. +func NewMaxDistanceToEdgeTarget(e Edge) *MaxDistanceToEdgeTarget { + m := maxDistance(0) + return &MaxDistanceToEdgeTarget{e: e, dist: m} +} + +// capBound returns a Cap that bounds the antipode of the target. (This +// is the set of points whose maxDistance to the target is maxDistance.zero) +func (m *MaxDistanceToEdgeTarget) capBound() Cap { + // The following computes a radius equal to half the edge length in an + // efficient and numerically stable way. + d2 := float64(ChordAngleBetweenPoints(m.e.V0, m.e.V1)) + r2 := (0.5 * d2) / (1 + math.Sqrt(1-0.25*d2)) + return CapFromCenterChordAngle(Point{m.e.V0.Add(m.e.V1.Vector).Mul(-1).Normalize()}, s1.ChordAngleFromSquaredLength(r2)) +} + +func (m *MaxDistanceToEdgeTarget) updateDistanceToPoint(p Point, dist distance) (distance, bool) { + if d, ok := UpdateMaxDistance(p, m.e.V0, m.e.V1, dist.chordAngle()); ok { + dist, _ = dist.updateDistance(maxDistance(d)) + return dist, true + } + return dist, false +} + +func (m *MaxDistanceToEdgeTarget) updateDistanceToEdge(edge Edge, dist distance) (distance, bool) { + if d, ok := updateEdgePairMaxDistance(m.e.V0, m.e.V1, edge.V0, edge.V1, dist.chordAngle()); ok { + dist, _ = dist.updateDistance(maxDistance(d)) + return dist, true + } + return dist, false +} + +func (m *MaxDistanceToEdgeTarget) updateDistanceToCell(cell Cell, dist distance) (distance, bool) { + return dist.updateDistance(maxDistance(cell.MaxDistanceToEdge(m.e.V0, m.e.V1))) +} + +func (m *MaxDistanceToEdgeTarget) visitContainingShapes(index *ShapeIndex, v shapePointVisitorFunc) bool { + // We only need to test one edge point. That is because the method *must* + // visit a polygon if it fully contains the target, and *is allowed* to + // visit a polygon if it intersects the target. If the tested vertex is not + // contained, we know the full edge is not contained; if the tested vertex is + // contained, then the edge either is fully contained (must be visited) or it + // intersects (is allowed to be visited). We visit the center of the edge so + // that edge AB gives identical results to BA. + target := NewMaxDistanceToPointTarget(Point{m.e.V0.Add(m.e.V1.Vector).Normalize()}) + return target.visitContainingShapes(index, v) +} + +func (m *MaxDistanceToEdgeTarget) setMaxError(maxErr s1.ChordAngle) bool { return false } +func (m *MaxDistanceToEdgeTarget) maxBruteForceIndexSize() int { return 30 } +func (m *MaxDistanceToEdgeTarget) distance() distance { return m.dist } + +// MaxDistanceToCellTarget is used for computing the maximum distance to a Cell. +type MaxDistanceToCellTarget struct { + cell Cell + dist distance +} + +// NewMaxDistanceToCellTarget returns a new target for the given Cell. +func NewMaxDistanceToCellTarget(cell Cell) *MaxDistanceToCellTarget { + m := maxDistance(0) + return &MaxDistanceToCellTarget{cell: cell, dist: m} +} + +func (m *MaxDistanceToCellTarget) capBound() Cap { + c := m.cell.CapBound() + return CapFromCenterAngle(Point{c.Center().Mul(-1)}, c.Radius()) +} + +func (m *MaxDistanceToCellTarget) updateDistanceToPoint(p Point, dist distance) (distance, bool) { + return dist.updateDistance(maxDistance(m.cell.MaxDistance(p))) +} + +func (m *MaxDistanceToCellTarget) updateDistanceToEdge(edge Edge, dist distance) (distance, bool) { + return dist.updateDistance(maxDistance(m.cell.MaxDistanceToEdge(edge.V0, edge.V1))) +} + +func (m *MaxDistanceToCellTarget) updateDistanceToCell(cell Cell, dist distance) (distance, bool) { + return dist.updateDistance(maxDistance(m.cell.MaxDistanceToCell(cell))) +} + +func (m *MaxDistanceToCellTarget) visitContainingShapes(index *ShapeIndex, v shapePointVisitorFunc) bool { + // We only need to check one point here - cell center is simplest. + // See comment at MaxDistanceToEdgeTarget's visitContainingShapes. + target := NewMaxDistanceToPointTarget(m.cell.Center()) + return target.visitContainingShapes(index, v) +} + +func (m *MaxDistanceToCellTarget) setMaxError(maxErr s1.ChordAngle) bool { return false } +func (m *MaxDistanceToCellTarget) maxBruteForceIndexSize() int { return 30 } +func (m *MaxDistanceToCellTarget) distance() distance { return m.dist } + +// MaxDistanceToShapeIndexTarget is used for computing the maximum distance to a ShapeIndex. +type MaxDistanceToShapeIndexTarget struct { + index *ShapeIndex + query *EdgeQuery + dist distance +} + +// NewMaxDistanceToShapeIndexTarget returns a new target for the given ShapeIndex. +func NewMaxDistanceToShapeIndexTarget(index *ShapeIndex) *MaxDistanceToShapeIndexTarget { + m := maxDistance(0) + return &MaxDistanceToShapeIndexTarget{ + index: index, + dist: m, + query: NewFurthestEdgeQuery(index, NewFurthestEdgeQueryOptions()), + } +} + +// capBound returns a Cap that bounds the antipode of the target. This +// is the set of points whose maxDistance to the target is maxDistance.zero() +func (m *MaxDistanceToShapeIndexTarget) capBound() Cap { + // TODO(roberts): Depends on ShapeIndexRegion + // c := makeShapeIndexRegion(m.index).CapBound() + // return CapFromCenterRadius(Point{c.Center.Mul(-1)}, c.Radius()) + panic("not implemented yet") +} + +func (m *MaxDistanceToShapeIndexTarget) updateDistanceToPoint(p Point, dist distance) (distance, bool) { + m.query.opts.distanceLimit = dist.chordAngle() + target := NewMaxDistanceToPointTarget(p) + r := m.query.findEdge(target, m.query.opts) + if r.shapeID < 0 { + return dist, false + } + return r.distance, true +} + +func (m *MaxDistanceToShapeIndexTarget) updateDistanceToEdge(edge Edge, dist distance) (distance, bool) { + m.query.opts.distanceLimit = dist.chordAngle() + target := NewMaxDistanceToEdgeTarget(edge) + r := m.query.findEdge(target, m.query.opts) + if r.shapeID < 0 { + return dist, false + } + return r.distance, true +} + +func (m *MaxDistanceToShapeIndexTarget) updateDistanceToCell(cell Cell, dist distance) (distance, bool) { + m.query.opts.distanceLimit = dist.chordAngle() + target := NewMaxDistanceToCellTarget(cell) + r := m.query.findEdge(target, m.query.opts) + if r.shapeID < 0 { + return dist, false + } + return r.distance, true +} + +// visitContainingShapes returns the polygons containing the antipodal +// reflection of *any* connected component for target types consisting of +// multiple connected components. It is sufficient to test containment of +// one vertex per connected component, since this allows us to also return +// any polygon whose boundary has distance.zero() to the target. +func (m *MaxDistanceToShapeIndexTarget) visitContainingShapes(index *ShapeIndex, v shapePointVisitorFunc) bool { + // It is sufficient to find the set of chain starts in the target index + // (i.e., one vertex per connected component of edges) that are contained by + // the query index, except for one special case to handle full polygons. + // + // TODO(roberts): Do this by merge-joining the two ShapeIndexes and share + // the code with BooleanOperation. + for _, shape := range m.index.shapes { + numChains := shape.NumChains() + // Shapes that don't have any edges require a special case (below). + testedPoint := false + for c := 0; c < numChains; c++ { + chain := shape.Chain(c) + if chain.Length == 0 { + continue + } + testedPoint = true + target := NewMaxDistanceToPointTarget(shape.ChainEdge(c, 0).V0) + if !target.visitContainingShapes(index, v) { + return false + } + } + if !testedPoint { + // Special case to handle full polygons. + ref := shape.ReferencePoint() + if !ref.Contained { + continue + } + target := NewMaxDistanceToPointTarget(ref.Point) + if !target.visitContainingShapes(index, v) { + return false + } + } + } + return true +} + +func (m *MaxDistanceToShapeIndexTarget) setMaxError(maxErr s1.ChordAngle) bool { + m.query.opts.maxError = maxErr + return true +} +func (m *MaxDistanceToShapeIndexTarget) maxBruteForceIndexSize() int { return 30 } +func (m *MaxDistanceToShapeIndexTarget) distance() distance { return m.dist } +func (m *MaxDistanceToShapeIndexTarget) setIncludeInteriors(b bool) { + m.query.opts.includeInteriors = b +} +func (m *MaxDistanceToShapeIndexTarget) setUseBruteForce(b bool) { m.query.opts.useBruteForce = b } + +// TODO(roberts): Remaining methods +// +// func (m *MaxDistanceToShapeIndexTarget) capBound() Cap { +// CellUnionTarget diff --git a/vendor/github.com/blevesearch/geo/s2/metric.go b/vendor/github.com/blevesearch/geo/s2/metric.go new file mode 100644 index 00000000..53db3d31 --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/metric.go @@ -0,0 +1,164 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +// This file implements functions for various S2 measurements. + +import "math" + +// A Metric is a measure for cells. It is used to describe the shape and size +// of cells. They are useful for deciding which cell level to use in order to +// satisfy a given condition (e.g. that cell vertices must be no further than +// "x" apart). You can use the Value(level) method to compute the corresponding +// length or area on the unit sphere for cells at a given level. The minimum +// and maximum bounds are valid for cells at all levels, but they may be +// somewhat conservative for very large cells (e.g. face cells). +type Metric struct { + // Dim is either 1 or 2, for a 1D or 2D metric respectively. + Dim int + // Deriv is the scaling factor for the metric. + Deriv float64 +} + +// Defined metrics. +// Of the projection methods defined in C++, Go only supports the quadratic projection. + +// Each cell is bounded by four planes passing through its four edges and +// the center of the sphere. These metrics relate to the angle between each +// pair of opposite bounding planes, or equivalently, between the planes +// corresponding to two different s-values or two different t-values. +var ( + MinAngleSpanMetric = Metric{1, 4.0 / 3} + AvgAngleSpanMetric = Metric{1, math.Pi / 2} + MaxAngleSpanMetric = Metric{1, 1.704897179199218452} +) + +// The width of geometric figure is defined as the distance between two +// parallel bounding lines in a given direction. For cells, the minimum +// width is always attained between two opposite edges, and the maximum +// width is attained between two opposite vertices. However, for our +// purposes we redefine the width of a cell as the perpendicular distance +// between a pair of opposite edges. A cell therefore has two widths, one +// in each direction. The minimum width according to this definition agrees +// with the classic geometric one, but the maximum width is different. (The +// maximum geometric width corresponds to MaxDiag defined below.) +// +// The average width in both directions for all cells at level k is approximately +// AvgWidthMetric.Value(k). +// +// The width is useful for bounding the minimum or maximum distance from a +// point on one edge of a cell to the closest point on the opposite edge. +// For example, this is useful when growing regions by a fixed distance. +var ( + MinWidthMetric = Metric{1, 2 * math.Sqrt2 / 3} + AvgWidthMetric = Metric{1, 1.434523672886099389} + MaxWidthMetric = Metric{1, MaxAngleSpanMetric.Deriv} +) + +// The edge length metrics can be used to bound the minimum, maximum, +// or average distance from the center of one cell to the center of one of +// its edge neighbors. In particular, it can be used to bound the distance +// between adjacent cell centers along the space-filling Hilbert curve for +// cells at any given level. +var ( + MinEdgeMetric = Metric{1, 2 * math.Sqrt2 / 3} + AvgEdgeMetric = Metric{1, 1.459213746386106062} + MaxEdgeMetric = Metric{1, MaxAngleSpanMetric.Deriv} + + // MaxEdgeAspect is the maximum edge aspect ratio over all cells at any level, + // where the edge aspect ratio of a cell is defined as the ratio of its longest + // edge length to its shortest edge length. + MaxEdgeAspect = 1.442615274452682920 + + MinAreaMetric = Metric{2, 8 * math.Sqrt2 / 9} + AvgAreaMetric = Metric{2, 4 * math.Pi / 6} + MaxAreaMetric = Metric{2, 2.635799256963161491} +) + +// The maximum diagonal is also the maximum diameter of any cell, +// and also the maximum geometric width (see the comment for widths). For +// example, the distance from an arbitrary point to the closest cell center +// at a given level is at most half the maximum diagonal length. +var ( + MinDiagMetric = Metric{1, 8 * math.Sqrt2 / 9} + AvgDiagMetric = Metric{1, 2.060422738998471683} + MaxDiagMetric = Metric{1, 2.438654594434021032} + + // MaxDiagAspect is the maximum diagonal aspect ratio over all cells at any + // level, where the diagonal aspect ratio of a cell is defined as the ratio + // of its longest diagonal length to its shortest diagonal length. + MaxDiagAspect = math.Sqrt(3) +) + +// Value returns the value of the metric at the given level. +func (m Metric) Value(level int) float64 { + return math.Ldexp(m.Deriv, -m.Dim*level) +} + +// MinLevel returns the minimum level such that the metric is at most +// the given value, or maxLevel (30) if there is no such level. +// +// For example, MinLevel(0.1) returns the minimum level such that all cell diagonal +// lengths are 0.1 or smaller. The returned value is always a valid level. +// +// In C++, this is called GetLevelForMaxValue. +func (m Metric) MinLevel(val float64) int { + if val < 0 { + return maxLevel + } + + level := -(math.Ilogb(val/m.Deriv) >> uint(m.Dim-1)) + if level > maxLevel { + level = maxLevel + } + if level < 0 { + level = 0 + } + return level +} + +// MaxLevel returns the maximum level such that the metric is at least +// the given value, or zero if there is no such level. +// +// For example, MaxLevel(0.1) returns the maximum level such that all cells have a +// minimum width of 0.1 or larger. The returned value is always a valid level. +// +// In C++, this is called GetLevelForMinValue. +func (m Metric) MaxLevel(val float64) int { + if val <= 0 { + return maxLevel + } + + level := math.Ilogb(m.Deriv/val) >> uint(m.Dim-1) + if level > maxLevel { + level = maxLevel + } + if level < 0 { + level = 0 + } + return level +} + +// ClosestLevel returns the level at which the metric has approximately the given +// value. The return value is always a valid level. For example, +// AvgEdgeMetric.ClosestLevel(0.1) returns the level at which the average cell edge +// length is approximately 0.1. +func (m Metric) ClosestLevel(val float64) int { + x := math.Sqrt2 + if m.Dim == 2 { + x = 2 + } + return m.MinLevel(x * val) +} diff --git a/vendor/github.com/blevesearch/geo/s2/min_distance_targets.go b/vendor/github.com/blevesearch/geo/s2/min_distance_targets.go new file mode 100644 index 00000000..b4cbd43e --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/min_distance_targets.go @@ -0,0 +1,362 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +import ( + "math" + + "github.com/golang/geo/s1" +) + +// minDistance implements distance interface to find closest distance types. +type minDistance s1.ChordAngle + +func (m minDistance) chordAngle() s1.ChordAngle { return s1.ChordAngle(m) } +func (m minDistance) zero() distance { return minDistance(0) } +func (m minDistance) negative() distance { return minDistance(s1.NegativeChordAngle) } +func (m minDistance) infinity() distance { return minDistance(s1.InfChordAngle()) } +func (m minDistance) less(other distance) bool { return m.chordAngle() < other.chordAngle() } +func (m minDistance) sub(other distance) distance { + return minDistance(m.chordAngle() - other.chordAngle()) +} +func (m minDistance) chordAngleBound() s1.ChordAngle { + return m.chordAngle().Expanded(m.chordAngle().MaxAngleError()) +} + +// updateDistance updates its own value if the other value is less() than it is, +// and reports if it updated. +func (m minDistance) updateDistance(dist distance) (distance, bool) { + if dist.less(m) { + m = minDistance(dist.chordAngle()) + return m, true + } + return m, false +} + +func (m minDistance) fromChordAngle(o s1.ChordAngle) distance { + return minDistance(o) +} + +// MinDistanceToPointTarget is a type for computing the minimum distance to a Point. +type MinDistanceToPointTarget struct { + point Point + dist distance +} + +// NewMinDistanceToPointTarget returns a new target for the given Point. +func NewMinDistanceToPointTarget(point Point) *MinDistanceToPointTarget { + m := minDistance(0) + return &MinDistanceToPointTarget{point: point, dist: &m} +} + +func (m *MinDistanceToPointTarget) capBound() Cap { + return CapFromCenterChordAngle(m.point, s1.ChordAngle(0)) +} + +func (m *MinDistanceToPointTarget) updateDistanceToPoint(p Point, dist distance) (distance, bool) { + var ok bool + dist, ok = dist.updateDistance(minDistance(ChordAngleBetweenPoints(p, m.point))) + return dist, ok +} + +func (m *MinDistanceToPointTarget) updateDistanceToEdge(edge Edge, dist distance) (distance, bool) { + if d, ok := UpdateMinDistance(m.point, edge.V0, edge.V1, dist.chordAngle()); ok { + dist, _ = dist.updateDistance(minDistance(d)) + return dist, true + } + return dist, false +} + +func (m *MinDistanceToPointTarget) updateDistanceToCell(cell Cell, dist distance) (distance, bool) { + var ok bool + dist, ok = dist.updateDistance(minDistance(cell.Distance(m.point))) + return dist, ok +} + +func (m *MinDistanceToPointTarget) visitContainingShapes(index *ShapeIndex, v shapePointVisitorFunc) bool { + // For furthest points, we visit the polygons whose interior contains + // the antipode of the target point. These are the polygons whose + // distance to the target is maxDistance.zero() + q := NewContainsPointQuery(index, VertexModelSemiOpen) + return q.visitContainingShapes(m.point, func(shape Shape) bool { + return v(shape, m.point) + }) +} + +func (m *MinDistanceToPointTarget) setMaxError(maxErr s1.ChordAngle) bool { return false } +func (m *MinDistanceToPointTarget) maxBruteForceIndexSize() int { return 30 } +func (m *MinDistanceToPointTarget) distance() distance { return m.dist } + +// ---------------------------------------------------------- + +// MinDistanceToEdgeTarget is a type for computing the minimum distance to an Edge. +type MinDistanceToEdgeTarget struct { + e Edge + dist distance +} + +// NewMinDistanceToEdgeTarget returns a new target for the given Edge. +func NewMinDistanceToEdgeTarget(e Edge) *MinDistanceToEdgeTarget { + m := minDistance(0) + return &MinDistanceToEdgeTarget{e: e, dist: m} +} + +// capBound returns a Cap that bounds the antipode of the target. (This +// is the set of points whose maxDistance to the target is maxDistance.zero) +func (m *MinDistanceToEdgeTarget) capBound() Cap { + // The following computes a radius equal to half the edge length in an + // efficient and numerically stable way. + d2 := float64(ChordAngleBetweenPoints(m.e.V0, m.e.V1)) + r2 := (0.5 * d2) / (1 + math.Sqrt(1-0.25*d2)) + return CapFromCenterChordAngle(Point{m.e.V0.Add(m.e.V1.Vector).Normalize()}, s1.ChordAngleFromSquaredLength(r2)) +} + +func (m *MinDistanceToEdgeTarget) updateDistanceToPoint(p Point, dist distance) (distance, bool) { + if d, ok := UpdateMinDistance(p, m.e.V0, m.e.V1, dist.chordAngle()); ok { + dist, _ = dist.updateDistance(minDistance(d)) + return dist, true + } + return dist, false +} + +func (m *MinDistanceToEdgeTarget) updateDistanceToEdge(edge Edge, dist distance) (distance, bool) { + if d, ok := updateEdgePairMinDistance(m.e.V0, m.e.V1, edge.V0, edge.V1, dist.chordAngle()); ok { + dist, _ = dist.updateDistance(minDistance(d)) + return dist, true + } + return dist, false +} + +func (m *MinDistanceToEdgeTarget) updateDistanceToCell(cell Cell, dist distance) (distance, bool) { + return dist.updateDistance(minDistance(cell.DistanceToEdge(m.e.V0, m.e.V1))) +} + +func (m *MinDistanceToEdgeTarget) visitContainingShapes(index *ShapeIndex, v shapePointVisitorFunc) bool { + // We test the center of the edge in order to ensure that edge targets AB + // and BA yield identical results (which is not guaranteed by the API but + // users might expect). Other options would be to test both endpoints, or + // return different results for AB and BA in some cases. + target := NewMinDistanceToPointTarget(Point{m.e.V0.Add(m.e.V1.Vector).Normalize()}) + return target.visitContainingShapes(index, v) +} + +func (m *MinDistanceToEdgeTarget) setMaxError(maxErr s1.ChordAngle) bool { return false } +func (m *MinDistanceToEdgeTarget) maxBruteForceIndexSize() int { return 30 } +func (m *MinDistanceToEdgeTarget) distance() distance { return m.dist } + +// ---------------------------------------------------------- + +// MinDistanceToCellTarget is a type for computing the minimum distance to a Cell. +type MinDistanceToCellTarget struct { + cell Cell + dist distance +} + +// NewMinDistanceToCellTarget returns a new target for the given Cell. +func NewMinDistanceToCellTarget(cell Cell) *MinDistanceToCellTarget { + m := minDistance(0) + return &MinDistanceToCellTarget{cell: cell, dist: m} +} + +func (m *MinDistanceToCellTarget) capBound() Cap { + return m.cell.CapBound() +} + +func (m *MinDistanceToCellTarget) updateDistanceToPoint(p Point, dist distance) (distance, bool) { + return dist.updateDistance(minDistance(m.cell.Distance(p))) +} + +func (m *MinDistanceToCellTarget) updateDistanceToEdge(edge Edge, dist distance) (distance, bool) { + return dist.updateDistance(minDistance(m.cell.DistanceToEdge(edge.V0, edge.V1))) +} + +func (m *MinDistanceToCellTarget) updateDistanceToCell(cell Cell, dist distance) (distance, bool) { + return dist.updateDistance(minDistance(m.cell.DistanceToCell(cell))) +} + +func (m *MinDistanceToCellTarget) visitContainingShapes(index *ShapeIndex, v shapePointVisitorFunc) bool { + // The simplest approach is simply to return the polygons that contain the + // cell center. Alternatively, if the index cell is smaller than the target + // cell then we could return all polygons that are present in the + // shapeIndexCell, but since the index is built conservatively this may + // include some polygons that don't quite intersect the cell. So we would + // either need to recheck for intersection more accurately, or weaken the + // VisitContainingShapes contract so that it only guarantees approximate + // intersection, neither of which seems like a good tradeoff. + target := NewMinDistanceToPointTarget(m.cell.Center()) + return target.visitContainingShapes(index, v) +} +func (m *MinDistanceToCellTarget) setMaxError(maxErr s1.ChordAngle) bool { return false } +func (m *MinDistanceToCellTarget) maxBruteForceIndexSize() int { return 30 } +func (m *MinDistanceToCellTarget) distance() distance { return m.dist } + +// ---------------------------------------------------------- + +/* +// MinDistanceToCellUnionTarget is a type for computing the minimum distance to a CellUnion. +type MinDistanceToCellUnionTarget struct { + cu CellUnion + query *ClosestCellQuery + dist distance +} + +// NewMinDistanceToCellUnionTarget returns a new target for the given CellUnion. +func NewMinDistanceToCellUnionTarget(cu CellUnion) *MinDistanceToCellUnionTarget { + m := minDistance(0) + return &MinDistanceToCellUnionTarget{cu: cu, dist: m} +} + +func (m *MinDistanceToCellUnionTarget) capBound() Cap { + return m.cu.CapBound() +} + +func (m *MinDistanceToCellUnionTarget) updateDistanceToCell(cell Cell, dist distance) (distance, bool) { + m.query.opts.DistanceLimit = dist.chordAngle() + target := NewMinDistanceToPointTarget(p) + r := m.query.findEdge(target) + if r.ShapeID < 0 { + return dist, false + } + return minDistance(r.Distance), true +} + +func (m *MinDistanceToCellUnionTarget) visitContainingShapes(index *ShapeIndex, v shapePointVisitorFunc) bool { + // We test the center of the edge in order to ensure that edge targets AB + // and BA yield identical results (which is not guaranteed by the API but + // users might expect). Other options would be to test both endpoints, or + // return different results for AB and BA in some cases. + target := NewMinDistanceToPointTarget(Point{m.e.V0.Add(m.e.V1.Vector).Normalize()}) + return target.visitContainingShapes(index, v) +} +func (m *MinDistanceToCellUnionTarget) setMaxError(maxErr s1.ChordAngle) bool { + m.query.opts.MaxError = maxErr + return true +} +func (m *MinDistanceToCellUnionTarget) maxBruteForceIndexSize() int { return 30 } +func (m *MinDistanceToCellUnionTarget) distance() distance { return m.dist } +*/ + +// ---------------------------------------------------------- + +// MinDistanceToShapeIndexTarget is a type for computing the minimum distance to a ShapeIndex. +type MinDistanceToShapeIndexTarget struct { + index *ShapeIndex + query *EdgeQuery + dist distance +} + +// NewMinDistanceToShapeIndexTarget returns a new target for the given ShapeIndex. +func NewMinDistanceToShapeIndexTarget(index *ShapeIndex) *MinDistanceToShapeIndexTarget { + m := minDistance(0) + return &MinDistanceToShapeIndexTarget{ + index: index, + dist: m, + query: NewClosestEdgeQuery(index, NewClosestEdgeQueryOptions()), + } +} + +func (m *MinDistanceToShapeIndexTarget) capBound() Cap { + // TODO(roberts): Depends on ShapeIndexRegion existing. + // c := makeS2ShapeIndexRegion(m.index).CapBound() + // return CapFromCenterRadius(Point{c.Center.Mul(-1)}, c.Radius()) + panic("not implemented yet") +} + +func (m *MinDistanceToShapeIndexTarget) updateDistanceToPoint(p Point, dist distance) (distance, bool) { + m.query.opts.distanceLimit = dist.chordAngle() + target := NewMinDistanceToPointTarget(p) + r := m.query.findEdge(target, m.query.opts) + if r.shapeID < 0 { + return dist, false + } + return r.distance, true +} + +func (m *MinDistanceToShapeIndexTarget) updateDistanceToEdge(edge Edge, dist distance) (distance, bool) { + m.query.opts.distanceLimit = dist.chordAngle() + target := NewMinDistanceToEdgeTarget(edge) + r := m.query.findEdge(target, m.query.opts) + if r.shapeID < 0 { + return dist, false + } + return r.distance, true +} + +func (m *MinDistanceToShapeIndexTarget) updateDistanceToCell(cell Cell, dist distance) (distance, bool) { + m.query.opts.distanceLimit = dist.chordAngle() + target := NewMinDistanceToCellTarget(cell) + r := m.query.findEdge(target, m.query.opts) + if r.shapeID < 0 { + return dist, false + } + return r.distance, true +} + +// For target types consisting of multiple connected components (such as this one), +// this method should return the polygons containing the antipodal reflection of +// *any* connected component. (It is sufficient to test containment of one vertex per +// connected component, since this allows us to also return any polygon whose +// boundary has distance.zero() to the target.) +func (m *MinDistanceToShapeIndexTarget) visitContainingShapes(index *ShapeIndex, v shapePointVisitorFunc) bool { + // It is sufficient to find the set of chain starts in the target index + // (i.e., one vertex per connected component of edges) that are contained by + // the query index, except for one special case to handle full polygons. + // + // TODO(roberts): Do this by merge-joining the two ShapeIndexes. + for _, shape := range m.index.shapes { + numChains := shape.NumChains() + // Shapes that don't have any edges require a special case (below). + testedPoint := false + for c := 0; c < numChains; c++ { + chain := shape.Chain(c) + if chain.Length == 0 { + continue + } + testedPoint = true + target := NewMinDistanceToPointTarget(shape.ChainEdge(c, 0).V0) + if !target.visitContainingShapes(index, v) { + return false + } + } + if !testedPoint { + // Special case to handle full polygons. + ref := shape.ReferencePoint() + if !ref.Contained { + continue + } + target := NewMinDistanceToPointTarget(ref.Point) + if !target.visitContainingShapes(index, v) { + return false + } + } + } + return true +} + +func (m *MinDistanceToShapeIndexTarget) setMaxError(maxErr s1.ChordAngle) bool { + m.query.opts.maxError = maxErr + return true +} +func (m *MinDistanceToShapeIndexTarget) maxBruteForceIndexSize() int { return 25 } +func (m *MinDistanceToShapeIndexTarget) distance() distance { return m.dist } +func (m *MinDistanceToShapeIndexTarget) setIncludeInteriors(b bool) { + m.query.opts.includeInteriors = b +} +func (m *MinDistanceToShapeIndexTarget) setUseBruteForce(b bool) { m.query.opts.useBruteForce = b } + +// TODO(roberts): Remaining methods +// +// func (m *MinDistanceToShapeIndexTarget) capBound() Cap { +// CellUnionTarget diff --git a/vendor/github.com/blevesearch/geo/s2/nthderivative.go b/vendor/github.com/blevesearch/geo/s2/nthderivative.go new file mode 100644 index 00000000..73445d6c --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/nthderivative.go @@ -0,0 +1,88 @@ +// Copyright 2017 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +// nthDerivativeCoder provides Nth Derivative Coding. +// (In signal processing disciplines, this is known as N-th Delta Coding.) +// +// Good for varint coding integer sequences with polynomial trends. +// +// Instead of coding a sequence of values directly, code its nth-order discrete +// derivative. Overflow in integer addition and subtraction makes this a +// lossless transform. +// +// constant linear quadratic +// trend trend trend +// / \ / \ / \_ +// input |0 0 0 0 1 2 3 4 9 16 25 36 +// 0th derivative(identity) |0 0 0 0 1 2 3 4 9 16 25 36 +// 1st derivative(delta coding) | 0 0 0 1 1 1 1 5 7 9 11 +// 2nd derivative(linear prediction) | 0 0 1 0 0 0 4 2 2 2 +// ------------------------------------- +// 0 1 2 3 4 5 6 7 8 9 10 11 +// n in sequence +// +// Higher-order codings can break even or be detrimental on other sequences. +// +// random oscillating +// / \ / \_ +// input |5 9 6 1 8 8 2 -2 4 -4 6 -6 +// 0th derivative(identity) |5 9 6 1 8 8 2 -2 4 -4 6 -6 +// 1st derivative(delta coding) | 4 -3 -5 7 0 -6 -4 6 -8 10 -12 +// 2nd derivative(linear prediction) | -7 -2 12 -7 -6 2 10 -14 18 -22 +// --------------------------------------- +// 0 1 2 3 4 5 6 7 8 9 10 11 +// n in sequence +// +// Note that the nth derivative isn't available until sequence item n. Earlier +// values are coded at lower order. For the above table, read 5 4 -7 -2 12 ... +type nthDerivativeCoder struct { + n, m int + memory [10]int32 +} + +// newNthDerivativeCoder returns a new coder, where n is the derivative order of the encoder (the N in NthDerivative). +// n must be within [0,10]. +func newNthDerivativeCoder(n int) *nthDerivativeCoder { + c := &nthDerivativeCoder{n: n} + if n < 0 || n > len(c.memory) { + panic("unsupported n. Must be within [0,10].") + } + return c +} + +func (c *nthDerivativeCoder) encode(k int32) int32 { + for i := 0; i < c.m; i++ { + delta := k - c.memory[i] + c.memory[i] = k + k = delta + } + if c.m < c.n { + c.memory[c.m] = k + c.m++ + } + return k +} + +func (c *nthDerivativeCoder) decode(k int32) int32 { + if c.m < c.n { + c.m++ + } + for i := c.m - 1; i >= 0; i-- { + c.memory[i] += k + k = c.memory[i] + } + return k +} diff --git a/vendor/github.com/blevesearch/geo/s2/paddedcell.go b/vendor/github.com/blevesearch/geo/s2/paddedcell.go new file mode 100644 index 00000000..ac304a6c --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/paddedcell.go @@ -0,0 +1,252 @@ +// Copyright 2016 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +import ( + "github.com/golang/geo/r1" + "github.com/golang/geo/r2" +) + +// PaddedCell represents a Cell whose (u,v)-range has been expanded on +// all sides by a given amount of "padding". Unlike Cell, its methods and +// representation are optimized for clipping edges against Cell boundaries +// to determine which cells are intersected by a given set of edges. +type PaddedCell struct { + id CellID + padding float64 + bound r2.Rect + middle r2.Rect // A rect in (u, v)-space that belongs to all four children. + iLo, jLo int // Minimum (i,j)-coordinates of this cell before padding + orientation int // Hilbert curve orientation of this cell. + level int +} + +// PaddedCellFromCellID constructs a padded cell with the given padding. +func PaddedCellFromCellID(id CellID, padding float64) *PaddedCell { + p := &PaddedCell{ + id: id, + padding: padding, + middle: r2.EmptyRect(), + } + + // Fast path for constructing a top-level face (the most common case). + if id.isFace() { + limit := padding + 1 + p.bound = r2.Rect{r1.Interval{-limit, limit}, r1.Interval{-limit, limit}} + p.middle = r2.Rect{r1.Interval{-padding, padding}, r1.Interval{-padding, padding}} + p.orientation = id.Face() & 1 + return p + } + + _, p.iLo, p.jLo, p.orientation = id.faceIJOrientation() + p.level = id.Level() + p.bound = ijLevelToBoundUV(p.iLo, p.jLo, p.level).ExpandedByMargin(padding) + ijSize := sizeIJ(p.level) + p.iLo &= -ijSize + p.jLo &= -ijSize + + return p +} + +// PaddedCellFromParentIJ constructs the child of parent with the given (i,j) index. +// The four child cells have indices of (0,0), (0,1), (1,0), (1,1), where the i and j +// indices correspond to increasing u- and v-values respectively. +func PaddedCellFromParentIJ(parent *PaddedCell, i, j int) *PaddedCell { + // Compute the position and orientation of the child incrementally from the + // orientation of the parent. + pos := ijToPos[parent.orientation][2*i+j] + + p := &PaddedCell{ + id: parent.id.Children()[pos], + padding: parent.padding, + bound: parent.bound, + orientation: parent.orientation ^ posToOrientation[pos], + level: parent.level + 1, + middle: r2.EmptyRect(), + } + + ijSize := sizeIJ(p.level) + p.iLo = parent.iLo + i*ijSize + p.jLo = parent.jLo + j*ijSize + + // For each child, one corner of the bound is taken directly from the parent + // while the diagonally opposite corner is taken from middle(). + middle := parent.Middle() + if i == 1 { + p.bound.X.Lo = middle.X.Lo + } else { + p.bound.X.Hi = middle.X.Hi + } + if j == 1 { + p.bound.Y.Lo = middle.Y.Lo + } else { + p.bound.Y.Hi = middle.Y.Hi + } + + return p +} + +// CellID returns the CellID this padded cell represents. +func (p PaddedCell) CellID() CellID { + return p.id +} + +// Padding returns the amount of padding on this cell. +func (p PaddedCell) Padding() float64 { + return p.padding +} + +// Level returns the level this cell is at. +func (p PaddedCell) Level() int { + return p.level +} + +// Center returns the center of this cell. +func (p PaddedCell) Center() Point { + ijSize := sizeIJ(p.level) + si := uint32(2*p.iLo + ijSize) + ti := uint32(2*p.jLo + ijSize) + return Point{faceSiTiToXYZ(p.id.Face(), si, ti).Normalize()} +} + +// Middle returns the rectangle in the middle of this cell that belongs to +// all four of its children in (u,v)-space. +func (p *PaddedCell) Middle() r2.Rect { + // We compute this field lazily because it is not needed the majority of the + // time (i.e., for cells where the recursion terminates). + if p.middle.IsEmpty() { + ijSize := sizeIJ(p.level) + u := stToUV(siTiToST(uint32(2*p.iLo + ijSize))) + v := stToUV(siTiToST(uint32(2*p.jLo + ijSize))) + p.middle = r2.Rect{ + r1.Interval{u - p.padding, u + p.padding}, + r1.Interval{v - p.padding, v + p.padding}, + } + } + return p.middle +} + +// Bound returns the bounds for this cell in (u,v)-space including padding. +func (p PaddedCell) Bound() r2.Rect { + return p.bound +} + +// ChildIJ returns the (i,j) coordinates for the child cell at the given traversal +// position. The traversal position corresponds to the order in which child +// cells are visited by the Hilbert curve. +func (p PaddedCell) ChildIJ(pos int) (i, j int) { + ij := posToIJ[p.orientation][pos] + return ij >> 1, ij & 1 +} + +// EntryVertex return the vertex where the space-filling curve enters this cell. +func (p PaddedCell) EntryVertex() Point { + // The curve enters at the (0,0) vertex unless the axis directions are + // reversed, in which case it enters at the (1,1) vertex. + i := p.iLo + j := p.jLo + if p.orientation&invertMask != 0 { + ijSize := sizeIJ(p.level) + i += ijSize + j += ijSize + } + return Point{faceSiTiToXYZ(p.id.Face(), uint32(2*i), uint32(2*j)).Normalize()} +} + +// ExitVertex returns the vertex where the space-filling curve exits this cell. +func (p PaddedCell) ExitVertex() Point { + // The curve exits at the (1,0) vertex unless the axes are swapped or + // inverted but not both, in which case it exits at the (0,1) vertex. + i := p.iLo + j := p.jLo + ijSize := sizeIJ(p.level) + if p.orientation == 0 || p.orientation == swapMask+invertMask { + i += ijSize + } else { + j += ijSize + } + return Point{faceSiTiToXYZ(p.id.Face(), uint32(2*i), uint32(2*j)).Normalize()} +} + +// ShrinkToFit returns the smallest CellID that contains all descendants of this +// padded cell whose bounds intersect the given rect. For algorithms that use +// recursive subdivision to find the cells that intersect a particular object, this +// method can be used to skip all of the initial subdivision steps where only +// one child needs to be expanded. +// +// Note that this method is not the same as returning the smallest cell that contains +// the intersection of this cell with rect. Because of the padding, even if one child +// completely contains rect it is still possible that a neighboring child may also +// intersect the given rect. +// +// The provided Rect must intersect the bounds of this cell. +func (p *PaddedCell) ShrinkToFit(rect r2.Rect) CellID { + // Quick rejection test: if rect contains the center of this cell along + // either axis, then no further shrinking is possible. + if p.level == 0 { + // Fast path (most calls to this function start with a face cell). + if rect.X.Contains(0) || rect.Y.Contains(0) { + return p.id + } + } + + ijSize := sizeIJ(p.level) + if rect.X.Contains(stToUV(siTiToST(uint32(2*p.iLo+ijSize)))) || + rect.Y.Contains(stToUV(siTiToST(uint32(2*p.jLo+ijSize)))) { + return p.id + } + + // Otherwise we expand rect by the given padding on all sides and find + // the range of coordinates that it spans along the i- and j-axes. We then + // compute the highest bit position at which the min and max coordinates + // differ. This corresponds to the first cell level at which at least two + // children intersect rect. + + // Increase the padding to compensate for the error in uvToST. + // (The constant below is a provable upper bound on the additional error.) + padded := rect.ExpandedByMargin(p.padding + 1.5*dblEpsilon) + iMin, jMin := p.iLo, p.jLo // Min i- or j- coordinate spanned by padded + var iXor, jXor int // XOR of the min and max i- or j-coordinates + + if iMin < stToIJ(uvToST(padded.X.Lo)) { + iMin = stToIJ(uvToST(padded.X.Lo)) + } + if a, b := p.iLo+ijSize-1, stToIJ(uvToST(padded.X.Hi)); a <= b { + iXor = iMin ^ a + } else { + iXor = iMin ^ b + } + + if jMin < stToIJ(uvToST(padded.Y.Lo)) { + jMin = stToIJ(uvToST(padded.Y.Lo)) + } + if a, b := p.jLo+ijSize-1, stToIJ(uvToST(padded.Y.Hi)); a <= b { + jXor = jMin ^ a + } else { + jXor = jMin ^ b + } + + // Compute the highest bit position where the two i- or j-endpoints differ, + // and then choose the cell level that includes both of these endpoints. So + // if both pairs of endpoints are equal we choose maxLevel; if they differ + // only at bit 0, we choose (maxLevel - 1), and so on. + levelMSB := uint64(((iXor | jXor) << 1) + 1) + level := maxLevel - findMSBSetNonZero64(levelMSB) + if level <= p.level { + return p.id + } + + return cellIDFromFaceIJ(p.id.Face(), iMin, jMin).Parent(level) +} diff --git a/vendor/github.com/blevesearch/geo/s2/point.go b/vendor/github.com/blevesearch/geo/s2/point.go new file mode 100644 index 00000000..89e7ae0e --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/point.go @@ -0,0 +1,258 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +import ( + "fmt" + "io" + "math" + "sort" + + "github.com/golang/geo/r3" + "github.com/golang/geo/s1" +) + +// Point represents a point on the unit sphere as a normalized 3D vector. +// Fields should be treated as read-only. Use one of the factory methods for creation. +type Point struct { + r3.Vector +} + +// sortPoints sorts the slice of Points in place. +func sortPoints(e []Point) { + sort.Sort(points(e)) +} + +// points implements the Sort interface for slices of Point. +type points []Point + +func (p points) Len() int { return len(p) } +func (p points) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p points) Less(i, j int) bool { return p[i].Cmp(p[j].Vector) == -1 } + +// PointFromCoords creates a new normalized point from coordinates. +// +// This always returns a valid point. If the given coordinates can not be normalized +// the origin point will be returned. +// +// This behavior is different from the C++ construction of a S2Point from coordinates +// (i.e. S2Point(x, y, z)) in that in C++ they do not Normalize. +func PointFromCoords(x, y, z float64) Point { + if x == 0 && y == 0 && z == 0 { + return OriginPoint() + } + return Point{r3.Vector{x, y, z}.Normalize()} +} + +// OriginPoint returns a unique "origin" on the sphere for operations that need a fixed +// reference point. In particular, this is the "point at infinity" used for +// point-in-polygon testing (by counting the number of edge crossings). +// +// It should *not* be a point that is commonly used in edge tests in order +// to avoid triggering code to handle degenerate cases (this rules out the +// north and south poles). It should also not be on the boundary of any +// low-level S2Cell for the same reason. +func OriginPoint() Point { + return Point{r3.Vector{-0.0099994664350250197, 0.0025924542609324121, 0.99994664350250195}} +} + +// PointCross returns a Point that is orthogonal to both p and op. This is similar to +// p.Cross(op) (the true cross product) except that it does a better job of +// ensuring orthogonality when the Point is nearly parallel to op, it returns +// a non-zero result even when p == op or p == -op and the result is a Point. +// +// It satisfies the following properties (f == PointCross): +// +// (1) f(p, op) != 0 for all p, op +// (2) f(op,p) == -f(p,op) unless p == op or p == -op +// (3) f(-p,op) == -f(p,op) unless p == op or p == -op +// (4) f(p,-op) == -f(p,op) unless p == op or p == -op +func (p Point) PointCross(op Point) Point { + // NOTE(dnadasi): In the C++ API the equivalent method here was known as "RobustCrossProd", + // but PointCross more accurately describes how this method is used. + x := p.Add(op.Vector).Cross(op.Sub(p.Vector)) + + // Compare exactly to the 0 vector. + if x == (r3.Vector{}) { + // The only result that makes sense mathematically is to return zero, but + // we find it more convenient to return an arbitrary orthogonal vector. + return Point{p.Ortho()} + } + + return Point{x} +} + +// OrderedCCW returns true if the edges OA, OB, and OC are encountered in that +// order while sweeping CCW around the point O. +// +// You can think of this as testing whether A <= B <= C with respect to the +// CCW ordering around O that starts at A, or equivalently, whether B is +// contained in the range of angles (inclusive) that starts at A and extends +// CCW to C. Properties: +// +// (1) If OrderedCCW(a,b,c,o) && OrderedCCW(b,a,c,o), then a == b +// (2) If OrderedCCW(a,b,c,o) && OrderedCCW(a,c,b,o), then b == c +// (3) If OrderedCCW(a,b,c,o) && OrderedCCW(c,b,a,o), then a == b == c +// (4) If a == b or b == c, then OrderedCCW(a,b,c,o) is true +// (5) Otherwise if a == c, then OrderedCCW(a,b,c,o) is false +func OrderedCCW(a, b, c, o Point) bool { + sum := 0 + if RobustSign(b, o, a) != Clockwise { + sum++ + } + if RobustSign(c, o, b) != Clockwise { + sum++ + } + if RobustSign(a, o, c) == CounterClockwise { + sum++ + } + return sum >= 2 +} + +// Distance returns the angle between two points. +func (p Point) Distance(b Point) s1.Angle { + return p.Vector.Angle(b.Vector) +} + +// ApproxEqual reports whether the two points are similar enough to be equal. +func (p Point) ApproxEqual(other Point) bool { + return p.approxEqual(other, s1.Angle(epsilon)) +} + +// approxEqual reports whether the two points are within the given epsilon. +func (p Point) approxEqual(other Point, eps s1.Angle) bool { + return p.Vector.Angle(other.Vector) <= eps +} + +// ChordAngleBetweenPoints constructs a ChordAngle corresponding to the distance +// between the two given points. The points must be unit length. +func ChordAngleBetweenPoints(x, y Point) s1.ChordAngle { + return s1.ChordAngle(math.Min(4.0, x.Sub(y.Vector).Norm2())) +} + +// regularPoints generates a slice of points shaped as a regular polygon with +// the numVertices vertices, all located on a circle of the specified angular radius +// around the center. The radius is the actual distance from center to each vertex. +func regularPoints(center Point, radius s1.Angle, numVertices int) []Point { + return regularPointsForFrame(getFrame(center), radius, numVertices) +} + +// regularPointsForFrame generates a slice of points shaped as a regular polygon +// with numVertices vertices, all on a circle of the specified angular radius around +// the center. The radius is the actual distance from the center to each vertex. +func regularPointsForFrame(frame matrix3x3, radius s1.Angle, numVertices int) []Point { + // We construct the loop in the given frame coordinates, with the center at + // (0, 0, 1). For a loop of radius r, the loop vertices have the form + // (x, y, z) where x^2 + y^2 = sin(r) and z = cos(r). The distance on the + // sphere (arc length) from each vertex to the center is acos(cos(r)) = r. + z := math.Cos(radius.Radians()) + r := math.Sin(radius.Radians()) + radianStep := 2 * math.Pi / float64(numVertices) + var vertices []Point + + for i := 0; i < numVertices; i++ { + angle := float64(i) * radianStep + p := Point{r3.Vector{r * math.Cos(angle), r * math.Sin(angle), z}} + vertices = append(vertices, Point{fromFrame(frame, p).Normalize()}) + } + + return vertices +} + +// CapBound returns a bounding cap for this point. +func (p Point) CapBound() Cap { + return CapFromPoint(p) +} + +// RectBound returns a bounding latitude-longitude rectangle from this point. +func (p Point) RectBound() Rect { + return RectFromLatLng(LatLngFromPoint(p)) +} + +// ContainsCell returns false as Points do not contain any other S2 types. +func (p Point) ContainsCell(c Cell) bool { return false } + +// IntersectsCell reports whether this Point intersects the given cell. +func (p Point) IntersectsCell(c Cell) bool { + return c.ContainsPoint(p) +} + +// ContainsPoint reports if this Point contains the other Point. +// (This method is named to satisfy the Region interface.) +func (p Point) ContainsPoint(other Point) bool { + return p.Contains(other) +} + +// CellUnionBound computes a covering of the Point. +func (p Point) CellUnionBound() []CellID { + return p.CapBound().CellUnionBound() +} + +// Contains reports if this Point contains the other Point. +// (This method matches all other s2 types where the reflexive Contains +// method does not contain the type's name.) +func (p Point) Contains(other Point) bool { return p == other } + +// Encode encodes the Point. +func (p Point) Encode(w io.Writer) error { + e := &encoder{w: w} + p.encode(e) + return e.err +} + +func (p Point) encode(e *encoder) { + e.writeInt8(encodingVersion) + e.writeFloat64(p.X) + e.writeFloat64(p.Y) + e.writeFloat64(p.Z) +} + +// Decode decodes the Point. +func (p *Point) Decode(r io.Reader) error { + d := &decoder{r: asByteReader(r)} + p.decode(d) + return d.err +} + +func (p *Point) decode(d *decoder) { + version := d.readInt8() + if d.err != nil { + return + } + if version != encodingVersion { + d.err = fmt.Errorf("only version %d is supported", encodingVersion) + return + } + p.X = d.readFloat64() + p.Y = d.readFloat64() + p.Z = d.readFloat64() +} + +// Rotate the given point about the given axis by the given angle. p and +// axis must be unit length; angle has no restrictions (e.g., it can be +// positive, negative, greater than 360 degrees, etc). +func Rotate(p, axis Point, angle s1.Angle) Point { + // Let M be the plane through P that is perpendicular to axis, and let + // center be the point where M intersects axis. We construct a + // right-handed orthogonal frame (dx, dy, center) such that dx is the + // vector from center to P, and dy has the same length as dx. The + // result can then be expressed as (cos(angle)*dx + sin(angle)*dy + center). + center := axis.Mul(p.Dot(axis.Vector)) + dx := p.Sub(center) + dy := axis.Cross(p.Vector) + // Mathematically the result is unit length, but normalization is necessary + // to ensure that numerical errors don't accumulate. + return Point{dx.Mul(math.Cos(angle.Radians())).Add(dy.Mul(math.Sin(angle.Radians()))).Add(center).Normalize()} +} diff --git a/vendor/github.com/blevesearch/geo/s2/point_measures.go b/vendor/github.com/blevesearch/geo/s2/point_measures.go new file mode 100644 index 00000000..6fa9b7ae --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/point_measures.go @@ -0,0 +1,149 @@ +// Copyright 2018 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +import ( + "math" + + "github.com/golang/geo/s1" +) + +// PointArea returns the area of triangle ABC. This method combines two different +// algorithms to get accurate results for both large and small triangles. +// The maximum error is about 5e-15 (about 0.25 square meters on the Earth's +// surface), the same as GirardArea below, but unlike that method it is +// also accurate for small triangles. Example: when the true area is 100 +// square meters, PointArea yields an error about 1 trillion times smaller than +// GirardArea. +// +// All points should be unit length, and no two points should be antipodal. +// The area is always positive. +func PointArea(a, b, c Point) float64 { + // This method is based on l'Huilier's theorem, + // + // tan(E/4) = sqrt(tan(s/2) tan((s-a)/2) tan((s-b)/2) tan((s-c)/2)) + // + // where E is the spherical excess of the triangle (i.e. its area), + // a, b, c are the side lengths, and + // s is the semiperimeter (a + b + c) / 2. + // + // The only significant source of error using l'Huilier's method is the + // cancellation error of the terms (s-a), (s-b), (s-c). This leads to a + // *relative* error of about 1e-16 * s / min(s-a, s-b, s-c). This compares + // to a relative error of about 1e-15 / E using Girard's formula, where E is + // the true area of the triangle. Girard's formula can be even worse than + // this for very small triangles, e.g. a triangle with a true area of 1e-30 + // might evaluate to 1e-5. + // + // So, we prefer l'Huilier's formula unless dmin < s * (0.1 * E), where + // dmin = min(s-a, s-b, s-c). This basically includes all triangles + // except for extremely long and skinny ones. + // + // Since we don't know E, we would like a conservative upper bound on + // the triangle area in terms of s and dmin. It's possible to show that + // E <= k1 * s * sqrt(s * dmin), where k1 = 2*sqrt(3)/Pi (about 1). + // Using this, it's easy to show that we should always use l'Huilier's + // method if dmin >= k2 * s^5, where k2 is about 1e-2. Furthermore, + // if dmin < k2 * s^5, the triangle area is at most k3 * s^4, where + // k3 is about 0.1. Since the best case error using Girard's formula + // is about 1e-15, this means that we shouldn't even consider it unless + // s >= 3e-4 or so. + sa := float64(b.Angle(c.Vector)) + sb := float64(c.Angle(a.Vector)) + sc := float64(a.Angle(b.Vector)) + s := 0.5 * (sa + sb + sc) + if s >= 3e-4 { + // Consider whether Girard's formula might be more accurate. + dmin := s - math.Max(sa, math.Max(sb, sc)) + if dmin < 1e-2*s*s*s*s*s { + // This triangle is skinny enough to use Girard's formula. + area := GirardArea(a, b, c) + if dmin < s*0.1*area { + return area + } + } + } + + // Use l'Huilier's formula. + return 4 * math.Atan(math.Sqrt(math.Max(0.0, math.Tan(0.5*s)*math.Tan(0.5*(s-sa))* + math.Tan(0.5*(s-sb))*math.Tan(0.5*(s-sc))))) +} + +// GirardArea returns the area of the triangle computed using Girard's formula. +// All points should be unit length, and no two points should be antipodal. +// +// This method is about twice as fast as PointArea() but has poor relative +// accuracy for small triangles. The maximum error is about 5e-15 (about +// 0.25 square meters on the Earth's surface) and the average error is about +// 1e-15. These bounds apply to triangles of any size, even as the maximum +// edge length of the triangle approaches 180 degrees. But note that for +// such triangles, tiny perturbations of the input points can change the +// true mathematical area dramatically. +func GirardArea(a, b, c Point) float64 { + // This is equivalent to the usual Girard's formula but is slightly more + // accurate, faster to compute, and handles a == b == c without a special + // case. PointCross is necessary to get good accuracy when two of + // the input points are very close together. + ab := a.PointCross(b) + bc := b.PointCross(c) + ac := a.PointCross(c) + + area := float64(ab.Angle(ac.Vector) - ab.Angle(bc.Vector) + bc.Angle(ac.Vector)) + if area < 0 { + area = 0 + } + return area +} + +// SignedArea returns a positive value for counterclockwise triangles and a negative +// value otherwise (similar to PointArea). +func SignedArea(a, b, c Point) float64 { + return float64(RobustSign(a, b, c)) * PointArea(a, b, c) +} + +// Angle returns the interior angle at the vertex B in the triangle ABC. The +// return value is always in the range [0, pi]. All points should be +// normalized. Ensures that Angle(a,b,c) == Angle(c,b,a) for all a,b,c. +// +// The angle is undefined if A or C is diametrically opposite from B, and +// becomes numerically unstable as the length of edge AB or BC approaches +// 180 degrees. +func Angle(a, b, c Point) s1.Angle { + // PointCross is necessary to get good accuracy when two of the input + // points are very close together. + return a.PointCross(b).Angle(c.PointCross(b).Vector) +} + +// TurnAngle returns the exterior angle at vertex B in the triangle ABC. The +// return value is positive if ABC is counterclockwise and negative otherwise. +// If you imagine an ant walking from A to B to C, this is the angle that the +// ant turns at vertex B (positive = left = CCW, negative = right = CW). +// This quantity is also known as the "geodesic curvature" at B. +// +// Ensures that TurnAngle(a,b,c) == -TurnAngle(c,b,a) for all distinct +// a,b,c. The result is undefined if (a == b || b == c), but is either +// -Pi or Pi if (a == c). All points should be normalized. +func TurnAngle(a, b, c Point) s1.Angle { + // We use PointCross to get good accuracy when two points are very + // close together, and RobustSign to ensure that the sign is correct for + // turns that are close to 180 degrees. + angle := a.PointCross(b).Angle(b.PointCross(c).Vector) + + // Don't return RobustSign * angle because it is legal to have (a == c). + if RobustSign(a, b, c) == CounterClockwise { + return angle + } + return -angle +} diff --git a/vendor/github.com/blevesearch/geo/s2/point_vector.go b/vendor/github.com/blevesearch/geo/s2/point_vector.go new file mode 100644 index 00000000..f8e6f65b --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/point_vector.go @@ -0,0 +1,42 @@ +// Copyright 2017 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +// Shape interface enforcement +var ( + _ Shape = (*PointVector)(nil) +) + +// PointVector is a Shape representing a set of Points. Each point +// is represented as a degenerate edge with the same starting and ending +// vertices. +// +// This type is useful for adding a collection of points to an ShapeIndex. +// +// Its methods are on *PointVector due to implementation details of ShapeIndex. +type PointVector []Point + +func (p *PointVector) NumEdges() int { return len(*p) } +func (p *PointVector) Edge(i int) Edge { return Edge{(*p)[i], (*p)[i]} } +func (p *PointVector) ReferencePoint() ReferencePoint { return OriginReferencePoint(false) } +func (p *PointVector) NumChains() int { return len(*p) } +func (p *PointVector) Chain(i int) Chain { return Chain{i, 1} } +func (p *PointVector) ChainEdge(i, j int) Edge { return Edge{(*p)[i], (*p)[j]} } +func (p *PointVector) ChainPosition(e int) ChainPosition { return ChainPosition{e, 0} } +func (p *PointVector) Dimension() int { return 0 } +func (p *PointVector) IsEmpty() bool { return defaultShapeIsEmpty(p) } +func (p *PointVector) IsFull() bool { return defaultShapeIsFull(p) } +func (p *PointVector) typeTag() typeTag { return typeTagPointVector } +func (p *PointVector) privateInterface() {} diff --git a/vendor/github.com/blevesearch/geo/s2/pointcompression.go b/vendor/github.com/blevesearch/geo/s2/pointcompression.go new file mode 100644 index 00000000..01838179 --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/pointcompression.go @@ -0,0 +1,319 @@ +// Copyright 2017 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +import ( + "errors" + "fmt" + + "github.com/golang/geo/r3" +) + +// maxEncodedVertices is the maximum number of vertices, in a row, to be encoded or decoded. +// On decode, this defends against malicious encodings that try and have us exceed RAM. +const maxEncodedVertices = 50000000 + +// xyzFaceSiTi represents the The XYZ and face,si,ti coordinates of a Point +// and, if this point is equal to the center of a Cell, the level of this cell +// (-1 otherwise). This is used for Loops and Polygons to store data in a more +// compressed format. +type xyzFaceSiTi struct { + xyz Point + face int + si, ti uint32 + level int +} + +const derivativeEncodingOrder = 2 + +func appendFace(faces []faceRun, face int) []faceRun { + if len(faces) == 0 || faces[len(faces)-1].face != face { + return append(faces, faceRun{face, 1}) + } + faces[len(faces)-1].count++ + return faces +} + +// encodePointsCompressed uses an optimized compressed format to encode the given values. +func encodePointsCompressed(e *encoder, vertices []xyzFaceSiTi, level int) { + var faces []faceRun + for _, v := range vertices { + faces = appendFace(faces, v.face) + } + encodeFaces(e, faces) + + type piQi struct { + pi, qi uint32 + } + verticesPiQi := make([]piQi, len(vertices)) + for i, v := range vertices { + verticesPiQi[i] = piQi{siTitoPiQi(v.si, level), siTitoPiQi(v.ti, level)} + } + piCoder, qiCoder := newNthDerivativeCoder(derivativeEncodingOrder), newNthDerivativeCoder(derivativeEncodingOrder) + for i, v := range verticesPiQi { + f := encodePointCompressed + if i == 0 { + // The first point will be just the (pi, qi) coordinates + // of the Point. NthDerivativeCoder will not save anything + // in that case, so we encode in fixed format rather than varint + // to avoid the varint overhead. + f = encodeFirstPointFixedLength + } + f(e, v.pi, v.qi, level, piCoder, qiCoder) + } + + var offCenter []int + for i, v := range vertices { + if v.level != level { + offCenter = append(offCenter, i) + } + } + e.writeUvarint(uint64(len(offCenter))) + for _, idx := range offCenter { + e.writeUvarint(uint64(idx)) + e.writeFloat64(vertices[idx].xyz.X) + e.writeFloat64(vertices[idx].xyz.Y) + e.writeFloat64(vertices[idx].xyz.Z) + } +} + +func encodeFirstPointFixedLength(e *encoder, pi, qi uint32, level int, piCoder, qiCoder *nthDerivativeCoder) { + // Do not ZigZagEncode the first point, since it cannot be negative. + codedPi, codedQi := piCoder.encode(int32(pi)), qiCoder.encode(int32(qi)) + // Interleave to reduce overhead from two partial bytes to one. + interleaved := interleaveUint32(uint32(codedPi), uint32(codedQi)) + + // Write as little endian. + bytesRequired := (level + 7) / 8 * 2 + for i := 0; i < bytesRequired; i++ { + e.writeUint8(uint8(interleaved)) + interleaved >>= 8 + } +} + +// encodePointCompressed encodes points into e. +// Given a sequence of Points assumed to be the center of level-k cells, +// compresses it into a stream using the following method: +// - decompose the points into (face, si, ti) tuples. +// - run-length encode the faces, combining face number and count into a +// varint32. See the faceRun struct. +// - right shift the (si, ti) to remove the part that's constant for all cells +// of level-k. The result is called the (pi, qi) space. +// - 2nd derivative encode the pi and qi sequences (linear prediction) +// - zig-zag encode all derivative values but the first, which cannot be +// negative +// - interleave the zig-zag encoded values +// - encode the first interleaved value in a fixed length encoding +// (varint would make this value larger) +// - encode the remaining interleaved values as varint64s, as the +// derivative encoding should make the values small. +// In addition, provides a lossless method to compress a sequence of points even +// if some points are not the center of level-k cells. These points are stored +// exactly, using 3 double precision values, after the above encoded string, +// together with their index in the sequence (this leads to some redundancy - it +// is expected that only a small fraction of the points are not cell centers). +// +// To encode leaf cells, this requires 8 bytes for the first vertex plus +// an average of 3.8 bytes for each additional vertex, when computed on +// Google's geographic repository. +func encodePointCompressed(e *encoder, pi, qi uint32, level int, piCoder, qiCoder *nthDerivativeCoder) { + // ZigZagEncode, as varint requires the maximum number of bytes for + // negative numbers. + zzPi := zigzagEncode(piCoder.encode(int32(pi))) + zzQi := zigzagEncode(qiCoder.encode(int32(qi))) + // Interleave to reduce overhead from two partial bytes to one. + interleaved := interleaveUint32(zzPi, zzQi) + e.writeUvarint(interleaved) +} + +type faceRun struct { + face, count int +} + +func decodeFaceRun(d *decoder) faceRun { + faceAndCount := d.readUvarint() + ret := faceRun{ + face: int(faceAndCount % numFaces), + count: int(faceAndCount / numFaces), + } + if ret.count <= 0 && d.err == nil { + d.err = errors.New("non-positive count for face run") + } + return ret +} + +func decodeFaces(numVertices int, d *decoder) []faceRun { + var frs []faceRun + for nparsed := 0; nparsed < numVertices; { + fr := decodeFaceRun(d) + if d.err != nil { + return nil + } + frs = append(frs, fr) + nparsed += fr.count + } + return frs +} + +// encodeFaceRun encodes each faceRun as a varint64 with value numFaces * count + face. +func encodeFaceRun(e *encoder, fr faceRun) { + // It isn't necessary to encode the number of faces left for the last run, + // but since this would only help if there were more than 21 faces, it will + // be a small overall savings, much smaller than the bound encoding. + coded := numFaces*uint64(fr.count) + uint64(fr.face) + e.writeUvarint(coded) +} + +func encodeFaces(e *encoder, frs []faceRun) { + for _, fr := range frs { + encodeFaceRun(e, fr) + } +} + +type facesIterator struct { + faces []faceRun + // How often have we yet shown the current face? + numCurrentFaceShown int + curFace int +} + +func (fi *facesIterator) next() (ok bool) { + if len(fi.faces) == 0 { + return false + } + fi.curFace = fi.faces[0].face + fi.numCurrentFaceShown++ + + // Advance fs if needed. + if fi.faces[0].count <= fi.numCurrentFaceShown { + fi.faces = fi.faces[1:] + fi.numCurrentFaceShown = 0 + } + + return true +} + +func decodePointsCompressed(d *decoder, level int, target []Point) { + faces := decodeFaces(len(target), d) + + piCoder := newNthDerivativeCoder(derivativeEncodingOrder) + qiCoder := newNthDerivativeCoder(derivativeEncodingOrder) + + iter := facesIterator{faces: faces} + for i := range target { + decodeFn := decodePointCompressed + if i == 0 { + decodeFn = decodeFirstPointFixedLength + } + pi, qi := decodeFn(d, level, piCoder, qiCoder) + if ok := iter.next(); !ok && d.err == nil { + d.err = fmt.Errorf("ran out of faces at target %d", i) + return + } + target[i] = Point{facePiQitoXYZ(iter.curFace, pi, qi, level)} + } + + numOffCenter := int(d.readUvarint()) + if d.err != nil { + return + } + if numOffCenter > len(target) { + d.err = fmt.Errorf("numOffCenter = %d, should be at most len(target) = %d", numOffCenter, len(target)) + return + } + for i := 0; i < numOffCenter; i++ { + idx := int(d.readUvarint()) + if d.err != nil { + return + } + if idx >= len(target) { + d.err = fmt.Errorf("off center index = %d, should be < len(target) = %d", idx, len(target)) + return + } + target[idx].X = d.readFloat64() + target[idx].Y = d.readFloat64() + target[idx].Z = d.readFloat64() + } +} + +func decodeFirstPointFixedLength(d *decoder, level int, piCoder, qiCoder *nthDerivativeCoder) (pi, qi uint32) { + bytesToRead := (level + 7) / 8 * 2 + var interleaved uint64 + for i := 0; i < bytesToRead; i++ { + rr := d.readUint8() + interleaved |= (uint64(rr) << uint(i*8)) + } + + piCoded, qiCoded := deinterleaveUint32(interleaved) + + return uint32(piCoder.decode(int32(piCoded))), uint32(qiCoder.decode(int32(qiCoded))) +} + +func zigzagEncode(x int32) uint32 { + return (uint32(x) << 1) ^ uint32(x>>31) +} + +func zigzagDecode(x uint32) int32 { + return int32((x >> 1) ^ uint32((int32(x&1)<<31)>>31)) +} + +func decodePointCompressed(d *decoder, level int, piCoder, qiCoder *nthDerivativeCoder) (pi, qi uint32) { + interleavedZigZagEncodedDerivPiQi := d.readUvarint() + piZigzag, qiZigzag := deinterleaveUint32(interleavedZigZagEncodedDerivPiQi) + return uint32(piCoder.decode(zigzagDecode(piZigzag))), uint32(qiCoder.decode(zigzagDecode(qiZigzag))) +} + +// We introduce a new coordinate system (pi, qi), which is (si, ti) +// with the bits that are constant for cells of that level shifted +// off to the right. +// si = round(s * 2^31) +// pi = si >> (31 - level) +// = floor(s * 2^level) +// If the point has been snapped to the level, the bits that are +// shifted off will be a 1 in the msb, then 0s after that, so the +// fractional part discarded by the cast is (close to) 0.5. + +// stToPiQi returns the value transformed to the PiQi coordinate space. +func stToPiQi(s float64, level uint) uint32 { + return uint32(s * float64(int(1)< max { + s = max + } + + return uint32(s >> (maxLevel + 1 - uint(level))) +} + +// piQiToST returns the value transformed to ST space. +func piQiToST(pi uint32, level int) float64 { + // We want to recover the position at the center of the cell. If the point + // was snapped to the center of the cell, then math.Modf(s * 2^level) == 0.5. + // Inverting STtoPiQi gives: + // s = (pi + 0.5) / 2^level. + return (float64(pi) + 0.5) / float64(int(1)< l.turningAngleMaxError() { + // Normalize the loop. + if angle < 0 { + l.Invert() + } + } else { + // Ensure that the loop does not contain the origin. + if l.ContainsOrigin() { + l.Invert() + } + } + } + + p := PolygonFromLoops(loops) + + if p.NumLoops() > 0 { + originLoop := p.Loop(0) + polygonContainsOrigin := false + for _, l := range p.Loops() { + if l.ContainsOrigin() { + polygonContainsOrigin = !polygonContainsOrigin + + originLoop = l + } + } + if containedOrigin[originLoop] != polygonContainsOrigin { + p.Invert() + } + } + + return p +} + +// Invert inverts the polygon (replaces it by its complement). +func (p *Polygon) Invert() { + // Inverting any one loop will invert the polygon. The best loop to invert + // is the one whose area is largest, since this yields the smallest area + // after inversion. The loop with the largest area is always at depth 0. + // The descendents of this loop all have their depth reduced by 1, while the + // former siblings of this loop all have their depth increased by 1. + + // The empty and full polygons are handled specially. + if p.IsEmpty() { + *p = *FullPolygon() + p.initLoopProperties() + return + } + if p.IsFull() { + *p = Polygon{} + p.initLoopProperties() + return + } + + // Find the loop whose area is largest (i.e., whose turning angle is + // smallest), minimizing calls to TurningAngle(). In particular, for + // polygons with a single shell at level 0 there is no need to call + // TurningAngle() at all. (This method is relatively expensive.) + best := 0 + const none = 10.0 // Flag that means "not computed yet" + bestAngle := none + for i := 1; i < p.NumLoops(); i++ { + if p.Loop(i).depth != 0 { + continue + } + // We defer computing the turning angle of loop 0 until we discover + // that the polygon has another top-level shell. + if bestAngle == none { + bestAngle = p.Loop(best).TurningAngle() + } + angle := p.Loop(i).TurningAngle() + // We break ties deterministically in order to avoid having the output + // depend on the input order of the loops. + if angle < bestAngle || (angle == bestAngle && compareLoops(p.Loop(i), p.Loop(best)) < 0) { + best = i + bestAngle = angle + } + } + // Build the new loops vector, starting with the inverted loop. + p.Loop(best).Invert() + newLoops := make([]*Loop, 0, p.NumLoops()) + // Add the former siblings of this loop as descendants. + lastBest := p.LastDescendant(best) + newLoops = append(newLoops, p.Loop(best)) + for i, l := range p.Loops() { + if i < best || i > lastBest { + l.depth++ + newLoops = append(newLoops, l) + } + } + // Add the former children of this loop as siblings. + for i, l := range p.Loops() { + if i > best && i <= lastBest { + l.depth-- + newLoops = append(newLoops, l) + } + } + + p.loops = newLoops + p.initLoopProperties() +} + +// Defines a total ordering on Loops that does not depend on the cyclic +// order of loop vertices. This function is used to choose which loop to +// invert in the case where several loops have exactly the same area. +func compareLoops(a, b *Loop) int { + if na, nb := a.NumVertices(), b.NumVertices(); na != nb { + return na - nb + } + ai, aDir := a.CanonicalFirstVertex() + bi, bDir := b.CanonicalFirstVertex() + if aDir != bDir { + return aDir - bDir + } + for n := a.NumVertices() - 1; n >= 0; n, ai, bi = n-1, ai+aDir, bi+bDir { + if cmp := a.Vertex(ai).Cmp(b.Vertex(bi).Vector); cmp != 0 { + return cmp + } + } + return 0 +} + +// PolygonFromCell returns a Polygon from a single loop created from the given Cell. +func PolygonFromCell(cell Cell) *Polygon { + return PolygonFromLoops([]*Loop{LoopFromCell(cell)}) +} + +// initNested takes the set of loops in this polygon and performs the nesting +// computations to set the proper nesting and parent/child relationships. +func (p *Polygon) initNested() { + if len(p.loops) == 1 { + p.initOneLoop() + return + } + + lm := make(loopMap) + + for _, l := range p.loops { + lm.insertLoop(l, nil) + } + // The loops have all been added to the loopMap for ordering. Clear the + // loops slice because we add all the loops in-order in initLoops. + p.loops = nil + + // Reorder the loops in depth-first traversal order. + p.initLoops(lm) + p.initLoopProperties() +} + +// loopMap is a map of a loop to its immediate children with respect to nesting. +// It is used to determine which loops are shells and which are holes. +type loopMap map[*Loop][]*Loop + +// insertLoop adds the given loop to the loop map under the specified parent. +// All children of the new entry are checked to see if the need to move up to +// a different level. +func (lm loopMap) insertLoop(newLoop, parent *Loop) { + var children []*Loop + for done := false; !done; { + children = lm[parent] + done = true + for _, child := range children { + if child.ContainsNested(newLoop) { + parent = child + done = false + break + } + } + } + + // Now, we have found a parent for this loop, it may be that some of the + // children of the parent of this loop may now be children of the new loop. + newChildren := lm[newLoop] + for i := 0; i < len(children); { + child := children[i] + if newLoop.ContainsNested(child) { + newChildren = append(newChildren, child) + children = append(children[0:i], children[i+1:]...) + } else { + i++ + } + } + + lm[newLoop] = newChildren + lm[parent] = append(children, newLoop) +} + +// loopStack simplifies access to the loops while being initialized. +type loopStack []*Loop + +func (s *loopStack) push(v *Loop) { + *s = append(*s, v) +} +func (s *loopStack) pop() *Loop { + l := len(*s) + r := (*s)[l-1] + *s = (*s)[:l-1] + return r +} + +// initLoops walks the mapping of loops to all of their children, and adds them in +// order into to the polygons set of loops. +func (p *Polygon) initLoops(lm loopMap) { + var stack loopStack + stack.push(nil) + depth := -1 + + for len(stack) > 0 { + loop := stack.pop() + if loop != nil { + depth = loop.depth + p.loops = append(p.loops, loop) + } + children := lm[loop] + for i := len(children) - 1; i >= 0; i-- { + child := children[i] + child.depth = depth + 1 + stack.push(child) + } + } +} + +// initOneLoop set the properties for a polygon made of a single loop. +// TODO(roberts): Can this be merged with initLoopProperties +func (p *Polygon) initOneLoop() { + p.hasHoles = false + p.numVertices = len(p.loops[0].vertices) + p.bound = p.loops[0].RectBound() + p.subregionBound = ExpandForSubregions(p.bound) + // Ensure the loops depth is set correctly. + p.loops[0].depth = 0 + + p.initEdgesAndIndex() +} + +// initLoopProperties sets the properties for polygons with multiple loops. +func (p *Polygon) initLoopProperties() { + p.numVertices = 0 + // the loops depths are set by initNested/initOriented prior to this. + p.bound = EmptyRect() + p.hasHoles = false + for _, l := range p.loops { + if l.IsHole() { + p.hasHoles = true + } else { + p.bound = p.bound.Union(l.RectBound()) + } + p.numVertices += l.NumVertices() + } + p.subregionBound = ExpandForSubregions(p.bound) + + p.initEdgesAndIndex() +} + +// initEdgesAndIndex performs the shape related initializations and adds the final +// polygon to the index. +func (p *Polygon) initEdgesAndIndex() { + p.numEdges = 0 + p.cumulativeEdges = nil + if p.IsFull() { + return + } + const maxLinearSearchLoops = 12 // Based on benchmarks. + if len(p.loops) > maxLinearSearchLoops { + p.cumulativeEdges = make([]int, 0, len(p.loops)) + } + + for _, l := range p.loops { + if p.cumulativeEdges != nil { + p.cumulativeEdges = append(p.cumulativeEdges, p.numEdges) + } + p.numEdges += len(l.vertices) + } + + p.index = NewShapeIndex() + p.index.Add(p) +} + +// FullPolygon returns a special "full" polygon. +func FullPolygon() *Polygon { + ret := &Polygon{ + loops: []*Loop{ + FullLoop(), + }, + numVertices: len(FullLoop().Vertices()), + bound: FullRect(), + subregionBound: FullRect(), + } + ret.initEdgesAndIndex() + return ret +} + +// Validate checks whether this is a valid polygon, +// including checking whether all the loops are themselves valid. +func (p *Polygon) Validate() error { + for i, l := range p.loops { + // Check for loop errors that don't require building a ShapeIndex. + if err := l.findValidationErrorNoIndex(); err != nil { + return fmt.Errorf("loop %d: %v", i, err) + } + // Check that no loop is empty, and that the full loop only appears in the + // full polygon. + if l.IsEmpty() { + return fmt.Errorf("loop %d: empty loops are not allowed", i) + } + if l.IsFull() && len(p.loops) > 1 { + return fmt.Errorf("loop %d: full loop appears in non-full polygon", i) + } + } + + // TODO(roberts): Uncomment the remaining checks when they are completed. + + // Check for loop self-intersections and loop pairs that cross + // (including duplicate edges and vertices). + // if findSelfIntersection(p.index) { + // return fmt.Errorf("polygon has loop pairs that cross") + // } + + // Check whether initOriented detected inconsistent loop orientations. + // if p.hasInconsistentLoopOrientations { + // return fmt.Errorf("inconsistent loop orientations detected") + // } + + // Finally, verify the loop nesting hierarchy. + return p.findLoopNestingError() +} + +// findLoopNestingError reports if there is an error in the loop nesting hierarchy. +func (p *Polygon) findLoopNestingError() error { + // First check that the loop depths make sense. + lastDepth := -1 + for i, l := range p.loops { + depth := l.depth + if depth < 0 || depth > lastDepth+1 { + return fmt.Errorf("loop %d: invalid loop depth (%d)", i, depth) + } + lastDepth = depth + } + // Then check that they correspond to the actual loop nesting. This test + // is quadratic in the number of loops but the cost per iteration is small. + for i, l := range p.loops { + last := p.LastDescendant(i) + for j, l2 := range p.loops { + if i == j { + continue + } + nested := (j >= i+1) && (j <= last) + const reverseB = false + + if l.containsNonCrossingBoundary(l2, reverseB) != nested { + nestedStr := "" + if !nested { + nestedStr = "not " + } + return fmt.Errorf("invalid nesting: loop %d should %scontain loop %d", i, nestedStr, j) + } + } + } + return nil +} + +// IsEmpty reports whether this is the special "empty" polygon (consisting of no loops). +func (p *Polygon) IsEmpty() bool { + return len(p.loops) == 0 +} + +// IsFull reports whether this is the special "full" polygon (consisting of a +// single loop that encompasses the entire sphere). +func (p *Polygon) IsFull() bool { + return len(p.loops) == 1 && p.loops[0].IsFull() +} + +// NumLoops returns the number of loops in this polygon. +func (p *Polygon) NumLoops() int { + return len(p.loops) +} + +// Loops returns the loops in this polygon. +func (p *Polygon) Loops() []*Loop { + return p.loops +} + +// Loop returns the loop at the given index. Note that during initialization, +// the given loops are reordered according to a pre-order traversal of the loop +// nesting hierarchy. This implies that every loop is immediately followed by +// its descendants. This hierarchy can be traversed using the methods Parent, +// LastDescendant, and Loop.depth. +func (p *Polygon) Loop(k int) *Loop { + return p.loops[k] +} + +// Parent returns the index of the parent of loop k. +// If the loop does not have a parent, ok=false is returned. +func (p *Polygon) Parent(k int) (index int, ok bool) { + // See where we are on the depth hierarchy. + depth := p.loops[k].depth + if depth == 0 { + return -1, false + } + + // There may be several loops at the same nesting level as us that share a + // parent loop with us. (Imagine a slice of swiss cheese, of which we are one loop. + // we don't know how many may be next to us before we get back to our parent loop.) + // Move up one position from us, and then begin traversing back through the set of loops + // until we find the one that is our parent or we get to the top of the polygon. + for k--; k >= 0 && p.loops[k].depth <= depth; k-- { + } + return k, true +} + +// LastDescendant returns the index of the last loop that is contained within loop k. +// If k is negative, it returns the last loop in the polygon. +// Note that loops are indexed according to a pre-order traversal of the nesting +// hierarchy, so the immediate children of loop k can be found by iterating over +// the loops (k+1)..LastDescendant(k) and selecting those whose depth is equal +// to Loop(k).depth+1. +func (p *Polygon) LastDescendant(k int) int { + if k < 0 { + return len(p.loops) - 1 + } + + depth := p.loops[k].depth + + // Find the next loop immediately past us in the set of loops, and then start + // moving down the list until we either get to the end or find the next loop + // that is higher up the hierarchy than we are. + for k++; k < len(p.loops) && p.loops[k].depth > depth; k++ { + } + return k - 1 +} + +// CapBound returns a bounding spherical cap. +func (p *Polygon) CapBound() Cap { return p.bound.CapBound() } + +// RectBound returns a bounding latitude-longitude rectangle. +func (p *Polygon) RectBound() Rect { return p.bound } + +// ContainsPoint reports whether the polygon contains the point. +func (p *Polygon) ContainsPoint(point Point) bool { + // NOTE: A bounds check slows down this function by about 50%. It is + // worthwhile only when it might allow us to delay building the index. + if !p.index.IsFresh() && !p.bound.ContainsPoint(point) { + return false + } + + // For small polygons, and during initial construction, it is faster to just + // check all the crossing. + const maxBruteForceVertices = 32 + if p.numVertices < maxBruteForceVertices || p.index == nil { + inside := false + for _, l := range p.loops { + // use loops bruteforce to avoid building the index on each loop. + inside = inside != l.bruteForceContainsPoint(point) + } + return inside + } + + // Otherwise we look up the ShapeIndex cell containing this point. + return NewContainsPointQuery(p.index, VertexModelSemiOpen).Contains(point) +} + +// ContainsCell reports whether the polygon contains the given cell. +func (p *Polygon) ContainsCell(cell Cell) bool { + it := p.index.Iterator() + relation := it.LocateCellID(cell.ID()) + + // If "cell" is disjoint from all index cells, it is not contained. + // Similarly, if "cell" is subdivided into one or more index cells then it + // is not contained, since index cells are subdivided only if they (nearly) + // intersect a sufficient number of edges. (But note that if "cell" itself + // is an index cell then it may be contained, since it could be a cell with + // no edges in the loop interior.) + if relation != Indexed { + return false + } + + // Otherwise check if any edges intersect "cell". + if p.boundaryApproxIntersects(it, cell) { + return false + } + + // Otherwise check if the loop contains the center of "cell". + return p.iteratorContainsPoint(it, cell.Center()) +} + +// IntersectsCell reports whether the polygon intersects the given cell. +func (p *Polygon) IntersectsCell(cell Cell) bool { + it := p.index.Iterator() + relation := it.LocateCellID(cell.ID()) + + // If cell does not overlap any index cell, there is no intersection. + if relation == Disjoint { + return false + } + // If cell is subdivided into one or more index cells, there is an + // intersection to within the S2ShapeIndex error bound (see Contains). + if relation == Subdivided { + return true + } + // If cell is an index cell, there is an intersection because index cells + // are created only if they have at least one edge or they are entirely + // contained by the loop. + if it.CellID() == cell.id { + return true + } + // Otherwise check if any edges intersect cell. + if p.boundaryApproxIntersects(it, cell) { + return true + } + // Otherwise check if the loop contains the center of cell. + return p.iteratorContainsPoint(it, cell.Center()) +} + +// CellUnionBound computes a covering of the Polygon. +func (p *Polygon) CellUnionBound() []CellID { + // TODO(roberts): Use ShapeIndexRegion when it's available. + return p.CapBound().CellUnionBound() +} + +// boundaryApproxIntersects reports whether the loop's boundary intersects cell. +// It may also return true when the loop boundary does not intersect cell but +// some edge comes within the worst-case error tolerance. +// +// This requires that it.Locate(cell) returned Indexed. +func (p *Polygon) boundaryApproxIntersects(it *ShapeIndexIterator, cell Cell) bool { + aClipped := it.IndexCell().findByShapeID(0) + + // If there are no edges, there is no intersection. + if len(aClipped.edges) == 0 { + return false + } + + // We can save some work if cell is the index cell itself. + if it.CellID() == cell.ID() { + return true + } + + // Otherwise check whether any of the edges intersect cell. + maxError := (faceClipErrorUVCoord + intersectsRectErrorUVDist) + bound := cell.BoundUV().ExpandedByMargin(maxError) + for _, e := range aClipped.edges { + edge := p.index.Shape(0).Edge(e) + v0, v1, ok := ClipToPaddedFace(edge.V0, edge.V1, cell.Face(), maxError) + if ok && edgeIntersectsRect(v0, v1, bound) { + return true + } + } + + return false +} + +// iteratorContainsPoint reports whether the iterator that is positioned at the +// ShapeIndexCell that may contain p, contains the point p. +func (p *Polygon) iteratorContainsPoint(it *ShapeIndexIterator, point Point) bool { + // Test containment by drawing a line segment from the cell center to the + // given point and counting edge crossings. + aClipped := it.IndexCell().findByShapeID(0) + inside := aClipped.containsCenter + + if len(aClipped.edges) == 0 { + return inside + } + + // This block requires ShapeIndex. + crosser := NewEdgeCrosser(it.Center(), point) + shape := p.index.Shape(0) + for _, e := range aClipped.edges { + edge := shape.Edge(e) + inside = inside != crosser.EdgeOrVertexCrossing(edge.V0, edge.V1) + } + + return inside +} + +// Shape Interface + +// NumEdges returns the number of edges in this shape. +func (p *Polygon) NumEdges() int { + return p.numEdges +} + +// Edge returns endpoints for the given edge index. +func (p *Polygon) Edge(e int) Edge { + var i int + + if len(p.cumulativeEdges) > 0 { + for i = range p.cumulativeEdges { + if i+1 >= len(p.cumulativeEdges) || e < p.cumulativeEdges[i+1] { + e -= p.cumulativeEdges[i] + break + } + } + } else { + // When the number of loops is small, use linear search. Most often + // there is exactly one loop and the code below executes zero times. + for i = 0; e >= len(p.Loop(i).vertices); i++ { + e -= len(p.Loop(i).vertices) + } + } + + return Edge{p.Loop(i).OrientedVertex(e), p.Loop(i).OrientedVertex(e + 1)} +} + +// ReferencePoint returns the reference point for this polygon. +func (p *Polygon) ReferencePoint() ReferencePoint { + containsOrigin := false + for _, l := range p.loops { + containsOrigin = containsOrigin != l.ContainsOrigin() + } + return OriginReferencePoint(containsOrigin) +} + +// NumChains reports the number of contiguous edge chains in the Polygon. +func (p *Polygon) NumChains() int { + return p.NumLoops() +} + +// Chain returns the i-th edge Chain (loop) in the Shape. +func (p *Polygon) Chain(chainID int) Chain { + if p.cumulativeEdges != nil { + return Chain{p.cumulativeEdges[chainID], len(p.Loop(chainID).vertices)} + } + e := 0 + for j := 0; j < chainID; j++ { + e += len(p.Loop(j).vertices) + } + + // Polygon represents a full loop as a loop with one vertex, while + // Shape represents a full loop as a chain with no vertices. + if numVertices := p.Loop(chainID).NumVertices(); numVertices != 1 { + return Chain{e, numVertices} + } + return Chain{e, 0} +} + +// ChainEdge returns the j-th edge of the i-th edge Chain (loop). +func (p *Polygon) ChainEdge(i, j int) Edge { + return Edge{p.Loop(i).OrientedVertex(j), p.Loop(i).OrientedVertex(j + 1)} +} + +// ChainPosition returns a pair (i, j) such that edgeID is the j-th edge +// of the i-th edge Chain. +func (p *Polygon) ChainPosition(edgeID int) ChainPosition { + var i int + + if len(p.cumulativeEdges) > 0 { + for i = range p.cumulativeEdges { + if i+1 >= len(p.cumulativeEdges) || edgeID < p.cumulativeEdges[i+1] { + edgeID -= p.cumulativeEdges[i] + break + } + } + } else { + // When the number of loops is small, use linear search. Most often + // there is exactly one loop and the code below executes zero times. + for i = 0; edgeID >= len(p.Loop(i).vertices); i++ { + edgeID -= len(p.Loop(i).vertices) + } + } + // TODO(roberts): unify this and Edge since they are mostly identical. + return ChainPosition{i, edgeID} +} + +// Dimension returns the dimension of the geometry represented by this Polygon. +func (p *Polygon) Dimension() int { return 2 } + +func (p *Polygon) typeTag() typeTag { return typeTagPolygon } + +func (p *Polygon) privateInterface() {} + +// Contains reports whether this polygon contains the other polygon. +// Specifically, it reports whether all the points in the other polygon +// are also in this polygon. +func (p *Polygon) Contains(o *Polygon) bool { + // If both polygons have one loop, use the more efficient Loop method. + // Note that Loop's Contains does its own bounding rectangle check. + if len(p.loops) == 1 && len(o.loops) == 1 { + return p.loops[0].Contains(o.loops[0]) + } + + // Otherwise if neither polygon has holes, we can still use the more + // efficient Loop's Contains method (rather than compareBoundary), + // but it's worthwhile to do our own bounds check first. + if !p.subregionBound.Contains(o.bound) { + // Even though Bound(A) does not contain Bound(B), it is still possible + // that A contains B. This can only happen when union of the two bounds + // spans all longitudes. For example, suppose that B consists of two + // shells with a longitude gap between them, while A consists of one shell + // that surrounds both shells of B but goes the other way around the + // sphere (so that it does not intersect the longitude gap). + if !p.bound.Lng.Union(o.bound.Lng).IsFull() { + return false + } + } + + if !p.hasHoles && !o.hasHoles { + for _, l := range o.loops { + if !p.anyLoopContains(l) { + return false + } + } + return true + } + + // Polygon A contains B iff B does not intersect the complement of A. From + // the intersection algorithm below, this means that the complement of A + // must exclude the entire boundary of B, and B must exclude all shell + // boundaries of the complement of A. (It can be shown that B must then + // exclude the entire boundary of the complement of A.) The first call + // below returns false if the boundaries cross, therefore the second call + // does not need to check for any crossing edges (which makes it cheaper). + return p.containsBoundary(o) && o.excludesNonCrossingComplementShells(p) +} + +// Intersects reports whether this polygon intersects the other polygon, i.e. +// if there is a point that is contained by both polygons. +func (p *Polygon) Intersects(o *Polygon) bool { + // If both polygons have one loop, use the more efficient Loop method. + // Note that Loop Intersects does its own bounding rectangle check. + if len(p.loops) == 1 && len(o.loops) == 1 { + return p.loops[0].Intersects(o.loops[0]) + } + + // Otherwise if neither polygon has holes, we can still use the more + // efficient Loop.Intersects method. The polygons intersect if and + // only if some pair of loop regions intersect. + if !p.bound.Intersects(o.bound) { + return false + } + + if !p.hasHoles && !o.hasHoles { + for _, l := range o.loops { + if p.anyLoopIntersects(l) { + return true + } + } + return false + } + + // Polygon A is disjoint from B if A excludes the entire boundary of B and B + // excludes all shell boundaries of A. (It can be shown that B must then + // exclude the entire boundary of A.) The first call below returns false if + // the boundaries cross, therefore the second call does not need to check + // for crossing edges. + return !p.excludesBoundary(o) || !o.excludesNonCrossingShells(p) +} + +// compareBoundary returns +1 if this polygon contains the boundary of B, -1 if A +// excludes the boundary of B, and 0 if the boundaries of A and B cross. +func (p *Polygon) compareBoundary(o *Loop) int { + result := -1 + for i := 0; i < len(p.loops) && result != 0; i++ { + // If B crosses any loop of A, the result is 0. Otherwise the result + // changes sign each time B is contained by a loop of A. + result *= -p.loops[i].compareBoundary(o) + } + return result +} + +// containsBoundary reports whether this polygon contains the entire boundary of B. +func (p *Polygon) containsBoundary(o *Polygon) bool { + for _, l := range o.loops { + if p.compareBoundary(l) <= 0 { + return false + } + } + return true +} + +// excludesBoundary reports whether this polygon excludes the entire boundary of B. +func (p *Polygon) excludesBoundary(o *Polygon) bool { + for _, l := range o.loops { + if p.compareBoundary(l) >= 0 { + return false + } + } + return true +} + +// containsNonCrossingBoundary reports whether polygon A contains the boundary of +// loop B. Shared edges are handled according to the rule described in loops +// containsNonCrossingBoundary. +func (p *Polygon) containsNonCrossingBoundary(o *Loop, reverse bool) bool { + var inside bool + for _, l := range p.loops { + x := l.containsNonCrossingBoundary(o, reverse) + inside = (inside != x) + } + return inside +} + +// excludesNonCrossingShells reports wheterh given two polygons A and B such that the +// boundary of A does not cross any loop of B, if A excludes all shell boundaries of B. +func (p *Polygon) excludesNonCrossingShells(o *Polygon) bool { + for _, l := range o.loops { + if l.IsHole() { + continue + } + if p.containsNonCrossingBoundary(l, false) { + return false + } + } + return true +} + +// excludesNonCrossingComplementShells reports whether given two polygons A and B +// such that the boundary of A does not cross any loop of B, if A excludes all +// shell boundaries of the complement of B. +func (p *Polygon) excludesNonCrossingComplementShells(o *Polygon) bool { + // Special case to handle the complement of the empty or full polygons. + if o.IsEmpty() { + return !p.IsFull() + } + if o.IsFull() { + return true + } + + // Otherwise the complement of B may be obtained by inverting loop(0) and + // then swapping the shell/hole status of all other loops. This implies + // that the shells of the complement consist of loop 0 plus all the holes of + // the original polygon. + for j, l := range o.loops { + if j > 0 && !l.IsHole() { + continue + } + + // The interior of the complement is to the right of loop 0, and to the + // left of the loops that were originally holes. + if p.containsNonCrossingBoundary(l, j == 0) { + return false + } + } + return true +} + +// anyLoopContains reports whether any loop in this polygon contains the given loop. +func (p *Polygon) anyLoopContains(o *Loop) bool { + for _, l := range p.loops { + if l.Contains(o) { + return true + } + } + return false +} + +// anyLoopIntersects reports whether any loop in this polygon intersects the given loop. +func (p *Polygon) anyLoopIntersects(o *Loop) bool { + for _, l := range p.loops { + if l.Intersects(o) { + return true + } + } + return false +} + +// Area returns the area of the polygon interior, i.e. the region on the left side +// of an odd number of loops. The return value is between 0 and 4*Pi. +func (p *Polygon) Area() float64 { + var area float64 + for _, loop := range p.loops { + area += float64(loop.Sign()) * loop.Area() + } + return area +} + +// Encode encodes the Polygon +func (p *Polygon) Encode(w io.Writer) error { + e := &encoder{w: w} + p.encode(e) + return e.err +} + +// encode only supports lossless encoding and not compressed format. +func (p *Polygon) encode(e *encoder) { + if p.numVertices == 0 { + p.encodeCompressed(e, maxLevel, nil) + return + } + + // Convert all the polygon vertices to XYZFaceSiTi format. + vs := make([]xyzFaceSiTi, 0, p.numVertices) + for _, l := range p.loops { + vs = append(vs, l.xyzFaceSiTiVertices()...) + } + + // Computes a histogram of the cell levels at which the vertices are snapped. + // (histogram[0] is the number of unsnapped vertices, histogram[i] the number + // of vertices snapped at level i-1). + histogram := make([]int, maxLevel+2) + for _, v := range vs { + histogram[v.level+1]++ + } + + // Compute the level at which most of the vertices are snapped. + // If multiple levels have the same maximum number of vertices + // snapped to it, the first one (lowest level number / largest + // area / smallest encoding length) will be chosen, so this + // is desired. + var snapLevel, numSnapped int + for level, h := range histogram[1:] { + if h > numSnapped { + snapLevel, numSnapped = level, h + } + } + + // Choose an encoding format based on the number of unsnapped vertices and a + // rough estimate of the encoded sizes. + numUnsnapped := p.numVertices - numSnapped // Number of vertices that won't be snapped at snapLevel. + const pointSize = 3 * 8 // s2.Point is an r3.Vector, which is 3 float64s. That's 3*8 = 24 bytes. + compressedSize := 4*p.numVertices + (pointSize+2)*numUnsnapped + losslessSize := pointSize * p.numVertices + if compressedSize < losslessSize { + p.encodeCompressed(e, snapLevel, vs) + } else { + p.encodeLossless(e) + } +} + +// encodeLossless encodes the polygon's Points as float64s. +func (p *Polygon) encodeLossless(e *encoder) { + e.writeInt8(encodingVersion) + e.writeBool(true) // a legacy c++ value. must be true. + e.writeBool(p.hasHoles) + e.writeUint32(uint32(len(p.loops))) + + if e.err != nil { + return + } + if len(p.loops) > maxEncodedLoops { + e.err = fmt.Errorf("too many loops (%d; max is %d)", len(p.loops), maxEncodedLoops) + return + } + for _, l := range p.loops { + l.encode(e) + } + + // Encode the bound. + p.bound.encode(e) +} + +func (p *Polygon) encodeCompressed(e *encoder, snapLevel int, vertices []xyzFaceSiTi) { + e.writeUint8(uint8(encodingCompressedVersion)) + e.writeUint8(uint8(snapLevel)) + e.writeUvarint(uint64(len(p.loops))) + + if e.err != nil { + return + } + if l := len(p.loops); l > maxEncodedLoops { + e.err = fmt.Errorf("too many loops to encode: %d; max is %d", l, maxEncodedLoops) + return + } + + for _, l := range p.loops { + l.encodeCompressed(e, snapLevel, vertices[:len(l.vertices)]) + vertices = vertices[len(l.vertices):] + } + // Do not write the bound, num_vertices, or has_holes_ as they can be + // cheaply recomputed by decodeCompressed. Microbenchmarks show the + // speed difference is inconsequential. +} + +// Decode decodes the Polygon. +func (p *Polygon) Decode(r io.Reader) error { + d := &decoder{r: asByteReader(r)} + version := int8(d.readUint8()) + var dec func(*decoder) + switch version { + case encodingVersion: + dec = p.decode + case encodingCompressedVersion: + dec = p.decodeCompressed + default: + return fmt.Errorf("unsupported version %d", version) + } + dec(d) + return d.err +} + +// maxEncodedLoops is the biggest supported number of loops in a polygon during encoding. +// Setting a maximum guards an allocation: it prevents an attacker from easily pushing us OOM. +const maxEncodedLoops = 10000000 + +func (p *Polygon) decode(d *decoder) { + *p = Polygon{BufPool: p.BufPool} + d.readUint8() // Ignore irrelevant serialized owns_loops_ value. + + p.hasHoles = d.readBool() + + // Polygons with no loops are explicitly allowed here: a newly created + // polygon has zero loops and such polygons encode and decode properly. + nloops := d.readUint32() + if d.err != nil { + return + } + if nloops > maxEncodedLoops { + d.err = fmt.Errorf("too many loops (%d; max is %d)", nloops, maxEncodedLoops) + return + } + p.loops = make([]*Loop, nloops) + for i := range p.loops { + p.loops[i] = new(Loop) + p.loops[i].BufPool = p.BufPool + p.loops[i].decode(d) + p.numVertices += len(p.loops[i].vertices) + } + + p.bound.decode(d) + if d.err != nil { + return + } + p.subregionBound = ExpandForSubregions(p.bound) + p.initEdgesAndIndex() +} + +func (p *Polygon) decodeCompressed(d *decoder) { + snapLevel := int(d.readUint8()) + + if snapLevel > maxLevel { + d.err = fmt.Errorf("snaplevel too big: %d", snapLevel) + return + } + // Polygons with no loops are explicitly allowed here: a newly created + // polygon has zero loops and such polygons encode and decode properly. + nloops := int(d.readUvarint()) + if nloops > maxEncodedLoops { + d.err = fmt.Errorf("too many loops (%d; max is %d)", nloops, maxEncodedLoops) + } + p.loops = make([]*Loop, nloops) + for i := range p.loops { + p.loops[i] = new(Loop) + p.loops[i].decodeCompressed(d, snapLevel) + } + p.initLoopProperties() +} + +func (p *Polygon) Project(point *Point) Point { + if p.ContainsPoint(*point) { + return *point + } + return p.ProjectToBoundary(point) +} + +func (p *Polygon) ProjectToBoundary(point *Point) Point { + options := NewClosestEdgeQueryOptions().MaxResults(1).IncludeInteriors(false) + q := NewClosestEdgeQuery(p.index, options) + target := NewMinDistanceToPointTarget(*point) + edges := q.FindEdges(target) + return q.Project(*point, edges[0]) +} + +// TODO(roberts): Differences from C++ +// Project - implemented in this fork. +// ProjectToBoundary - implemented in this fork. + +// Centroid +// SnapLevel +// DistanceToPoint +// DistanceToBoundary +// ApproxContains/ApproxDisjoint for Polygons +// InitTo{Intersection/ApproxIntersection/Union/ApproxUnion/Diff/ApproxDiff} +// InitToSimplified +// InitToSnapped +// IntersectWithPolyline +// ApproxIntersectWithPolyline +// SubtractFromPolyline +// ApproxSubtractFromPolyline +// DestructiveUnion +// DestructiveApproxUnion +// InitToCellUnionBorder +// IsNormalized +// Equal/BoundaryEqual/BoundaryApproxEqual/BoundaryNear Polygons +// BreakEdgesAndAddToBuilder +// +// clearLoops +// findLoopNestingError +// initToSimplifiedInternal +// internalClipPolyline +// clipBoundary diff --git a/vendor/github.com/blevesearch/geo/s2/polyline.go b/vendor/github.com/blevesearch/geo/s2/polyline.go new file mode 100644 index 00000000..51796834 --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/polyline.go @@ -0,0 +1,589 @@ +// Copyright 2016 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +import ( + "fmt" + "io" + "math" + + "github.com/golang/geo/s1" +) + +// Polyline represents a sequence of zero or more vertices connected by +// straight edges (geodesics). Edges of length 0 and 180 degrees are not +// allowed, i.e. adjacent vertices should not be identical or antipodal. +type Polyline []Point + +// PolylineFromLatLngs creates a new Polyline from the given LatLngs. +func PolylineFromLatLngs(points []LatLng) *Polyline { + p := make(Polyline, len(points)) + for k, v := range points { + p[k] = PointFromLatLng(v) + } + return &p +} + +// Reverse reverses the order of the Polyline vertices. +func (p *Polyline) Reverse() { + for i := 0; i < len(*p)/2; i++ { + (*p)[i], (*p)[len(*p)-i-1] = (*p)[len(*p)-i-1], (*p)[i] + } +} + +// Length returns the length of this Polyline. +func (p *Polyline) Length() s1.Angle { + var length s1.Angle + + for i := 1; i < len(*p); i++ { + length += (*p)[i-1].Distance((*p)[i]) + } + return length +} + +// Centroid returns the true centroid of the polyline multiplied by the length of the +// polyline. The result is not unit length, so you may wish to normalize it. +// +// Scaling by the Polyline length makes it easy to compute the centroid +// of several Polylines (by simply adding up their centroids). +func (p *Polyline) Centroid() Point { + var centroid Point + for i := 1; i < len(*p); i++ { + // The centroid (multiplied by length) is a vector toward the midpoint + // of the edge, whose length is twice the sin of half the angle between + // the two vertices. Defining theta to be this angle, we have: + vSum := (*p)[i-1].Add((*p)[i].Vector) // Length == 2*cos(theta) + vDiff := (*p)[i-1].Sub((*p)[i].Vector) // Length == 2*sin(theta) + + // Length == 2*sin(theta) + centroid = Point{centroid.Add(vSum.Mul(math.Sqrt(vDiff.Norm2() / vSum.Norm2())))} + } + return centroid +} + +// Equal reports whether the given Polyline is exactly the same as this one. +func (p *Polyline) Equal(b *Polyline) bool { + if len(*p) != len(*b) { + return false + } + for i, v := range *p { + if v != (*b)[i] { + return false + } + } + + return true +} + +// ApproxEqual reports whether two polylines have the same number of vertices, +// and corresponding vertex pairs are separated by no more the standard margin. +func (p *Polyline) ApproxEqual(o *Polyline) bool { + return p.approxEqual(o, s1.Angle(epsilon)) +} + +// approxEqual reports whether two polylines are equal within the given margin. +func (p *Polyline) approxEqual(o *Polyline, maxError s1.Angle) bool { + if len(*p) != len(*o) { + return false + } + for offset, val := range *p { + if !val.approxEqual((*o)[offset], maxError) { + return false + } + } + return true +} + +// CapBound returns the bounding Cap for this Polyline. +func (p *Polyline) CapBound() Cap { + return p.RectBound().CapBound() +} + +// RectBound returns the bounding Rect for this Polyline. +func (p *Polyline) RectBound() Rect { + rb := NewRectBounder() + for _, v := range *p { + rb.AddPoint(v) + } + return rb.RectBound() +} + +// ContainsCell reports whether this Polyline contains the given Cell. Always returns false +// because "containment" is not numerically well-defined except at the Polyline vertices. +func (p *Polyline) ContainsCell(cell Cell) bool { + return false +} + +// IntersectsCell reports whether this Polyline intersects the given Cell. +func (p *Polyline) IntersectsCell(cell Cell) bool { + if len(*p) == 0 { + return false + } + + // We only need to check whether the cell contains vertex 0 for correctness, + // but these tests are cheap compared to edge crossings so we might as well + // check all the vertices. + for _, v := range *p { + if cell.ContainsPoint(v) { + return true + } + } + + cellVertices := []Point{ + cell.Vertex(0), + cell.Vertex(1), + cell.Vertex(2), + cell.Vertex(3), + } + + for j := 0; j < 4; j++ { + crosser := NewChainEdgeCrosser(cellVertices[j], cellVertices[(j+1)&3], (*p)[0]) + for i := 1; i < len(*p); i++ { + if crosser.ChainCrossingSign((*p)[i]) != DoNotCross { + // There is a proper crossing, or two vertices were the same. + return true + } + } + } + return false +} + +// ContainsPoint returns false since Polylines are not closed. +func (p *Polyline) ContainsPoint(point Point) bool { + return false +} + +// CellUnionBound computes a covering of the Polyline. +func (p *Polyline) CellUnionBound() []CellID { + return p.CapBound().CellUnionBound() +} + +// NumEdges returns the number of edges in this shape. +func (p *Polyline) NumEdges() int { + if len(*p) == 0 { + return 0 + } + return len(*p) - 1 +} + +// Edge returns endpoints for the given edge index. +func (p *Polyline) Edge(i int) Edge { + return Edge{(*p)[i], (*p)[i+1]} +} + +// ReferencePoint returns the default reference point with negative containment because Polylines are not closed. +func (p *Polyline) ReferencePoint() ReferencePoint { + return OriginReferencePoint(false) +} + +// NumChains reports the number of contiguous edge chains in this Polyline. +func (p *Polyline) NumChains() int { + return minInt(1, p.NumEdges()) +} + +// Chain returns the i-th edge Chain in the Shape. +func (p *Polyline) Chain(chainID int) Chain { + return Chain{0, p.NumEdges()} +} + +// ChainEdge returns the j-th edge of the i-th edge Chain. +func (p *Polyline) ChainEdge(chainID, offset int) Edge { + return Edge{(*p)[offset], (*p)[offset+1]} +} + +// ChainPosition returns a pair (i, j) such that edgeID is the j-th edge +func (p *Polyline) ChainPosition(edgeID int) ChainPosition { + return ChainPosition{0, edgeID} +} + +// Dimension returns the dimension of the geometry represented by this Polyline. +func (p *Polyline) Dimension() int { return 1 } + +// IsEmpty reports whether this shape contains no points. +func (p *Polyline) IsEmpty() bool { return defaultShapeIsEmpty(p) } + +// IsFull reports whether this shape contains all points on the sphere. +func (p *Polyline) IsFull() bool { return defaultShapeIsFull(p) } + +func (p *Polyline) typeTag() typeTag { return typeTagPolyline } + +func (p *Polyline) privateInterface() {} + +// findEndVertex reports the maximal end index such that the line segment between +// the start index and this one such that the line segment between these two +// vertices passes within the given tolerance of all interior vertices, in order. +func findEndVertex(p Polyline, tolerance s1.Angle, index int) int { + // The basic idea is to keep track of the "pie wedge" of angles + // from the starting vertex such that a ray from the starting + // vertex at that angle will pass through the discs of radius + // tolerance centered around all vertices processed so far. + // + // First we define a coordinate frame for the tangent and normal + // spaces at the starting vertex. Essentially this means picking + // three orthonormal vectors X,Y,Z such that X and Y span the + // tangent plane at the starting vertex, and Z is up. We use + // the coordinate frame to define a mapping from 3D direction + // vectors to a one-dimensional ray angle in the range (-π, + // π]. The angle of a direction vector is computed by + // transforming it into the X,Y,Z basis, and then calculating + // atan2(y,x). This mapping allows us to represent a wedge of + // angles as a 1D interval. Since the interval wraps around, we + // represent it as an Interval, i.e. an interval on the unit + // circle. + origin := p[index] + frame := getFrame(origin) + + // As we go along, we keep track of the current wedge of angles + // and the distance to the last vertex (which must be + // non-decreasing). + currentWedge := s1.FullInterval() + var lastDistance s1.Angle + + for index++; index < len(p); index++ { + candidate := p[index] + distance := origin.Distance(candidate) + + // We don't allow simplification to create edges longer than + // 90 degrees, to avoid numeric instability as lengths + // approach 180 degrees. We do need to allow for original + // edges longer than 90 degrees, though. + if distance > math.Pi/2 && lastDistance > 0 { + break + } + + // Vertices must be in increasing order along the ray, except + // for the initial disc around the origin. + if distance < lastDistance && lastDistance > tolerance { + break + } + + lastDistance = distance + + // Points that are within the tolerance distance of the origin + // do not constrain the ray direction, so we can ignore them. + if distance <= tolerance { + continue + } + + // If the current wedge of angles does not contain the angle + // to this vertex, then stop right now. Note that the wedge + // of possible ray angles is not necessarily empty yet, but we + // can't continue unless we are willing to backtrack to the + // last vertex that was contained within the wedge (since we + // don't create new vertices). This would be more complicated + // and also make the worst-case running time more than linear. + direction := toFrame(frame, candidate) + center := math.Atan2(direction.Y, direction.X) + if !currentWedge.Contains(center) { + break + } + + // To determine how this vertex constrains the possible ray + // angles, consider the triangle ABC where A is the origin, B + // is the candidate vertex, and C is one of the two tangent + // points between A and the spherical cap of radius + // tolerance centered at B. Then from the spherical law of + // sines, sin(a)/sin(A) = sin(c)/sin(C), where a and c are + // the lengths of the edges opposite A and C. In our case C + // is a 90 degree angle, therefore A = asin(sin(a) / sin(c)). + // Angle A is the half-angle of the allowable wedge. + halfAngle := math.Asin(math.Sin(tolerance.Radians()) / math.Sin(distance.Radians())) + target := s1.IntervalFromPointPair(center, center).Expanded(halfAngle) + currentWedge = currentWedge.Intersection(target) + } + + // We break out of the loop when we reach a vertex index that + // can't be included in the line segment, so back up by one + // vertex. + return index - 1 +} + +// SubsampleVertices returns a subsequence of vertex indices such that the +// polyline connecting these vertices is never further than the given tolerance from +// the original polyline. Provided the first and last vertices are distinct, +// they are always preserved; if they are not, the subsequence may contain +// only a single index. +// +// Some useful properties of the algorithm: +// +// - It runs in linear time. +// +// - The output always represents a valid polyline. In particular, adjacent +// output vertices are never identical or antipodal. +// +// - The method is not optimal, but it tends to produce 2-3% fewer +// vertices than the Douglas-Peucker algorithm with the same tolerance. +// +// - The output is parametrically equivalent to the original polyline to +// within the given tolerance. For example, if a polyline backtracks on +// itself and then proceeds onwards, the backtracking will be preserved +// (to within the given tolerance). This is different than the +// Douglas-Peucker algorithm which only guarantees geometric equivalence. +func (p *Polyline) SubsampleVertices(tolerance s1.Angle) []int { + var result []int + + if len(*p) < 1 { + return result + } + + result = append(result, 0) + clampedTolerance := s1.Angle(math.Max(tolerance.Radians(), 0)) + + for index := 0; index+1 < len(*p); { + nextIndex := findEndVertex(*p, clampedTolerance, index) + // Don't create duplicate adjacent vertices. + if (*p)[nextIndex] != (*p)[index] { + result = append(result, nextIndex) + } + index = nextIndex + } + + return result +} + +// Encode encodes the Polyline. +func (p Polyline) Encode(w io.Writer) error { + e := &encoder{w: w} + p.encode(e) + return e.err +} + +func (p Polyline) encode(e *encoder) { + e.writeInt8(encodingVersion) + e.writeUint32(uint32(len(p))) + for _, v := range p { + e.writeFloat64(v.X) + e.writeFloat64(v.Y) + e.writeFloat64(v.Z) + } +} + +// Decode decodes the polyline. +func (p *Polyline) Decode(r io.Reader) error { + d := decoder{r: asByteReader(r)} + p.decode(d) + return d.err +} + +func (p *Polyline) decode(d decoder) { + version := d.readInt8() + if d.err != nil { + return + } + if int(version) != int(encodingVersion) { + d.err = fmt.Errorf("can't decode version %d; my version: %d", version, encodingVersion) + return + } + nvertices := d.readUint32() + if d.err != nil { + return + } + if nvertices > maxEncodedVertices { + d.err = fmt.Errorf("too many vertices (%d; max is %d)", nvertices, maxEncodedVertices) + return + } + *p = make([]Point, nvertices) + for i := range *p { + (*p)[i].X = d.readFloat64() + (*p)[i].Y = d.readFloat64() + (*p)[i].Z = d.readFloat64() + } +} + +// Project returns a point on the polyline that is closest to the given point, +// and the index of the next vertex after the projected point. The +// value of that index is always in the range [1, len(polyline)]. +// The polyline must not be empty. +func (p *Polyline) Project(point Point) (Point, int) { + if len(*p) == 1 { + // If there is only one vertex, it is always closest to any given point. + return (*p)[0], 1 + } + + // Initial value larger than any possible distance on the unit sphere. + minDist := 10 * s1.Radian + minIndex := -1 + + // Find the line segment in the polyline that is closest to the point given. + for i := 1; i < len(*p); i++ { + if dist := DistanceFromSegment(point, (*p)[i-1], (*p)[i]); dist < minDist { + minDist = dist + minIndex = i + } + } + + // Compute the point on the segment found that is closest to the point given. + closest := Project(point, (*p)[minIndex-1], (*p)[minIndex]) + if closest == (*p)[minIndex] { + minIndex++ + } + + return closest, minIndex +} + +// IsOnRight reports whether the point given is on the right hand side of the +// polyline, using a naive definition of "right-hand-sideness" where the point +// is on the RHS of the polyline iff the point is on the RHS of the line segment +// in the polyline which it is closest to. +// The polyline must have at least 2 vertices. +func (p *Polyline) IsOnRight(point Point) bool { + // If the closest point C is an interior vertex of the polyline, let B and D + // be the previous and next vertices. The given point P is on the right of + // the polyline (locally) if B, P, D are ordered CCW around vertex C. + closest, next := p.Project(point) + if closest == (*p)[next-1] && next > 1 && next < len(*p) { + if point == (*p)[next-1] { + // Polyline vertices are not on the RHS. + return false + } + return OrderedCCW((*p)[next-2], point, (*p)[next], (*p)[next-1]) + } + // Otherwise, the closest point C is incident to exactly one polyline edge. + // We test the point P against that edge. + if next == len(*p) { + next-- + } + return Sign(point, (*p)[next], (*p)[next-1]) +} + +// Validate checks whether this is a valid polyline or not. +func (p *Polyline) Validate() error { + // All vertices must be unit length. + for i, pt := range *p { + if !pt.IsUnit() { + return fmt.Errorf("vertex %d is not unit length", i) + } + } + + // Adjacent vertices must not be identical or antipodal. + for i := 1; i < len(*p); i++ { + prev, cur := (*p)[i-1], (*p)[i] + if prev == cur { + return fmt.Errorf("vertices %d and %d are identical", i-1, i) + } + if prev == (Point{cur.Mul(-1)}) { + return fmt.Errorf("vertices %d and %d are antipodal", i-1, i) + } + } + + return nil +} + +// Intersects reports whether this polyline intersects the given polyline. If +// the polylines share a vertex they are considered to be intersecting. When a +// polyline endpoint is the only intersection with the other polyline, the +// function may return true or false arbitrarily. +// +// The running time is quadratic in the number of vertices. +func (p *Polyline) Intersects(o *Polyline) bool { + if len(*p) == 0 || len(*o) == 0 { + return false + } + + if !p.RectBound().Intersects(o.RectBound()) { + return false + } + + // TODO(roberts): Use ShapeIndex here. + for i := 1; i < len(*p); i++ { + crosser := NewChainEdgeCrosser((*p)[i-1], (*p)[i], (*o)[0]) + for j := 1; j < len(*o); j++ { + if crosser.ChainCrossingSign((*o)[j]) != DoNotCross { + return true + } + } + } + return false +} + +// Interpolate returns the point whose distance from vertex 0 along the polyline is +// the given fraction of the polyline's total length, and the index of +// the next vertex after the interpolated point P. Fractions less than zero +// or greater than one are clamped. The return value is unit length. The cost of +// this function is currently linear in the number of vertices. +// +// This method allows the caller to easily construct a given suffix of the +// polyline by concatenating P with the polyline vertices starting at that next +// vertex. Note that P is guaranteed to be different than the point at the next +// vertex, so this will never result in a duplicate vertex. +// +// The polyline must not be empty. Note that if fraction >= 1.0, then the next +// vertex will be set to len(p) (indicating that no vertices from the polyline +// need to be appended). The value of the next vertex is always between 1 and +// len(p). +// +// This method can also be used to construct a prefix of the polyline, by +// taking the polyline vertices up to next vertex-1 and appending the +// returned point P if it is different from the last vertex (since in this +// case there is no guarantee of distinctness). +func (p *Polyline) Interpolate(fraction float64) (Point, int) { + // We intentionally let the (fraction >= 1) case fall through, since + // we need to handle it in the loop below in any case because of + // possible roundoff errors. + if fraction <= 0 { + return (*p)[0], 1 + } + target := s1.Angle(fraction) * p.Length() + + for i := 1; i < len(*p); i++ { + length := (*p)[i-1].Distance((*p)[i]) + if target < length { + // This interpolates with respect to arc length rather than + // straight-line distance, and produces a unit-length result. + result := InterpolateAtDistance(target, (*p)[i-1], (*p)[i]) + + // It is possible that (result == vertex(i)) due to rounding errors. + if result == (*p)[i] { + return result, i + 1 + } + return result, i + } + target -= length + } + + return (*p)[len(*p)-1], len(*p) +} + +// Uninterpolate is the inverse operation of Interpolate. Given a point on the +// polyline, it returns the ratio of the distance to the point from the +// beginning of the polyline over the length of the polyline. The return +// value is always betwen 0 and 1 inclusive. +// +// The polyline should not be empty. If it has fewer than 2 vertices, the +// return value is zero. +func (p *Polyline) Uninterpolate(point Point, nextVertex int) float64 { + if len(*p) < 2 { + return 0 + } + + var sum s1.Angle + for i := 1; i < nextVertex; i++ { + sum += (*p)[i-1].Distance((*p)[i]) + } + lengthToPoint := sum + (*p)[nextVertex-1].Distance(point) + for i := nextVertex; i < len(*p); i++ { + sum += (*p)[i-1].Distance((*p)[i]) + } + // The ratio can be greater than 1.0 due to rounding errors or because the + // point is not exactly on the polyline. + return minFloat64(1.0, float64(lengthToPoint/sum)) +} + +// TODO(roberts): Differences from C++. +// NearlyCoversPolyline +// InitToSnapped +// InitToSimplified +// SnapLevel +// encode/decode compressed diff --git a/vendor/github.com/blevesearch/geo/s2/polyline_measures.go b/vendor/github.com/blevesearch/geo/s2/polyline_measures.go new file mode 100644 index 00000000..38ce991b --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/polyline_measures.go @@ -0,0 +1,53 @@ +// Copyright 2018 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +// This file defines various measures for polylines on the sphere. These are +// low-level methods that work directly with arrays of Points. They are used to +// implement the methods in various other measures files. + +import ( + "github.com/golang/geo/r3" + "github.com/golang/geo/s1" +) + +// polylineLength returns the length of the given Polyline. +// It returns 0 for polylines with fewer than two vertices. +func polylineLength(p []Point) s1.Angle { + var length s1.Angle + + for i := 1; i < len(p); i++ { + length += p[i-1].Distance(p[i]) + } + return length +} + +// polylineCentroid returns the true centroid of the polyline multiplied by the +// length of the polyline. The result is not unit length, so you may wish to +// normalize it. +// +// Scaling by the Polyline length makes it easy to compute the centroid +// of several Polylines (by simply adding up their centroids). +// +// Note that for degenerate Polylines (e.g., AA) this returns Point(0, 0, 0). +// (This answer is correct; the result of this function is a line integral over +// the polyline, whose value is always zero if the polyline is degenerate.) +func polylineCentroid(p []Point) Point { + var centroid r3.Vector + for i := 1; i < len(p); i++ { + centroid = centroid.Add(EdgeTrueCentroid(p[i-1], p[i]).Vector) + } + return Point{centroid} +} diff --git a/vendor/github.com/blevesearch/geo/s2/predicates.go b/vendor/github.com/blevesearch/geo/s2/predicates.go new file mode 100644 index 00000000..9fc5e175 --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/predicates.go @@ -0,0 +1,701 @@ +// Copyright 2016 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +// This file contains various predicates that are guaranteed to produce +// correct, consistent results. They are also relatively efficient. This is +// achieved by computing conservative error bounds and falling back to high +// precision or even exact arithmetic when the result is uncertain. Such +// predicates are useful in implementing robust algorithms. +// +// See also EdgeCrosser, which implements various exact +// edge-crossing predicates more efficiently than can be done here. + +import ( + "math" + "math/big" + + "github.com/golang/geo/r3" + "github.com/golang/geo/s1" +) + +const ( + // If any other machine architectures need to be suppported, these next three + // values will need to be updated. + + // epsilon is a small number that represents a reasonable level of noise between two + // values that can be considered to be equal. + epsilon = 1e-15 + // dblEpsilon is a smaller number for values that require more precision. + // This is the C++ DBL_EPSILON equivalent. + dblEpsilon = 2.220446049250313e-16 + // dblError is the C++ value for S2 rounding_epsilon(). + dblError = 1.110223024625156e-16 + + // maxDeterminantError is the maximum error in computing (AxB).C where all vectors + // are unit length. Using standard inequalities, it can be shown that + // + // fl(AxB) = AxB + D where |D| <= (|AxB| + (2/sqrt(3))*|A|*|B|) * e + // + // where "fl()" denotes a calculation done in floating-point arithmetic, + // |x| denotes either absolute value or the L2-norm as appropriate, and + // e is a reasonably small value near the noise level of floating point + // number accuracy. Similarly, + // + // fl(B.C) = B.C + d where |d| <= (|B.C| + 2*|B|*|C|) * e . + // + // Applying these bounds to the unit-length vectors A,B,C and neglecting + // relative error (which does not affect the sign of the result), we get + // + // fl((AxB).C) = (AxB).C + d where |d| <= (3 + 2/sqrt(3)) * e + maxDeterminantError = 1.8274 * dblEpsilon + + // detErrorMultiplier is the factor to scale the magnitudes by when checking + // for the sign of set of points with certainty. Using a similar technique to + // the one used for maxDeterminantError, the error is at most: + // + // |d| <= (3 + 6/sqrt(3)) * |A-C| * |B-C| * e + // + // If the determinant magnitude is larger than this value then we know + // its sign with certainty. + detErrorMultiplier = 3.2321 * dblEpsilon +) + +// Direction is an indication of the ordering of a set of points. +type Direction int + +// These are the three options for the direction of a set of points. +const ( + Clockwise Direction = -1 + Indeterminate Direction = 0 + CounterClockwise Direction = 1 +) + +// newBigFloat constructs a new big.Float with maximum precision. +func newBigFloat() *big.Float { return new(big.Float).SetPrec(big.MaxPrec) } + +// Sign returns true if the points A, B, C are strictly counterclockwise, +// and returns false if the points are clockwise or collinear (i.e. if they are all +// contained on some great circle). +// +// Due to numerical errors, situations may arise that are mathematically +// impossible, e.g. ABC may be considered strictly CCW while BCA is not. +// However, the implementation guarantees the following: +// +// If Sign(a,b,c), then !Sign(c,b,a) for all a,b,c. +func Sign(a, b, c Point) bool { + // NOTE(dnadasi): In the C++ API the equivalent method here was known as "SimpleSign". + + // We compute the signed volume of the parallelepiped ABC. The usual + // formula for this is (A ⨯ B) · C, but we compute it here using (C ⨯ A) · B + // in order to ensure that ABC and CBA are not both CCW. This follows + // from the following identities (which are true numerically, not just + // mathematically): + // + // (1) x ⨯ y == -(y ⨯ x) + // (2) -x · y == -(x · y) + return c.Cross(a.Vector).Dot(b.Vector) > 0 +} + +// RobustSign returns a Direction representing the ordering of the points. +// CounterClockwise is returned if the points are in counter-clockwise order, +// Clockwise for clockwise, and Indeterminate if any two points are the same (collinear), +// or the sign could not completely be determined. +// +// This function has additional logic to make sure that the above properties hold even +// when the three points are coplanar, and to deal with the limitations of +// floating-point arithmetic. +// +// RobustSign satisfies the following conditions: +// +// (1) RobustSign(a,b,c) == Indeterminate if and only if a == b, b == c, or c == a +// (2) RobustSign(b,c,a) == RobustSign(a,b,c) for all a,b,c +// (3) RobustSign(c,b,a) == -RobustSign(a,b,c) for all a,b,c +// +// In other words: +// +// (1) The result is Indeterminate if and only if two points are the same. +// (2) Rotating the order of the arguments does not affect the result. +// (3) Exchanging any two arguments inverts the result. +// +// On the other hand, note that it is not true in general that +// RobustSign(-a,b,c) == -RobustSign(a,b,c), or any similar identities +// involving antipodal points. +func RobustSign(a, b, c Point) Direction { + sign := triageSign(a, b, c) + if sign == Indeterminate { + sign = expensiveSign(a, b, c) + } + return sign +} + +// stableSign reports the direction sign of the points in a numerically stable way. +// Unlike triageSign, this method can usually compute the correct determinant sign +// even when all three points are as collinear as possible. For example if three +// points are spaced 1km apart along a random line on the Earth's surface using +// the nearest representable points, there is only a 0.4% chance that this method +// will not be able to find the determinant sign. The probability of failure +// decreases as the points get closer together; if the collinear points are 1 meter +// apart, the failure rate drops to 0.0004%. +// +// This method could be extended to also handle nearly-antipodal points, but antipodal +// points are rare in practice so it seems better to simply fall back to +// exact arithmetic in that case. +func stableSign(a, b, c Point) Direction { + ab := b.Sub(a.Vector) + ab2 := ab.Norm2() + bc := c.Sub(b.Vector) + bc2 := bc.Norm2() + ca := a.Sub(c.Vector) + ca2 := ca.Norm2() + + // Now compute the determinant ((A-C)x(B-C)).C, where the vertices have been + // cyclically permuted if necessary so that AB is the longest edge. (This + // minimizes the magnitude of cross product.) At the same time we also + // compute the maximum error in the determinant. + + // The two shortest edges, pointing away from their common point. + var e1, e2, op r3.Vector + if ab2 >= bc2 && ab2 >= ca2 { + // AB is the longest edge. + e1, e2, op = ca, bc, c.Vector + } else if bc2 >= ca2 { + // BC is the longest edge. + e1, e2, op = ab, ca, a.Vector + } else { + // CA is the longest edge. + e1, e2, op = bc, ab, b.Vector + } + + det := -e1.Cross(e2).Dot(op) + maxErr := detErrorMultiplier * math.Sqrt(e1.Norm2()*e2.Norm2()) + + // If the determinant isn't zero, within maxErr, we know definitively the point ordering. + if det > maxErr { + return CounterClockwise + } + if det < -maxErr { + return Clockwise + } + return Indeterminate +} + +// triageSign returns the direction sign of the points. It returns Indeterminate if two +// points are identical or the result is uncertain. Uncertain cases can be resolved, if +// desired, by calling expensiveSign. +// +// The purpose of this method is to allow additional cheap tests to be done without +// calling expensiveSign. +func triageSign(a, b, c Point) Direction { + det := a.Cross(b.Vector).Dot(c.Vector) + if det > maxDeterminantError { + return CounterClockwise + } + if det < -maxDeterminantError { + return Clockwise + } + return Indeterminate +} + +// expensiveSign reports the direction sign of the points. It returns Indeterminate +// if two of the input points are the same. It uses multiple-precision arithmetic +// to ensure that its results are always self-consistent. +func expensiveSign(a, b, c Point) Direction { + // Return Indeterminate if and only if two points are the same. + // This ensures RobustSign(a,b,c) == Indeterminate if and only if a == b, b == c, or c == a. + // ie. Property 1 of RobustSign. + if a == b || b == c || c == a { + return Indeterminate + } + + // Next we try recomputing the determinant still using floating-point + // arithmetic but in a more precise way. This is more expensive than the + // simple calculation done by triageSign, but it is still *much* cheaper + // than using arbitrary-precision arithmetic. This optimization is able to + // compute the correct determinant sign in virtually all cases except when + // the three points are truly collinear (e.g., three points on the equator). + detSign := stableSign(a, b, c) + if detSign != Indeterminate { + return detSign + } + + // Otherwise fall back to exact arithmetic and symbolic permutations. + return exactSign(a, b, c, true) +} + +// exactSign reports the direction sign of the points computed using high-precision +// arithmetic and/or symbolic perturbations. +func exactSign(a, b, c Point, perturb bool) Direction { + // Sort the three points in lexicographic order, keeping track of the sign + // of the permutation. (Each exchange inverts the sign of the determinant.) + permSign := CounterClockwise + pa := &a + pb := &b + pc := &c + if pa.Cmp(pb.Vector) > 0 { + pa, pb = pb, pa + permSign = -permSign + } + if pb.Cmp(pc.Vector) > 0 { + pb, pc = pc, pb + permSign = -permSign + } + if pa.Cmp(pb.Vector) > 0 { + pa, pb = pb, pa + permSign = -permSign + } + + // Construct multiple-precision versions of the sorted points and compute + // their precise 3x3 determinant. + xa := r3.PreciseVectorFromVector(pa.Vector) + xb := r3.PreciseVectorFromVector(pb.Vector) + xc := r3.PreciseVectorFromVector(pc.Vector) + xbCrossXc := xb.Cross(xc) + det := xa.Dot(xbCrossXc) + + // The precision of big.Float is high enough that the result should always + // be exact enough (no rounding was performed). + + // If the exact determinant is non-zero, we're done. + detSign := Direction(det.Sign()) + if detSign == Indeterminate && perturb { + // Otherwise, we need to resort to symbolic perturbations to resolve the + // sign of the determinant. + detSign = symbolicallyPerturbedSign(xa, xb, xc, xbCrossXc) + } + return permSign * detSign +} + +// symbolicallyPerturbedSign reports the sign of the determinant of three points +// A, B, C under a model where every possible Point is slightly perturbed by +// a unique infinitesmal amount such that no three perturbed points are +// collinear and no four points are coplanar. The perturbations are so small +// that they do not change the sign of any determinant that was non-zero +// before the perturbations, and therefore can be safely ignored unless the +// determinant of three points is exactly zero (using multiple-precision +// arithmetic). This returns CounterClockwise or Clockwise according to the +// sign of the determinant after the symbolic perturbations are taken into account. +// +// Since the symbolic perturbation of a given point is fixed (i.e., the +// perturbation is the same for all calls to this method and does not depend +// on the other two arguments), the results of this method are always +// self-consistent. It will never return results that would correspond to an +// impossible configuration of non-degenerate points. +// +// This requires that the 3x3 determinant of A, B, C must be exactly zero. +// And the points must be distinct, with A < B < C in lexicographic order. +// +// Reference: +// "Simulation of Simplicity" (Edelsbrunner and Muecke, ACM Transactions on +// Graphics, 1990). +// +func symbolicallyPerturbedSign(a, b, c, bCrossC r3.PreciseVector) Direction { + // This method requires that the points are sorted in lexicographically + // increasing order. This is because every possible Point has its own + // symbolic perturbation such that if A < B then the symbolic perturbation + // for A is much larger than the perturbation for B. + // + // Alternatively, we could sort the points in this method and keep track of + // the sign of the permutation, but it is more efficient to do this before + // converting the inputs to the multi-precision representation, and this + // also lets us re-use the result of the cross product B x C. + // + // Every input coordinate x[i] is assigned a symbolic perturbation dx[i]. + // We then compute the sign of the determinant of the perturbed points, + // i.e. + // | a.X+da.X a.Y+da.Y a.Z+da.Z | + // | b.X+db.X b.Y+db.Y b.Z+db.Z | + // | c.X+dc.X c.Y+dc.Y c.Z+dc.Z | + // + // The perturbations are chosen such that + // + // da.Z > da.Y > da.X > db.Z > db.Y > db.X > dc.Z > dc.Y > dc.X + // + // where each perturbation is so much smaller than the previous one that we + // don't even need to consider it unless the coefficients of all previous + // perturbations are zero. In fact, it is so small that we don't need to + // consider it unless the coefficient of all products of the previous + // perturbations are zero. For example, we don't need to consider the + // coefficient of db.Y unless the coefficient of db.Z *da.X is zero. + // + // The follow code simply enumerates the coefficients of the perturbations + // (and products of perturbations) that appear in the determinant above, in + // order of decreasing perturbation magnitude. The first non-zero + // coefficient determines the sign of the result. The easiest way to + // enumerate the coefficients in the correct order is to pretend that each + // perturbation is some tiny value "eps" raised to a power of two: + // + // eps** 1 2 4 8 16 32 64 128 256 + // da.Z da.Y da.X db.Z db.Y db.X dc.Z dc.Y dc.X + // + // Essentially we can then just count in binary and test the corresponding + // subset of perturbations at each step. So for example, we must test the + // coefficient of db.Z*da.X before db.Y because eps**12 > eps**16. + // + // Of course, not all products of these perturbations appear in the + // determinant above, since the determinant only contains the products of + // elements in distinct rows and columns. Thus we don't need to consider + // da.Z*da.Y, db.Y *da.Y, etc. Furthermore, sometimes different pairs of + // perturbations have the same coefficient in the determinant; for example, + // da.Y*db.X and db.Y*da.X have the same coefficient (c.Z). Therefore + // we only need to test this coefficient the first time we encounter it in + // the binary order above (which will be db.Y*da.X). + // + // The sequence of tests below also appears in Table 4-ii of the paper + // referenced above, if you just want to look it up, with the following + // translations: [a,b,c] -> [i,j,k] and [0,1,2] -> [1,2,3]. Also note that + // some of the signs are different because the opposite cross product is + // used (e.g., B x C rather than C x B). + + detSign := bCrossC.Z.Sign() // da.Z + if detSign != 0 { + return Direction(detSign) + } + detSign = bCrossC.Y.Sign() // da.Y + if detSign != 0 { + return Direction(detSign) + } + detSign = bCrossC.X.Sign() // da.X + if detSign != 0 { + return Direction(detSign) + } + + detSign = newBigFloat().Sub(newBigFloat().Mul(c.X, a.Y), newBigFloat().Mul(c.Y, a.X)).Sign() // db.Z + if detSign != 0 { + return Direction(detSign) + } + detSign = c.X.Sign() // db.Z * da.Y + if detSign != 0 { + return Direction(detSign) + } + detSign = -(c.Y.Sign()) // db.Z * da.X + if detSign != 0 { + return Direction(detSign) + } + + detSign = newBigFloat().Sub(newBigFloat().Mul(c.Z, a.X), newBigFloat().Mul(c.X, a.Z)).Sign() // db.Y + if detSign != 0 { + return Direction(detSign) + } + detSign = c.Z.Sign() // db.Y * da.X + if detSign != 0 { + return Direction(detSign) + } + + // The following test is listed in the paper, but it is redundant because + // the previous tests guarantee that C == (0, 0, 0). + // (c.Y*a.Z - c.Z*a.Y).Sign() // db.X + + detSign = newBigFloat().Sub(newBigFloat().Mul(a.X, b.Y), newBigFloat().Mul(a.Y, b.X)).Sign() // dc.Z + if detSign != 0 { + return Direction(detSign) + } + detSign = -(b.X.Sign()) // dc.Z * da.Y + if detSign != 0 { + return Direction(detSign) + } + detSign = b.Y.Sign() // dc.Z * da.X + if detSign != 0 { + return Direction(detSign) + } + detSign = a.X.Sign() // dc.Z * db.Y + if detSign != 0 { + return Direction(detSign) + } + return CounterClockwise // dc.Z * db.Y * da.X +} + +// CompareDistances returns -1, 0, or +1 according to whether AX < BX, A == B, +// or AX > BX respectively. Distances are measured with respect to the positions +// of X, A, and B as though they were reprojected to lie exactly on the surface of +// the unit sphere. Furthermore, this method uses symbolic perturbations to +// ensure that the result is non-zero whenever A != B, even when AX == BX +// exactly, or even when A and B project to the same point on the sphere. +// Such results are guaranteed to be self-consistent, i.e. if AB < BC and +// BC < AC, then AB < AC. +func CompareDistances(x, a, b Point) int { + // We start by comparing distances using dot products (i.e., cosine of the + // angle), because (1) this is the cheapest technique, and (2) it is valid + // over the entire range of possible angles. (We can only use the sin^2 + // technique if both angles are less than 90 degrees or both angles are + // greater than 90 degrees.) + sign := triageCompareCosDistances(x, a, b) + if sign != 0 { + return sign + } + + // Optimization for (a == b) to avoid falling back to exact arithmetic. + if a == b { + return 0 + } + + // It is much better numerically to compare distances using cos(angle) if + // the distances are near 90 degrees and sin^2(angle) if the distances are + // near 0 or 180 degrees. We only need to check one of the two angles when + // making this decision because the fact that the test above failed means + // that angles "a" and "b" are very close together. + cosAX := a.Dot(x.Vector) + if cosAX > 1/math.Sqrt2 { + // Angles < 45 degrees. + sign = triageCompareSin2Distances(x, a, b) + } else if cosAX < -1/math.Sqrt2 { + // Angles > 135 degrees. sin^2(angle) is decreasing in this range. + sign = -triageCompareSin2Distances(x, a, b) + } + // C++ adds an additional check here using 80-bit floats. + // This is skipped in Go because we only have 32 and 64 bit floats. + + if sign != 0 { + return sign + } + + sign = exactCompareDistances(r3.PreciseVectorFromVector(x.Vector), r3.PreciseVectorFromVector(a.Vector), r3.PreciseVectorFromVector(b.Vector)) + if sign != 0 { + return sign + } + return symbolicCompareDistances(x, a, b) +} + +// cosDistance returns cos(XY) where XY is the angle between X and Y, and the +// maximum error amount in the result. This requires X and Y be normalized. +func cosDistance(x, y Point) (cos, err float64) { + cos = x.Dot(y.Vector) + return cos, 9.5*dblError*math.Abs(cos) + 1.5*dblError +} + +// sin2Distance returns sin**2(XY), where XY is the angle between X and Y, +// and the maximum error amount in the result. This requires X and Y be normalized. +func sin2Distance(x, y Point) (sin2, err float64) { + // The (x-y).Cross(x+y) trick eliminates almost all of error due to x + // and y being not quite unit length. This method is extremely accurate + // for small distances; the *relative* error in the result is O(dblError) for + // distances as small as dblError. + n := x.Sub(y.Vector).Cross(x.Add(y.Vector)) + sin2 = 0.25 * n.Norm2() + err = ((21+4*math.Sqrt(3))*dblError*sin2 + + 32*math.Sqrt(3)*dblError*dblError*math.Sqrt(sin2) + + 768*dblError*dblError*dblError*dblError) + return sin2, err +} + +// triageCompareCosDistances returns -1, 0, or +1 according to whether AX < BX, +// A == B, or AX > BX by comparing the distances between them using cosDistance. +func triageCompareCosDistances(x, a, b Point) int { + cosAX, cosAXerror := cosDistance(a, x) + cosBX, cosBXerror := cosDistance(b, x) + diff := cosAX - cosBX + err := cosAXerror + cosBXerror + if diff > err { + return -1 + } + if diff < -err { + return 1 + } + return 0 +} + +// triageCompareSin2Distances returns -1, 0, or +1 according to whether AX < BX, +// A == B, or AX > BX by comparing the distances between them using sin2Distance. +func triageCompareSin2Distances(x, a, b Point) int { + sin2AX, sin2AXerror := sin2Distance(a, x) + sin2BX, sin2BXerror := sin2Distance(b, x) + diff := sin2AX - sin2BX + err := sin2AXerror + sin2BXerror + if diff > err { + return 1 + } + if diff < -err { + return -1 + } + return 0 +} + +// exactCompareDistances returns -1, 0, or 1 after comparing using the values as +// PreciseVectors. +func exactCompareDistances(x, a, b r3.PreciseVector) int { + // This code produces the same result as though all points were reprojected + // to lie exactly on the surface of the unit sphere. It is based on testing + // whether x.Dot(a.Normalize()) < x.Dot(b.Normalize()), reformulated + // so that it can be evaluated using exact arithmetic. + cosAX := x.Dot(a) + cosBX := x.Dot(b) + + // If the two values have different signs, we need to handle that case now + // before squaring them below. + aSign := cosAX.Sign() + bSign := cosBX.Sign() + if aSign != bSign { + // If cos(AX) > cos(BX), then AX < BX. + if aSign > bSign { + return -1 + } + return 1 + } + cosAX2 := newBigFloat().Mul(cosAX, cosAX) + cosBX2 := newBigFloat().Mul(cosBX, cosBX) + cmp := newBigFloat().Sub(cosBX2.Mul(cosBX2, a.Norm2()), cosAX2.Mul(cosAX2, b.Norm2())) + return aSign * cmp.Sign() +} + +// symbolicCompareDistances returns -1, 0, or +1 given three points such that AX == BX +// (exactly) according to whether AX < BX, AX == BX, or AX > BX after symbolic +// perturbations are taken into account. +func symbolicCompareDistances(x, a, b Point) int { + // Our symbolic perturbation strategy is based on the following model. + // Similar to "simulation of simplicity", we assign a perturbation to every + // point such that if A < B, then the symbolic perturbation for A is much, + // much larger than the symbolic perturbation for B. We imagine that + // rather than projecting every point to lie exactly on the unit sphere, + // instead each point is positioned on its own tiny pedestal that raises it + // just off the surface of the unit sphere. This means that the distance AX + // is actually the true distance AX plus the (symbolic) heights of the + // pedestals for A and X. The pedestals are infinitesmally thin, so they do + // not affect distance measurements except at the two endpoints. If several + // points project to exactly the same point on the unit sphere, we imagine + // that they are placed on separate pedestals placed close together, where + // the distance between pedestals is much, much less than the height of any + // pedestal. (There are a finite number of Points, and therefore a finite + // number of pedestals, so this is possible.) + // + // If A < B, then A is on a higher pedestal than B, and therefore AX > BX. + switch a.Cmp(b.Vector) { + case -1: + return 1 + case 1: + return -1 + default: + return 0 + } +} + +var ( + // ca45Degrees is a predefined ChordAngle representing (approximately) 45 degrees. + ca45Degrees = s1.ChordAngleFromSquaredLength(2 - math.Sqrt2) +) + +// CompareDistance returns -1, 0, or +1 according to whether the distance XY is +// respectively less than, equal to, or greater than the provided chord angle. Distances are measured +// with respect to the positions of all points as though they are projected to lie +// exactly on the surface of the unit sphere. +func CompareDistance(x, y Point, r s1.ChordAngle) int { + // As with CompareDistances, we start by comparing dot products because + // the sin^2 method is only valid when the distance XY and the limit "r" are + // both less than 90 degrees. + sign := triageCompareCosDistance(x, y, float64(r)) + if sign != 0 { + return sign + } + + // Unlike with CompareDistances, it's not worth using the sin^2 method + // when the distance limit is near 180 degrees because the ChordAngle + // representation itself has has a rounding error of up to 2e-8 radians for + // distances near 180 degrees. + if r < ca45Degrees { + sign = triageCompareSin2Distance(x, y, float64(r)) + if sign != 0 { + return sign + } + } + return exactCompareDistance(r3.PreciseVectorFromVector(x.Vector), r3.PreciseVectorFromVector(y.Vector), big.NewFloat(float64(r)).SetPrec(big.MaxPrec)) +} + +// triageCompareCosDistance returns -1, 0, or +1 according to whether the distance XY is +// less than, equal to, or greater than r2 respectively using cos distance. +func triageCompareCosDistance(x, y Point, r2 float64) int { + cosXY, cosXYError := cosDistance(x, y) + cosR := 1.0 - 0.5*r2 + cosRError := 2.0 * dblError * cosR + diff := cosXY - cosR + err := cosXYError + cosRError + if diff > err { + return -1 + } + if diff < -err { + return 1 + } + return 0 +} + +// triageCompareSin2Distance returns -1, 0, or +1 according to whether the distance XY is +// less than, equal to, or greater than r2 respectively using sin^2 distance. +func triageCompareSin2Distance(x, y Point, r2 float64) int { + // Only valid for distance limits < 90 degrees. + sin2XY, sin2XYError := sin2Distance(x, y) + sin2R := r2 * (1.0 - 0.25*r2) + sin2RError := 3.0 * dblError * sin2R + diff := sin2XY - sin2R + err := sin2XYError + sin2RError + if diff > err { + return 1 + } + if diff < -err { + return -1 + } + return 0 +} + +var ( + bigOne = big.NewFloat(1.0).SetPrec(big.MaxPrec) + bigHalf = big.NewFloat(0.5).SetPrec(big.MaxPrec) +) + +// exactCompareDistance returns -1, 0, or +1 after comparing using PreciseVectors. +func exactCompareDistance(x, y r3.PreciseVector, r2 *big.Float) int { + // This code produces the same result as though all points were reprojected + // to lie exactly on the surface of the unit sphere. It is based on + // comparing the cosine of the angle XY (when both points are projected to + // lie exactly on the sphere) to the given threshold. + cosXY := x.Dot(y) + cosR := newBigFloat().Sub(bigOne, newBigFloat().Mul(bigHalf, r2)) + + // If the two values have different signs, we need to handle that case now + // before squaring them below. + xySign := cosXY.Sign() + rSign := cosR.Sign() + if xySign != rSign { + if xySign > rSign { + return -1 + } + return 1 // If cos(XY) > cos(r), then XY < r. + } + cmp := newBigFloat().Sub( + newBigFloat().Mul( + newBigFloat().Mul(cosR, cosR), newBigFloat().Mul(x.Norm2(), y.Norm2())), + newBigFloat().Mul(cosXY, cosXY)) + return xySign * cmp.Sign() +} + +// TODO(roberts): Differences from C++ +// CompareEdgeDistance +// CompareEdgeDirections +// EdgeCircumcenterSign +// GetVoronoiSiteExclusion +// GetClosestVertex +// TriageCompareLineSin2Distance +// TriageCompareLineCos2Distance +// TriageCompareLineDistance +// TriageCompareEdgeDistance +// ExactCompareLineDistance +// ExactCompareEdgeDistance +// TriageCompareEdgeDirections +// ExactCompareEdgeDirections +// ArePointsAntipodal +// ArePointsLinearlyDependent +// GetCircumcenter +// TriageEdgeCircumcenterSign +// ExactEdgeCircumcenterSign +// UnperturbedSign +// SymbolicEdgeCircumcenterSign +// ExactVoronoiSiteExclusion diff --git a/vendor/github.com/blevesearch/geo/s2/projections.go b/vendor/github.com/blevesearch/geo/s2/projections.go new file mode 100644 index 00000000..f7273609 --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/projections.go @@ -0,0 +1,241 @@ +// Copyright 2018 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +import ( + "math" + + "github.com/golang/geo/r2" + "github.com/golang/geo/s1" +) + +// Projection defines an interface for different ways of mapping between s2 and r2 Points. +// It can also define the coordinate wrapping behavior along each axis. +type Projection interface { + // Project converts a point on the sphere to a projected 2D point. + Project(p Point) r2.Point + + // Unproject converts a projected 2D point to a point on the sphere. + // + // If wrapping is defined for a given axis (see below), then this method + // should accept any real number for the corresponding coordinate. + Unproject(p r2.Point) Point + + // FromLatLng is a convenience function equivalent to Project(LatLngToPoint(ll)), + // but the implementation is more efficient. + FromLatLng(ll LatLng) r2.Point + + // ToLatLng is a convenience function equivalent to LatLngFromPoint(Unproject(p)), + // but the implementation is more efficient. + ToLatLng(p r2.Point) LatLng + + // Interpolate returns the point obtained by interpolating the given + // fraction of the distance along the line from A to B. + // Fractions < 0 or > 1 result in extrapolation instead. + Interpolate(f float64, a, b r2.Point) r2.Point + + // WrapDistance reports the coordinate wrapping distance along each axis. + // If this value is non-zero for a given axis, the coordinates are assumed + // to "wrap" with the given period. For example, if WrapDistance.Y == 360 + // then (x, y) and (x, y + 360) should map to the same Point. + // + // This information is used to ensure that edges takes the shortest path + // between two given points. For example, if coordinates represent + // (latitude, longitude) pairs in degrees and WrapDistance().Y == 360, + // then the edge (5:179, 5:-179) would be interpreted as spanning 2 degrees + // of longitude rather than 358 degrees. + // + // If a given axis does not wrap, its WrapDistance should be set to zero. + WrapDistance() r2.Point + + // WrapDestination that wraps the coordinates of B if necessary in order to + // obtain the shortest edge AB. For example, suppose that A = [170, 20], + // B = [-170, 20], and the projection wraps so that [x, y] == [x + 360, y]. + // Then this function would return [190, 20] for point B (reducing the edge + // length in the "x" direction from 340 to 20). + WrapDestination(a, b r2.Point) r2.Point + + // We do not support implementations of this interface outside this package. + privateInterface() +} + +// PlateCarreeProjection defines the "plate carree" (square plate) projection, +// which converts points on the sphere to (longitude, latitude) pairs. +// Coordinates can be scaled so that they represent radians, degrees, etc, but +// the projection is always centered around (latitude=0, longitude=0). +// +// Note that (x, y) coordinates are backwards compared to the usual (latitude, +// longitude) ordering, in order to match the usual convention for graphs in +// which "x" is horizontal and "y" is vertical. +type PlateCarreeProjection struct { + xWrap float64 + toRadians float64 // Multiplier to convert coordinates to radians. + fromRadians float64 // Multiplier to convert coordinates from radians. +} + +// NewPlateCarreeProjection constructs a plate carree projection where the +// x-coordinates (lng) span [-xScale, xScale] and the y coordinates (lat) +// span [-xScale/2, xScale/2]. For example if xScale==180 then the x +// range is [-180, 180] and the y range is [-90, 90]. +// +// By default coordinates are expressed in radians, i.e. the x range is +// [-Pi, Pi] and the y range is [-Pi/2, Pi/2]. +func NewPlateCarreeProjection(xScale float64) Projection { + return &PlateCarreeProjection{ + xWrap: 2 * xScale, + toRadians: math.Pi / xScale, + fromRadians: xScale / math.Pi, + } +} + +// Project converts a point on the sphere to a projected 2D point. +func (p *PlateCarreeProjection) Project(pt Point) r2.Point { + return p.FromLatLng(LatLngFromPoint(pt)) +} + +// Unproject converts a projected 2D point to a point on the sphere. +func (p *PlateCarreeProjection) Unproject(pt r2.Point) Point { + return PointFromLatLng(p.ToLatLng(pt)) +} + +// FromLatLng returns the LatLng projected into an R2 Point. +func (p *PlateCarreeProjection) FromLatLng(ll LatLng) r2.Point { + return r2.Point{ + X: p.fromRadians * ll.Lng.Radians(), + Y: p.fromRadians * ll.Lat.Radians(), + } +} + +// ToLatLng returns the LatLng projected from the given R2 Point. +func (p *PlateCarreeProjection) ToLatLng(pt r2.Point) LatLng { + return LatLng{ + Lat: s1.Angle(p.toRadians * pt.Y), + Lng: s1.Angle(p.toRadians * math.Remainder(pt.X, p.xWrap)), + } +} + +// Interpolate returns the point obtained by interpolating the given +// fraction of the distance along the line from A to B. +func (p *PlateCarreeProjection) Interpolate(f float64, a, b r2.Point) r2.Point { + return a.Mul(1 - f).Add(b.Mul(f)) +} + +// WrapDistance reports the coordinate wrapping distance along each axis. +func (p *PlateCarreeProjection) WrapDistance() r2.Point { + return r2.Point{p.xWrap, 0} +} + +// WrapDestination wraps the points if needed to get the shortest edge. +func (p *PlateCarreeProjection) WrapDestination(a, b r2.Point) r2.Point { + return wrapDestination(a, b, p.WrapDistance) +} + +func (p *PlateCarreeProjection) privateInterface() {} + +// MercatorProjection defines the spherical Mercator projection. Google Maps +// uses this projection together with WGS84 coordinates, in which case it is +// known as the "Web Mercator" projection (see Wikipedia). This class makes +// no assumptions regarding the coordinate system of its input points, but +// simply applies the spherical Mercator projection to them. +// +// The Mercator projection is finite in width (x) but infinite in height (y). +// "x" corresponds to longitude, and spans a finite range such as [-180, 180] +// (with coordinate wrapping), while "y" is a function of latitude and spans +// an infinite range. (As "y" coordinates get larger, points get closer to +// the north pole but never quite reach it.) The north and south poles have +// infinite "y" values. (Note that this will cause problems if you tessellate +// a Mercator edge where one endpoint is a pole. If you need to do this, clip +// the edge first so that the "y" coordinate is no more than about 5 * maxX.) +type MercatorProjection struct { + xWrap float64 + toRadians float64 // Multiplier to convert coordinates to radians. + fromRadians float64 // Multiplier to convert coordinates from radians. +} + +// NewMercatorProjection constructs a Mercator projection with the given maximum +// longitude axis value corresponding to a range of [-maxLng, maxLng]. +// The horizontal and vertical axes are scaled equally. +func NewMercatorProjection(maxLng float64) Projection { + return &MercatorProjection{ + xWrap: 2 * maxLng, + toRadians: math.Pi / maxLng, + fromRadians: maxLng / math.Pi, + } +} + +// Project converts a point on the sphere to a projected 2D point. +func (p *MercatorProjection) Project(pt Point) r2.Point { + return p.FromLatLng(LatLngFromPoint(pt)) +} + +// Unproject converts a projected 2D point to a point on the sphere. +func (p *MercatorProjection) Unproject(pt r2.Point) Point { + return PointFromLatLng(p.ToLatLng(pt)) +} + +// FromLatLng returns the LatLng projected into an R2 Point. +func (p *MercatorProjection) FromLatLng(ll LatLng) r2.Point { + // This formula is more accurate near zero than the log(tan()) version. + // Note that latitudes of +/- 90 degrees yield "y" values of +/- infinity. + sinPhi := math.Sin(float64(ll.Lat)) + y := 0.5 * math.Log((1+sinPhi)/(1-sinPhi)) + return r2.Point{p.fromRadians * float64(ll.Lng), p.fromRadians * y} +} + +// ToLatLng returns the LatLng projected from the given R2 Point. +func (p *MercatorProjection) ToLatLng(pt r2.Point) LatLng { + // This formula is more accurate near zero than the atan(exp()) version. + x := p.toRadians * math.Remainder(pt.X, p.xWrap) + k := math.Exp(2 * p.toRadians * pt.Y) + var y float64 + if math.IsInf(k, 0) { + y = math.Pi / 2 + } else { + y = math.Asin((k - 1) / (k + 1)) + } + return LatLng{s1.Angle(y), s1.Angle(x)} +} + +// Interpolate returns the point obtained by interpolating the given +// fraction of the distance along the line from A to B. +func (p *MercatorProjection) Interpolate(f float64, a, b r2.Point) r2.Point { + return a.Mul(1 - f).Add(b.Mul(f)) +} + +// WrapDistance reports the coordinate wrapping distance along each axis. +func (p *MercatorProjection) WrapDistance() r2.Point { + return r2.Point{p.xWrap, 0} +} + +// WrapDestination wraps the points if needed to get the shortest edge. +func (p *MercatorProjection) WrapDestination(a, b r2.Point) r2.Point { + return wrapDestination(a, b, p.WrapDistance) +} + +func (p *MercatorProjection) privateInterface() {} + +func wrapDestination(a, b r2.Point, wrapDistance func() r2.Point) r2.Point { + wrap := wrapDistance() + x := b.X + y := b.Y + // The code below ensures that "b" is unmodified unless wrapping is required. + if wrap.X > 0 && math.Abs(x-a.X) > 0.5*wrap.X { + x = a.X + math.Remainder(x-a.X, wrap.X) + } + if wrap.Y > 0 && math.Abs(y-a.Y) > 0.5*wrap.Y { + y = a.Y + math.Remainder(y-a.Y, wrap.Y) + } + return r2.Point{x, y} +} diff --git a/vendor/github.com/blevesearch/geo/s2/query_entry.go b/vendor/github.com/blevesearch/geo/s2/query_entry.go new file mode 100644 index 00000000..65e819e3 --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/query_entry.go @@ -0,0 +1,93 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +import "container/heap" + +// A queryQueueEntry stores CellIDs and distance from a target. It is used by the +// different S2 Query types to efficiently build their internal priority queue +// in the optimized algorithm implementations. +type queryQueueEntry struct { + // A lower bound on the distance from the target to ID. This is the key + // of the priority queue. + distance distance + + // The cell being queued. + id CellID + + // If the CellID belongs to a ShapeIndex, this field stores the + // corresponding ShapeIndexCell. Otherwise ID is a proper ancestor of + // one or more ShapeIndexCells and this field stores is nil. + indexCell *ShapeIndexCell +} + +// queryQueue is used by the optimized algorithm to maintain a priority queue of +// unprocessed CellIDs, sorted in increasing order of distance from the target. +type queryQueue struct { + queue queryPQ +} + +// newQueryQueue returns a new initialized queryQueue. +func newQueryQueue() *queryQueue { + q := &queryQueue{ + queue: make(queryPQ, 0), + } + heap.Init(&q.queue) + return q +} + +// push adds the given entry to the top of this queue. +func (q *queryQueue) push(e *queryQueueEntry) { + heap.Push(&q.queue, e) +} + +// pop returns the top element of this queue. +func (q *queryQueue) pop() *queryQueueEntry { + return heap.Pop(&q.queue).(*queryQueueEntry) +} + +func (q *queryQueue) size() int { + return q.queue.Len() +} + +func (q *queryQueue) reset() { + q.queue = q.queue[:0] +} + +// queryPQ is a priority queue that implements the heap interface. +type queryPQ []*queryQueueEntry + +func (q queryPQ) Len() int { return len(q) } +func (q queryPQ) Less(i, j int) bool { + return q[i].distance.less(q[j].distance) +} + +// Swap swaps the two entries. +func (q queryPQ) Swap(i, j int) { + q[i], q[j] = q[j], q[i] +} + +// Push adds the given entry to the queue. +func (q *queryPQ) Push(x interface{}) { + item := x.(*queryQueueEntry) + *q = append(*q, item) +} + +// Pop returns the top element of the queue. +func (q *queryPQ) Pop() interface{} { + item := (*q)[len(*q)-1] + *q = (*q)[:len(*q)-1] + return item +} diff --git a/vendor/github.com/blevesearch/geo/s2/query_options.go b/vendor/github.com/blevesearch/geo/s2/query_options.go new file mode 100644 index 00000000..9b7e38d6 --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/query_options.go @@ -0,0 +1,196 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +import ( + "math" + + "github.com/golang/geo/s1" +) + +const maxQueryResults = math.MaxInt32 + +// queryOptions represents the set of all configurable parameters used by all of +// the Query types. Most of these fields have non-zero defaults, so initialization +// is handled within each Query type. All of the exported methods accept user +// supplied sets of options to set or adjust as necessary. +// +// Several of the defaults depend on the distance interface type being used +// (e.g. minDistance, maxDistance, etc.) +// +// If a user sets an option value that a given query type doesn't use, it is ignored. +type queryOptions struct { + // maxResults specifies that at most MaxResults edges should be returned. + // This must be at least 1. + // + // The default value is to return all results. + maxResults int + + // distanceLimit specifies that only edges whose distance to the target is + // within this distance should be returned. + // + // Note that edges whose distance is exactly equal to this are + // not returned. In most cases this doesn't matter (since distances are + // not computed exactly in the first place), but if such edges are needed + // then you can retrieve them by specifying the distance as the next + // largest representable distance. i.e. distanceLimit.Successor(). + // + // The default value is the infinity value, such that all results will be + // returned. + distanceLimit s1.ChordAngle + + // maxError specifies that edges up to MaxError further away than the true + // closest edges may be substituted in the result set, as long as such + // edges satisfy all the remaining search criteria (such as DistanceLimit). + // This option only has an effect if MaxResults is also specified; + // otherwise all edges closer than MaxDistance will always be returned. + // + // Note that this does not affect how the distance between edges is + // computed; it simply gives the algorithm permission to stop the search + // early as soon as the best possible improvement drops below MaxError. + // + // This can be used to implement distance predicates efficiently. For + // example, to determine whether the minimum distance is less than D, set + // MaxResults == 1 and MaxDistance == MaxError == D. This causes + // the algorithm to terminate as soon as it finds any edge whose distance + // is less than D, rather than continuing to search for an edge that is + // even closer. + // + // The default value is zero. + maxError s1.ChordAngle + + // includeInteriors specifies that polygon interiors should be included + // when measuring distances. In other words, polygons that contain the target + // should have a distance of zero. (For targets consisting of multiple connected + // components, the distance is zero if any component is contained.) This + // is indicated in the results by returning a (ShapeID, EdgeID) pair + // with EdgeID == -1, i.e. this value denotes the polygons's interior. + // + // Note that for efficiency, any polygon that intersects the target may or + // may not have an EdgeID == -1 result. Such results are optional + // because in that case the distance to the polygon is already zero. + // + // The default value is true. + includeInteriors bool + + // specifies that distances should be computed by examining every edge + // rather than using the ShapeIndex. + // + // TODO(roberts): When optimized is implemented, update the default to false. + // The default value is true. + useBruteForce bool + + // region specifies that results must intersect the given Region. + // + // Note that if you want to set the region to a disc around a target + // point, it is faster to use a PointTarget with distanceLimit set + // instead. You can also set a distance limit and also require that results + // lie within a given rectangle. + // + // The default is nil (no region limits). + region Region +} + +// UseBruteForce sets or disables the use of brute force in a query. +func (q *queryOptions) UseBruteForce(x bool) *queryOptions { + q.useBruteForce = x + return q +} + +// IncludeInteriors specifies whether polygon interiors should be +// included when measuring distances. +func (q *queryOptions) IncludeInteriors(x bool) *queryOptions { + q.includeInteriors = x + return q +} + +// MaxError specifies that edges up to dist away than the true +// matching edges may be substituted in the result set, as long as such +// edges satisfy all the remaining search criteria (such as DistanceLimit). +// This option only has an effect if MaxResults is also specified; +// otherwise all edges closer than MaxDistance will always be returned. +func (q *queryOptions) MaxError(x s1.ChordAngle) *queryOptions { + q.maxError = x + return q +} + +// MaxResults specifies that at most MaxResults edges should be returned. +// This must be at least 1. +func (q *queryOptions) MaxResults(x int) *queryOptions { + // TODO(roberts): What should be done if the value is <= 0? + q.maxResults = int(x) + return q +} + +// DistanceLimit specifies that only edges whose distance to the target is +// within, this distance should be returned. Edges whose distance is equal +// are not returned. +// +// To include values that are equal, specify the limit with the next largest +// representable distance such as limit.Successor(), or set the option with +// Furthest/ClosestInclusiveDistanceLimit. +func (q *queryOptions) DistanceLimit(x s1.ChordAngle) *queryOptions { + q.distanceLimit = x + return q +} + +// ClosestInclusiveDistanceLimit sets the distance limit such that results whose +// distance is exactly equal to the limit are also returned. +func (q *queryOptions) ClosestInclusiveDistanceLimit(limit s1.ChordAngle) *queryOptions { + q.distanceLimit = limit.Successor() + return q +} + +// FurthestInclusiveDistanceLimit sets the distance limit such that results whose +// distance is exactly equal to the limit are also returned. +func (q *queryOptions) FurthestInclusiveDistanceLimit(limit s1.ChordAngle) *queryOptions { + q.distanceLimit = limit.Predecessor() + return q +} + +// ClosestConservativeDistanceLimit sets the distance limit such that results +// also incorporates the error in distance calculations. This ensures that all +// edges whose true distance is less than or equal to limit will be returned +// (along with some edges whose true distance is slightly greater). +// +// Algorithms that need to do exact distance comparisons can use this +// option to find a set of candidate edges that can then be filtered +// further (e.g., using CompareDistance). +func (q *queryOptions) ClosestConservativeDistanceLimit(limit s1.ChordAngle) *queryOptions { + q.distanceLimit = limit.Expanded(minUpdateDistanceMaxError(limit)) + return q +} + +// FurthestConservativeDistanceLimit sets the distance limit such that results +// also incorporates the error in distance calculations. This ensures that all +// edges whose true distance is greater than or equal to limit will be returned +// (along with some edges whose true distance is slightly less). +func (q *queryOptions) FurthestConservativeDistanceLimit(limit s1.ChordAngle) *queryOptions { + q.distanceLimit = limit.Expanded(-minUpdateDistanceMaxError(limit)) + return q +} + +// newQueryOptions returns a set of options using the given distance type +// with the proper default values. +func newQueryOptions(d distance) *queryOptions { + return &queryOptions{ + maxResults: maxQueryResults, + distanceLimit: d.infinity().chordAngle(), + maxError: 0, + includeInteriors: true, + useBruteForce: false, + region: nil, + } +} diff --git a/vendor/github.com/blevesearch/geo/s2/rect.go b/vendor/github.com/blevesearch/geo/s2/rect.go new file mode 100644 index 00000000..d7b9aa5e --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/rect.go @@ -0,0 +1,726 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +import ( + "fmt" + "io" + "math" + + "github.com/golang/geo/r1" + "github.com/golang/geo/r3" + "github.com/golang/geo/s1" +) + +// Rect represents a closed latitude-longitude rectangle. +type Rect struct { + Lat r1.Interval + Lng s1.Interval +} + +var ( + validRectLatRange = r1.Interval{-math.Pi / 2, math.Pi / 2} + validRectLngRange = s1.FullInterval() +) + +// EmptyRect returns the empty rectangle. +func EmptyRect() Rect { return Rect{r1.EmptyInterval(), s1.EmptyInterval()} } + +// FullRect returns the full rectangle. +func FullRect() Rect { return Rect{validRectLatRange, validRectLngRange} } + +// RectFromLatLng constructs a rectangle containing a single point p. +func RectFromLatLng(p LatLng) Rect { + return Rect{ + Lat: r1.Interval{p.Lat.Radians(), p.Lat.Radians()}, + Lng: s1.Interval{p.Lng.Radians(), p.Lng.Radians()}, + } +} + +// RectFromCenterSize constructs a rectangle with the given size and center. +// center needs to be normalized, but size does not. The latitude +// interval of the result is clamped to [-90,90] degrees, and the longitude +// interval of the result is FullRect() if and only if the longitude size is +// 360 degrees or more. +// +// Examples of clamping (in degrees): +// center=(80,170), size=(40,60) -> lat=[60,90], lng=[140,-160] +// center=(10,40), size=(210,400) -> lat=[-90,90], lng=[-180,180] +// center=(-90,180), size=(20,50) -> lat=[-90,-80], lng=[155,-155] +func RectFromCenterSize(center, size LatLng) Rect { + half := LatLng{size.Lat / 2, size.Lng / 2} + return RectFromLatLng(center).expanded(half) +} + +// IsValid returns true iff the rectangle is valid. +// This requires Lat ⊆ [-π/2,π/2] and Lng ⊆ [-π,π], and Lat = ∅ iff Lng = ∅ +func (r Rect) IsValid() bool { + return math.Abs(r.Lat.Lo) <= math.Pi/2 && + math.Abs(r.Lat.Hi) <= math.Pi/2 && + r.Lng.IsValid() && + r.Lat.IsEmpty() == r.Lng.IsEmpty() +} + +// IsEmpty reports whether the rectangle is empty. +func (r Rect) IsEmpty() bool { return r.Lat.IsEmpty() } + +// IsFull reports whether the rectangle is full. +func (r Rect) IsFull() bool { return r.Lat.Equal(validRectLatRange) && r.Lng.IsFull() } + +// IsPoint reports whether the rectangle is a single point. +func (r Rect) IsPoint() bool { return r.Lat.Lo == r.Lat.Hi && r.Lng.Lo == r.Lng.Hi } + +// Vertex returns the i-th vertex of the rectangle (i = 0,1,2,3) in CCW order +// (lower left, lower right, upper right, upper left). +func (r Rect) Vertex(i int) LatLng { + var lat, lng float64 + + switch i { + case 0: + lat = r.Lat.Lo + lng = r.Lng.Lo + case 1: + lat = r.Lat.Lo + lng = r.Lng.Hi + case 2: + lat = r.Lat.Hi + lng = r.Lng.Hi + case 3: + lat = r.Lat.Hi + lng = r.Lng.Lo + } + return LatLng{s1.Angle(lat) * s1.Radian, s1.Angle(lng) * s1.Radian} +} + +// Lo returns one corner of the rectangle. +func (r Rect) Lo() LatLng { + return LatLng{s1.Angle(r.Lat.Lo) * s1.Radian, s1.Angle(r.Lng.Lo) * s1.Radian} +} + +// Hi returns the other corner of the rectangle. +func (r Rect) Hi() LatLng { + return LatLng{s1.Angle(r.Lat.Hi) * s1.Radian, s1.Angle(r.Lng.Hi) * s1.Radian} +} + +// Center returns the center of the rectangle. +func (r Rect) Center() LatLng { + return LatLng{s1.Angle(r.Lat.Center()) * s1.Radian, s1.Angle(r.Lng.Center()) * s1.Radian} +} + +// Size returns the size of the Rect. +func (r Rect) Size() LatLng { + return LatLng{s1.Angle(r.Lat.Length()) * s1.Radian, s1.Angle(r.Lng.Length()) * s1.Radian} +} + +// Area returns the surface area of the Rect. +func (r Rect) Area() float64 { + if r.IsEmpty() { + return 0 + } + capDiff := math.Abs(math.Sin(r.Lat.Hi) - math.Sin(r.Lat.Lo)) + return r.Lng.Length() * capDiff +} + +// AddPoint increases the size of the rectangle to include the given point. +func (r Rect) AddPoint(ll LatLng) Rect { + if !ll.IsValid() { + return r + } + return Rect{ + Lat: r.Lat.AddPoint(ll.Lat.Radians()), + Lng: r.Lng.AddPoint(ll.Lng.Radians()), + } +} + +// expanded returns a rectangle that has been expanded by margin.Lat on each side +// in the latitude direction, and by margin.Lng on each side in the longitude +// direction. If either margin is negative, then it shrinks the rectangle on +// the corresponding sides instead. The resulting rectangle may be empty. +// +// The latitude-longitude space has the topology of a cylinder. Longitudes +// "wrap around" at +/-180 degrees, while latitudes are clamped to range [-90, 90]. +// This means that any expansion (positive or negative) of the full longitude range +// remains full (since the "rectangle" is actually a continuous band around the +// cylinder), while expansion of the full latitude range remains full only if the +// margin is positive. +// +// If either the latitude or longitude interval becomes empty after +// expansion by a negative margin, the result is empty. +// +// Note that if an expanded rectangle contains a pole, it may not contain +// all possible lat/lng representations of that pole, e.g., both points [π/2,0] +// and [π/2,1] represent the same pole, but they might not be contained by the +// same Rect. +// +// If you are trying to grow a rectangle by a certain distance on the +// sphere (e.g. 5km), refer to the ExpandedByDistance() C++ method implementation +// instead. +func (r Rect) expanded(margin LatLng) Rect { + lat := r.Lat.Expanded(margin.Lat.Radians()) + lng := r.Lng.Expanded(margin.Lng.Radians()) + + if lat.IsEmpty() || lng.IsEmpty() { + return EmptyRect() + } + + return Rect{ + Lat: lat.Intersection(validRectLatRange), + Lng: lng, + } +} + +func (r Rect) String() string { return fmt.Sprintf("[Lo%v, Hi%v]", r.Lo(), r.Hi()) } + +// PolarClosure returns the rectangle unmodified if it does not include either pole. +// If it includes either pole, PolarClosure returns an expansion of the rectangle along +// the longitudinal range to include all possible representations of the contained poles. +func (r Rect) PolarClosure() Rect { + if r.Lat.Lo == -math.Pi/2 || r.Lat.Hi == math.Pi/2 { + return Rect{r.Lat, s1.FullInterval()} + } + return r +} + +// Union returns the smallest Rect containing the union of this rectangle and the given rectangle. +func (r Rect) Union(other Rect) Rect { + return Rect{ + Lat: r.Lat.Union(other.Lat), + Lng: r.Lng.Union(other.Lng), + } +} + +// Intersection returns the smallest rectangle containing the intersection of +// this rectangle and the given rectangle. Note that the region of intersection +// may consist of two disjoint rectangles, in which case a single rectangle +// spanning both of them is returned. +func (r Rect) Intersection(other Rect) Rect { + lat := r.Lat.Intersection(other.Lat) + lng := r.Lng.Intersection(other.Lng) + + if lat.IsEmpty() || lng.IsEmpty() { + return EmptyRect() + } + return Rect{lat, lng} +} + +// Intersects reports whether this rectangle and the other have any points in common. +func (r Rect) Intersects(other Rect) bool { + return r.Lat.Intersects(other.Lat) && r.Lng.Intersects(other.Lng) +} + +// CapBound returns a cap that contains Rect. +func (r Rect) CapBound() Cap { + // We consider two possible bounding caps, one whose axis passes + // through the center of the lat-long rectangle and one whose axis + // is the north or south pole. We return the smaller of the two caps. + + if r.IsEmpty() { + return EmptyCap() + } + + var poleZ, poleAngle float64 + if r.Lat.Hi+r.Lat.Lo < 0 { + // South pole axis yields smaller cap. + poleZ = -1 + poleAngle = math.Pi/2 + r.Lat.Hi + } else { + poleZ = 1 + poleAngle = math.Pi/2 - r.Lat.Lo + } + poleCap := CapFromCenterAngle(Point{r3.Vector{0, 0, poleZ}}, s1.Angle(poleAngle)*s1.Radian) + + // For bounding rectangles that span 180 degrees or less in longitude, the + // maximum cap size is achieved at one of the rectangle vertices. For + // rectangles that are larger than 180 degrees, we punt and always return a + // bounding cap centered at one of the two poles. + if math.Remainder(r.Lng.Hi-r.Lng.Lo, 2*math.Pi) >= 0 && r.Lng.Hi-r.Lng.Lo < 2*math.Pi { + midCap := CapFromPoint(PointFromLatLng(r.Center())).AddPoint(PointFromLatLng(r.Lo())).AddPoint(PointFromLatLng(r.Hi())) + if midCap.Height() < poleCap.Height() { + return midCap + } + } + return poleCap +} + +// RectBound returns itself. +func (r Rect) RectBound() Rect { + return r +} + +// Contains reports whether this Rect contains the other Rect. +func (r Rect) Contains(other Rect) bool { + return r.Lat.ContainsInterval(other.Lat) && r.Lng.ContainsInterval(other.Lng) +} + +// ContainsCell reports whether the given Cell is contained by this Rect. +func (r Rect) ContainsCell(c Cell) bool { + // A latitude-longitude rectangle contains a cell if and only if it contains + // the cell's bounding rectangle. This test is exact from a mathematical + // point of view, assuming that the bounds returned by Cell.RectBound() + // are tight. However, note that there can be a loss of precision when + // converting between representations -- for example, if an s2.Cell is + // converted to a polygon, the polygon's bounding rectangle may not contain + // the cell's bounding rectangle. This has some slightly unexpected side + // effects; for instance, if one creates an s2.Polygon from an s2.Cell, the + // polygon will contain the cell, but the polygon's bounding box will not. + return r.Contains(c.RectBound()) +} + +// ContainsLatLng reports whether the given LatLng is within the Rect. +func (r Rect) ContainsLatLng(ll LatLng) bool { + if !ll.IsValid() { + return false + } + return r.Lat.Contains(ll.Lat.Radians()) && r.Lng.Contains(ll.Lng.Radians()) +} + +// ContainsPoint reports whether the given Point is within the Rect. +func (r Rect) ContainsPoint(p Point) bool { + return r.ContainsLatLng(LatLngFromPoint(p)) +} + +// CellUnionBound computes a covering of the Rect. +func (r Rect) CellUnionBound() []CellID { + return r.CapBound().CellUnionBound() +} + +// intersectsLatEdge reports whether the edge AB intersects the given edge of constant +// latitude. Requires the points to have unit length. +func intersectsLatEdge(a, b Point, lat s1.Angle, lng s1.Interval) bool { + // Unfortunately, lines of constant latitude are curves on + // the sphere. They can intersect a straight edge in 0, 1, or 2 points. + + // First, compute the normal to the plane AB that points vaguely north. + z := Point{a.PointCross(b).Normalize()} + if z.Z < 0 { + z = Point{z.Mul(-1)} + } + + // Extend this to an orthonormal frame (x,y,z) where x is the direction + // where the great circle through AB achieves its maximium latitude. + y := Point{z.PointCross(PointFromCoords(0, 0, 1)).Normalize()} + x := y.Cross(z.Vector) + + // Compute the angle "theta" from the x-axis (in the x-y plane defined + // above) where the great circle intersects the given line of latitude. + sinLat := math.Sin(float64(lat)) + if math.Abs(sinLat) >= x.Z { + // The great circle does not reach the given latitude. + return false + } + + cosTheta := sinLat / x.Z + sinTheta := math.Sqrt(1 - cosTheta*cosTheta) + theta := math.Atan2(sinTheta, cosTheta) + + // The candidate intersection points are located +/- theta in the x-y + // plane. For an intersection to be valid, we need to check that the + // intersection point is contained in the interior of the edge AB and + // also that it is contained within the given longitude interval "lng". + + // Compute the range of theta values spanned by the edge AB. + abTheta := s1.IntervalFromPointPair( + math.Atan2(a.Dot(y.Vector), a.Dot(x)), + math.Atan2(b.Dot(y.Vector), b.Dot(x))) + + if abTheta.Contains(theta) { + // Check if the intersection point is also in the given lng interval. + isect := x.Mul(cosTheta).Add(y.Mul(sinTheta)) + if lng.Contains(math.Atan2(isect.Y, isect.X)) { + return true + } + } + + if abTheta.Contains(-theta) { + // Check if the other intersection point is also in the given lng interval. + isect := x.Mul(cosTheta).Sub(y.Mul(sinTheta)) + if lng.Contains(math.Atan2(isect.Y, isect.X)) { + return true + } + } + return false +} + +// intersectsLngEdge reports whether the edge AB intersects the given edge of constant +// longitude. Requires the points to have unit length. +func intersectsLngEdge(a, b Point, lat r1.Interval, lng s1.Angle) bool { + // The nice thing about edges of constant longitude is that + // they are straight lines on the sphere (geodesics). + return CrossingSign(a, b, PointFromLatLng(LatLng{s1.Angle(lat.Lo), lng}), + PointFromLatLng(LatLng{s1.Angle(lat.Hi), lng})) == Cross +} + +// IntersectsCell reports whether this rectangle intersects the given cell. This is an +// exact test and may be fairly expensive. +func (r Rect) IntersectsCell(c Cell) bool { + // First we eliminate the cases where one region completely contains the + // other. Once these are disposed of, then the regions will intersect + // if and only if their boundaries intersect. + if r.IsEmpty() { + return false + } + if r.ContainsPoint(Point{c.id.rawPoint()}) { + return true + } + if c.ContainsPoint(PointFromLatLng(r.Center())) { + return true + } + + // Quick rejection test (not required for correctness). + if !r.Intersects(c.RectBound()) { + return false + } + + // Precompute the cell vertices as points and latitude-longitudes. We also + // check whether the Cell contains any corner of the rectangle, or + // vice-versa, since the edge-crossing tests only check the edge interiors. + vertices := [4]Point{} + latlngs := [4]LatLng{} + + for i := range vertices { + vertices[i] = c.Vertex(i) + latlngs[i] = LatLngFromPoint(vertices[i]) + if r.ContainsLatLng(latlngs[i]) { + return true + } + if c.ContainsPoint(PointFromLatLng(r.Vertex(i))) { + return true + } + } + + // Now check whether the boundaries intersect. Unfortunately, a + // latitude-longitude rectangle does not have straight edges: two edges + // are curved, and at least one of them is concave. + for i := range vertices { + edgeLng := s1.IntervalFromEndpoints(latlngs[i].Lng.Radians(), latlngs[(i+1)&3].Lng.Radians()) + if !r.Lng.Intersects(edgeLng) { + continue + } + + a := vertices[i] + b := vertices[(i+1)&3] + if edgeLng.Contains(r.Lng.Lo) && intersectsLngEdge(a, b, r.Lat, s1.Angle(r.Lng.Lo)) { + return true + } + if edgeLng.Contains(r.Lng.Hi) && intersectsLngEdge(a, b, r.Lat, s1.Angle(r.Lng.Hi)) { + return true + } + if intersectsLatEdge(a, b, s1.Angle(r.Lat.Lo), r.Lng) { + return true + } + if intersectsLatEdge(a, b, s1.Angle(r.Lat.Hi), r.Lng) { + return true + } + } + return false +} + +// Encode encodes the Rect. +func (r Rect) Encode(w io.Writer) error { + e := &encoder{w: w} + r.encode(e) + return e.err +} + +func (r Rect) encode(e *encoder) { + e.writeInt8(encodingVersion) + e.writeFloat64(r.Lat.Lo) + e.writeFloat64(r.Lat.Hi) + e.writeFloat64(r.Lng.Lo) + e.writeFloat64(r.Lng.Hi) +} + +// Decode decodes a rectangle. +func (r *Rect) Decode(rd io.Reader) error { + d := &decoder{r: asByteReader(rd)} + r.decode(d) + return d.err +} + +func (r *Rect) decode(d *decoder) { + if version := d.readUint8(); int8(version) != encodingVersion && d.err == nil { + d.err = fmt.Errorf("can't decode version %d; my version: %d", version, encodingVersion) + return + } + r.Lat.Lo = d.readFloat64() + r.Lat.Hi = d.readFloat64() + r.Lng.Lo = d.readFloat64() + r.Lng.Hi = d.readFloat64() + return +} + +// DistanceToLatLng returns the minimum distance (measured along the surface of the sphere) +// from a given point to the rectangle (both its boundary and its interior). +// If r is empty, the result is meaningless. +// The latlng must be valid. +func (r Rect) DistanceToLatLng(ll LatLng) s1.Angle { + if r.Lng.Contains(float64(ll.Lng)) { + return maxAngle(0, ll.Lat-s1.Angle(r.Lat.Hi), s1.Angle(r.Lat.Lo)-ll.Lat) + } + + i := s1.IntervalFromEndpoints(r.Lng.Hi, r.Lng.ComplementCenter()) + rectLng := r.Lng.Lo + if i.Contains(float64(ll.Lng)) { + rectLng = r.Lng.Hi + } + + lo := LatLng{s1.Angle(r.Lat.Lo) * s1.Radian, s1.Angle(rectLng) * s1.Radian} + hi := LatLng{s1.Angle(r.Lat.Hi) * s1.Radian, s1.Angle(rectLng) * s1.Radian} + return DistanceFromSegment(PointFromLatLng(ll), PointFromLatLng(lo), PointFromLatLng(hi)) +} + +// DirectedHausdorffDistance returns the directed Hausdorff distance (measured along the +// surface of the sphere) to the given Rect. The directed Hausdorff +// distance from rectangle A to rectangle B is given by +// h(A, B) = max_{p in A} min_{q in B} d(p, q). +func (r Rect) DirectedHausdorffDistance(other Rect) s1.Angle { + if r.IsEmpty() { + return 0 * s1.Radian + } + if other.IsEmpty() { + return math.Pi * s1.Radian + } + + lng := r.Lng.DirectedHausdorffDistance(other.Lng) + return directedHausdorffDistance(lng, r.Lat, other.Lat) +} + +// HausdorffDistance returns the undirected Hausdorff distance (measured along the +// surface of the sphere) to the given Rect. +// The Hausdorff distance between rectangle A and rectangle B is given by +// H(A, B) = max{h(A, B), h(B, A)}. +func (r Rect) HausdorffDistance(other Rect) s1.Angle { + return maxAngle(r.DirectedHausdorffDistance(other), + other.DirectedHausdorffDistance(r)) +} + +// ApproxEqual reports whether the latitude and longitude intervals of the two rectangles +// are the same up to a small tolerance. +func (r Rect) ApproxEqual(other Rect) bool { + return r.Lat.ApproxEqual(other.Lat) && r.Lng.ApproxEqual(other.Lng) +} + +// directedHausdorffDistance returns the directed Hausdorff distance +// from one longitudinal edge spanning latitude range 'a' to the other +// longitudinal edge spanning latitude range 'b', with their longitudinal +// difference given by 'lngDiff'. +func directedHausdorffDistance(lngDiff s1.Angle, a, b r1.Interval) s1.Angle { + // By symmetry, we can assume a's longitude is 0 and b's longitude is + // lngDiff. Call b's two endpoints bLo and bHi. Let H be the hemisphere + // containing a and delimited by the longitude line of b. The Voronoi diagram + // of b on H has three edges (portions of great circles) all orthogonal to b + // and meeting at bLo cross bHi. + // E1: (bLo, bLo cross bHi) + // E2: (bHi, bLo cross bHi) + // E3: (-bMid, bLo cross bHi), where bMid is the midpoint of b + // + // They subdivide H into three Voronoi regions. Depending on how longitude 0 + // (which contains edge a) intersects these regions, we distinguish two cases: + // Case 1: it intersects three regions. This occurs when lngDiff <= π/2. + // Case 2: it intersects only two regions. This occurs when lngDiff > π/2. + // + // In the first case, the directed Hausdorff distance to edge b can only be + // realized by the following points on a: + // A1: two endpoints of a. + // A2: intersection of a with the equator, if b also intersects the equator. + // + // In the second case, the directed Hausdorff distance to edge b can only be + // realized by the following points on a: + // B1: two endpoints of a. + // B2: intersection of a with E3 + // B3: farthest point from bLo to the interior of D, and farthest point from + // bHi to the interior of U, if any, where D (resp. U) is the portion + // of edge a below (resp. above) the intersection point from B2. + + if lngDiff < 0 { + panic("impossible: negative lngDiff") + } + if lngDiff > math.Pi { + panic("impossible: lngDiff > Pi") + } + + if lngDiff == 0 { + return s1.Angle(a.DirectedHausdorffDistance(b)) + } + + // Assumed longitude of b. + bLng := lngDiff + // Two endpoints of b. + bLo := PointFromLatLng(LatLng{s1.Angle(b.Lo), bLng}) + bHi := PointFromLatLng(LatLng{s1.Angle(b.Hi), bLng}) + + // Cases A1 and B1. + aLo := PointFromLatLng(LatLng{s1.Angle(a.Lo), 0}) + aHi := PointFromLatLng(LatLng{s1.Angle(a.Hi), 0}) + maxDistance := maxAngle( + DistanceFromSegment(aLo, bLo, bHi), + DistanceFromSegment(aHi, bLo, bHi)) + + if lngDiff <= math.Pi/2 { + // Case A2. + if a.Contains(0) && b.Contains(0) { + maxDistance = maxAngle(maxDistance, lngDiff) + } + return maxDistance + } + + // Case B2. + p := bisectorIntersection(b, bLng) + pLat := LatLngFromPoint(p).Lat + if a.Contains(float64(pLat)) { + maxDistance = maxAngle(maxDistance, p.Angle(bLo.Vector)) + } + + // Case B3. + if pLat > s1.Angle(a.Lo) { + intDist, ok := interiorMaxDistance(r1.Interval{a.Lo, math.Min(float64(pLat), a.Hi)}, bLo) + if ok { + maxDistance = maxAngle(maxDistance, intDist) + } + } + if pLat < s1.Angle(a.Hi) { + intDist, ok := interiorMaxDistance(r1.Interval{math.Max(float64(pLat), a.Lo), a.Hi}, bHi) + if ok { + maxDistance = maxAngle(maxDistance, intDist) + } + } + + return maxDistance +} + +// interiorMaxDistance returns the max distance from a point b to the segment spanning latitude range +// aLat on longitude 0 if the max occurs in the interior of aLat. Otherwise, returns (0, false). +func interiorMaxDistance(aLat r1.Interval, b Point) (a s1.Angle, ok bool) { + // Longitude 0 is in the y=0 plane. b.X >= 0 implies that the maximum + // does not occur in the interior of aLat. + if aLat.IsEmpty() || b.X >= 0 { + return 0, false + } + + // Project b to the y=0 plane. The antipodal of the normalized projection is + // the point at which the maxium distance from b occurs, if it is contained + // in aLat. + intersectionPoint := PointFromCoords(-b.X, 0, -b.Z) + if !aLat.InteriorContains(float64(LatLngFromPoint(intersectionPoint).Lat)) { + return 0, false + } + return b.Angle(intersectionPoint.Vector), true +} + +// bisectorIntersection return the intersection of longitude 0 with the bisector of an edge +// on longitude 'lng' and spanning latitude range 'lat'. +func bisectorIntersection(lat r1.Interval, lng s1.Angle) Point { + lng = s1.Angle(math.Abs(float64(lng))) + latCenter := s1.Angle(lat.Center()) + + // A vector orthogonal to the bisector of the given longitudinal edge. + orthoBisector := LatLng{latCenter - math.Pi/2, lng} + if latCenter < 0 { + orthoBisector = LatLng{-latCenter - math.Pi/2, lng - math.Pi} + } + + // A vector orthogonal to longitude 0. + orthoLng := Point{r3.Vector{0, -1, 0}} + + return orthoLng.PointCross(PointFromLatLng(orthoBisector)) +} + +// Centroid returns the true centroid of the given Rect multiplied by its +// surface area. The result is not unit length, so you may want to normalize it. +// Note that in general the centroid is *not* at the center of the rectangle, and +// in fact it may not even be contained by the rectangle. (It is the "center of +// mass" of the rectangle viewed as subset of the unit sphere, i.e. it is the +// point in space about which this curved shape would rotate.) +// +// The reason for multiplying the result by the rectangle area is to make it +// easier to compute the centroid of more complicated shapes. The centroid +// of a union of disjoint regions can be computed simply by adding their +// Centroid results. +func (r Rect) Centroid() Point { + // When a sphere is divided into slices of constant thickness by a set + // of parallel planes, all slices have the same surface area. This + // implies that the z-component of the centroid is simply the midpoint + // of the z-interval spanned by the Rect. + // + // Similarly, it is easy to see that the (x,y) of the centroid lies in + // the plane through the midpoint of the rectangle's longitude interval. + // We only need to determine the distance "d" of this point from the + // z-axis. + // + // Let's restrict our attention to a particular z-value. In this + // z-plane, the Rect is a circular arc. The centroid of this arc + // lies on a radial line through the midpoint of the arc, and at a + // distance from the z-axis of + // + // r * (sin(alpha) / alpha) + // + // where r = sqrt(1-z^2) is the radius of the arc, and "alpha" is half + // of the arc length (i.e., the arc covers longitudes [-alpha, alpha]). + // + // To find the centroid distance from the z-axis for the entire + // rectangle, we just need to integrate over the z-interval. This gives + // + // d = Integrate[sqrt(1-z^2)*sin(alpha)/alpha, z1..z2] / (z2 - z1) + // + // where [z1, z2] is the range of z-values covered by the rectangle. + // This simplifies to + // + // d = sin(alpha)/(2*alpha*(z2-z1))*(z2*r2 - z1*r1 + theta2 - theta1) + // + // where [theta1, theta2] is the latitude interval, z1=sin(theta1), + // z2=sin(theta2), r1=cos(theta1), and r2=cos(theta2). + // + // Finally, we want to return not the centroid itself, but the centroid + // scaled by the area of the rectangle. The area of the rectangle is + // + // A = 2 * alpha * (z2 - z1) + // + // which fortunately appears in the denominator of "d". + + if r.IsEmpty() { + return Point{} + } + + z1 := math.Sin(r.Lat.Lo) + z2 := math.Sin(r.Lat.Hi) + r1 := math.Cos(r.Lat.Lo) + r2 := math.Cos(r.Lat.Hi) + + alpha := 0.5 * r.Lng.Length() + r0 := math.Sin(alpha) * (r2*z2 - r1*z1 + r.Lat.Length()) + lng := r.Lng.Center() + z := alpha * (z2 + z1) * (z2 - z1) // scaled by the area + + return Point{r3.Vector{r0 * math.Cos(lng), r0 * math.Sin(lng), z}} +} + +// BUG: The major differences from the C++ version are: +// - Get*Distance, Vertex, InteriorContains(LatLng|Rect|Point) + +func RectFromDegrees(latLo, lngLo, latHi, lngHi float64) Rect { + // Convenience method to construct a rectangle. This method is + // intentionally *not* in the S2LatLngRect interface because the + // argument order is ambiguous, but is fine for the test. + return Rect{ + Lat: r1.Interval{ + Lo: (s1.Angle(latLo) * s1.Degree).Radians(), + Hi: (s1.Angle(latHi) * s1.Degree).Radians(), + }, + Lng: s1.IntervalFromEndpoints( + (s1.Angle(lngLo) * s1.Degree).Radians(), + (s1.Angle(lngHi) * s1.Degree).Radians(), + ), + } +} diff --git a/vendor/github.com/blevesearch/geo/s2/rect_bounder.go b/vendor/github.com/blevesearch/geo/s2/rect_bounder.go new file mode 100644 index 00000000..419dea0c --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/rect_bounder.go @@ -0,0 +1,352 @@ +// Copyright 2017 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +import ( + "math" + + "github.com/golang/geo/r1" + "github.com/golang/geo/r3" + "github.com/golang/geo/s1" +) + +// RectBounder is used to compute a bounding rectangle that contains all edges +// defined by a vertex chain (v0, v1, v2, ...). All vertices must be unit length. +// Note that the bounding rectangle of an edge can be larger than the bounding +// rectangle of its endpoints, e.g. consider an edge that passes through the North Pole. +// +// The bounds are calculated conservatively to account for numerical errors +// when points are converted to LatLngs. More precisely, this function +// guarantees the following: +// Let L be a closed edge chain (Loop) such that the interior of the loop does +// not contain either pole. Now if P is any point such that L.ContainsPoint(P), +// then RectBound(L).ContainsPoint(LatLngFromPoint(P)). +type RectBounder struct { + // The previous vertex in the chain. + a Point + // The previous vertex latitude longitude. + aLL LatLng + bound Rect +} + +// NewRectBounder returns a new instance of a RectBounder. +func NewRectBounder() *RectBounder { + return &RectBounder{ + bound: EmptyRect(), + } +} + +// maxErrorForTests returns the maximum error in RectBound provided that the +// result does not include either pole. It is only used for testing purposes +func (r *RectBounder) maxErrorForTests() LatLng { + // The maximum error in the latitude calculation is + // 3.84 * dblEpsilon for the PointCross calculation + // 0.96 * dblEpsilon for the Latitude calculation + // 5 * dblEpsilon added by AddPoint/RectBound to compensate for error + // ----------------- + // 9.80 * dblEpsilon maximum error in result + // + // The maximum error in the longitude calculation is dblEpsilon. RectBound + // does not do any expansion because this isn't necessary in order to + // bound the *rounded* longitudes of contained points. + return LatLng{10 * dblEpsilon * s1.Radian, 1 * dblEpsilon * s1.Radian} +} + +// AddPoint adds the given point to the chain. The Point must be unit length. +func (r *RectBounder) AddPoint(b Point) { + bLL := LatLngFromPoint(b) + + if r.bound.IsEmpty() { + r.a = b + r.aLL = bLL + r.bound = r.bound.AddPoint(bLL) + return + } + + // First compute the cross product N = A x B robustly. This is the normal + // to the great circle through A and B. We don't use RobustSign + // since that method returns an arbitrary vector orthogonal to A if the two + // vectors are proportional, and we want the zero vector in that case. + n := r.a.Sub(b.Vector).Cross(r.a.Add(b.Vector)) // N = 2 * (A x B) + + // The relative error in N gets large as its norm gets very small (i.e., + // when the two points are nearly identical or antipodal). We handle this + // by choosing a maximum allowable error, and if the error is greater than + // this we fall back to a different technique. Since it turns out that + // the other sources of error in converting the normal to a maximum + // latitude add up to at most 1.16 * dblEpsilon, and it is desirable to + // have the total error be a multiple of dblEpsilon, we have chosen to + // limit the maximum error in the normal to be 3.84 * dblEpsilon. + // It is possible to show that the error is less than this when + // + // n.Norm() >= 8 * sqrt(3) / (3.84 - 0.5 - sqrt(3)) * dblEpsilon + // = 1.91346e-15 (about 8.618 * dblEpsilon) + nNorm := n.Norm() + if nNorm < 1.91346e-15 { + // A and B are either nearly identical or nearly antipodal (to within + // 4.309 * dblEpsilon, or about 6 nanometers on the earth's surface). + if r.a.Dot(b.Vector) < 0 { + // The two points are nearly antipodal. The easiest solution is to + // assume that the edge between A and B could go in any direction + // around the sphere. + r.bound = FullRect() + } else { + // The two points are nearly identical (to within 4.309 * dblEpsilon). + // In this case we can just use the bounding rectangle of the points, + // since after the expansion done by GetBound this Rect is + // guaranteed to include the (lat,lng) values of all points along AB. + r.bound = r.bound.Union(RectFromLatLng(r.aLL).AddPoint(bLL)) + } + r.a = b + r.aLL = bLL + return + } + + // Compute the longitude range spanned by AB. + lngAB := s1.EmptyInterval().AddPoint(r.aLL.Lng.Radians()).AddPoint(bLL.Lng.Radians()) + if lngAB.Length() >= math.Pi-2*dblEpsilon { + // The points lie on nearly opposite lines of longitude to within the + // maximum error of the calculation. The easiest solution is to assume + // that AB could go on either side of the pole. + lngAB = s1.FullInterval() + } + + // Next we compute the latitude range spanned by the edge AB. We start + // with the range spanning the two endpoints of the edge: + latAB := r1.IntervalFromPoint(r.aLL.Lat.Radians()).AddPoint(bLL.Lat.Radians()) + + // This is the desired range unless the edge AB crosses the plane + // through N and the Z-axis (which is where the great circle through A + // and B attains its minimum and maximum latitudes). To test whether AB + // crosses this plane, we compute a vector M perpendicular to this + // plane and then project A and B onto it. + m := n.Cross(r3.Vector{0, 0, 1}) + mA := m.Dot(r.a.Vector) + mB := m.Dot(b.Vector) + + // We want to test the signs of "mA" and "mB", so we need to bound + // the error in these calculations. It is possible to show that the + // total error is bounded by + // + // (1 + sqrt(3)) * dblEpsilon * nNorm + 8 * sqrt(3) * (dblEpsilon**2) + // = 6.06638e-16 * nNorm + 6.83174e-31 + + mError := 6.06638e-16*nNorm + 6.83174e-31 + if mA*mB < 0 || math.Abs(mA) <= mError || math.Abs(mB) <= mError { + // Minimum/maximum latitude *may* occur in the edge interior. + // + // The maximum latitude is 90 degrees minus the latitude of N. We + // compute this directly using atan2 in order to get maximum accuracy + // near the poles. + // + // Our goal is compute a bound that contains the computed latitudes of + // all S2Points P that pass the point-in-polygon containment test. + // There are three sources of error we need to consider: + // - the directional error in N (at most 3.84 * dblEpsilon) + // - converting N to a maximum latitude + // - computing the latitude of the test point P + // The latter two sources of error are at most 0.955 * dblEpsilon + // individually, but it is possible to show by a more complex analysis + // that together they can add up to at most 1.16 * dblEpsilon, for a + // total error of 5 * dblEpsilon. + // + // We add 3 * dblEpsilon to the bound here, and GetBound() will pad + // the bound by another 2 * dblEpsilon. + maxLat := math.Min( + math.Atan2(math.Sqrt(n.X*n.X+n.Y*n.Y), math.Abs(n.Z))+3*dblEpsilon, + math.Pi/2) + + // In order to get tight bounds when the two points are close together, + // we also bound the min/max latitude relative to the latitudes of the + // endpoints A and B. First we compute the distance between A and B, + // and then we compute the maximum change in latitude between any two + // points along the great circle that are separated by this distance. + // This gives us a latitude change "budget". Some of this budget must + // be spent getting from A to B; the remainder bounds the round-trip + // distance (in latitude) from A or B to the min or max latitude + // attained along the edge AB. + latBudget := 2 * math.Asin(0.5*(r.a.Sub(b.Vector)).Norm()*math.Sin(maxLat)) + maxDelta := 0.5*(latBudget-latAB.Length()) + dblEpsilon + + // Test whether AB passes through the point of maximum latitude or + // minimum latitude. If the dot product(s) are small enough then the + // result may be ambiguous. + if mA <= mError && mB >= -mError { + latAB.Hi = math.Min(maxLat, latAB.Hi+maxDelta) + } + if mB <= mError && mA >= -mError { + latAB.Lo = math.Max(-maxLat, latAB.Lo-maxDelta) + } + } + r.a = b + r.aLL = bLL + r.bound = r.bound.Union(Rect{latAB, lngAB}) +} + +// RectBound returns the bounding rectangle of the edge chain that connects the +// vertices defined so far. This bound satisfies the guarantee made +// above, i.e. if the edge chain defines a Loop, then the bound contains +// the LatLng coordinates of all Points contained by the loop. +func (r *RectBounder) RectBound() Rect { + return r.bound.expanded(LatLng{s1.Angle(2 * dblEpsilon), 0}).PolarClosure() +} + +// ExpandForSubregions expands a bounding Rect so that it is guaranteed to +// contain the bounds of any subregion whose bounds are computed using +// ComputeRectBound. For example, consider a loop L that defines a square. +// GetBound ensures that if a point P is contained by this square, then +// LatLngFromPoint(P) is contained by the bound. But now consider a diamond +// shaped loop S contained by L. It is possible that GetBound returns a +// *larger* bound for S than it does for L, due to rounding errors. This +// method expands the bound for L so that it is guaranteed to contain the +// bounds of any subregion S. +// +// More precisely, if L is a loop that does not contain either pole, and S +// is a loop such that L.Contains(S), then +// +// ExpandForSubregions(L.RectBound).Contains(S.RectBound). +// +func ExpandForSubregions(bound Rect) Rect { + // Empty bounds don't need expansion. + if bound.IsEmpty() { + return bound + } + + // First we need to check whether the bound B contains any nearly-antipodal + // points (to within 4.309 * dblEpsilon). If so then we need to return + // FullRect, since the subregion might have an edge between two + // such points, and AddPoint returns Full for such edges. Note that + // this can happen even if B is not Full for example, consider a loop + // that defines a 10km strip straddling the equator extending from + // longitudes -100 to +100 degrees. + // + // It is easy to check whether B contains any antipodal points, but checking + // for nearly-antipodal points is trickier. Essentially we consider the + // original bound B and its reflection through the origin B', and then test + // whether the minimum distance between B and B' is less than 4.309 * dblEpsilon. + + // lngGap is a lower bound on the longitudinal distance between B and its + // reflection B'. (2.5 * dblEpsilon is the maximum combined error of the + // endpoint longitude calculations and the Length call.) + lngGap := math.Max(0, math.Pi-bound.Lng.Length()-2.5*dblEpsilon) + + // minAbsLat is the minimum distance from B to the equator (if zero or + // negative, then B straddles the equator). + minAbsLat := math.Max(bound.Lat.Lo, -bound.Lat.Hi) + + // latGapSouth and latGapNorth measure the minimum distance from B to the + // south and north poles respectively. + latGapSouth := math.Pi/2 + bound.Lat.Lo + latGapNorth := math.Pi/2 - bound.Lat.Hi + + if minAbsLat >= 0 { + // The bound B does not straddle the equator. In this case the minimum + // distance is between one endpoint of the latitude edge in B closest to + // the equator and the other endpoint of that edge in B'. The latitude + // distance between these two points is 2*minAbsLat, and the longitude + // distance is lngGap. We could compute the distance exactly using the + // Haversine formula, but then we would need to bound the errors in that + // calculation. Since we only need accuracy when the distance is very + // small (close to 4.309 * dblEpsilon), we substitute the Euclidean + // distance instead. This gives us a right triangle XYZ with two edges of + // length x = 2*minAbsLat and y ~= lngGap. The desired distance is the + // length of the third edge z, and we have + // + // z ~= sqrt(x^2 + y^2) >= (x + y) / sqrt(2) + // + // Therefore the region may contain nearly antipodal points only if + // + // 2*minAbsLat + lngGap < sqrt(2) * 4.309 * dblEpsilon + // ~= 1.354e-15 + // + // Note that because the given bound B is conservative, minAbsLat and + // lngGap are both lower bounds on their true values so we do not need + // to make any adjustments for their errors. + if 2*minAbsLat+lngGap < 1.354e-15 { + return FullRect() + } + } else if lngGap >= math.Pi/2 { + // B spans at most Pi/2 in longitude. The minimum distance is always + // between one corner of B and the diagonally opposite corner of B'. We + // use the same distance approximation that we used above; in this case + // we have an obtuse triangle XYZ with two edges of length x = latGapSouth + // and y = latGapNorth, and angle Z >= Pi/2 between them. We then have + // + // z >= sqrt(x^2 + y^2) >= (x + y) / sqrt(2) + // + // Unlike the case above, latGapSouth and latGapNorth are not lower bounds + // (because of the extra addition operation, and because math.Pi/2 is not + // exactly equal to Pi/2); they can exceed their true values by up to + // 0.75 * dblEpsilon. Putting this all together, the region may contain + // nearly antipodal points only if + // + // latGapSouth + latGapNorth < (sqrt(2) * 4.309 + 1.5) * dblEpsilon + // ~= 1.687e-15 + if latGapSouth+latGapNorth < 1.687e-15 { + return FullRect() + } + } else { + // Otherwise we know that (1) the bound straddles the equator and (2) its + // width in longitude is at least Pi/2. In this case the minimum + // distance can occur either between a corner of B and the diagonally + // opposite corner of B' (as in the case above), or between a corner of B + // and the opposite longitudinal edge reflected in B'. It is sufficient + // to only consider the corner-edge case, since this distance is also a + // lower bound on the corner-corner distance when that case applies. + + // Consider the spherical triangle XYZ where X is a corner of B with + // minimum absolute latitude, Y is the closest pole to X, and Z is the + // point closest to X on the opposite longitudinal edge of B'. This is a + // right triangle (Z = Pi/2), and from the spherical law of sines we have + // + // sin(z) / sin(Z) = sin(y) / sin(Y) + // sin(maxLatGap) / 1 = sin(dMin) / sin(lngGap) + // sin(dMin) = sin(maxLatGap) * sin(lngGap) + // + // where "maxLatGap" = max(latGapSouth, latGapNorth) and "dMin" is the + // desired minimum distance. Now using the facts that sin(t) >= (2/Pi)*t + // for 0 <= t <= Pi/2, that we only need an accurate approximation when + // at least one of "maxLatGap" or lngGap is extremely small (in which + // case sin(t) ~= t), and recalling that "maxLatGap" has an error of up + // to 0.75 * dblEpsilon, we want to test whether + // + // maxLatGap * lngGap < (4.309 + 0.75) * (Pi/2) * dblEpsilon + // ~= 1.765e-15 + if math.Max(latGapSouth, latGapNorth)*lngGap < 1.765e-15 { + return FullRect() + } + } + // Next we need to check whether the subregion might contain any edges that + // span (math.Pi - 2 * dblEpsilon) radians or more in longitude, since AddPoint + // sets the longitude bound to Full in that case. This corresponds to + // testing whether (lngGap <= 0) in lngExpansion below. + + // Otherwise, the maximum latitude error in AddPoint is 4.8 * dblEpsilon. + // In the worst case, the errors when computing the latitude bound for a + // subregion could go in the opposite direction as the errors when computing + // the bound for the original region, so we need to double this value. + // (More analysis shows that it's okay to round down to a multiple of + // dblEpsilon.) + // + // For longitude, we rely on the fact that atan2 is correctly rounded and + // therefore no additional bounds expansion is necessary. + + latExpansion := 9 * dblEpsilon + lngExpansion := 0.0 + if lngGap <= 0 { + lngExpansion = math.Pi + } + return bound.expanded(LatLng{s1.Angle(latExpansion), s1.Angle(lngExpansion)}).PolarClosure() +} diff --git a/vendor/github.com/blevesearch/geo/s2/region.go b/vendor/github.com/blevesearch/geo/s2/region.go new file mode 100644 index 00000000..9ea3de1c --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/region.go @@ -0,0 +1,71 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +// A Region represents a two-dimensional region on the unit sphere. +// +// The purpose of this interface is to allow complex regions to be +// approximated as simpler regions. The interface is restricted to methods +// that are useful for computing approximations. +type Region interface { + // CapBound returns a bounding spherical cap. This is not guaranteed to be exact. + CapBound() Cap + + // RectBound returns a bounding latitude-longitude rectangle that contains + // the region. The bounds are not guaranteed to be tight. + RectBound() Rect + + // ContainsCell reports whether the region completely contains the given region. + // It returns false if containment could not be determined. + ContainsCell(c Cell) bool + + // IntersectsCell reports whether the region intersects the given cell or + // if intersection could not be determined. It returns false if the region + // does not intersect. + IntersectsCell(c Cell) bool + + // ContainsPoint reports whether the region contains the given point or not. + // The point should be unit length, although some implementations may relax + // this restriction. + ContainsPoint(p Point) bool + + // CellUnionBound returns a small collection of CellIDs whose union covers + // the region. The cells are not sorted, may have redundancies (such as cells + // that contain other cells), and may cover much more area than necessary. + // + // This method is not intended for direct use by client code. Clients + // should typically use Covering, which has options to control the size and + // accuracy of the covering. Alternatively, if you want a fast covering and + // don't care about accuracy, consider calling FastCovering (which returns a + // cleaned-up version of the covering computed by this method). + // + // CellUnionBound implementations should attempt to return a small + // covering (ideally 4 cells or fewer) that covers the region and can be + // computed quickly. The result is used by RegionCoverer as a starting + // point for further refinement. + CellUnionBound() []CellID +} + +// Enforce Region interface satisfaction. +var ( + _ Region = Cap{} + _ Region = Cell{} + _ Region = (*CellUnion)(nil) + _ Region = (*Loop)(nil) + _ Region = Point{} + _ Region = (*Polygon)(nil) + _ Region = (*Polyline)(nil) + _ Region = Rect{} +) diff --git a/vendor/github.com/blevesearch/geo/s2/region_term_indexer.go b/vendor/github.com/blevesearch/geo/s2/region_term_indexer.go new file mode 100644 index 00000000..aa0b6aae --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/region_term_indexer.go @@ -0,0 +1,441 @@ +// Copyright 2021 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +// Indexing Strategy +// ----------------- +// +// Given a query region, we want to find all of the document regions that +// intersect it. The first step is to represent all the regions as S2Cell +// coverings (see S2RegionCoverer). We then split the problem into two parts, +// namely finding the document regions that are "smaller" than the query +// region and those that are "larger" than the query region. +// +// We do this by defining two terms for each S2CellId: a "covering term" and +// an "ancestor term". (In the implementation below, covering terms are +// distinguished by prefixing a '$' to them.) For each document region, we +// insert a covering term for every cell in the region's covering, and we +// insert an ancestor term for these cells *and* all of their ancestors. +// +// Then given a query region, we can look up all the document regions that +// intersect its covering by querying the union of the following terms: +// +// 1. An "ancestor term" for each cell in the query region. These terms +// ensure that we find all document regions that are "smaller" than the +// query region, i.e. where the query region contains a cell that is either +// a cell of a document region or one of its ancestors. +// +// 2. A "covering term" for every ancestor of the cells in the query region. +// These terms ensure that we find all the document regions that are +// "larger" than the query region, i.e. where document region contains a +// cell that is a (proper) ancestor of a cell in the query region. +// +// Together, these terms find all of the document regions that intersect the +// query region. Furthermore, the number of terms to be indexed and queried +// are both fairly small, and can be bounded in terms of max_cells() and the +// number of cell levels used. +// +// Optimizations +// ------------- +// +// + Cells at the maximum level being indexed (max_level()) have the special +// property that they will never be an ancestor of a cell in the query +// region. Therefore we can safely skip generating "covering terms" for +// these cells (see query step 2 above). +// +// + If the index will contain only points (rather than general regions), then +// we can skip all the covering terms mentioned above because there will +// never be any document regions larger than the query region. This can +// significantly reduce the size of queries. +// +// + If it is more important to optimize index size rather than query speed, +// the number of index terms can be reduced by creating ancestor terms only +// for the *proper* ancestors of the cells in a document region, and +// compensating for this by including covering terms for all cells in the +// query region (in addition to their ancestors). +// +// Effectively, when the query region and a document region contain exactly +// the same cell, we have a choice about whether to treat this match as a +// "covering term" or an "ancestor term". One choice minimizes query size +// while the other minimizes index size. + +package s2 + +import ( + "strings" + + "github.com/golang/geo/s1" +) + +type TermType int + +var marker = string('$') + +const ( + ANCESTOR TermType = iota + 1 + COVERING +) + +var defaultMaxCells = int(8) + +type Options struct { + ///////////////// Options Inherited From S2RegionCoverer //////////////// + + // maxCells controls the maximum number of cells when approximating + // each region. This parameter value may be changed as often as desired. + // e.g. to approximate some regions more accurately than others. + // + // Increasing this value during indexing will make indexes more accurate + // but larger. Increasing this value for queries will make queries more + // accurate but slower. (See regioncoverer.go for details on how this + // parameter affects accuracy.) For example, if you don't mind large + // indexes but want fast serving, it might be reasonable to set + // max_cells() == 100 during indexing and max_cells() == 8 for queries. + // + // DEFAULT: 8 (coarse approximations) + maxCells int + + // minLevel and maxLevel control the minimum and maximum size of the + // S2Cells used to approximate regions. Setting these parameters + // appropriately can reduce the size of the index and speed up queries by + // reducing the number of terms needed. For example, if you know that + // your query regions will rarely be less than 100 meters in width, then + // you could set maxLevel to 100. + // + // This restricts the index to S2Cells that are approximately 100 meters + // across or larger. Similar, if you know that query regions will rarely + // be larger than 1000km across, then you could set minLevel similarly. + // + // If minLevel is set too high, then large regions may generate too + // many query terms. If maxLevel() set too low, then small query + // regions will not be able to discriminate which regions they intersect + // very precisely and may return many more candidates than necessary. + // + // If you have no idea about the scale of the regions being queried, + // it is perfectly fine to set minLevel to 0 and maxLevel to 30. + // The only drawback is that may result in a larger index and slower queries. + // + // The default parameter values are suitable for query regions ranging + // from about 100 meters to 3000 km across. + // + // DEFAULT: 4 (average cell width == 600km) + minLevel int + + // DEFAULT: 16 (average cell width == 150m) + maxLevel int + + // Setting levelMod to a value greater than 1 increases the effective + // branching factor of the S2Cell hierarchy by skipping some levels. For + // example, if levelMod to 2 then every second level is skipped (which + // increases the effective branching factor to 16). You might want to + // consider doing this if your query regions are typically very small + // (e.g., single points) and you don't mind increasing the index size + // (since skipping levels will reduce the accuracy of cell coverings for a + // given maxCells limit). + // + // DEFAULT: 1 (don't skip any cell levels) + levelMod int + + // If your index will only contain points (rather than regions), be sure + // to set this flag. This will generate smaller and faster queries that + // are specialized for the points-only case. + // + // With the default quality settings, this flag reduces the number of + // query terms by about a factor of two. (The improvement gets smaller + // as maxCells is increased, but there is really no reason not to use + // this flag if your index consists entirely of points.) + // + // DEFAULT: false + pointsOnly bool + + // If true, the index will be optimized for space rather than for query + // time. With the default quality settings, this flag reduces the number + // of index terms and increases the number of query terms by the same + // factor (approximately 1.3). The factor increases up to a limiting + // ratio of 2.0 as maxCells is increased. + // + // CAVEAT: This option has no effect if the index contains only points. + // + // DEFAULT: false + optimizeSpace bool +} + +func (o *Options) MaxCells() int { + return o.maxCells +} + +func (o *Options) SetMaxCells(mc int) { + o.maxCells = mc +} + +func (o *Options) MinLevel() int { + return o.minLevel +} + +func (o *Options) SetMinLevel(ml int) { + o.minLevel = ml +} + +func (o *Options) MaxLevel() int { + return o.maxLevel +} + +func (o *Options) SetMaxLevel(ml int) { + o.maxLevel = ml +} + +func (o *Options) LevelMod() int { + return o.levelMod +} + +func (o *Options) SetLevelMod(lm int) { + o.levelMod = lm +} + +func (o *Options) SetPointsOnly(v bool) { + o.pointsOnly = v +} + +func (o *Options) SetOptimizeSpace(v bool) { + o.optimizeSpace = v +} + +func (o *Options) trueMaxLevel() int { + trueMax := o.maxLevel + if o.levelMod != 1 { + trueMax = o.maxLevel - (o.maxLevel-o.minLevel)%o.levelMod + } + return trueMax +} + +// RegionTermIndexer is a helper struct for adding spatial data to an +// information retrieval system. Such systems work by converting documents +// into a collection of "index terms" (e.g., representing words or phrases), +// and then building an "inverted index" that maps each term to a list of +// documents (and document positions) where that term occurs. +// +// This class deals with the problem of converting spatial data into index +// terms, which can then be indexed along with the other document information. +// +// Spatial data is represented using the S2Region type. Useful S2Region +// subtypes include: +// +// S2Cap +// - a disc-shaped region +// +// S2LatLngRect +// - a rectangle in latitude-longitude coordinates +// +// S2Polyline +// - a polyline +// +// S2Polygon +// - a polygon, possibly with multiple holes and/or shells +// +// S2CellUnion +// - a region approximated as a collection of S2CellIds +// +// S2ShapeIndexRegion +// - an arbitrary collection of points, polylines, and polygons +// +// S2ShapeIndexBufferedRegion +// - like the above, but expanded by a given radius +// +// S2RegionUnion, S2RegionIntersection +// - the union or intersection of arbitrary other regions +// +// So for example, if you want to query documents that are within 500 meters +// of a polyline, you could use an S2ShapeIndexBufferedRegion containing the +// polyline with a radius of 500 meters. +// +// For example usage refer: +// https://github.com/google/s2geometry/blob/ad1489e898f369ca09e2099353ccd55bd0fd7a26/src/s2/s2region_term_indexer.h#L58 + +type RegionTermIndexer struct { + options Options + regionCoverer RegionCoverer +} + +func NewRegionTermIndexer() *RegionTermIndexer { + rv := &RegionTermIndexer{ + options: Options{ + maxCells: 8, + minLevel: 4, + maxLevel: 16, + levelMod: 1, + }, + } + return rv +} + +func NewRegionTermIndexerWithOptions(option Options) *RegionTermIndexer { + return &RegionTermIndexer{options: option} +} + +func (rti *RegionTermIndexer) GetTerm(termTyp TermType, id CellID, + prefix string) string { + if termTyp == ANCESTOR { + return prefix + id.ToToken() + } + return prefix + marker + id.ToToken() +} + +func (rti *RegionTermIndexer) GetIndexTermsForPoint(p Point, prefix string) []string { + // See the top of this file for an overview of the indexing strategy. + // + // The last cell generated by this loop is effectively the covering for + // the given point. You might expect that this cell would be indexed as a + // covering term, but as an optimization we always index these cells as + // ancestor terms only. This is possible because query regions will never + // contain a descendant of such cells. Note that this is true even when + // max_level() != true_max_level() (see S2RegionCoverer::Options). + cellID := cellIDFromPoint(p) + var rv []string + for l := rti.options.minLevel; l <= rti.options.maxLevel; l += rti.options.levelMod { + rv = append(rv, rti.GetTerm(ANCESTOR, cellID.Parent(l), prefix)) + } + return rv +} + +func (rti *RegionTermIndexer) GetIndexTermsForRegion(region Region, + prefix string) []string { + rti.regionCoverer.LevelMod = rti.options.levelMod + rti.regionCoverer.MaxLevel = rti.options.maxLevel + rti.regionCoverer.MinLevel = rti.options.minLevel + rti.regionCoverer.MaxCells = rti.options.maxCells + + covering := rti.regionCoverer.Covering(region) + return rti.GetIndexTermsForCanonicalCovering(covering, prefix) +} + +func (rti *RegionTermIndexer) GetIndexTermsForCanonicalCovering( + covering CellUnion, prefix string) []string { + // See the top of this file for an overview of the indexing strategy. + // + // Cells in the covering are normally indexed as covering terms. If we are + // optimizing for query time rather than index space, they are also indexed + // as ancestor terms (since this lets us reduce the number of terms in the + // query). Finally, as an optimization we always index true_max_level() + // cells as ancestor cells only, since these cells have the special property + // that query regions will never contain a descendant of these cells. + var rv []string + prevID := CellID(0) + tml := rti.options.trueMaxLevel() + + for _, cellID := range covering { + level := cellID.Level() + if level < tml { + rv = append(rv, rti.GetTerm(COVERING, cellID, prefix)) + } + + if level == tml || !rti.options.optimizeSpace { + rv = append(rv, rti.GetTerm(ANCESTOR, cellID.Parent(level), prefix)) + } + + for (level - rti.options.levelMod) >= rti.options.minLevel { + level -= rti.options.levelMod + ancestorID := cellID.Parent(level) + if prevID != CellID(0) && prevID.Level() > level && + prevID.Parent(level) == ancestorID { + break + } + rv = append(rv, rti.GetTerm(ANCESTOR, ancestorID, prefix)) + } + prevID = cellID + } + + return rv +} + +func (rti *RegionTermIndexer) GetQueryTermsForPoint(p Point, prefix string) []string { + cellID := cellIDFromPoint(p) + var rv []string + + level := rti.options.trueMaxLevel() + rv = append(rv, rti.GetTerm(ANCESTOR, cellID.Parent(level), prefix)) + if rti.options.pointsOnly { + return rv + } + + for level >= rti.options.minLevel { + rv = append(rv, rti.GetTerm(COVERING, cellID.Parent(level), prefix)) + level -= rti.options.levelMod + } + + return rv +} + +func (rti *RegionTermIndexer) GetQueryTermsForRegion(region Region, + prefix string) []string { + rti.regionCoverer.LevelMod = rti.options.levelMod + rti.regionCoverer.MaxLevel = rti.options.maxLevel + rti.regionCoverer.MinLevel = rti.options.minLevel + rti.regionCoverer.MaxCells = rti.options.maxCells + + covering := rti.regionCoverer.Covering(region) + return rti.GetQueryTermsForCanonicalCovering(covering, prefix) + +} + +func (rti *RegionTermIndexer) GetQueryTermsForCanonicalCovering( + covering CellUnion, prefix string) []string { + var rv []string + prevID := CellID(0) + tml := rti.options.trueMaxLevel() + for _, cellID := range covering { + level := cellID.Level() + rv = append(rv, rti.GetTerm(ANCESTOR, cellID, prefix)) + + if rti.options.pointsOnly { + continue + } + + if rti.options.optimizeSpace && level < tml { + rv = append(rv, rti.GetTerm(COVERING, cellID, prefix)) + } + + for level-rti.options.levelMod >= rti.options.minLevel { + level -= rti.options.levelMod + ancestorID := cellID.Parent(level) + if prevID != CellID(0) && prevID.Level() > level && + prevID.Parent(level) == ancestorID { + break + } + rv = append(rv, rti.GetTerm(COVERING, ancestorID, prefix)) + } + + prevID = cellID + } + + return rv +} + +func CapFromCenterAndRadius(centerLat, centerLon, dist float64) Cap { + return CapFromCenterAngle(PointFromLatLng( + LatLngFromDegrees(centerLat, centerLon)), s1.Angle((dist/1000)/6378)) +} + +// FilterOutCoveringTerms filters out the covering terms so that +// it helps to reduce the search terms while searching in a one +// dimensional space. (point only indexing usecase) +func FilterOutCoveringTerms(terms []string) []string { + rv := make([]string, 0, len(terms)) + for _, term := range terms { + if strings.HasPrefix(term, marker) { + continue + } + rv = append(rv, term) + } + return rv +} diff --git a/vendor/github.com/blevesearch/geo/s2/regioncoverer.go b/vendor/github.com/blevesearch/geo/s2/regioncoverer.go new file mode 100644 index 00000000..de5b0c20 --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/regioncoverer.go @@ -0,0 +1,615 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +import ( + "container/heap" + "sort" +) + +// RegionCoverer allows arbitrary regions to be approximated as unions of cells (CellUnion). +// This is useful for implementing various sorts of search and precomputation operations. +// +// Typical usage: +// +// rc := &s2.RegionCoverer{MaxLevel: 30, MaxCells: 5} +// r := s2.Region(CapFromCenterArea(center, area)) +// covering := rc.Covering(r) +// +// This yields a CellUnion of at most 5 cells that is guaranteed to cover the +// given region (a disc-shaped region on the sphere). +// +// For covering, only cells where (level - MinLevel) is a multiple of LevelMod will be used. +// This effectively allows the branching factor of the S2 CellID hierarchy to be increased. +// Currently the only parameter values allowed are 1, 2, or 3, corresponding to +// branching factors of 4, 16, and 64 respectively. +// +// Note the following: +// +// - MinLevel takes priority over MaxCells, i.e. cells below the given level will +// never be used even if this causes a large number of cells to be returned. +// +// - For any setting of MaxCells, up to 6 cells may be returned if that +// is the minimum number of cells required (e.g. if the region intersects +// all six face cells). Up to 3 cells may be returned even for very tiny +// convex regions if they happen to be located at the intersection of +// three cube faces. +// +// - For any setting of MaxCells, an arbitrary number of cells may be +// returned if MinLevel is too high for the region being approximated. +// +// - If MaxCells is less than 4, the area of the covering may be +// arbitrarily large compared to the area of the original region even if +// the region is convex (e.g. a Cap or Rect). +// +// The approximation algorithm is not optimal but does a pretty good job in +// practice. The output does not always use the maximum number of cells +// allowed, both because this would not always yield a better approximation, +// and because MaxCells is a limit on how much work is done exploring the +// possible covering as well as a limit on the final output size. +// +// Because it is an approximation algorithm, one should not rely on the +// stability of the output. In particular, the output of the covering algorithm +// may change across different versions of the library. +// +// One can also generate interior coverings, which are sets of cells which +// are entirely contained within a region. Interior coverings can be +// empty, even for non-empty regions, if there are no cells that satisfy +// the provided constraints and are contained by the region. Note that for +// performance reasons, it is wise to specify a MaxLevel when computing +// interior coverings - otherwise for regions with small or zero area, the +// algorithm may spend a lot of time subdividing cells all the way to leaf +// level to try to find contained cells. +type RegionCoverer struct { + MinLevel int // the minimum cell level to be used. + MaxLevel int // the maximum cell level to be used. + LevelMod int // the LevelMod to be used. + MaxCells int // the maximum desired number of cells in the approximation. +} + +// NewRegionCoverer returns a region coverer with the appropriate defaults. +func NewRegionCoverer() *RegionCoverer { + return &RegionCoverer{ + MinLevel: 0, + MaxLevel: maxLevel, + LevelMod: 1, + MaxCells: 8, + } +} + +type coverer struct { + minLevel int // the minimum cell level to be used. + maxLevel int // the maximum cell level to be used. + levelMod int // the LevelMod to be used. + maxCells int // the maximum desired number of cells in the approximation. + region Region + result CellUnion + pq priorityQueue + interiorCovering bool +} + +type candidate struct { + cell Cell + terminal bool // Cell should not be expanded further. + numChildren int // Number of children that intersect the region. + children []*candidate // Actual size may be 0, 4, 16, or 64 elements. + priority int // Priority of the candidate. +} + +type priorityQueue []*candidate + +func (pq priorityQueue) Len() int { + return len(pq) +} + +func (pq priorityQueue) Less(i, j int) bool { + // We want Pop to give us the highest, not lowest, priority so we use greater than here. + return pq[i].priority > pq[j].priority +} + +func (pq priorityQueue) Swap(i, j int) { + pq[i], pq[j] = pq[j], pq[i] +} + +func (pq *priorityQueue) Push(x interface{}) { + item := x.(*candidate) + *pq = append(*pq, item) +} + +func (pq *priorityQueue) Pop() interface{} { + item := (*pq)[len(*pq)-1] + *pq = (*pq)[:len(*pq)-1] + return item +} + +func (pq *priorityQueue) Reset() { + *pq = (*pq)[:0] +} + +// newCandidate returns a new candidate with no children if the cell intersects the given region. +// The candidate is marked as terminal if it should not be expanded further. +func (c *coverer) newCandidate(cell Cell) *candidate { + if !c.region.IntersectsCell(cell) { + return nil + } + cand := &candidate{cell: cell} + level := int(cell.level) + if level >= c.minLevel { + if c.interiorCovering { + if c.region.ContainsCell(cell) { + cand.terminal = true + } else if level+c.levelMod > c.maxLevel { + return nil + } + } else if level+c.levelMod > c.maxLevel || c.region.ContainsCell(cell) { + cand.terminal = true + } + } + return cand +} + +// expandChildren populates the children of the candidate by expanding the given number of +// levels from the given cell. Returns the number of children that were marked "terminal". +func (c *coverer) expandChildren(cand *candidate, cell Cell, numLevels int) int { + numLevels-- + var numTerminals int + last := cell.id.ChildEnd() + for ci := cell.id.ChildBegin(); ci != last; ci = ci.Next() { + childCell := CellFromCellID(ci) + if numLevels > 0 { + if c.region.IntersectsCell(childCell) { + numTerminals += c.expandChildren(cand, childCell, numLevels) + } + continue + } + if child := c.newCandidate(childCell); child != nil { + cand.children = append(cand.children, child) + cand.numChildren++ + if child.terminal { + numTerminals++ + } + } + } + return numTerminals +} + +// addCandidate adds the given candidate to the result if it is marked as "terminal", +// otherwise expands its children and inserts it into the priority queue. +// Passing an argument of nil does nothing. +func (c *coverer) addCandidate(cand *candidate) { + if cand == nil { + return + } + + if cand.terminal { + c.result = append(c.result, cand.cell.id) + return + } + + // Expand one level at a time until we hit minLevel to ensure that we don't skip over it. + numLevels := c.levelMod + level := int(cand.cell.level) + if level < c.minLevel { + numLevels = 1 + } + + numTerminals := c.expandChildren(cand, cand.cell, numLevels) + maxChildrenShift := uint(2 * c.levelMod) + if cand.numChildren == 0 { + return + } else if !c.interiorCovering && numTerminals == 1<= c.minLevel { + // Optimization: add the parent cell rather than all of its children. + // We can't do this for interior coverings, since the children just + // intersect the region, but may not be contained by it - we need to + // subdivide them further. + cand.terminal = true + c.addCandidate(cand) + } else { + // We negate the priority so that smaller absolute priorities are returned + // first. The heuristic is designed to refine the largest cells first, + // since those are where we have the largest potential gain. Among cells + // of the same size, we prefer the cells with the fewest children. + // Finally, among cells with equal numbers of children we prefer those + // with the smallest number of children that cannot be refined further. + cand.priority = -(((level< 1 && level > c.minLevel { + level -= (level - c.minLevel) % c.levelMod + } + return level +} + +// adjustCellLevels ensures that all cells with level > minLevel also satisfy levelMod, +// by replacing them with an ancestor if necessary. Cell levels smaller +// than minLevel are not modified (see AdjustLevel). The output is +// then normalized to ensure that no redundant cells are present. +func (c *coverer) adjustCellLevels(cells *CellUnion) { + if c.levelMod == 1 { + return + } + + var out int + for _, ci := range *cells { + level := ci.Level() + newLevel := c.adjustLevel(level) + if newLevel != level { + ci = ci.Parent(newLevel) + } + if out > 0 && (*cells)[out-1].Contains(ci) { + continue + } + for out > 0 && ci.Contains((*cells)[out-1]) { + out-- + } + (*cells)[out] = ci + out++ + } + *cells = (*cells)[:out] +} + +// initialCandidates computes a set of initial candidates that cover the given region. +func (c *coverer) initialCandidates() { + // Optimization: start with a small (usually 4 cell) covering of the region's bounding cap. + temp := &RegionCoverer{MaxLevel: c.maxLevel, LevelMod: 1, MaxCells: minInt(4, c.maxCells)} + + cells := temp.FastCovering(c.region) + c.adjustCellLevels(&cells) + for _, ci := range cells { + c.addCandidate(c.newCandidate(CellFromCellID(ci))) + } +} + +// coveringInternal generates a covering and stores it in result. +// Strategy: Start with the 6 faces of the cube. Discard any +// that do not intersect the shape. Then repeatedly choose the +// largest cell that intersects the shape and subdivide it. +// +// result contains the cells that will be part of the output, while pq +// contains cells that we may still subdivide further. Cells that are +// entirely contained within the region are immediately added to the output, +// while cells that do not intersect the region are immediately discarded. +// Therefore pq only contains cells that partially intersect the region. +// Candidates are prioritized first according to cell size (larger cells +// first), then by the number of intersecting children they have (fewest +// children first), and then by the number of fully contained children +// (fewest children first). +func (c *coverer) coveringInternal(region Region) { + c.region = region + + c.initialCandidates() + for c.pq.Len() > 0 && (!c.interiorCovering || len(c.result) < c.maxCells) { + cand := heap.Pop(&c.pq).(*candidate) + + // For interior covering we keep subdividing no matter how many children + // candidate has. If we reach MaxCells before expanding all children, + // we will just use some of them. + // For exterior covering we cannot do this, because result has to cover the + // whole region, so all children have to be used. + // candidate.numChildren == 1 case takes care of the situation when we + // already have more than MaxCells in result (minLevel is too high). + // Subdividing of the candidate with one child does no harm in this case. + if c.interiorCovering || int(cand.cell.level) < c.minLevel || cand.numChildren == 1 || len(c.result)+c.pq.Len()+cand.numChildren <= c.maxCells { + for _, child := range cand.children { + if !c.interiorCovering || len(c.result) < c.maxCells { + c.addCandidate(child) + } + } + } else { + cand.terminal = true + c.addCandidate(cand) + } + } + + c.pq.Reset() + c.region = nil + + // Rather than just returning the raw list of cell ids, we construct a cell + // union and then denormalize it. This has the effect of replacing four + // child cells with their parent whenever this does not violate the covering + // parameters specified (min_level, level_mod, etc). This significantly + // reduces the number of cells returned in many cases, and it is cheap + // compared to computing the covering in the first place. + c.result.Normalize() + if c.minLevel > 0 || c.levelMod > 1 { + c.result.Denormalize(c.minLevel, c.levelMod) + } +} + +// newCoverer returns an instance of coverer. +func (rc *RegionCoverer) newCoverer() *coverer { + return &coverer{ + minLevel: maxInt(0, minInt(maxLevel, rc.MinLevel)), + maxLevel: maxInt(0, minInt(maxLevel, rc.MaxLevel)), + levelMod: maxInt(1, minInt(3, rc.LevelMod)), + maxCells: rc.MaxCells, + } +} + +// Covering returns a CellUnion that covers the given region and satisfies the various restrictions. +func (rc *RegionCoverer) Covering(region Region) CellUnion { + covering := rc.CellUnion(region) + covering.Denormalize(maxInt(0, minInt(maxLevel, rc.MinLevel)), maxInt(1, minInt(3, rc.LevelMod))) + return covering +} + +// InteriorCovering returns a CellUnion that is contained within the given region and satisfies the various restrictions. +func (rc *RegionCoverer) InteriorCovering(region Region) CellUnion { + intCovering := rc.InteriorCellUnion(region) + intCovering.Denormalize(maxInt(0, minInt(maxLevel, rc.MinLevel)), maxInt(1, minInt(3, rc.LevelMod))) + return intCovering +} + +// CellUnion returns a normalized CellUnion that covers the given region and +// satisfies the restrictions except for minLevel and levelMod. These criteria +// cannot be satisfied using a cell union because cell unions are +// automatically normalized by replacing four child cells with their parent +// whenever possible. (Note that the list of cell ids passed to the CellUnion +// constructor does in fact satisfy all the given restrictions.) +func (rc *RegionCoverer) CellUnion(region Region) CellUnion { + c := rc.newCoverer() + c.coveringInternal(region) + cu := c.result + cu.Normalize() + return cu +} + +// InteriorCellUnion returns a normalized CellUnion that is contained within the given region and +// satisfies the restrictions except for minLevel and levelMod. These criteria +// cannot be satisfied using a cell union because cell unions are +// automatically normalized by replacing four child cells with their parent +// whenever possible. (Note that the list of cell ids passed to the CellUnion +// constructor does in fact satisfy all the given restrictions.) +func (rc *RegionCoverer) InteriorCellUnion(region Region) CellUnion { + c := rc.newCoverer() + c.interiorCovering = true + c.coveringInternal(region) + cu := c.result + cu.Normalize() + return cu +} + +// FastCovering returns a CellUnion that covers the given region similar to Covering, +// except that this method is much faster and the coverings are not as tight. +// All of the usual parameters are respected (MaxCells, MinLevel, MaxLevel, and LevelMod), +// except that the implementation makes no attempt to take advantage of large values of +// MaxCells. (A small number of cells will always be returned.) +// +// This function is useful as a starting point for algorithms that +// recursively subdivide cells. +func (rc *RegionCoverer) FastCovering(region Region) CellUnion { + c := rc.newCoverer() + cu := CellUnion(region.CellUnionBound()) + c.normalizeCovering(&cu) + return cu +} + +// IsCanonical reports whether the given CellUnion represents a valid covering +// that conforms to the current covering parameters. In particular: +// +// - All CellIDs must be valid. +// +// - CellIDs must be sorted and non-overlapping. +// +// - CellID levels must satisfy MinLevel, MaxLevel, and LevelMod. +// +// - If the covering has more than MaxCells, there must be no two cells with +// a common ancestor at MinLevel or higher. +// +// - There must be no sequence of cells that could be replaced by an +// ancestor (i.e. with LevelMod == 1, the 4 child cells of a parent). +func (rc *RegionCoverer) IsCanonical(covering CellUnion) bool { + return rc.newCoverer().isCanonical(covering) +} + +// normalizeCovering normalizes the "covering" so that it conforms to the +// current covering parameters (maxCells, minLevel, maxLevel, and levelMod). +// This method makes no attempt to be optimal. In particular, if +// minLevel > 0 or levelMod > 1 then it may return more than the +// desired number of cells even when this isn't necessary. +// +// Note that when the covering parameters have their default values, almost +// all of the code in this function is skipped. +func (c *coverer) normalizeCovering(covering *CellUnion) { + // If any cells are too small, or don't satisfy levelMod, then replace them with ancestors. + if c.maxLevel < maxLevel || c.levelMod > 1 { + for i, ci := range *covering { + level := ci.Level() + newLevel := c.adjustLevel(minInt(level, c.maxLevel)) + if newLevel != level { + (*covering)[i] = ci.Parent(newLevel) + } + } + } + // Sort the cells and simplify them. + covering.Normalize() + + // Make sure that the covering satisfies minLevel and levelMod, + // possibly at the expense of satisfying MaxCells. + if c.minLevel > 0 || c.levelMod > 1 { + covering.Denormalize(c.minLevel, c.levelMod) + } + + // If there are too many cells and the covering is very large, use the + // RegionCoverer to compute a new covering. (This avoids possible O(n^2) + // behavior of the simpler algorithm below.) + excess := len(*covering) - c.maxCells + if excess <= 0 || c.isCanonical(*covering) { + return + } + if excess*len(*covering) > 10000 { + rc := NewRegionCoverer() + (*covering) = rc.Covering(covering) + return + } + + // If there are still too many cells, then repeatedly replace two adjacent + // cells in CellID order by their lowest common ancestor. + for len(*covering) > c.maxCells { + bestIndex := -1 + bestLevel := -1 + for i := 0; i+1 < len(*covering); i++ { + level, ok := (*covering)[i].CommonAncestorLevel((*covering)[i+1]) + if !ok { + continue + } + level = c.adjustLevel(level) + if level > bestLevel { + bestLevel = level + bestIndex = i + } + } + + if bestLevel < c.minLevel { + break + } + + // Replace all cells contained by the new ancestor cell. + id := (*covering)[bestIndex].Parent(bestLevel) + (*covering) = c.replaceCellsWithAncestor(*covering, id) + + // Now repeatedly check whether all children of the parent cell are + // present, in which case we can replace those cells with their parent. + for bestLevel > c.minLevel { + bestLevel -= c.levelMod + id = id.Parent(bestLevel) + if !c.containsAllChildren(*covering, id) { + break + } + (*covering) = c.replaceCellsWithAncestor(*covering, id) + } + } +} + +// isCanonical reports whether the covering is canonical. +func (c *coverer) isCanonical(covering CellUnion) bool { + trueMax := c.maxLevel + if c.levelMod != 1 { + trueMax = c.maxLevel - (c.maxLevel-c.minLevel)%c.levelMod + } + tooManyCells := len(covering) > c.maxCells + sameParentCount := 1 + + prevID := CellID(0) + for _, id := range covering { + if !id.IsValid() { + return false + } + + // Check that the CellID level is acceptable. + level := id.Level() + if level < c.minLevel || level > trueMax { + return false + } + if c.levelMod > 1 && (level-c.minLevel)%c.levelMod != 0 { + return false + } + + if prevID != 0 { + // Check that cells are sorted and non-overlapping. + if prevID.RangeMax() >= id.RangeMin() { + return false + } + + lev, ok := id.CommonAncestorLevel(prevID) + // If there are too many cells, check that no pair of adjacent cells + // could be replaced by an ancestor. + if tooManyCells && (ok && lev >= c.minLevel) { + return false + } + + // Check that there are no sequences of (4 ** level_mod) cells that all + // have the same parent (considering only multiples of "level_mod"). + pLevel := level - c.levelMod + if pLevel < c.minLevel || level != prevID.Level() || + id.Parent(pLevel) != prevID.Parent(pLevel) { + sameParentCount = 1 + } else { + sameParentCount++ + if sameParentCount == 1<= id.RangeMin() }) + level := id.Level() + c.levelMod + for child := id.ChildBeginAtLevel(level); child != id.ChildEndAtLevel(level); child = child.Next() { + if pos == len(covering) || covering[pos] != child { + return false + } + pos++ + } + return true +} + +// replaceCellsWithAncestor replaces all descendants of the given id in covering +// with id. This requires the covering contains at least one descendant of id. +func (c *coverer) replaceCellsWithAncestor(covering []CellID, id CellID) []CellID { + begin := sort.Search(len(covering), func(i int) bool { return covering[i] > id.RangeMin() }) + end := sort.Search(len(covering), func(i int) bool { return covering[i] > id.RangeMax() }) + + return append(append(covering[:begin], id), covering[end:]...) +} + +// SimpleRegionCovering returns a set of cells at the given level that cover +// the connected region and a starting point on the boundary or inside the +// region. The cells are returned in arbitrary order. +// +// Note that this method is not faster than the regular Covering +// method for most region types, such as Cap or Polygon, and in fact it +// can be much slower when the output consists of a large number of cells. +// Currently it can be faster at generating coverings of long narrow regions +// such as polylines, but this may change in the future. +func SimpleRegionCovering(region Region, start Point, level int) []CellID { + return FloodFillRegionCovering(region, cellIDFromPoint(start).Parent(level)) +} + +// FloodFillRegionCovering returns all edge-connected cells at the same level as +// the given CellID that intersect the given region, in arbitrary order. +func FloodFillRegionCovering(region Region, start CellID) []CellID { + var output []CellID + all := map[CellID]bool{ + start: true, + } + frontier := []CellID{start} + for len(frontier) > 0 { + id := frontier[len(frontier)-1] + frontier = frontier[:len(frontier)-1] + if !region.IntersectsCell(CellFromCellID(id)) { + continue + } + output = append(output, id) + for _, nbr := range id.EdgeNeighbors() { + if !all[nbr] { + all[nbr] = true + frontier = append(frontier, nbr) + } + } + } + + return output +} diff --git a/vendor/github.com/blevesearch/geo/s2/regionunion.go b/vendor/github.com/blevesearch/geo/s2/regionunion.go new file mode 100644 index 00000000..915b7c33 --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/regionunion.go @@ -0,0 +1,66 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +// A RegionUnion represents a union of possibly overlapping regions. +// It is convenient for computing a covering of a set of regions. +type RegionUnion []Region + +// CapBound returns a bounding cap for this RegionUnion. +func (ru RegionUnion) CapBound() Cap { return ru.RectBound().CapBound() } + +// RectBound returns a bounding latitude-longitude rectangle for this RegionUnion. +func (ru RegionUnion) RectBound() Rect { + ret := EmptyRect() + for _, reg := range ru { + ret = ret.Union(reg.RectBound()) + } + return ret +} + +// ContainsCell reports whether the given Cell is contained by this RegionUnion. +func (ru RegionUnion) ContainsCell(c Cell) bool { + for _, reg := range ru { + if reg.ContainsCell(c) { + return true + } + } + return false +} + +// IntersectsCell reports whether this RegionUnion intersects the given cell. +func (ru RegionUnion) IntersectsCell(c Cell) bool { + for _, reg := range ru { + if reg.IntersectsCell(c) { + return true + } + } + return false +} + +// ContainsPoint reports whether this RegionUnion contains the Point. +func (ru RegionUnion) ContainsPoint(p Point) bool { + for _, reg := range ru { + if reg.ContainsPoint(p) { + return true + } + } + return false +} + +// CellUnionBound computes a covering of the RegionUnion. +func (ru RegionUnion) CellUnionBound() []CellID { + return ru.CapBound().CellUnionBound() +} diff --git a/vendor/github.com/blevesearch/geo/s2/shape.go b/vendor/github.com/blevesearch/geo/s2/shape.go new file mode 100644 index 00000000..2cbf170c --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/shape.go @@ -0,0 +1,263 @@ +// Copyright 2017 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +import ( + "sort" +) + +// Edge represents a geodesic edge consisting of two vertices. Zero-length edges are +// allowed, and can be used to represent points. +type Edge struct { + V0, V1 Point +} + +// Cmp compares the two edges using the underlying Points Cmp method and returns +// +// -1 if e < other +// 0 if e == other +// +1 if e > other +// +// The two edges are compared by first vertex, and then by the second vertex. +func (e Edge) Cmp(other Edge) int { + if v0cmp := e.V0.Cmp(other.V0.Vector); v0cmp != 0 { + return v0cmp + } + return e.V1.Cmp(other.V1.Vector) +} + +// sortEdges sorts the slice of Edges in place. +func sortEdges(e []Edge) { + sort.Sort(edges(e)) +} + +// edges implements the Sort interface for slices of Edge. +type edges []Edge + +func (e edges) Len() int { return len(e) } +func (e edges) Swap(i, j int) { e[i], e[j] = e[j], e[i] } +func (e edges) Less(i, j int) bool { return e[i].Cmp(e[j]) == -1 } + +// ShapeEdgeID is a unique identifier for an Edge within an ShapeIndex, +// consisting of a (shapeID, edgeID) pair. +type ShapeEdgeID struct { + ShapeID int32 + EdgeID int32 +} + +// Cmp compares the two ShapeEdgeIDs and returns +// +// -1 if s < other +// 0 if s == other +// +1 if s > other +// +// The two are compared first by shape id and then by edge id. +func (s ShapeEdgeID) Cmp(other ShapeEdgeID) int { + switch { + case s.ShapeID < other.ShapeID: + return -1 + case s.ShapeID > other.ShapeID: + return 1 + } + switch { + case s.EdgeID < other.EdgeID: + return -1 + case s.EdgeID > other.EdgeID: + return 1 + } + return 0 +} + +// ShapeEdge represents a ShapeEdgeID with the two endpoints of that Edge. +type ShapeEdge struct { + ID ShapeEdgeID + Edge Edge +} + +// Chain represents a range of edge IDs corresponding to a chain of connected +// edges, specified as a (start, length) pair. The chain is defined to consist of +// edge IDs {start, start + 1, ..., start + length - 1}. +type Chain struct { + Start, Length int +} + +// ChainPosition represents the position of an edge within a given edge chain, +// specified as a (chainID, offset) pair. Chains are numbered sequentially +// starting from zero, and offsets are measured from the start of each chain. +type ChainPosition struct { + ChainID, Offset int +} + +// A ReferencePoint consists of a point and a boolean indicating whether the point +// is contained by a particular shape. +type ReferencePoint struct { + Point Point + Contained bool +} + +// OriginReferencePoint returns a ReferencePoint with the given value for +// contained and the origin point. It should be used when all points or no +// points are contained. +func OriginReferencePoint(contained bool) ReferencePoint { + return ReferencePoint{Point: OriginPoint(), Contained: contained} +} + +// typeTag is a 32-bit tag that can be used to identify the type of an encoded +// Shape. All encodable types have a non-zero type tag. The tag associated with +type typeTag uint32 + +const ( + // Indicates that a given Shape type cannot be encoded. + typeTagNone typeTag = 0 + typeTagPolygon typeTag = 1 + typeTagPolyline typeTag = 2 + typeTagPointVector typeTag = 3 + typeTagLaxPolyline typeTag = 4 + typeTagLaxPolygon typeTag = 5 + + // The minimum allowable tag for future user-defined Shape types. + typeTagMinUser typeTag = 8192 +) + +// Shape represents polygonal geometry in a flexible way. It is organized as a +// collection of edges that optionally defines an interior. All geometry +// represented by a given Shape must have the same dimension, which means that +// an Shape can represent either a set of points, a set of polylines, or a set +// of polygons. +// +// Shape is defined as an interface in order to give clients control over the +// underlying data representation. Sometimes an Shape does not have any data of +// its own, but instead wraps some other type. +// +// Shape operations are typically defined on a ShapeIndex rather than +// individual shapes. An ShapeIndex is simply a collection of Shapes, +// possibly of different dimensions (e.g. 10 points and 3 polygons), organized +// into a data structure for efficient edge access. +// +// The edges of a Shape are indexed by a contiguous range of edge IDs +// starting at 0. The edges are further subdivided into chains, where each +// chain consists of a sequence of edges connected end-to-end (a polyline). +// For example, a Shape representing two polylines AB and CDE would have +// three edges (AB, CD, DE) grouped into two chains: (AB) and (CD, DE). +// Similarly, an Shape representing 5 points would have 5 chains consisting +// of one edge each. +// +// Shape has methods that allow edges to be accessed either using the global +// numbering (edge ID) or within a particular chain. The global numbering is +// sufficient for most purposes, but the chain representation is useful for +// certain algorithms such as intersection (see BooleanOperation). +type Shape interface { + // NumEdges returns the number of edges in this shape. + NumEdges() int + + // Edge returns the edge for the given edge index. + Edge(i int) Edge + + // ReferencePoint returns an arbitrary reference point for the shape. (The + // containment boolean value must be false for shapes that do not have an interior.) + // + // This reference point may then be used to compute the containment of other + // points by counting edge crossings. + ReferencePoint() ReferencePoint + + // NumChains reports the number of contiguous edge chains in the shape. + // For example, a shape whose edges are [AB, BC, CD, AE, EF] would consist + // of two chains (AB,BC,CD and AE,EF). Every chain is assigned a chain Id + // numbered sequentially starting from zero. + // + // Note that it is always acceptable to implement this method by returning + // NumEdges, i.e. every chain consists of a single edge, but this may + // reduce the efficiency of some algorithms. + NumChains() int + + // Chain returns the range of edge IDs corresponding to the given edge chain. + // Edge chains must form contiguous, non-overlapping ranges that cover + // the entire range of edge IDs. This is spelled out more formally below: + // + // 0 <= i < NumChains() + // Chain(i).length > 0, for all i + // Chain(0).start == 0 + // Chain(i).start + Chain(i).length == Chain(i+1).start, for i < NumChains()-1 + // Chain(i).start + Chain(i).length == NumEdges(), for i == NumChains()-1 + Chain(chainID int) Chain + + // ChainEdgeReturns the edge at offset "offset" within edge chain "chainID". + // Equivalent to "shape.Edge(shape.Chain(chainID).start + offset)" + // but more efficient. + ChainEdge(chainID, offset int) Edge + + // ChainPosition finds the chain containing the given edge, and returns the + // position of that edge as a ChainPosition(chainID, offset) pair. + // + // shape.Chain(pos.chainID).start + pos.offset == edgeID + // shape.Chain(pos.chainID+1).start > edgeID + // + // where pos == shape.ChainPosition(edgeID). + ChainPosition(edgeID int) ChainPosition + + // Dimension returns the dimension of the geometry represented by this shape, + // either 0, 1 or 2 for point, polyline and polygon geometry respectively. + // + // 0 - Point geometry. Each point is represented as a degenerate edge. + // + // 1 - Polyline geometry. Polyline edges may be degenerate. A shape may + // represent any number of polylines. Polylines edges may intersect. + // + // 2 - Polygon geometry. Edges should be oriented such that the polygon + // interior is always on the left. In theory the edges may be returned + // in any order, but typically the edges are organized as a collection + // of edge chains where each chain represents one polygon loop. + // Polygons may have degeneracies (e.g., degenerate edges or sibling + // pairs consisting of an edge and its corresponding reversed edge). + // A polygon loop may also be full (containing all points on the + // sphere); by convention this is represented as a chain with no edges. + // (See laxPolygon for details.) + // + // This method allows degenerate geometry of different dimensions + // to be distinguished, e.g. it allows a point to be distinguished from a + // polyline or polygon that has been simplified to a single point. + Dimension() int + + // IsEmpty reports whether the Shape contains no points. (Note that the full + // polygon is represented as a chain with zero edges.) + IsEmpty() bool + + // IsFull reports whether the Shape contains all points on the sphere. + IsFull() bool + + // typeTag returns a value that can be used to identify the type of an + // encoded Shape. + typeTag() typeTag + + // We do not support implementations of this interface outside this package. + privateInterface() +} + +// defaultShapeIsEmpty reports whether this shape contains no points. +func defaultShapeIsEmpty(s Shape) bool { + return s.NumEdges() == 0 && (s.Dimension() != 2 || s.NumChains() == 0) +} + +// defaultShapeIsFull reports whether this shape contains all points on the sphere. +func defaultShapeIsFull(s Shape) bool { + return s.NumEdges() == 0 && s.Dimension() == 2 && s.NumChains() > 0 +} + +// A minimal check for types that should satisfy the Shape interface. +var ( + _ Shape = &Loop{} + _ Shape = &Polygon{} + _ Shape = &Polyline{} +) diff --git a/vendor/github.com/blevesearch/geo/s2/shapeindex.go b/vendor/github.com/blevesearch/geo/s2/shapeindex.go new file mode 100644 index 00000000..6efa213a --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/shapeindex.go @@ -0,0 +1,1526 @@ +// Copyright 2016 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +import ( + "math" + "sort" + "sync" + "sync/atomic" + + "github.com/golang/geo/r1" + "github.com/golang/geo/r2" +) + +// CellRelation describes the possible relationships between a target cell +// and the cells of the ShapeIndex. If the target is an index cell or is +// contained by an index cell, it is Indexed. If the target is subdivided +// into one or more index cells, it is Subdivided. Otherwise it is Disjoint. +type CellRelation int + +// The possible CellRelations for a ShapeIndex. +const ( + Indexed CellRelation = iota + Subdivided + Disjoint +) + +const ( + // cellPadding defines the total error when clipping an edge which comes + // from two sources: + // (1) Clipping the original spherical edge to a cube face (the face edge). + // The maximum error in this step is faceClipErrorUVCoord. + // (2) Clipping the face edge to the u- or v-coordinate of a cell boundary. + // The maximum error in this step is edgeClipErrorUVCoord. + // Finally, since we encounter the same errors when clipping query edges, we + // double the total error so that we only need to pad edges during indexing + // and not at query time. + cellPadding = 2.0 * (faceClipErrorUVCoord + edgeClipErrorUVCoord) + + // cellSizeToLongEdgeRatio defines the cell size relative to the length of an + // edge at which it is first considered to be long. Long edges do not + // contribute toward the decision to subdivide a cell further. For example, + // a value of 2.0 means that the cell must be at least twice the size of the + // edge in order for that edge to be counted. There are two reasons for not + // counting long edges: (1) such edges typically need to be propagated to + // several children, which increases time and memory costs without much benefit, + // and (2) in pathological cases, many long edges close together could force + // subdivision to continue all the way to the leaf cell level. + cellSizeToLongEdgeRatio = 1.0 +) + +// clippedShape represents the part of a shape that intersects a Cell. +// It consists of the set of edge IDs that intersect that cell and a boolean +// indicating whether the center of the cell is inside the shape (for shapes +// that have an interior). +// +// Note that the edges themselves are not clipped; we always use the original +// edges for intersection tests so that the results will be the same as the +// original shape. +type clippedShape struct { + // shapeID is the index of the shape this clipped shape is a part of. + shapeID int32 + + // containsCenter indicates if the center of the CellID this shape has been + // clipped to falls inside this shape. This is false for shapes that do not + // have an interior. + containsCenter bool + + // edges is the ordered set of ShapeIndex original edge IDs. Edges + // are stored in increasing order of edge ID. + edges []int +} + +// newClippedShape returns a new clipped shape for the given shapeID and number of expected edges. +func newClippedShape(id int32, numEdges int) *clippedShape { + return &clippedShape{ + shapeID: id, + edges: make([]int, numEdges), + } +} + +// numEdges returns the number of edges that intersect the CellID of the Cell this was clipped to. +func (c *clippedShape) numEdges() int { + return len(c.edges) +} + +// containsEdge reports if this clipped shape contains the given edge ID. +func (c *clippedShape) containsEdge(id int) bool { + // Linear search is fast because the number of edges per shape is typically + // very small (less than 10). + for _, e := range c.edges { + if e == id { + return true + } + } + return false +} + +// ShapeIndexCell stores the index contents for a particular CellID. +type ShapeIndexCell struct { + shapes []*clippedShape +} + +// NewShapeIndexCell creates a new cell that is sized to hold the given number of shapes. +func NewShapeIndexCell(numShapes int) *ShapeIndexCell { + return &ShapeIndexCell{ + shapes: make([]*clippedShape, numShapes), + } +} + +// numEdges reports the total number of edges in all clipped shapes in this cell. +func (s *ShapeIndexCell) numEdges() int { + var e int + for _, cs := range s.shapes { + e += cs.numEdges() + } + return e +} + +// add adds the given clipped shape to this index cell. +func (s *ShapeIndexCell) add(c *clippedShape) { + // C++ uses a set, so it's ordered and unique. We don't currently catch + // the case when a duplicate value is added. + s.shapes = append(s.shapes, c) +} + +// findByShapeID returns the clipped shape that contains the given shapeID, +// or nil if none of the clipped shapes contain it. +func (s *ShapeIndexCell) findByShapeID(shapeID int32) *clippedShape { + // Linear search is fine because the number of shapes per cell is typically + // very small (most often 1), and is large only for pathological inputs + // (e.g. very deeply nested loops). + for _, clipped := range s.shapes { + if clipped.shapeID == shapeID { + return clipped + } + } + return nil +} + +// faceEdge and clippedEdge store temporary edge data while the index is being +// updated. +// +// While it would be possible to combine all the edge information into one +// structure, there are two good reasons for separating it: +// +// - Memory usage. Separating the two means that we only need to +// store one copy of the per-face data no matter how many times an edge is +// subdivided, and it also lets us delay computing bounding boxes until +// they are needed for processing each face (when the dataset spans +// multiple faces). +// +// - Performance. UpdateEdges is significantly faster on large polygons when +// the data is separated, because it often only needs to access the data in +// clippedEdge and this data is cached more successfully. + +// faceEdge represents an edge that has been projected onto a given face, +type faceEdge struct { + shapeID int32 // The ID of shape that this edge belongs to + edgeID int // Edge ID within that shape + maxLevel int // Not desirable to subdivide this edge beyond this level + hasInterior bool // Belongs to a shape that has a dimension of 2 + a, b r2.Point // The edge endpoints, clipped to a given face + edge Edge // The original edge. +} + +// clippedEdge represents the portion of that edge that has been clipped to a given Cell. +type clippedEdge struct { + faceEdge *faceEdge // The original unclipped edge + bound r2.Rect // Bounding box for the clipped portion +} + +// ShapeIndexIteratorPos defines the set of possible iterator starting positions. By +// default iterators are unpositioned, since this avoids an extra seek in this +// situation where one of the seek methods (such as Locate) is immediately called. +type ShapeIndexIteratorPos int + +const ( + // IteratorBegin specifies the iterator should be positioned at the beginning of the index. + IteratorBegin ShapeIndexIteratorPos = iota + // IteratorEnd specifies the iterator should be positioned at the end of the index. + IteratorEnd +) + +// ShapeIndexIterator is an iterator that provides low-level access to +// the cells of the index. Cells are returned in increasing order of CellID. +// +// for it := index.Iterator(); !it.Done(); it.Next() { +// fmt.Print(it.CellID()) +// } +// +type ShapeIndexIterator struct { + index *ShapeIndex + position int + id CellID + cell *ShapeIndexCell +} + +// NewShapeIndexIterator creates a new iterator for the given index. If a starting +// position is specified, the iterator is positioned at the given spot. +func NewShapeIndexIterator(index *ShapeIndex, pos ...ShapeIndexIteratorPos) *ShapeIndexIterator { + s := &ShapeIndexIterator{ + index: index, + } + + if len(pos) > 0 { + if len(pos) > 1 { + panic("too many ShapeIndexIteratorPos arguments") + } + switch pos[0] { + case IteratorBegin: + s.Begin() + case IteratorEnd: + s.End() + default: + panic("unknown ShapeIndexIteratorPos value") + } + } + + return s +} + +func (s *ShapeIndexIterator) clone() *ShapeIndexIterator { + return &ShapeIndexIterator{ + index: s.index, + position: s.position, + id: s.id, + cell: s.cell, + } +} + +// CellID returns the CellID of the current index cell. +// If s.Done() is true, a value larger than any valid CellID is returned. +func (s *ShapeIndexIterator) CellID() CellID { + return s.id +} + +// IndexCell returns the current index cell. +func (s *ShapeIndexIterator) IndexCell() *ShapeIndexCell { + // TODO(roberts): C++ has this call a virtual method to allow subclasses + // of ShapeIndexIterator to do other work before returning the cell. Do + // we need such a thing? + return s.cell +} + +// Center returns the Point at the center of the current position of the iterator. +func (s *ShapeIndexIterator) Center() Point { + return s.CellID().Point() +} + +// Begin positions the iterator at the beginning of the index. +func (s *ShapeIndexIterator) Begin() { + if !s.index.IsFresh() { + s.index.maybeApplyUpdates() + } + s.position = 0 + s.refresh() +} + +// Next positions the iterator at the next index cell. +func (s *ShapeIndexIterator) Next() { + s.position++ + s.refresh() +} + +// Prev advances the iterator to the previous cell in the index and returns true to +// indicate it was not yet at the beginning of the index. If the iterator is at the +// first cell the call does nothing and returns false. +func (s *ShapeIndexIterator) Prev() bool { + if s.position <= 0 { + return false + } + + s.position-- + s.refresh() + return true +} + +// End positions the iterator at the end of the index. +func (s *ShapeIndexIterator) End() { + s.position = len(s.index.cells) + s.refresh() +} + +// Done reports if the iterator is positioned at or after the last index cell. +func (s *ShapeIndexIterator) Done() bool { + return s.id == SentinelCellID +} + +// refresh updates the stored internal iterator values. +func (s *ShapeIndexIterator) refresh() { + if s.position < len(s.index.cells) { + s.id = s.index.cells[s.position] + s.cell = s.index.cellMap[s.CellID()] + } else { + s.id = SentinelCellID + s.cell = nil + } +} + +// seek positions the iterator at the first cell whose ID >= target, or at the +// end of the index if no such cell exists. +func (s *ShapeIndexIterator) seek(target CellID) { + s.position = sort.Search(len(s.index.cells), func(i int) bool { + return s.index.cells[i] >= target + }) + s.refresh() +} + +// LocatePoint positions the iterator at the cell that contains the given Point. +// If no such cell exists, the iterator position is unspecified, and false is returned. +// The cell at the matched position is guaranteed to contain all edges that might +// intersect the line segment between target and the cell's center. +func (s *ShapeIndexIterator) LocatePoint(p Point) bool { + // Let I = cellMap.LowerBound(T), where T is the leaf cell containing + // point P. Then if T is contained by an index cell, then the + // containing cell is either I or I'. We test for containment by comparing + // the ranges of leaf cells spanned by T, I, and I'. + target := cellIDFromPoint(p) + s.seek(target) + if !s.Done() && s.CellID().RangeMin() <= target { + return true + } + + if s.Prev() && s.CellID().RangeMax() >= target { + return true + } + return false +} + +// LocateCellID attempts to position the iterator at the first matching index cell +// in the index that has some relation to the given CellID. Let T be the target CellID. +// If T is contained by (or equal to) some index cell I, then the iterator is positioned +// at I and returns Indexed. Otherwise if T contains one or more (smaller) index cells, +// then the iterator is positioned at the first such cell I and return Subdivided. +// Otherwise Disjoint is returned and the iterator position is undefined. +func (s *ShapeIndexIterator) LocateCellID(target CellID) CellRelation { + // Let T be the target, let I = cellMap.LowerBound(T.RangeMin()), and + // let I' be the predecessor of I. If T contains any index cells, then T + // contains I. Similarly, if T is contained by an index cell, then the + // containing cell is either I or I'. We test for containment by comparing + // the ranges of leaf cells spanned by T, I, and I'. + s.seek(target.RangeMin()) + if !s.Done() { + if s.CellID() >= target && s.CellID().RangeMin() <= target { + return Indexed + } + if s.CellID() <= target.RangeMax() { + return Subdivided + } + } + if s.Prev() && s.CellID().RangeMax() >= target { + return Indexed + } + return Disjoint +} + +// tracker keeps track of which shapes in a given set contain a particular point +// (the focus). It provides an efficient way to move the focus from one point +// to another and incrementally update the set of shapes which contain it. We use +// this to compute which shapes contain the center of every CellID in the index, +// by advancing the focus from one cell center to the next. +// +// Initially the focus is at the start of the CellID space-filling curve. We then +// visit all the cells that are being added to the ShapeIndex in increasing order +// of CellID. For each cell, we draw two edges: one from the entry vertex to the +// center, and another from the center to the exit vertex (where entry and exit +// refer to the points where the space-filling curve enters and exits the cell). +// By counting edge crossings we can incrementally compute which shapes contain +// the cell center. Note that the same set of shapes will always contain the exit +// point of one cell and the entry point of the next cell in the index, because +// either (a) these two points are actually the same, or (b) the intervening +// cells in CellID order are all empty, and therefore there are no edge crossings +// if we follow this path from one cell to the other. +// +// In C++, this is S2ShapeIndex::InteriorTracker. +type tracker struct { + isActive bool + a Point + b Point + nextCellID CellID + crosser *EdgeCrosser + shapeIDs []int32 + + // Shape ids saved by saveAndClearStateBefore. The state is never saved + // recursively so we don't need to worry about maintaining a stack. + savedIDs []int32 +} + +// newTracker returns a new tracker with the appropriate defaults. +func newTracker() *tracker { + // As shapes are added, we compute which ones contain the start of the + // CellID space-filling curve by drawing an edge from OriginPoint to this + // point and counting how many shape edges cross this edge. + t := &tracker{ + isActive: false, + b: trackerOrigin(), + nextCellID: CellIDFromFace(0).ChildBeginAtLevel(maxLevel), + } + t.drawTo(Point{faceUVToXYZ(0, -1, -1).Normalize()}) // CellID curve start + + return t +} + +// trackerOrigin returns the initial focus point when the tracker is created +// (corresponding to the start of the CellID space-filling curve). +func trackerOrigin() Point { + // The start of the S2CellId space-filling curve. + return Point{faceUVToXYZ(0, -1, -1).Normalize()} +} + +// focus returns the current focus point of the tracker. +func (t *tracker) focus() Point { return t.b } + +// addShape adds a shape whose interior should be tracked. containsOrigin indicates +// whether the current focus point is inside the shape. Alternatively, if +// the focus point is in the process of being moved (via moveTo/drawTo), you +// can also specify containsOrigin at the old focus point and call testEdge +// for every edge of the shape that might cross the current drawTo line. +// This updates the state to correspond to the new focus point. +// +// This requires shape.HasInterior +func (t *tracker) addShape(shapeID int32, containsFocus bool) { + t.isActive = true + if containsFocus { + t.toggleShape(shapeID) + } +} + +// moveTo moves the focus of the tracker to the given point. This method should +// only be used when it is known that there are no edge crossings between the old +// and new focus locations; otherwise use drawTo. +func (t *tracker) moveTo(b Point) { t.b = b } + +// drawTo moves the focus of the tracker to the given point. After this method is +// called, testEdge should be called with all edges that may cross the line +// segment between the old and new focus locations. +func (t *tracker) drawTo(b Point) { + t.a = t.b + t.b = b + // TODO: the edge crosser may need an in-place Init method if this gets expensive + t.crosser = NewEdgeCrosser(t.a, t.b) +} + +// testEdge checks if the given edge crosses the current edge, and if so, then +// toggle the state of the given shapeID. +// This requires shape to have an interior. +func (t *tracker) testEdge(shapeID int32, edge Edge) { + if t.crosser.EdgeOrVertexCrossing(edge.V0, edge.V1) { + t.toggleShape(shapeID) + } +} + +// setNextCellID is used to indicate that the last argument to moveTo or drawTo +// was the entry vertex of the given CellID, i.e. the tracker is positioned at the +// start of this cell. By using this method together with atCellID, the caller +// can avoid calling moveTo in cases where the exit vertex of the previous cell +// is the same as the entry vertex of the current cell. +func (t *tracker) setNextCellID(nextCellID CellID) { + t.nextCellID = nextCellID.RangeMin() +} + +// atCellID reports if the focus is already at the entry vertex of the given +// CellID (provided that the caller calls setNextCellID as each cell is processed). +func (t *tracker) atCellID(cellid CellID) bool { + return cellid.RangeMin() == t.nextCellID +} + +// toggleShape adds or removes the given shapeID from the set of IDs it is tracking. +func (t *tracker) toggleShape(shapeID int32) { + // Most shapeIDs slices are small, so special case the common steps. + + // If there is nothing here, add it. + if len(t.shapeIDs) == 0 { + t.shapeIDs = append(t.shapeIDs, shapeID) + return + } + + // If it's the first element, drop it from the slice. + if t.shapeIDs[0] == shapeID { + t.shapeIDs = t.shapeIDs[1:] + return + } + + for i, s := range t.shapeIDs { + if s < shapeID { + continue + } + + // If it's in the set, cut it out. + if s == shapeID { + copy(t.shapeIDs[i:], t.shapeIDs[i+1:]) // overwrite the ith element + t.shapeIDs = t.shapeIDs[:len(t.shapeIDs)-1] + return + } + + // We've got to a point in the slice where we should be inserted. + // (the given shapeID is now less than the current positions id.) + t.shapeIDs = append(t.shapeIDs[0:i], + append([]int32{shapeID}, t.shapeIDs[i:len(t.shapeIDs)]...)...) + return + } + + // We got to the end and didn't find it, so add it to the list. + t.shapeIDs = append(t.shapeIDs, shapeID) +} + +// saveAndClearStateBefore makes an internal copy of the state for shape ids below +// the given limit, and then clear the state for those shapes. This is used during +// incremental updates to track the state of added and removed shapes separately. +func (t *tracker) saveAndClearStateBefore(limitShapeID int32) { + limit := t.lowerBound(limitShapeID) + t.savedIDs = append([]int32(nil), t.shapeIDs[:limit]...) + t.shapeIDs = t.shapeIDs[limit:] +} + +// restoreStateBefore restores the state previously saved by saveAndClearStateBefore. +// This only affects the state for shapeIDs below "limitShapeID". +func (t *tracker) restoreStateBefore(limitShapeID int32) { + limit := t.lowerBound(limitShapeID) + t.shapeIDs = append(append([]int32(nil), t.savedIDs...), t.shapeIDs[limit:]...) + t.savedIDs = nil +} + +// lowerBound returns the shapeID of the first entry x where x >= shapeID. +func (t *tracker) lowerBound(shapeID int32) int32 { + panic("not implemented") +} + +// removedShape represents a set of edges from the given shape that is queued for removal. +type removedShape struct { + shapeID int32 + hasInterior bool + containsTrackerOrigin bool + edges []Edge +} + +// There are three basic states the index can be in. +const ( + stale int32 = iota // There are pending updates. + updating // Updates are currently being applied. + fresh // There are no pending updates. +) + +// ShapeIndex indexes a set of Shapes, where a Shape is some collection of edges +// that optionally defines an interior. It can be used to represent a set of +// points, a set of polylines, or a set of polygons. For Shapes that have +// interiors, the index makes it very fast to determine which Shape(s) contain +// a given point or region. +// +// The index can be updated incrementally by adding or removing shapes. It is +// designed to handle up to hundreds of millions of edges. All data structures +// are designed to be small, so the index is compact; generally it is smaller +// than the underlying data being indexed. The index is also fast to construct. +// +// Polygon, Loop, and Polyline implement Shape which allows these objects to +// be indexed easily. You can find useful query methods in CrossingEdgeQuery +// and ClosestEdgeQuery (Not yet implemented in Go). +// +// Example showing how to build an index of Polylines: +// +// index := NewShapeIndex() +// for _, polyline := range polylines { +// index.Add(polyline); +// } +// // Now you can use a CrossingEdgeQuery or ClosestEdgeQuery here. +// +type ShapeIndex struct { + // shapes is a map of shape ID to shape. + shapes map[int32]Shape + + // The maximum number of edges per cell. + // TODO(roberts): Update the comments when the usage of this is implemented. + maxEdgesPerCell int + + // nextID tracks the next ID to hand out. IDs are not reused when shapes + // are removed from the index. + nextID int32 + + // cellMap is a map from CellID to the set of clipped shapes that intersect that + // cell. The cell IDs cover a set of non-overlapping regions on the sphere. + // In C++, this is a BTree, so the cells are ordered naturally by the data structure. + cellMap map[CellID]*ShapeIndexCell + // Track the ordered list of cell IDs. + cells []CellID + + // The current status of the index; accessed atomically. + status int32 + + // Additions and removals are queued and processed on the first subsequent + // query. There are several reasons to do this: + // + // - It is significantly more efficient to process updates in batches if + // the amount of entities added grows. + // - Often the index will never be queried, in which case we can save both + // the time and memory required to build it. Examples: + // + Loops that are created simply to pass to an Polygon. (We don't + // need the Loop index, because Polygon builds its own index.) + // + Applications that load a database of geometry and then query only + // a small fraction of it. + // + // The main drawback is that we need to go to some extra work to ensure that + // some methods are still thread-safe. Note that the goal is *not* to + // make this thread-safe in general, but simply to hide the fact that + // we defer some of the indexing work until query time. + // + // This mutex protects all of following fields in the index. + mu sync.RWMutex + + // pendingAdditionsPos is the index of the first entry that has not been processed + // via applyUpdatesInternal. + pendingAdditionsPos int32 + + // The set of shapes that have been queued for removal but not processed yet by + // applyUpdatesInternal. + pendingRemovals []*removedShape +} + +// NewShapeIndex creates a new ShapeIndex. +func NewShapeIndex() *ShapeIndex { + return &ShapeIndex{ + maxEdgesPerCell: 10, + shapes: make(map[int32]Shape), + cellMap: make(map[CellID]*ShapeIndexCell), + cells: nil, + status: fresh, + } +} + +// Iterator returns an iterator for this index. +func (s *ShapeIndex) Iterator() *ShapeIndexIterator { + s.maybeApplyUpdates() + return NewShapeIndexIterator(s, IteratorBegin) +} + +// Begin positions the iterator at the first cell in the index. +func (s *ShapeIndex) Begin() *ShapeIndexIterator { + s.maybeApplyUpdates() + return NewShapeIndexIterator(s, IteratorBegin) +} + +// End positions the iterator at the last cell in the index. +func (s *ShapeIndex) End() *ShapeIndexIterator { + // TODO(roberts): It's possible that updates could happen to the index between + // the time this is called and the time the iterators position is used and this + // will be invalid or not the end. For now, things will be undefined if this + // happens. See about referencing the IsFresh to guard for this in the future. + s.maybeApplyUpdates() + return NewShapeIndexIterator(s, IteratorEnd) +} + +// Len reports the number of Shapes in this index. +func (s *ShapeIndex) Len() int { + return len(s.shapes) +} + +// Reset resets the index to its original state. +func (s *ShapeIndex) Reset() { + s.shapes = make(map[int32]Shape) + s.nextID = 0 + s.cellMap = make(map[CellID]*ShapeIndexCell) + s.cells = nil + atomic.StoreInt32(&s.status, fresh) +} + +// NumEdges returns the number of edges in this index. +func (s *ShapeIndex) NumEdges() int { + numEdges := 0 + for _, shape := range s.shapes { + numEdges += shape.NumEdges() + } + return numEdges +} + +// NumEdgesUpTo returns the number of edges in the given index, up to the given +// limit. If the limit is encountered, the current running total is returned, +// which may be more than the limit. +func (s *ShapeIndex) NumEdgesUpTo(limit int) int { + var numEdges int + // We choose to iterate over the shapes in order to match the counting + // up behavior in C++ and for test compatibility instead of using a + // more idiomatic range over the shape map. + for i := int32(0); i <= s.nextID; i++ { + s := s.Shape(i) + if s == nil { + continue + } + numEdges += s.NumEdges() + if numEdges >= limit { + break + } + } + + return numEdges +} + +// Shape returns the shape with the given ID, or nil if the shape has been removed from the index. +func (s *ShapeIndex) Shape(id int32) Shape { return s.shapes[id] } + +// idForShape returns the id of the given shape in this index, or -1 if it is +// not in the index. +// +// TODO(roberts): Need to figure out an appropriate way to expose this on a Shape. +// C++ allows a given S2 type (Loop, Polygon, etc) to be part of multiple indexes. +// By having each type extend S2Shape which has an id element, they all inherit their +// own id field rather than having to track it themselves. +func (s *ShapeIndex) idForShape(shape Shape) int32 { + for k, v := range s.shapes { + if v == shape { + return k + } + } + return -1 +} + +// Add adds the given shape to the index and returns the assigned ID.. +func (s *ShapeIndex) Add(shape Shape) int32 { + s.shapes[s.nextID] = shape + s.nextID++ + atomic.StoreInt32(&s.status, stale) + return s.nextID - 1 +} + +// Remove removes the given shape from the index. +func (s *ShapeIndex) Remove(shape Shape) { + // The index updates itself lazily because it is much more efficient to + // process additions and removals in batches. + id := s.idForShape(shape) + + // If the shape wasn't found, it's already been removed or was not in the index. + if s.shapes[id] == nil { + return + } + + // Remove the shape from the shapes map. + delete(s.shapes, id) + + // We are removing a shape that has not yet been added to the index, + // so there is nothing else to do. + if id >= s.pendingAdditionsPos { + return + } + + numEdges := shape.NumEdges() + removed := &removedShape{ + shapeID: id, + hasInterior: shape.Dimension() == 2, + containsTrackerOrigin: shape.ReferencePoint().Contained, + edges: make([]Edge, numEdges), + } + + for e := 0; e < numEdges; e++ { + removed.edges[e] = shape.Edge(e) + } + + s.pendingRemovals = append(s.pendingRemovals, removed) + atomic.StoreInt32(&s.status, stale) +} + +// Build triggers the update of the index. Calls to Add and Release are normally +// queued and processed on the first subsequent query. This has many advantages, +// the most important of which is that sometimes there *is* no subsequent +// query, which lets us avoid building the index completely. +// +// This method forces any pending updates to be applied immediately. +func (s *ShapeIndex) Build() { + s.maybeApplyUpdates() +} + +// IsFresh reports if there are no pending updates that need to be applied. +// This can be useful to avoid building the index unnecessarily, or for +// choosing between two different algorithms depending on whether the index +// is available. +// +// The returned index status may be slightly out of date if the index was +// built in a different thread. This is fine for the intended use (as an +// efficiency hint), but it should not be used by internal methods. +func (s *ShapeIndex) IsFresh() bool { + return atomic.LoadInt32(&s.status) == fresh +} + +// isFirstUpdate reports if this is the first update to the index. +func (s *ShapeIndex) isFirstUpdate() bool { + // Note that it is not sufficient to check whether cellMap is empty, since + // entries are added to it during the update process. + return s.pendingAdditionsPos == 0 +} + +// isShapeBeingRemoved reports if the shape with the given ID is currently slated for removal. +func (s *ShapeIndex) isShapeBeingRemoved(shapeID int32) bool { + // All shape ids being removed fall below the index position of shapes being added. + return shapeID < s.pendingAdditionsPos +} + +// maybeApplyUpdates checks if the index pieces have changed, and if so, applies pending updates. +func (s *ShapeIndex) maybeApplyUpdates() { + // TODO(roberts): To avoid acquiring and releasing the mutex on every + // query, we should use atomic operations when testing whether the status + // is fresh and when updating the status to be fresh. This guarantees + // that any thread that sees a status of fresh will also see the + // corresponding index updates. + if atomic.LoadInt32(&s.status) != fresh { + s.mu.Lock() + s.applyUpdatesInternal() + atomic.StoreInt32(&s.status, fresh) + s.mu.Unlock() + } +} + +// applyUpdatesInternal does the actual work of updating the index by applying all +// pending additions and removals. It does *not* update the indexes status. +func (s *ShapeIndex) applyUpdatesInternal() { + // TODO(roberts): Building the index can use up to 20x as much memory per + // edge as the final index memory size. If this causes issues, add in + // batched updating to limit the amount of items per batch to a + // configurable memory footprint overhead. + t := newTracker() + + // allEdges maps a Face to a collection of faceEdges. + allEdges := make([][]faceEdge, 6) + + for _, p := range s.pendingRemovals { + s.removeShapeInternal(p, allEdges, t) + } + + for id := s.pendingAdditionsPos; id < int32(len(s.shapes)); id++ { + s.addShapeInternal(id, allEdges, t) + } + + for face := 0; face < 6; face++ { + s.updateFaceEdges(face, allEdges[face], t) + } + + s.pendingRemovals = s.pendingRemovals[:0] + s.pendingAdditionsPos = int32(len(s.shapes)) + // It is the caller's responsibility to update the index status. +} + +// addShapeInternal clips all edges of the given shape to the six cube faces, +// adds the clipped edges to the set of allEdges, and starts tracking its +// interior if necessary. +func (s *ShapeIndex) addShapeInternal(shapeID int32, allEdges [][]faceEdge, t *tracker) { + shape, ok := s.shapes[shapeID] + if !ok { + // This shape has already been removed. + return + } + + faceEdge := faceEdge{ + shapeID: shapeID, + hasInterior: shape.Dimension() == 2, + } + + if faceEdge.hasInterior { + t.addShape(shapeID, containsBruteForce(shape, t.focus())) + } + + numEdges := shape.NumEdges() + for e := 0; e < numEdges; e++ { + edge := shape.Edge(e) + + faceEdge.edgeID = e + faceEdge.edge = edge + faceEdge.maxLevel = maxLevelForEdge(edge) + s.addFaceEdge(faceEdge, allEdges) + } +} + +// addFaceEdge adds the given faceEdge into the collection of all edges. +func (s *ShapeIndex) addFaceEdge(fe faceEdge, allEdges [][]faceEdge) { + aFace := face(fe.edge.V0.Vector) + // See if both endpoints are on the same face, and are far enough from + // the edge of the face that they don't intersect any (padded) adjacent face. + if aFace == face(fe.edge.V1.Vector) { + x, y := validFaceXYZToUV(aFace, fe.edge.V0.Vector) + fe.a = r2.Point{x, y} + x, y = validFaceXYZToUV(aFace, fe.edge.V1.Vector) + fe.b = r2.Point{x, y} + + maxUV := 1 - cellPadding + if math.Abs(fe.a.X) <= maxUV && math.Abs(fe.a.Y) <= maxUV && + math.Abs(fe.b.X) <= maxUV && math.Abs(fe.b.Y) <= maxUV { + allEdges[aFace] = append(allEdges[aFace], fe) + return + } + } + + // Otherwise, we simply clip the edge to all six faces. + for face := 0; face < 6; face++ { + if aClip, bClip, intersects := ClipToPaddedFace(fe.edge.V0, fe.edge.V1, face, cellPadding); intersects { + fe.a = aClip + fe.b = bClip + allEdges[face] = append(allEdges[face], fe) + } + } +} + +// updateFaceEdges adds or removes the various edges from the index. +// An edge is added if shapes[id] is not nil, and removed otherwise. +func (s *ShapeIndex) updateFaceEdges(face int, faceEdges []faceEdge, t *tracker) { + numEdges := len(faceEdges) + if numEdges == 0 && len(t.shapeIDs) == 0 { + return + } + + // Create the initial clippedEdge for each faceEdge. Additional clipped + // edges are created when edges are split between child cells. We create + // two arrays, one containing the edge data and another containing pointers + // to those edges, so that during the recursion we only need to copy + // pointers in order to propagate an edge to the correct child. + clippedEdges := make([]*clippedEdge, numEdges) + bound := r2.EmptyRect() + for e := 0; e < numEdges; e++ { + clipped := &clippedEdge{ + faceEdge: &faceEdges[e], + } + clipped.bound = r2.RectFromPoints(faceEdges[e].a, faceEdges[e].b) + clippedEdges[e] = clipped + bound = bound.AddRect(clipped.bound) + } + + // Construct the initial face cell containing all the edges, and then update + // all the edges in the index recursively. + faceID := CellIDFromFace(face) + pcell := PaddedCellFromCellID(faceID, cellPadding) + + disjointFromIndex := s.isFirstUpdate() + if numEdges > 0 { + shrunkID := s.shrinkToFit(pcell, bound) + if shrunkID != pcell.id { + // All the edges are contained by some descendant of the face cell. We + // can save a lot of work by starting directly with that cell, but if we + // are in the interior of at least one shape then we need to create + // index entries for the cells we are skipping over. + s.skipCellRange(faceID.RangeMin(), shrunkID.RangeMin(), t, disjointFromIndex) + pcell = PaddedCellFromCellID(shrunkID, cellPadding) + s.updateEdges(pcell, clippedEdges, t, disjointFromIndex) + s.skipCellRange(shrunkID.RangeMax().Next(), faceID.RangeMax().Next(), t, disjointFromIndex) + return + } + } + + // Otherwise (no edges, or no shrinking is possible), subdivide normally. + s.updateEdges(pcell, clippedEdges, t, disjointFromIndex) +} + +// shrinkToFit shrinks the PaddedCell to fit within the given bounds. +func (s *ShapeIndex) shrinkToFit(pcell *PaddedCell, bound r2.Rect) CellID { + shrunkID := pcell.ShrinkToFit(bound) + + if !s.isFirstUpdate() && shrunkID != pcell.CellID() { + // Don't shrink any smaller than the existing index cells, since we need + // to combine the new edges with those cells. + iter := s.Iterator() + if iter.LocateCellID(shrunkID) == Indexed { + shrunkID = iter.CellID() + } + } + return shrunkID +} + +// skipCellRange skips over the cells in the given range, creating index cells if we are +// currently in the interior of at least one shape. +func (s *ShapeIndex) skipCellRange(begin, end CellID, t *tracker, disjointFromIndex bool) { + // If we aren't in the interior of a shape, then skipping over cells is easy. + if len(t.shapeIDs) == 0 { + return + } + + // Otherwise generate the list of cell ids that we need to visit, and create + // an index entry for each one. + skipped := CellUnionFromRange(begin, end) + for _, cell := range skipped { + var clippedEdges []*clippedEdge + s.updateEdges(PaddedCellFromCellID(cell, cellPadding), clippedEdges, t, disjointFromIndex) + } +} + +// updateEdges adds or removes the given edges whose bounding boxes intersect a +// given cell. disjointFromIndex is an optimization hint indicating that cellMap +// does not contain any entries that overlap the given cell. +func (s *ShapeIndex) updateEdges(pcell *PaddedCell, edges []*clippedEdge, t *tracker, disjointFromIndex bool) { + // This function is recursive with a maximum recursion depth of 30 (maxLevel). + + // Incremental updates are handled as follows. All edges being added or + // removed are combined together in edges, and all shapes with interiors + // are tracked using tracker. We subdivide recursively as usual until we + // encounter an existing index cell. At this point we absorb the index + // cell as follows: + // + // - Edges and shapes that are being removed are deleted from edges and + // tracker. + // - All remaining edges and shapes from the index cell are added to + // edges and tracker. + // - Continue subdividing recursively, creating new index cells as needed. + // - When the recursion gets back to the cell that was absorbed, we + // restore edges and tracker to their previous state. + // + // Note that the only reason that we include removed shapes in the recursive + // subdivision process is so that we can find all of the index cells that + // contain those shapes efficiently, without maintaining an explicit list of + // index cells for each shape (which would be expensive in terms of memory). + indexCellAbsorbed := false + if !disjointFromIndex { + // There may be existing index cells contained inside pcell. If we + // encounter such a cell, we need to combine the edges being updated with + // the existing cell contents by absorbing the cell. + iter := s.Iterator() + r := iter.LocateCellID(pcell.id) + if r == Disjoint { + disjointFromIndex = true + } else if r == Indexed { + // Absorb the index cell by transferring its contents to edges and + // deleting it. We also start tracking the interior of any new shapes. + s.absorbIndexCell(pcell, iter, edges, t) + indexCellAbsorbed = true + disjointFromIndex = true + } else { + // DCHECK_EQ(SUBDIVIDED, r) + } + } + + // If there are existing index cells below us, then we need to keep + // subdividing so that we can merge with those cells. Otherwise, + // makeIndexCell checks if the number of edges is small enough, and creates + // an index cell if possible (returning true when it does so). + if !disjointFromIndex || !s.makeIndexCell(pcell, edges, t) { + // TODO(roberts): If it turns out to have memory problems when there + // are 10M+ edges in the index, look into pre-allocating space so we + // are not always appending. + childEdges := [2][2][]*clippedEdge{} // [i][j] + + // Compute the middle of the padded cell, defined as the rectangle in + // (u,v)-space that belongs to all four (padded) children. By comparing + // against the four boundaries of middle we can determine which children + // each edge needs to be propagated to. + middle := pcell.Middle() + + // Build up a vector edges to be passed to each child cell. The (i,j) + // directions are left (i=0), right (i=1), lower (j=0), and upper (j=1). + // Note that the vast majority of edges are propagated to a single child. + for _, edge := range edges { + if edge.bound.X.Hi <= middle.X.Lo { + // Edge is entirely contained in the two left children. + a, b := s.clipVAxis(edge, middle.Y) + if a != nil { + childEdges[0][0] = append(childEdges[0][0], a) + } + if b != nil { + childEdges[0][1] = append(childEdges[0][1], b) + } + } else if edge.bound.X.Lo >= middle.X.Hi { + // Edge is entirely contained in the two right children. + a, b := s.clipVAxis(edge, middle.Y) + if a != nil { + childEdges[1][0] = append(childEdges[1][0], a) + } + if b != nil { + childEdges[1][1] = append(childEdges[1][1], b) + } + } else if edge.bound.Y.Hi <= middle.Y.Lo { + // Edge is entirely contained in the two lower children. + if a := s.clipUBound(edge, 1, middle.X.Hi); a != nil { + childEdges[0][0] = append(childEdges[0][0], a) + } + if b := s.clipUBound(edge, 0, middle.X.Lo); b != nil { + childEdges[1][0] = append(childEdges[1][0], b) + } + } else if edge.bound.Y.Lo >= middle.Y.Hi { + // Edge is entirely contained in the two upper children. + if a := s.clipUBound(edge, 1, middle.X.Hi); a != nil { + childEdges[0][1] = append(childEdges[0][1], a) + } + if b := s.clipUBound(edge, 0, middle.X.Lo); b != nil { + childEdges[1][1] = append(childEdges[1][1], b) + } + } else { + // The edge bound spans all four children. The edge + // itself intersects either three or four padded children. + left := s.clipUBound(edge, 1, middle.X.Hi) + a, b := s.clipVAxis(left, middle.Y) + if a != nil { + childEdges[0][0] = append(childEdges[0][0], a) + } + if b != nil { + childEdges[0][1] = append(childEdges[0][1], b) + } + right := s.clipUBound(edge, 0, middle.X.Lo) + a, b = s.clipVAxis(right, middle.Y) + if a != nil { + childEdges[1][0] = append(childEdges[1][0], a) + } + if b != nil { + childEdges[1][1] = append(childEdges[1][1], b) + } + } + } + + // Now recursively update the edges in each child. We call the children in + // increasing order of CellID so that when the index is first constructed, + // all insertions into cellMap are at the end (which is much faster). + for pos := 0; pos < 4; pos++ { + i, j := pcell.ChildIJ(pos) + if len(childEdges[i][j]) > 0 || len(t.shapeIDs) > 0 { + s.updateEdges(PaddedCellFromParentIJ(pcell, i, j), childEdges[i][j], + t, disjointFromIndex) + } + } + } + + if indexCellAbsorbed { + // Restore the state for any edges being removed that we are tracking. + t.restoreStateBefore(s.pendingAdditionsPos) + } +} + +// makeIndexCell builds an indexCell from the given padded cell and set of edges and adds +// it to the index. If the cell or edges are empty, no cell is added. +func (s *ShapeIndex) makeIndexCell(p *PaddedCell, edges []*clippedEdge, t *tracker) bool { + // If the cell is empty, no index cell is needed. (In most cases this + // situation is detected before we get to this point, but this can happen + // when all shapes in a cell are removed.) + if len(edges) == 0 && len(t.shapeIDs) == 0 { + return true + } + + // Count the number of edges that have not reached their maximum level yet. + // Return false if there are too many such edges. + count := 0 + for _, ce := range edges { + if p.Level() < ce.faceEdge.maxLevel { + count++ + } + + if count > s.maxEdgesPerCell { + return false + } + } + + // Possible optimization: Continue subdividing as long as exactly one child + // of the padded cell intersects the given edges. This can be done by finding + // the bounding box of all the edges and calling ShrinkToFit: + // + // cellID = p.ShrinkToFit(RectBound(edges)); + // + // Currently this is not beneficial; it slows down construction by 4-25% + // (mainly computing the union of the bounding rectangles) and also slows + // down queries (since more recursive clipping is required to get down to + // the level of a spatial index cell). But it may be worth trying again + // once containsCenter is computed and all algorithms are modified to + // take advantage of it. + + // We update the InteriorTracker as follows. For every Cell in the index + // we construct two edges: one edge from entry vertex of the cell to its + // center, and one from the cell center to its exit vertex. Here entry + // and exit refer the CellID ordering, i.e. the order in which points + // are encountered along the 2 space-filling curve. The exit vertex then + // becomes the entry vertex for the next cell in the index, unless there are + // one or more empty intervening cells, in which case the InteriorTracker + // state is unchanged because the intervening cells have no edges. + + // Shift the InteriorTracker focus point to the center of the current cell. + if t.isActive && len(edges) != 0 { + if !t.atCellID(p.id) { + t.moveTo(p.EntryVertex()) + } + t.drawTo(p.Center()) + s.testAllEdges(edges, t) + } + + // Allocate and fill a new index cell. To get the total number of shapes we + // need to merge the shapes associated with the intersecting edges together + // with the shapes that happen to contain the cell center. + cshapeIDs := t.shapeIDs + numShapes := s.countShapes(edges, cshapeIDs) + cell := NewShapeIndexCell(numShapes) + + // To fill the index cell we merge the two sources of shapes: edge shapes + // (those that have at least one edge that intersects this cell), and + // containing shapes (those that contain the cell center). We keep track + // of the index of the next intersecting edge and the next containing shape + // as we go along. Both sets of shape ids are already sorted. + eNext := 0 + cNextIdx := 0 + for i := 0; i < numShapes; i++ { + var clipped *clippedShape + // advance to next value base + i + eshapeID := int32(s.Len()) + cshapeID := eshapeID // Sentinels + + if eNext != len(edges) { + eshapeID = edges[eNext].faceEdge.shapeID + } + if cNextIdx < len(cshapeIDs) { + cshapeID = cshapeIDs[cNextIdx] + } + eBegin := eNext + if cshapeID < eshapeID { + // The entire cell is in the shape interior. + clipped = newClippedShape(cshapeID, 0) + clipped.containsCenter = true + cNextIdx++ + } else { + // Count the number of edges for this shape and allocate space for them. + for eNext < len(edges) && edges[eNext].faceEdge.shapeID == eshapeID { + eNext++ + } + clipped = newClippedShape(eshapeID, eNext-eBegin) + for e := eBegin; e < eNext; e++ { + clipped.edges[e-eBegin] = edges[e].faceEdge.edgeID + } + if cshapeID == eshapeID { + clipped.containsCenter = true + cNextIdx++ + } + } + cell.shapes[i] = clipped + } + + // Add this cell to the map. + s.cellMap[p.id] = cell + s.cells = append(s.cells, p.id) + + // Shift the tracker focus point to the exit vertex of this cell. + if t.isActive && len(edges) != 0 { + t.drawTo(p.ExitVertex()) + s.testAllEdges(edges, t) + t.setNextCellID(p.id.Next()) + } + return true +} + +// updateBound updates the specified endpoint of the given clipped edge and returns the +// resulting clipped edge. +func (s *ShapeIndex) updateBound(edge *clippedEdge, uEnd int, u float64, vEnd int, v float64) *clippedEdge { + c := &clippedEdge{faceEdge: edge.faceEdge} + if uEnd == 0 { + c.bound.X.Lo = u + c.bound.X.Hi = edge.bound.X.Hi + } else { + c.bound.X.Lo = edge.bound.X.Lo + c.bound.X.Hi = u + } + + if vEnd == 0 { + c.bound.Y.Lo = v + c.bound.Y.Hi = edge.bound.Y.Hi + } else { + c.bound.Y.Lo = edge.bound.Y.Lo + c.bound.Y.Hi = v + } + + return c +} + +// clipUBound clips the given endpoint (lo=0, hi=1) of the u-axis so that +// it does not extend past the given value of the given edge. +func (s *ShapeIndex) clipUBound(edge *clippedEdge, uEnd int, u float64) *clippedEdge { + // First check whether the edge actually requires any clipping. (Sometimes + // this method is called when clipping is not necessary, e.g. when one edge + // endpoint is in the overlap area between two padded child cells.) + if uEnd == 0 { + if edge.bound.X.Lo >= u { + return edge + } + } else { + if edge.bound.X.Hi <= u { + return edge + } + } + // We interpolate the new v-value from the endpoints of the original edge. + // This has two advantages: (1) we don't need to store the clipped endpoints + // at all, just their bounding box; and (2) it avoids the accumulation of + // roundoff errors due to repeated interpolations. The result needs to be + // clamped to ensure that it is in the appropriate range. + e := edge.faceEdge + v := edge.bound.Y.ClampPoint(interpolateFloat64(u, e.a.X, e.b.X, e.a.Y, e.b.Y)) + + // Determine which endpoint of the v-axis bound to update. If the edge + // slope is positive we update the same endpoint, otherwise we update the + // opposite endpoint. + var vEnd int + positiveSlope := (e.a.X > e.b.X) == (e.a.Y > e.b.Y) + if (uEnd == 1) == positiveSlope { + vEnd = 1 + } + return s.updateBound(edge, uEnd, u, vEnd, v) +} + +// clipVBound clips the given endpoint (lo=0, hi=1) of the v-axis so that +// it does not extend past the given value of the given edge. +func (s *ShapeIndex) clipVBound(edge *clippedEdge, vEnd int, v float64) *clippedEdge { + if vEnd == 0 { + if edge.bound.Y.Lo >= v { + return edge + } + } else { + if edge.bound.Y.Hi <= v { + return edge + } + } + + // We interpolate the new v-value from the endpoints of the original edge. + // This has two advantages: (1) we don't need to store the clipped endpoints + // at all, just their bounding box; and (2) it avoids the accumulation of + // roundoff errors due to repeated interpolations. The result needs to be + // clamped to ensure that it is in the appropriate range. + e := edge.faceEdge + u := edge.bound.X.ClampPoint(interpolateFloat64(v, e.a.Y, e.b.Y, e.a.X, e.b.X)) + + // Determine which endpoint of the v-axis bound to update. If the edge + // slope is positive we update the same endpoint, otherwise we update the + // opposite endpoint. + var uEnd int + positiveSlope := (e.a.X > e.b.X) == (e.a.Y > e.b.Y) + if (vEnd == 1) == positiveSlope { + uEnd = 1 + } + return s.updateBound(edge, uEnd, u, vEnd, v) +} + +// cliupVAxis returns the given edge clipped to within the boundaries of the middle +// interval along the v-axis, and adds the result to its children. +func (s *ShapeIndex) clipVAxis(edge *clippedEdge, middle r1.Interval) (a, b *clippedEdge) { + if edge.bound.Y.Hi <= middle.Lo { + // Edge is entirely contained in the lower child. + return edge, nil + } else if edge.bound.Y.Lo >= middle.Hi { + // Edge is entirely contained in the upper child. + return nil, edge + } + // The edge bound spans both children. + return s.clipVBound(edge, 1, middle.Hi), s.clipVBound(edge, 0, middle.Lo) +} + +// absorbIndexCell absorbs an index cell by transferring its contents to edges +// and/or "tracker", and then delete this cell from the index. If edges includes +// any edges that are being removed, this method also updates their +// InteriorTracker state to correspond to the exit vertex of this cell. +func (s *ShapeIndex) absorbIndexCell(p *PaddedCell, iter *ShapeIndexIterator, edges []*clippedEdge, t *tracker) { + // When we absorb a cell, we erase all the edges that are being removed. + // However when we are finished with this cell, we want to restore the state + // of those edges (since that is how we find all the index cells that need + // to be updated). The edges themselves are restored automatically when + // UpdateEdges returns from its recursive call, but the InteriorTracker + // state needs to be restored explicitly. + // + // Here we first update the InteriorTracker state for removed edges to + // correspond to the exit vertex of this cell, and then save the + // InteriorTracker state. This state will be restored by UpdateEdges when + // it is finished processing the contents of this cell. + if t.isActive && len(edges) != 0 && s.isShapeBeingRemoved(edges[0].faceEdge.shapeID) { + // We probably need to update the tracker. ("Probably" because + // it's possible that all shapes being removed do not have interiors.) + if !t.atCellID(p.id) { + t.moveTo(p.EntryVertex()) + } + t.drawTo(p.ExitVertex()) + t.setNextCellID(p.id.Next()) + for _, edge := range edges { + fe := edge.faceEdge + if !s.isShapeBeingRemoved(fe.shapeID) { + break // All shapes being removed come first. + } + if fe.hasInterior { + t.testEdge(fe.shapeID, fe.edge) + } + } + } + + // Save the state of the edges being removed, so that it can be restored + // when we are finished processing this cell and its children. We don't + // need to save the state of the edges being added because they aren't being + // removed from "edges" and will therefore be updated normally as we visit + // this cell and its children. + t.saveAndClearStateBefore(s.pendingAdditionsPos) + + // Create a faceEdge for each edge in this cell that isn't being removed. + var faceEdges []*faceEdge + trackerMoved := false + + cell := iter.IndexCell() + for _, clipped := range cell.shapes { + shapeID := clipped.shapeID + shape := s.Shape(shapeID) + if shape == nil { + continue // This shape is being removed. + } + + numClipped := clipped.numEdges() + + // If this shape has an interior, start tracking whether we are inside the + // shape. updateEdges wants to know whether the entry vertex of this + // cell is inside the shape, but we only know whether the center of the + // cell is inside the shape, so we need to test all the edges against the + // line segment from the cell center to the entry vertex. + edge := &faceEdge{ + shapeID: shapeID, + hasInterior: shape.Dimension() == 2, + } + + if edge.hasInterior { + t.addShape(shapeID, clipped.containsCenter) + // There might not be any edges in this entire cell (i.e., it might be + // in the interior of all shapes), so we delay updating the tracker + // until we see the first edge. + if !trackerMoved && numClipped > 0 { + t.moveTo(p.Center()) + t.drawTo(p.EntryVertex()) + t.setNextCellID(p.id) + trackerMoved = true + } + } + for i := 0; i < numClipped; i++ { + edgeID := clipped.edges[i] + edge.edgeID = edgeID + edge.edge = shape.Edge(edgeID) + edge.maxLevel = maxLevelForEdge(edge.edge) + if edge.hasInterior { + t.testEdge(shapeID, edge.edge) + } + var ok bool + edge.a, edge.b, ok = ClipToPaddedFace(edge.edge.V0, edge.edge.V1, p.id.Face(), cellPadding) + if !ok { + panic("invariant failure in ShapeIndex") + } + faceEdges = append(faceEdges, edge) + } + } + // Now create a clippedEdge for each faceEdge, and put them in "new_edges". + var newEdges []*clippedEdge + for _, faceEdge := range faceEdges { + clipped := &clippedEdge{ + faceEdge: faceEdge, + bound: clippedEdgeBound(faceEdge.a, faceEdge.b, p.bound), + } + newEdges = append(newEdges, clipped) + } + + // Discard any edges from "edges" that are being removed, and append the + // remainder to "newEdges" (This keeps the edges sorted by shape id.) + for i, clipped := range edges { + if !s.isShapeBeingRemoved(clipped.faceEdge.shapeID) { + newEdges = append(newEdges, edges[i:]...) + break + } + } + + // Update the edge list and delete this cell from the index. + edges, newEdges = newEdges, edges + delete(s.cellMap, p.id) + // TODO(roberts): delete from s.Cells +} + +// testAllEdges calls the trackers testEdge on all edges from shapes that have interiors. +func (s *ShapeIndex) testAllEdges(edges []*clippedEdge, t *tracker) { + for _, edge := range edges { + if edge.faceEdge.hasInterior { + t.testEdge(edge.faceEdge.shapeID, edge.faceEdge.edge) + } + } +} + +// countShapes reports the number of distinct shapes that are either associated with the +// given edges, or that are currently stored in the InteriorTracker. +func (s *ShapeIndex) countShapes(edges []*clippedEdge, shapeIDs []int32) int { + count := 0 + lastShapeID := int32(-1) + + // next clipped shape id in the shapeIDs list. + clippedNext := int32(0) + // index of the current element in the shapeIDs list. + shapeIDidx := 0 + for _, edge := range edges { + if edge.faceEdge.shapeID == lastShapeID { + continue + } + + count++ + lastShapeID = edge.faceEdge.shapeID + + // Skip over any containing shapes up to and including this one, + // updating count as appropriate. + for ; shapeIDidx < len(shapeIDs); shapeIDidx++ { + clippedNext = shapeIDs[shapeIDidx] + if clippedNext > lastShapeID { + break + } + if clippedNext < lastShapeID { + count++ + } + } + } + + // Count any remaining containing shapes. + count += len(shapeIDs) - shapeIDidx + return count +} + +// maxLevelForEdge reports the maximum level for a given edge. +func maxLevelForEdge(edge Edge) int { + // Compute the maximum cell size for which this edge is considered long. + // The calculation does not need to be perfectly accurate, so we use Norm + // rather than Angle for speed. + cellSize := edge.V0.Sub(edge.V1.Vector).Norm() * cellSizeToLongEdgeRatio + // Now return the first level encountered during subdivision where the + // average cell size is at most cellSize. + return AvgEdgeMetric.MinLevel(cellSize) +} + +// removeShapeInternal does the actual work for removing a given shape from the index. +func (s *ShapeIndex) removeShapeInternal(removed *removedShape, allEdges [][]faceEdge, t *tracker) { + // TODO(roberts): finish the implementation of this. +} diff --git a/vendor/github.com/blevesearch/geo/s2/shapeutil.go b/vendor/github.com/blevesearch/geo/s2/shapeutil.go new file mode 100644 index 00000000..64245dfa --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/shapeutil.go @@ -0,0 +1,228 @@ +// Copyright 2017 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +// CrossingType defines different ways of reporting edge intersections. +type CrossingType int + +const ( + // CrossingTypeInterior reports intersections that occur at a point + // interior to both edges (i.e., not at a vertex). + CrossingTypeInterior CrossingType = iota + + // CrossingTypeAll reports all intersections, even those where two edges + // intersect only because they share a common vertex. + CrossingTypeAll + + // CrossingTypeNonAdjacent reports all intersections except for pairs of + // the form (AB, BC) where both edges are from the same ShapeIndex. + CrossingTypeNonAdjacent +) + +// rangeIterator is a wrapper over ShapeIndexIterator with extra methods +// that are useful for merging the contents of two or more ShapeIndexes. +type rangeIterator struct { + it *ShapeIndexIterator + // The min and max leaf cell ids covered by the current cell. If done() is + // true, these methods return a value larger than any valid cell id. + rangeMin CellID + rangeMax CellID +} + +// newRangeIterator creates a new rangeIterator positioned at the first cell of the given index. +func newRangeIterator(index *ShapeIndex) *rangeIterator { + r := &rangeIterator{ + it: index.Iterator(), + } + r.refresh() + return r +} + +func (r *rangeIterator) cellID() CellID { return r.it.CellID() } +func (r *rangeIterator) indexCell() *ShapeIndexCell { return r.it.IndexCell() } +func (r *rangeIterator) next() { r.it.Next(); r.refresh() } +func (r *rangeIterator) done() bool { return r.it.Done() } + +// seekTo positions the iterator at the first cell that overlaps or follows +// the current range minimum of the target iterator, i.e. such that its +// rangeMax >= target.rangeMin. +func (r *rangeIterator) seekTo(target *rangeIterator) { + r.it.seek(target.rangeMin) + // If the current cell does not overlap target, it is possible that the + // previous cell is the one we are looking for. This can only happen when + // the previous cell contains target but has a smaller CellID. + if r.it.Done() || r.it.CellID().RangeMin() > target.rangeMax { + if r.it.Prev() && r.it.CellID().RangeMax() < target.cellID() { + r.it.Next() + } + } + r.refresh() +} + +// seekBeyond positions the iterator at the first cell that follows the current +// range minimum of the target iterator. i.e. the first cell such that its +// rangeMin > target.rangeMax. +func (r *rangeIterator) seekBeyond(target *rangeIterator) { + r.it.seek(target.rangeMax.Next()) + if !r.it.Done() && r.it.CellID().RangeMin() <= target.rangeMax { + r.it.Next() + } + r.refresh() +} + +// refresh updates the iterators min and max values. +func (r *rangeIterator) refresh() { + r.rangeMin = r.cellID().RangeMin() + r.rangeMax = r.cellID().RangeMax() +} + +// referencePointForShape is a helper function for implementing various Shapes +// ReferencePoint functions. +// +// Given a shape consisting of closed polygonal loops, the interior of the +// shape is defined as the region to the left of all edges (which must be +// oriented consistently). This function then chooses an arbitrary point and +// returns true if that point is contained by the shape. +// +// Unlike Loop and Polygon, this method allows duplicate vertices and +// edges, which requires some extra care with definitions. The rule that we +// apply is that an edge and its reverse edge cancel each other: the result +// is the same as if that edge pair were not present. Therefore shapes that +// consist only of degenerate loop(s) are either empty or full; by convention, +// the shape is considered full if and only if it contains an empty loop (see +// laxPolygon for details). +// +// Determining whether a loop on the sphere contains a point is harder than +// the corresponding problem in 2D plane geometry. It cannot be implemented +// just by counting edge crossings because there is no such thing as a point +// at infinity that is guaranteed to be outside the loop. +// +// This function requires that the given Shape have an interior. +func referencePointForShape(shape Shape) ReferencePoint { + if shape.NumEdges() == 0 { + // A shape with no edges is defined to be full if and only if it + // contains at least one chain. + return OriginReferencePoint(shape.NumChains() > 0) + } + // Define a "matched" edge as one that can be paired with a corresponding + // reversed edge. Define a vertex as "balanced" if all of its edges are + // matched. In order to determine containment, we must find an unbalanced + // vertex. Often every vertex is unbalanced, so we start by trying an + // arbitrary vertex. + edge := shape.Edge(0) + + if ref, ok := referencePointAtVertex(shape, edge.V0); ok { + return ref + } + + // That didn't work, so now we do some extra work to find an unbalanced + // vertex (if any). Essentially we gather a list of edges and a list of + // reversed edges, and then sort them. The first edge that appears in one + // list but not the other is guaranteed to be unmatched. + n := shape.NumEdges() + var edges = make([]Edge, n) + var revEdges = make([]Edge, n) + for i := 0; i < n; i++ { + edge := shape.Edge(i) + edges[i] = edge + revEdges[i] = Edge{V0: edge.V1, V1: edge.V0} + } + + sortEdges(edges) + sortEdges(revEdges) + + for i := 0; i < n; i++ { + if edges[i].Cmp(revEdges[i]) == -1 { // edges[i] is unmatched + if ref, ok := referencePointAtVertex(shape, edges[i].V0); ok { + return ref + } + } + if revEdges[i].Cmp(edges[i]) == -1 { // revEdges[i] is unmatched + if ref, ok := referencePointAtVertex(shape, revEdges[i].V0); ok { + return ref + } + } + } + + // All vertices are balanced, so this polygon is either empty or full except + // for degeneracies. By convention it is defined to be full if it contains + // any chain with no edges. + for i := 0; i < shape.NumChains(); i++ { + if shape.Chain(i).Length == 0 { + return OriginReferencePoint(true) + } + } + + return OriginReferencePoint(false) +} + +// referencePointAtVertex reports whether the given vertex is unbalanced, and +// returns a ReferencePoint indicating if the point is contained. +// Otherwise returns false. +func referencePointAtVertex(shape Shape, vTest Point) (ReferencePoint, bool) { + var ref ReferencePoint + + // Let P be an unbalanced vertex. Vertex P is defined to be inside the + // region if the region contains a particular direction vector starting from + // P, namely the direction p.Ortho(). This can be calculated using + // ContainsVertexQuery. + + containsQuery := NewContainsVertexQuery(vTest) + n := shape.NumEdges() + for e := 0; e < n; e++ { + edge := shape.Edge(e) + if edge.V0 == vTest { + containsQuery.AddEdge(edge.V1, 1) + } + if edge.V1 == vTest { + containsQuery.AddEdge(edge.V0, -1) + } + } + containsSign := containsQuery.ContainsVertex() + if containsSign == 0 { + return ref, false // There are no unmatched edges incident to this vertex. + } + ref.Point = vTest + ref.Contained = containsSign > 0 + + return ref, true +} + +// containsBruteForce reports whether the given shape contains the given point. +// Most clients should not use this method, since its running time is linear in +// the number of shape edges. Instead clients should create a ShapeIndex and use +// ContainsPointQuery, since this strategy is much more efficient when many +// points need to be tested. +// +// Polygon boundaries are treated as being semi-open (see ContainsPointQuery +// and VertexModel for other options). +func containsBruteForce(shape Shape, point Point) bool { + if shape.Dimension() != 2 { + return false + } + + refPoint := shape.ReferencePoint() + if refPoint.Point == point { + return refPoint.Contained + } + + crosser := NewEdgeCrosser(refPoint.Point, point) + inside := refPoint.Contained + for e := 0; e < shape.NumEdges(); e++ { + edge := shape.Edge(e) + inside = inside != crosser.EdgeOrVertexCrossing(edge.V0, edge.V1) + } + return inside +} diff --git a/vendor/github.com/blevesearch/geo/s2/shapeutil_edge_iterator.go b/vendor/github.com/blevesearch/geo/s2/shapeutil_edge_iterator.go new file mode 100644 index 00000000..2a0d8236 --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/shapeutil_edge_iterator.go @@ -0,0 +1,72 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +// EdgeIterator is an iterator that advances through all edges in an ShapeIndex. +// This is different to the ShapeIndexIterator, which advances through the cells in the +// ShapeIndex. +type EdgeIterator struct { + index *ShapeIndex + shapeID int32 + numEdges int32 + edgeID int32 +} + +// NewEdgeIterator creates a new edge iterator for the given index. +func NewEdgeIterator(index *ShapeIndex) *EdgeIterator { + e := &EdgeIterator{ + index: index, + shapeID: -1, + edgeID: -1, + } + + e.Next() + return e +} + +// ShapeID returns the current shape ID. +func (e *EdgeIterator) ShapeID() int32 { return e.shapeID } + +// EdgeID returns the current edge ID. +func (e *EdgeIterator) EdgeID() int32 { return e.edgeID } + +// ShapeEdgeID returns the current (shapeID, edgeID). +func (e *EdgeIterator) ShapeEdgeID() ShapeEdgeID { return ShapeEdgeID{e.shapeID, e.edgeID} } + +// Edge returns the current edge. +func (e *EdgeIterator) Edge() Edge { + return e.index.Shape(e.shapeID).Edge(int(e.edgeID)) +} + +// Done reports if the iterator is positioned at or after the last index edge. +func (e *EdgeIterator) Done() bool { return e.shapeID >= int32(len(e.index.shapes)) } + +// Next positions the iterator at the next index edge. +func (e *EdgeIterator) Next() { + e.edgeID++ + for ; e.edgeID >= e.numEdges; e.edgeID++ { + e.shapeID++ + if e.shapeID >= int32(len(e.index.shapes)) { + break + } + shape := e.index.Shape(e.shapeID) + if shape == nil { + e.numEdges = 0 + } else { + e.numEdges = int32(shape.NumEdges()) + } + e.edgeID = -1 + } +} diff --git a/vendor/github.com/blevesearch/geo/s2/stuv.go b/vendor/github.com/blevesearch/geo/s2/stuv.go new file mode 100644 index 00000000..7663bb39 --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/stuv.go @@ -0,0 +1,427 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +import ( + "math" + + "github.com/golang/geo/r3" +) + +// +// This file contains documentation of the various coordinate systems used +// throughout the library. Most importantly, S2 defines a framework for +// decomposing the unit sphere into a hierarchy of "cells". Each cell is a +// quadrilateral bounded by four geodesics. The top level of the hierarchy is +// obtained by projecting the six faces of a cube onto the unit sphere, and +// lower levels are obtained by subdividing each cell into four children +// recursively. Cells are numbered such that sequentially increasing cells +// follow a continuous space-filling curve over the entire sphere. The +// transformation is designed to make the cells at each level fairly uniform +// in size. +// +////////////////////////// S2 Cell Decomposition ///////////////////////// +// +// The following methods define the cube-to-sphere projection used by +// the Cell decomposition. +// +// In the process of converting a latitude-longitude pair to a 64-bit cell +// id, the following coordinate systems are used: +// +// (id) +// An CellID is a 64-bit encoding of a face and a Hilbert curve position +// on that face. The Hilbert curve position implicitly encodes both the +// position of a cell and its subdivision level (see s2cellid.go). +// +// (face, i, j) +// Leaf-cell coordinates. "i" and "j" are integers in the range +// [0,(2**30)-1] that identify a particular leaf cell on the given face. +// The (i, j) coordinate system is right-handed on each face, and the +// faces are oriented such that Hilbert curves connect continuously from +// one face to the next. +// +// (face, s, t) +// Cell-space coordinates. "s" and "t" are real numbers in the range +// [0,1] that identify a point on the given face. For example, the point +// (s, t) = (0.5, 0.5) corresponds to the center of the top-level face +// cell. This point is also a vertex of exactly four cells at each +// subdivision level greater than zero. +// +// (face, si, ti) +// Discrete cell-space coordinates. These are obtained by multiplying +// "s" and "t" by 2**31 and rounding to the nearest unsigned integer. +// Discrete coordinates lie in the range [0,2**31]. This coordinate +// system can represent the edge and center positions of all cells with +// no loss of precision (including non-leaf cells). In binary, each +// coordinate of a level-k cell center ends with a 1 followed by +// (30 - k) 0s. The coordinates of its edges end with (at least) +// (31 - k) 0s. +// +// (face, u, v) +// Cube-space coordinates in the range [-1,1]. To make the cells at each +// level more uniform in size after they are projected onto the sphere, +// we apply a nonlinear transformation of the form u=f(s), v=f(t). +// The (u, v) coordinates after this transformation give the actual +// coordinates on the cube face (modulo some 90 degree rotations) before +// it is projected onto the unit sphere. +// +// (face, u, v, w) +// Per-face coordinate frame. This is an extension of the (face, u, v) +// cube-space coordinates that adds a third axis "w" in the direction of +// the face normal. It is always a right-handed 3D coordinate system. +// Cube-space coordinates can be converted to this frame by setting w=1, +// while (u,v,w) coordinates can be projected onto the cube face by +// dividing by w, i.e. (face, u/w, v/w). +// +// (x, y, z) +// Direction vector (Point). Direction vectors are not necessarily unit +// length, and are often chosen to be points on the biunit cube +// [-1,+1]x[-1,+1]x[-1,+1]. They can be be normalized to obtain the +// corresponding point on the unit sphere. +// +// (lat, lng) +// Latitude and longitude (LatLng). Latitudes must be between -90 and +// 90 degrees inclusive, and longitudes must be between -180 and 180 +// degrees inclusive. +// +// Note that the (i, j), (s, t), (si, ti), and (u, v) coordinate systems are +// right-handed on all six faces. +// +// +// There are a number of different projections from cell-space (s,t) to +// cube-space (u,v): linear, quadratic, and tangent. They have the following +// tradeoffs: +// +// Linear - This is the fastest transformation, but also produces the least +// uniform cell sizes. Cell areas vary by a factor of about 5.2, with the +// largest cells at the center of each face and the smallest cells in +// the corners. +// +// Tangent - Transforming the coordinates via Atan makes the cell sizes +// more uniform. The areas vary by a maximum ratio of 1.4 as opposed to a +// maximum ratio of 5.2. However, each call to Atan is about as expensive +// as all of the other calculations combined when converting from points to +// cell ids, i.e. it reduces performance by a factor of 3. +// +// Quadratic - This is an approximation of the tangent projection that +// is much faster and produces cells that are almost as uniform in size. +// It is about 3 times faster than the tangent projection for converting +// cell ids to points or vice versa. Cell areas vary by a maximum ratio of +// about 2.1. +// +// Here is a table comparing the cell uniformity using each projection. Area +// Ratio is the maximum ratio over all subdivision levels of the largest cell +// area to the smallest cell area at that level, Edge Ratio is the maximum +// ratio of the longest edge of any cell to the shortest edge of any cell at +// the same level, and Diag Ratio is the ratio of the longest diagonal of +// any cell to the shortest diagonal of any cell at the same level. +// +// Area Edge Diag +// Ratio Ratio Ratio +// ----------------------------------- +// Linear: 5.200 2.117 2.959 +// Tangent: 1.414 1.414 1.704 +// Quadratic: 2.082 1.802 1.932 +// +// The worst-case cell aspect ratios are about the same with all three +// projections. The maximum ratio of the longest edge to the shortest edge +// within the same cell is about 1.4 and the maximum ratio of the diagonals +// within the same cell is about 1.7. +// +// For Go we have chosen to use only the Quadratic approach. Other language +// implementations may offer other choices. + +const ( + // maxSiTi is the maximum value of an si- or ti-coordinate. + // It is one shift more than maxSize. The range of valid (si,ti) + // values is [0..maxSiTi]. + maxSiTi = maxSize << 1 +) + +// siTiToST converts an si- or ti-value to the corresponding s- or t-value. +// Value is capped at 1.0 because there is no DCHECK in Go. +func siTiToST(si uint32) float64 { + if si > maxSiTi { + return 1.0 + } + return float64(si) / float64(maxSiTi) +} + +// stToSiTi converts the s- or t-value to the nearest si- or ti-coordinate. +// The result may be outside the range of valid (si,ti)-values. Value of +// 0.49999999999999994 (math.NextAfter(0.5, -1)), will be incorrectly rounded up. +func stToSiTi(s float64) uint32 { + if s < 0 { + return uint32(s*maxSiTi - 0.5) + } + return uint32(s*maxSiTi + 0.5) +} + +// stToUV converts an s or t value to the corresponding u or v value. +// This is a non-linear transformation from [-1,1] to [-1,1] that +// attempts to make the cell sizes more uniform. +// This uses what the C++ version calls 'the quadratic transform'. +func stToUV(s float64) float64 { + if s >= 0.5 { + return (1 / 3.) * (4*s*s - 1) + } + return (1 / 3.) * (1 - 4*(1-s)*(1-s)) +} + +// uvToST is the inverse of the stToUV transformation. Note that it +// is not always true that uvToST(stToUV(x)) == x due to numerical +// errors. +func uvToST(u float64) float64 { + if u >= 0 { + return 0.5 * math.Sqrt(1+3*u) + } + return 1 - 0.5*math.Sqrt(1-3*u) +} + +// face returns face ID from 0 to 5 containing the r. For points on the +// boundary between faces, the result is arbitrary but deterministic. +func face(r r3.Vector) int { + f := r.LargestComponent() + switch { + case f == r3.XAxis && r.X < 0: + f += 3 + case f == r3.YAxis && r.Y < 0: + f += 3 + case f == r3.ZAxis && r.Z < 0: + f += 3 + } + return int(f) +} + +// validFaceXYZToUV given a valid face for the given point r (meaning that +// dot product of r with the face normal is positive), returns +// the corresponding u and v values, which may lie outside the range [-1,1]. +func validFaceXYZToUV(face int, r r3.Vector) (float64, float64) { + switch face { + case 0: + return r.Y / r.X, r.Z / r.X + case 1: + return -r.X / r.Y, r.Z / r.Y + case 2: + return -r.X / r.Z, -r.Y / r.Z + case 3: + return r.Z / r.X, r.Y / r.X + case 4: + return r.Z / r.Y, -r.X / r.Y + } + return -r.Y / r.Z, -r.X / r.Z +} + +// xyzToFaceUV converts a direction vector (not necessarily unit length) to +// (face, u, v) coordinates. +func xyzToFaceUV(r r3.Vector) (f int, u, v float64) { + f = face(r) + u, v = validFaceXYZToUV(f, r) + return f, u, v +} + +// faceUVToXYZ turns face and UV coordinates into an unnormalized 3 vector. +func faceUVToXYZ(face int, u, v float64) r3.Vector { + switch face { + case 0: + return r3.Vector{1, u, v} + case 1: + return r3.Vector{-u, 1, v} + case 2: + return r3.Vector{-u, -v, 1} + case 3: + return r3.Vector{-1, -v, -u} + case 4: + return r3.Vector{v, -1, -u} + default: + return r3.Vector{v, u, -1} + } +} + +// faceXYZToUV returns the u and v values (which may lie outside the range +// [-1, 1]) if the dot product of the point p with the given face normal is positive. +func faceXYZToUV(face int, p Point) (u, v float64, ok bool) { + switch face { + case 0: + if p.X <= 0 { + return 0, 0, false + } + case 1: + if p.Y <= 0 { + return 0, 0, false + } + case 2: + if p.Z <= 0 { + return 0, 0, false + } + case 3: + if p.X >= 0 { + return 0, 0, false + } + case 4: + if p.Y >= 0 { + return 0, 0, false + } + default: + if p.Z >= 0 { + return 0, 0, false + } + } + + u, v = validFaceXYZToUV(face, p.Vector) + return u, v, true +} + +// faceXYZtoUVW transforms the given point P to the (u,v,w) coordinate frame of the given +// face where the w-axis represents the face normal. +func faceXYZtoUVW(face int, p Point) Point { + // The result coordinates are simply the dot products of P with the (u,v,w) + // axes for the given face (see faceUVWAxes). + switch face { + case 0: + return Point{r3.Vector{p.Y, p.Z, p.X}} + case 1: + return Point{r3.Vector{-p.X, p.Z, p.Y}} + case 2: + return Point{r3.Vector{-p.X, -p.Y, p.Z}} + case 3: + return Point{r3.Vector{-p.Z, -p.Y, -p.X}} + case 4: + return Point{r3.Vector{-p.Z, p.X, -p.Y}} + default: + return Point{r3.Vector{p.Y, p.X, -p.Z}} + } +} + +// faceSiTiToXYZ transforms the (si, ti) coordinates to a (not necessarily +// unit length) Point on the given face. +func faceSiTiToXYZ(face int, si, ti uint32) Point { + return Point{faceUVToXYZ(face, stToUV(siTiToST(si)), stToUV(siTiToST(ti)))} +} + +// xyzToFaceSiTi transforms the (not necessarily unit length) Point to +// (face, si, ti) coordinates and the level the Point is at. +func xyzToFaceSiTi(p Point) (face int, si, ti uint32, level int) { + face, u, v := xyzToFaceUV(p.Vector) + si = stToSiTi(uvToST(u)) + ti = stToSiTi(uvToST(v)) + + // If the levels corresponding to si,ti are not equal, then p is not a cell + // center. The si,ti values of 0 and maxSiTi need to be handled specially + // because they do not correspond to cell centers at any valid level; they + // are mapped to level -1 by the code at the end. + level = maxLevel - findLSBSetNonZero64(uint64(si|maxSiTi)) + if level < 0 || level != maxLevel-findLSBSetNonZero64(uint64(ti|maxSiTi)) { + return face, si, ti, -1 + } + + // In infinite precision, this test could be changed to ST == SiTi. However, + // due to rounding errors, uvToST(xyzToFaceUV(faceUVToXYZ(stToUV(...)))) is + // not idempotent. On the other hand, the center is computed exactly the same + // way p was originally computed (if it is indeed the center of a Cell); + // the comparison can be exact. + if p.Vector == faceSiTiToXYZ(face, si, ti).Normalize() { + return face, si, ti, level + } + + return face, si, ti, -1 +} + +// uNorm returns the right-handed normal (not necessarily unit length) for an +// edge in the direction of the positive v-axis at the given u-value on +// the given face. (This vector is perpendicular to the plane through +// the sphere origin that contains the given edge.) +func uNorm(face int, u float64) r3.Vector { + switch face { + case 0: + return r3.Vector{u, -1, 0} + case 1: + return r3.Vector{1, u, 0} + case 2: + return r3.Vector{1, 0, u} + case 3: + return r3.Vector{-u, 0, 1} + case 4: + return r3.Vector{0, -u, 1} + default: + return r3.Vector{0, -1, -u} + } +} + +// vNorm returns the right-handed normal (not necessarily unit length) for an +// edge in the direction of the positive u-axis at the given v-value on +// the given face. +func vNorm(face int, v float64) r3.Vector { + switch face { + case 0: + return r3.Vector{-v, 0, 1} + case 1: + return r3.Vector{0, -v, 1} + case 2: + return r3.Vector{0, -1, -v} + case 3: + return r3.Vector{v, -1, 0} + case 4: + return r3.Vector{1, v, 0} + default: + return r3.Vector{1, 0, v} + } +} + +// faceUVWAxes are the U, V, and W axes for each face. +var faceUVWAxes = [6][3]Point{ + {Point{r3.Vector{0, 1, 0}}, Point{r3.Vector{0, 0, 1}}, Point{r3.Vector{1, 0, 0}}}, + {Point{r3.Vector{-1, 0, 0}}, Point{r3.Vector{0, 0, 1}}, Point{r3.Vector{0, 1, 0}}}, + {Point{r3.Vector{-1, 0, 0}}, Point{r3.Vector{0, -1, 0}}, Point{r3.Vector{0, 0, 1}}}, + {Point{r3.Vector{0, 0, -1}}, Point{r3.Vector{0, -1, 0}}, Point{r3.Vector{-1, 0, 0}}}, + {Point{r3.Vector{0, 0, -1}}, Point{r3.Vector{1, 0, 0}}, Point{r3.Vector{0, -1, 0}}}, + {Point{r3.Vector{0, 1, 0}}, Point{r3.Vector{1, 0, 0}}, Point{r3.Vector{0, 0, -1}}}, +} + +// faceUVWFaces are the precomputed neighbors of each face. +var faceUVWFaces = [6][3][2]int{ + {{4, 1}, {5, 2}, {3, 0}}, + {{0, 3}, {5, 2}, {4, 1}}, + {{0, 3}, {1, 4}, {5, 2}}, + {{2, 5}, {1, 4}, {0, 3}}, + {{2, 5}, {3, 0}, {1, 4}}, + {{4, 1}, {3, 0}, {2, 5}}, +} + +// uvwAxis returns the given axis of the given face. +func uvwAxis(face, axis int) Point { + return faceUVWAxes[face][axis] +} + +// uvwFaces returns the face in the (u,v,w) coordinate system on the given axis +// in the given direction. +func uvwFace(face, axis, direction int) int { + return faceUVWFaces[face][axis][direction] +} + +// uAxis returns the u-axis for the given face. +func uAxis(face int) Point { + return uvwAxis(face, 0) +} + +// vAxis returns the v-axis for the given face. +func vAxis(face int) Point { + return uvwAxis(face, 1) +} + +// Return the unit-length normal for the given face. +func unitNorm(face int) Point { + return uvwAxis(face, 2) +} diff --git a/vendor/github.com/blevesearch/geo/s2/util.go b/vendor/github.com/blevesearch/geo/s2/util.go new file mode 100644 index 00000000..7cab746d --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/util.go @@ -0,0 +1,125 @@ +// Copyright 2017 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +import "github.com/golang/geo/s1" + +// roundAngle returns the value rounded to nearest as an int32. +// This does not match C++ exactly for the case of x.5. +func roundAngle(val s1.Angle) int32 { + if val < 0 { + return int32(val - 0.5) + } + return int32(val + 0.5) +} + +// minAngle returns the smallest of the given values. +func minAngle(x s1.Angle, others ...s1.Angle) s1.Angle { + min := x + for _, y := range others { + if y < min { + min = y + } + } + return min +} + +// maxAngle returns the largest of the given values. +func maxAngle(x s1.Angle, others ...s1.Angle) s1.Angle { + max := x + for _, y := range others { + if y > max { + max = y + } + } + return max +} + +// minChordAngle returns the smallest of the given values. +func minChordAngle(x s1.ChordAngle, others ...s1.ChordAngle) s1.ChordAngle { + min := x + for _, y := range others { + if y < min { + min = y + } + } + return min +} + +// maxChordAngle returns the largest of the given values. +func maxChordAngle(x s1.ChordAngle, others ...s1.ChordAngle) s1.ChordAngle { + max := x + for _, y := range others { + if y > max { + max = y + } + } + return max +} + +// minFloat64 returns the smallest of the given values. +func minFloat64(x float64, others ...float64) float64 { + min := x + for _, y := range others { + if y < min { + min = y + } + } + return min +} + +// maxFloat64 returns the largest of the given values. +func maxFloat64(x float64, others ...float64) float64 { + max := x + for _, y := range others { + if y > max { + max = y + } + } + return max +} + +// minInt returns the smallest of the given values. +func minInt(x int, others ...int) int { + min := x + for _, y := range others { + if y < min { + min = y + } + } + return min +} + +// maxInt returns the largest of the given values. +func maxInt(x int, others ...int) int { + max := x + for _, y := range others { + if y > max { + max = y + } + } + return max +} + +// clampInt returns the number closest to x within the range min..max. +func clampInt(x, min, max int) int { + if x < min { + return min + } + if x > max { + return max + } + return x +} diff --git a/vendor/github.com/blevesearch/geo/s2/wedge_relations.go b/vendor/github.com/blevesearch/geo/s2/wedge_relations.go new file mode 100644 index 00000000..d637bb68 --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/wedge_relations.go @@ -0,0 +1,97 @@ +// Copyright 2017 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +// WedgeRel enumerates the possible relation between two wedges A and B. +type WedgeRel int + +// Define the different possible relationships between two wedges. +// +// Given an edge chain (x0, x1, x2), the wedge at x1 is the region to the +// left of the edges. More precisely, it is the set of all rays from x1x0 +// (inclusive) to x1x2 (exclusive) in the *clockwise* direction. +const ( + WedgeEquals WedgeRel = iota // A and B are equal. + WedgeProperlyContains // A is a strict superset of B. + WedgeIsProperlyContained // A is a strict subset of B. + WedgeProperlyOverlaps // A-B, B-A, and A intersect B are non-empty. + WedgeIsDisjoint // A and B are disjoint. +) + +// WedgeRelation reports the relation between two non-empty wedges +// A=(a0, ab1, a2) and B=(b0, ab1, b2). +func WedgeRelation(a0, ab1, a2, b0, b2 Point) WedgeRel { + // There are 6 possible edge orderings at a shared vertex (all + // of these orderings are circular, i.e. abcd == bcda): + // + // (1) a2 b2 b0 a0: A contains B + // (2) a2 a0 b0 b2: B contains A + // (3) a2 a0 b2 b0: A and B are disjoint + // (4) a2 b0 a0 b2: A and B intersect in one wedge + // (5) a2 b2 a0 b0: A and B intersect in one wedge + // (6) a2 b0 b2 a0: A and B intersect in two wedges + // + // We do not distinguish between 4, 5, and 6. + // We pay extra attention when some of the edges overlap. When edges + // overlap, several of these orderings can be satisfied, and we take + // the most specific. + if a0 == b0 && a2 == b2 { + return WedgeEquals + } + + // Cases 1, 2, 5, and 6 + if OrderedCCW(a0, a2, b2, ab1) { + // The cases with this vertex ordering are 1, 5, and 6, + if OrderedCCW(b2, b0, a0, ab1) { + return WedgeProperlyContains + } + + // We are in case 5 or 6, or case 2 if a2 == b2. + if a2 == b2 { + return WedgeIsProperlyContained + } + return WedgeProperlyOverlaps + + } + // We are in case 2, 3, or 4. + if OrderedCCW(a0, b0, b2, ab1) { + return WedgeIsProperlyContained + } + + if OrderedCCW(a0, b0, a2, ab1) { + return WedgeIsDisjoint + } + return WedgeProperlyOverlaps +} + +// WedgeContains reports whether non-empty wedge A=(a0, ab1, a2) contains B=(b0, ab1, b2). +// Equivalent to WedgeRelation == WedgeProperlyContains || WedgeEquals. +func WedgeContains(a0, ab1, a2, b0, b2 Point) bool { + // For A to contain B (where each loop interior is defined to be its left + // side), the CCW edge order around ab1 must be a2 b2 b0 a0. We split + // this test into two parts that test three vertices each. + return OrderedCCW(a2, b2, b0, ab1) && OrderedCCW(b0, a0, a2, ab1) +} + +// WedgeIntersects reports whether non-empty wedge A=(a0, ab1, a2) intersects B=(b0, ab1, b2). +// Equivalent but faster than WedgeRelation != WedgeIsDisjoint +func WedgeIntersects(a0, ab1, a2, b0, b2 Point) bool { + // For A not to intersect B (where each loop interior is defined to be + // its left side), the CCW edge order around ab1 must be a0 b2 b0 a2. + // Note that it's important to write these conditions as negatives + // (!OrderedCCW(a,b,c,o) rather than Ordered(c,b,a,o)) to get correct + // results when two vertices are the same. + return !(OrderedCCW(a0, b2, b0, ab1) && OrderedCCW(b0, a2, a0, ab1)) +} diff --git a/vendor/github.com/steveyen/gtreap/.gitignore b/vendor/github.com/blevesearch/gtreap/.gitignore similarity index 100% rename from vendor/github.com/steveyen/gtreap/.gitignore rename to vendor/github.com/blevesearch/gtreap/.gitignore diff --git a/vendor/github.com/steveyen/gtreap/LICENSE b/vendor/github.com/blevesearch/gtreap/LICENSE similarity index 100% rename from vendor/github.com/steveyen/gtreap/LICENSE rename to vendor/github.com/blevesearch/gtreap/LICENSE diff --git a/vendor/github.com/steveyen/gtreap/README.md b/vendor/github.com/blevesearch/gtreap/README.md similarity index 100% rename from vendor/github.com/steveyen/gtreap/README.md rename to vendor/github.com/blevesearch/gtreap/README.md diff --git a/vendor/github.com/steveyen/gtreap/treap.go b/vendor/github.com/blevesearch/gtreap/treap.go similarity index 83% rename from vendor/github.com/steveyen/gtreap/treap.go rename to vendor/github.com/blevesearch/gtreap/treap.go index f758ffe4..1f8002c2 100644 --- a/vendor/github.com/steveyen/gtreap/treap.go +++ b/vendor/github.com/blevesearch/gtreap/treap.go @@ -165,9 +165,10 @@ func (t *Treap) join(this *node, that *node) *node { } } +// ItemVistor callback should return true to keep going on the visitation. type ItemVisitor func(i Item) bool -// Visit items greater-than-or-equal to the pivot. +// Visit items greater-than-or-equal to the pivot, in ascending order. func (t *Treap) VisitAscend(pivot Item, visitor ItemVisitor) { t.visitAscend(t.root, pivot, visitor) } @@ -176,13 +177,31 @@ func (t *Treap) visitAscend(n *node, pivot Item, visitor ItemVisitor) bool { if n == nil { return true } - if t.compare(pivot, n.item) <= 0 { - if !t.visitAscend(n.left, pivot, visitor) { - return false - } - if !visitor(n.item) { - return false - } + c := t.compare(pivot, n.item) + if c < 0 && !t.visitAscend(n.left, pivot, visitor) { + return false + } + if c <= 0 && !visitor(n.item) { + return false } return t.visitAscend(n.right, pivot, visitor) } + +// Visit items less-than-or-equal to the pivot, in descending order. +func (t *Treap) VisitDescend(pivot Item, visitor ItemVisitor) { + t.visitDescend(t.root, pivot, visitor) +} + +func (t *Treap) visitDescend(n *node, pivot Item, visitor ItemVisitor) bool { + if n == nil { + return true + } + c := t.compare(pivot, n.item) + if c > 0 && !t.visitDescend(n.right, pivot, visitor) { + return false + } + if c >= 0 && !visitor(n.item) { + return false + } + return t.visitDescend(n.left, pivot, visitor) +} diff --git a/vendor/github.com/blevesearch/scorch_segment_api/v2/.golangci.yml b/vendor/github.com/blevesearch/scorch_segment_api/v2/.golangci.yml new file mode 100644 index 00000000..664f35f2 --- /dev/null +++ b/vendor/github.com/blevesearch/scorch_segment_api/v2/.golangci.yml @@ -0,0 +1,42 @@ +linters: + # please, do not use `enable-all`: it's deprecated and will be removed soon. + # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint + disable-all: true + enable: + - bodyclose + - deadcode + - depguard + - dogsled + - dupl + - errcheck + - funlen + - gochecknoinits + - goconst + - gocritic + - gocyclo + - gofmt + - goimports + - golint + - gomnd + - goprintffuncname + - gosec + - gosimple + - govet + - ineffassign + - interfacer + - lll + - misspell + - nakedret + - nolintlint + - rowserrcheck + - scopelint + - staticcheck + - structcheck + - stylecheck + - typecheck + - unconvert + - unparam + - unused + - varcheck + - whitespace + diff --git a/vendor/github.com/blevesearch/zap/v12/LICENSE b/vendor/github.com/blevesearch/scorch_segment_api/v2/LICENSE similarity index 100% rename from vendor/github.com/blevesearch/zap/v12/LICENSE rename to vendor/github.com/blevesearch/scorch_segment_api/v2/LICENSE diff --git a/vendor/github.com/blevesearch/scorch_segment_api/v2/README.md b/vendor/github.com/blevesearch/scorch_segment_api/v2/README.md new file mode 100644 index 00000000..dc33b004 --- /dev/null +++ b/vendor/github.com/blevesearch/scorch_segment_api/v2/README.md @@ -0,0 +1,11 @@ +# Scorch Segment API + +[![PkgGoDev](https://pkg.go.dev/badge/github.com/blevesearch/scorch_segment_api)](https://pkg.go.dev/github.com/blevesearch/scorch_segment_api) +[![Tests](https://github.com/blevesearch/scorch_segment_api/workflows/Tests/badge.svg?branch=master&event=push)](https://github.com/blevesearch/scorch_segment_api/actions?query=workflow%3ATests+event%3Apush+branch%3Amaster) +[![Lint](https://github.com/blevesearch/scorch_segment_api/workflows/Lint/badge.svg?branch=master&event=push)](https://github.com/blevesearch/scorch_segment_api/actions?query=workflow%3ALint+event%3Apush+branch%3Amaster) + +Scorch supports a pluggable Segment interface. + +By placing these interfaces in their own, *hopefully* slowly evolving module, it frees up Scorch and the underlying segment to each introduce new major versions without interfering with one another. + +With that in mind, we anticipate introducing non-breaking changes only to this module, and keeping the major version at 1.x for some time. diff --git a/vendor/github.com/blevesearch/scorch_segment_api/v2/automaton.go b/vendor/github.com/blevesearch/scorch_segment_api/v2/automaton.go new file mode 100644 index 00000000..4577ceb2 --- /dev/null +++ b/vendor/github.com/blevesearch/scorch_segment_api/v2/automaton.go @@ -0,0 +1,36 @@ +// Copyright (c) 2021 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package segment + +// Automaton represents the general contract of a byte-based finite automaton +type Automaton interface { + + // Start returns the start state + Start() int + + // IsMatch returns true if and only if the state is a match + IsMatch(int) bool + + // CanMatch returns true if and only if it is possible to reach a match + // in zero or more steps + CanMatch(int) bool + + // WillAlwaysMatch returns true if and only if the current state matches + // and will always match no matter what steps are taken + WillAlwaysMatch(int) bool + + // Accept returns the next state given the input to the specified state + Accept(int, byte) int +} diff --git a/vendor/github.com/blevesearch/scorch_segment_api/v2/segment.go b/vendor/github.com/blevesearch/scorch_segment_api/v2/segment.go new file mode 100644 index 00000000..37d04c47 --- /dev/null +++ b/vendor/github.com/blevesearch/scorch_segment_api/v2/segment.go @@ -0,0 +1,170 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package segment + +import ( + "fmt" + + "github.com/RoaringBitmap/roaring" + index "github.com/blevesearch/bleve_index_api" +) + +var ErrClosed = fmt.Errorf("index closed") + +// StoredFieldValueVisitor defines a callback to be visited for each +// stored field value. The return value determines if the visitor +// should keep going. Returning true continues visiting, false stops. +type StoredFieldValueVisitor func(field string, typ byte, value []byte, pos []uint64) bool + +type Segment interface { + DiskStatsReporter + + Dictionary(field string) (TermDictionary, error) + + VisitStoredFields(num uint64, visitor StoredFieldValueVisitor) error + + DocID(num uint64) ([]byte, error) + + Count() uint64 + + DocNumbers([]string) (*roaring.Bitmap, error) + + Fields() []string + + Close() error + + Size() int + + AddRef() + DecRef() error +} + +type UnpersistedSegment interface { + Segment + Persist(path string) error +} + +type PersistedSegment interface { + Segment + Path() string +} + +type TermDictionary interface { + PostingsList(term []byte, except *roaring.Bitmap, prealloc PostingsList) (PostingsList, error) + + AutomatonIterator(a Automaton, + startKeyInclusive, endKeyExclusive []byte) DictionaryIterator + + Contains(key []byte) (bool, error) +} + +type DictionaryIterator interface { + Next() (*index.DictEntry, error) +} + +type PostingsList interface { + DiskStatsReporter + + Iterator(includeFreq, includeNorm, includeLocations bool, prealloc PostingsIterator) PostingsIterator + + Size() int + + Count() uint64 + + // NOTE deferred for future work + + // And(other PostingsList) PostingsList + // Or(other PostingsList) PostingsList +} + +type PostingsIterator interface { + DiskStatsReporter + + // The caller is responsible for copying whatever it needs from + // the returned Posting instance before calling Next(), as some + // implementations may return a shared instance to reduce memory + // allocations. + Next() (Posting, error) + + // Advance will return the posting with the specified doc number + // or if there is no such posting, the next posting. + // Callers MUST NOT attempt to pass a docNum that is less than or + // equal to the currently visited posting doc Num. + Advance(docNum uint64) (Posting, error) + + Size() int +} + +type DiskStatsReporter interface { + // BytesRead returns the bytes read from the disk as + // part of the current running query. + BytesRead() uint64 + + // ResetBytesRead is used by the parent layer + // to reset the bytes read value to a consistent + // value during operations such as merging of segments. + ResetBytesRead(uint64) + + // BytesWritten returns the bytes written to disk while + // building an index + BytesWritten() uint64 +} + +type OptimizablePostingsIterator interface { + ActualBitmap() *roaring.Bitmap + DocNum1Hit() (uint64, bool) + ReplaceActual(*roaring.Bitmap) +} + +type Posting interface { + Number() uint64 + + Frequency() uint64 + Norm() float64 + + Locations() []Location + + Size() int +} + +type Location interface { + Field() string + Start() uint64 + End() uint64 + Pos() uint64 + ArrayPositions() []uint64 + Size() int +} + +// DocValueVisitable is implemented by various scorch segment +// implementations with persistence for the un inverting of the +// postings or other indexed values. +type DocValueVisitable interface { + VisitDocValues(localDocNum uint64, fields []string, + visitor index.DocValueVisitor, optional DocVisitState) (DocVisitState, error) + + // VisitableDocValueFields implementation should return + // the list of fields which are document value persisted and + // therefore visitable by the above VisitDocValues method. + VisitableDocValueFields() ([]string, error) +} + +type DocVisitState interface { + DiskStatsReporter +} + +type StatsReporter interface { + ReportBytesWritten(bytesWritten uint64) +} diff --git a/vendor/github.com/blevesearch/segment/.travis.yml b/vendor/github.com/blevesearch/segment/.travis.yml deleted file mode 100644 index b9d58e7c..00000000 --- a/vendor/github.com/blevesearch/segment/.travis.yml +++ /dev/null @@ -1,15 +0,0 @@ -language: go - -go: - - 1.7 - -script: - - go get golang.org/x/tools/cmd/cover - - go get github.com/mattn/goveralls - - go test -v -covermode=count -coverprofile=profile.out - - go vet - - goveralls -service drone.io -coverprofile=profile.out -repotoken $COVERALLS - -notifications: - email: - - marty.schoch@gmail.com diff --git a/vendor/github.com/blevesearch/segment/README.md b/vendor/github.com/blevesearch/segment/README.md index 0840962d..b24dfb46 100644 --- a/vendor/github.com/blevesearch/segment/README.md +++ b/vendor/github.com/blevesearch/segment/README.md @@ -1,5 +1,7 @@ # segment +[![Tests](https://github.com/blevesearch/segment/workflows/Tests/badge.svg?branch=master&event=push)](https://github.com/blevesearch/segment/actions?query=workflow%3ATests+event%3Apush+branch%3Amaster) + A Go library for performing Unicode Text Segmentation as described in [Unicode Standard Annex #29](http://www.unicode.org/reports/tr29/) diff --git a/vendor/github.com/blevesearch/zap/v13/LICENSE b/vendor/github.com/blevesearch/upsidedown_store_api/LICENSE similarity index 100% rename from vendor/github.com/blevesearch/zap/v13/LICENSE rename to vendor/github.com/blevesearch/upsidedown_store_api/LICENSE diff --git a/vendor/github.com/blevesearch/upsidedown_store_api/README.md b/vendor/github.com/blevesearch/upsidedown_store_api/README.md new file mode 100644 index 00000000..91928194 --- /dev/null +++ b/vendor/github.com/blevesearch/upsidedown_store_api/README.md @@ -0,0 +1,7 @@ +# Upsidedown Store API + +Upsidedown supports a pluggable Key/Value storage interface. + +By placing these interfaces in their own, *hopefully* slowly evolving module, it frees up Upsidedown and the underlying storage implementations to each introduce new major versions without interfering with one another. + +With that in mind, we anticipate introducing non-breaking changes only to this module, and keeping the major version at 1.x for some time. diff --git a/vendor/github.com/blevesearch/bleve/index/store/batch.go b/vendor/github.com/blevesearch/upsidedown_store_api/batch.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/index/store/batch.go rename to vendor/github.com/blevesearch/upsidedown_store_api/batch.go diff --git a/vendor/github.com/blevesearch/bleve/index/store/kvstore.go b/vendor/github.com/blevesearch/upsidedown_store_api/kvstore.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/index/store/kvstore.go rename to vendor/github.com/blevesearch/upsidedown_store_api/kvstore.go diff --git a/vendor/github.com/blevesearch/bleve/index/store/merge.go b/vendor/github.com/blevesearch/upsidedown_store_api/merge.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/index/store/merge.go rename to vendor/github.com/blevesearch/upsidedown_store_api/merge.go diff --git a/vendor/github.com/blevesearch/bleve/index/store/multiget.go b/vendor/github.com/blevesearch/upsidedown_store_api/multiget.go similarity index 100% rename from vendor/github.com/blevesearch/bleve/index/store/multiget.go rename to vendor/github.com/blevesearch/upsidedown_store_api/multiget.go diff --git a/vendor/github.com/couchbase/vellum/CONTRIBUTING.md b/vendor/github.com/blevesearch/vellum/CONTRIBUTING.md similarity index 100% rename from vendor/github.com/couchbase/vellum/CONTRIBUTING.md rename to vendor/github.com/blevesearch/vellum/CONTRIBUTING.md diff --git a/vendor/github.com/blevesearch/zap/v14/LICENSE b/vendor/github.com/blevesearch/vellum/LICENSE similarity index 100% rename from vendor/github.com/blevesearch/zap/v14/LICENSE rename to vendor/github.com/blevesearch/vellum/LICENSE diff --git a/vendor/github.com/couchbase/vellum/README.md b/vendor/github.com/blevesearch/vellum/README.md similarity index 100% rename from vendor/github.com/couchbase/vellum/README.md rename to vendor/github.com/blevesearch/vellum/README.md diff --git a/vendor/github.com/couchbase/vellum/automaton.go b/vendor/github.com/blevesearch/vellum/automaton.go similarity index 100% rename from vendor/github.com/couchbase/vellum/automaton.go rename to vendor/github.com/blevesearch/vellum/automaton.go diff --git a/vendor/github.com/blevesearch/vellum/builder.go b/vendor/github.com/blevesearch/vellum/builder.go new file mode 100644 index 00000000..7e545cbe --- /dev/null +++ b/vendor/github.com/blevesearch/vellum/builder.go @@ -0,0 +1,447 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vellum + +import ( + "bytes" + "io" +) + +var defaultBuilderOpts = &BuilderOpts{ + Encoder: 1, + RegistryTableSize: 10000, + RegistryMRUSize: 2, +} + +// A Builder is used to build a new FST. When possible data is +// streamed out to the underlying Writer as soon as possible. +type Builder struct { + unfinished *unfinishedNodes + registry *registry + last []byte + len int + + lastAddr int + + encoder encoder + opts *BuilderOpts + + builderNodePool *builderNodePool +} + +const noneAddr = 1 +const emptyAddr = 0 + +// NewBuilder returns a new Builder which will stream out the +// underlying representation to the provided Writer as the set is built. +func newBuilder(w io.Writer, opts *BuilderOpts) (*Builder, error) { + if opts == nil { + opts = defaultBuilderOpts + } + builderNodePool := &builderNodePool{} + rv := &Builder{ + unfinished: newUnfinishedNodes(builderNodePool), + registry: newRegistry(builderNodePool, opts.RegistryTableSize, opts.RegistryMRUSize), + builderNodePool: builderNodePool, + opts: opts, + lastAddr: noneAddr, + } + + var err error + rv.encoder, err = loadEncoder(opts.Encoder, w) + if err != nil { + return nil, err + } + err = rv.encoder.start() + if err != nil { + return nil, err + } + return rv, nil +} + +func (b *Builder) Reset(w io.Writer) error { + b.unfinished.Reset() + b.registry.Reset() + b.lastAddr = noneAddr + b.encoder.reset(w) + b.last = nil + b.len = 0 + + err := b.encoder.start() + if err != nil { + return err + } + return nil +} + +// Insert the provided value to the set being built. +// NOTE: values must be inserted in lexicographical order. +func (b *Builder) Insert(key []byte, val uint64) error { + // ensure items are added in lexicographic order + if bytes.Compare(key, b.last) < 0 { + return ErrOutOfOrder + } + if len(key) == 0 { + b.len = 1 + b.unfinished.setRootOutput(val) + return nil + } + + prefixLen, out := b.unfinished.findCommonPrefixAndSetOutput(key, val) + b.len++ + err := b.compileFrom(prefixLen) + if err != nil { + return err + } + b.copyLastKey(key) + b.unfinished.addSuffix(key[prefixLen:], out) + + return nil +} + +func (b *Builder) copyLastKey(key []byte) { + if b.last == nil { + b.last = make([]byte, 0, 64) + } else { + b.last = b.last[:0] + } + b.last = append(b.last, key...) +} + +// Close MUST be called after inserting all values. +func (b *Builder) Close() error { + err := b.compileFrom(0) + if err != nil { + return err + } + root := b.unfinished.popRoot() + rootAddr, err := b.compile(root) + if err != nil { + return err + } + return b.encoder.finish(b.len, rootAddr) +} + +func (b *Builder) compileFrom(iState int) error { + addr := noneAddr + for iState+1 < len(b.unfinished.stack) { + var node *builderNode + if addr == noneAddr { + node = b.unfinished.popEmpty() + } else { + node = b.unfinished.popFreeze(addr) + } + var err error + addr, err = b.compile(node) + if err != nil { + return nil + } + } + b.unfinished.topLastFreeze(addr) + return nil +} + +func (b *Builder) compile(node *builderNode) (int, error) { + if node.final && len(node.trans) == 0 && + node.finalOutput == 0 { + return 0, nil + } + found, addr, entry := b.registry.entry(node) + if found { + return addr, nil + } + addr, err := b.encoder.encodeState(node, b.lastAddr) + if err != nil { + return 0, err + } + + b.lastAddr = addr + entry.addr = addr + return addr, nil +} + +type unfinishedNodes struct { + stack []*builderNodeUnfinished + + // cache allocates a reasonable number of builderNodeUnfinished + // objects up front and tries to keep reusing them + // because the main data structure is a stack, we assume the + // same access pattern, and don't track items separately + // this means calls get() and pushXYZ() must be paired, + // as well as calls put() and popXYZ() + cache []builderNodeUnfinished + + builderNodePool *builderNodePool +} + +func (u *unfinishedNodes) Reset() { + u.stack = u.stack[:0] + for i := 0; i < len(u.cache); i++ { + u.cache[i] = builderNodeUnfinished{} + } + u.pushEmpty(false) +} + +func newUnfinishedNodes(p *builderNodePool) *unfinishedNodes { + rv := &unfinishedNodes{ + stack: make([]*builderNodeUnfinished, 0, 64), + cache: make([]builderNodeUnfinished, 64), + builderNodePool: p, + } + rv.pushEmpty(false) + return rv +} + +// get new builderNodeUnfinished, reusing cache if possible +func (u *unfinishedNodes) get() *builderNodeUnfinished { + if len(u.stack) < len(u.cache) { + return &u.cache[len(u.stack)] + } + // full now allocate a new one + return &builderNodeUnfinished{} +} + +// return builderNodeUnfinished, clearing it for reuse +func (u *unfinishedNodes) put() { + if len(u.stack) >= len(u.cache) { + return + // do nothing, not part of cache + } + u.cache[len(u.stack)] = builderNodeUnfinished{} +} + +func (u *unfinishedNodes) findCommonPrefixAndSetOutput(key []byte, + out uint64) (int, uint64) { + var i int + for i < len(key) { + if i >= len(u.stack) { + break + } + var addPrefix uint64 + if !u.stack[i].hasLastT { + break + } + if u.stack[i].lastIn == key[i] { + commonPre := outputPrefix(u.stack[i].lastOut, out) + addPrefix = outputSub(u.stack[i].lastOut, commonPre) + out = outputSub(out, commonPre) + u.stack[i].lastOut = commonPre + i++ + } else { + break + } + + if addPrefix != 0 { + u.stack[i].addOutputPrefix(addPrefix) + } + } + + return i, out +} + +func (u *unfinishedNodes) pushEmpty(final bool) { + next := u.get() + next.node = u.builderNodePool.Get() + next.node.final = final + u.stack = append(u.stack, next) +} + +func (u *unfinishedNodes) popRoot() *builderNode { + l := len(u.stack) + var unfinished *builderNodeUnfinished + u.stack, unfinished = u.stack[:l-1], u.stack[l-1] + rv := unfinished.node + u.put() + return rv +} + +func (u *unfinishedNodes) popFreeze(addr int) *builderNode { + l := len(u.stack) + var unfinished *builderNodeUnfinished + u.stack, unfinished = u.stack[:l-1], u.stack[l-1] + unfinished.lastCompiled(addr) + rv := unfinished.node + u.put() + return rv +} + +func (u *unfinishedNodes) popEmpty() *builderNode { + l := len(u.stack) + var unfinished *builderNodeUnfinished + u.stack, unfinished = u.stack[:l-1], u.stack[l-1] + rv := unfinished.node + u.put() + return rv +} + +func (u *unfinishedNodes) setRootOutput(out uint64) { + u.stack[0].node.final = true + u.stack[0].node.finalOutput = out +} + +func (u *unfinishedNodes) topLastFreeze(addr int) { + last := len(u.stack) - 1 + u.stack[last].lastCompiled(addr) +} + +func (u *unfinishedNodes) addSuffix(bs []byte, out uint64) { + if len(bs) == 0 { + return + } + last := len(u.stack) - 1 + u.stack[last].hasLastT = true + u.stack[last].lastIn = bs[0] + u.stack[last].lastOut = out + for _, b := range bs[1:] { + next := u.get() + next.node = u.builderNodePool.Get() + next.hasLastT = true + next.lastIn = b + next.lastOut = 0 + u.stack = append(u.stack, next) + } + u.pushEmpty(true) +} + +type builderNodeUnfinished struct { + node *builderNode + lastOut uint64 + lastIn byte + hasLastT bool +} + +func (b *builderNodeUnfinished) lastCompiled(addr int) { + if b.hasLastT { + transIn := b.lastIn + transOut := b.lastOut + b.hasLastT = false + b.lastOut = 0 + b.node.trans = append(b.node.trans, transition{ + in: transIn, + out: transOut, + addr: addr, + }) + } +} + +func (b *builderNodeUnfinished) addOutputPrefix(prefix uint64) { + if b.node.final { + b.node.finalOutput = outputCat(prefix, b.node.finalOutput) + } + for i := range b.node.trans { + b.node.trans[i].out = outputCat(prefix, b.node.trans[i].out) + } + if b.hasLastT { + b.lastOut = outputCat(prefix, b.lastOut) + } +} + +type builderNode struct { + finalOutput uint64 + trans []transition + final bool + + // intrusive linked list + next *builderNode +} + +// reset resets the receiver builderNode to a re-usable state. +func (n *builderNode) reset() { + n.final = false + n.finalOutput = 0 + n.trans = n.trans[:0] + n.next = nil +} + +func (n *builderNode) equiv(o *builderNode) bool { + if n.final != o.final { + return false + } + if n.finalOutput != o.finalOutput { + return false + } + if len(n.trans) != len(o.trans) { + return false + } + for i, ntrans := range n.trans { + otrans := o.trans[i] + if ntrans.in != otrans.in { + return false + } + if ntrans.addr != otrans.addr { + return false + } + if ntrans.out != otrans.out { + return false + } + } + return true +} + +type transition struct { + out uint64 + addr int + in byte +} + +func outputPrefix(l, r uint64) uint64 { + if l < r { + return l + } + return r +} + +func outputSub(l, r uint64) uint64 { + return l - r +} + +func outputCat(l, r uint64) uint64 { + return l + r +} + +// builderNodePool pools builderNodes using a singly linked list. +// +// NB: builderNode lifecylce is described by the following interactions - +// +------------------------+ +----------------------+ +// | Unfinished Nodes | Transfer once | Registry | +// |(not frozen builderNode)|-----builderNode is ------->| (frozen builderNode) | +// +------------------------+ marked frozen +----------------------+ +// ^ | +// | | +// | Put() +// | Get() on +-------------------+ when +// +-new char--------| builderNode Pool |<-----------evicted +// +-------------------+ +type builderNodePool struct { + head *builderNode +} + +func (p *builderNodePool) Get() *builderNode { + if p.head == nil { + return &builderNode{} + } + head := p.head + p.head = p.head.next + return head +} + +func (p *builderNodePool) Put(v *builderNode) { + if v == nil { + return + } + v.reset() + v.next = p.head + p.head = v +} diff --git a/vendor/github.com/couchbase/vellum/common.go b/vendor/github.com/blevesearch/vellum/common.go similarity index 100% rename from vendor/github.com/couchbase/vellum/common.go rename to vendor/github.com/blevesearch/vellum/common.go diff --git a/vendor/github.com/couchbase/vellum/decoder_v1.go b/vendor/github.com/blevesearch/vellum/decoder_v1.go similarity index 100% rename from vendor/github.com/couchbase/vellum/decoder_v1.go rename to vendor/github.com/blevesearch/vellum/decoder_v1.go diff --git a/vendor/github.com/couchbase/vellum/encoder_v1.go b/vendor/github.com/blevesearch/vellum/encoder_v1.go similarity index 100% rename from vendor/github.com/couchbase/vellum/encoder_v1.go rename to vendor/github.com/blevesearch/vellum/encoder_v1.go diff --git a/vendor/github.com/couchbase/vellum/encoding.go b/vendor/github.com/blevesearch/vellum/encoding.go similarity index 100% rename from vendor/github.com/couchbase/vellum/encoding.go rename to vendor/github.com/blevesearch/vellum/encoding.go diff --git a/vendor/github.com/couchbase/vellum/fst.go b/vendor/github.com/blevesearch/vellum/fst.go similarity index 99% rename from vendor/github.com/couchbase/vellum/fst.go rename to vendor/github.com/blevesearch/vellum/fst.go index 64ee21a4..3140042b 100644 --- a/vendor/github.com/couchbase/vellum/fst.go +++ b/vendor/github.com/blevesearch/vellum/fst.go @@ -17,7 +17,7 @@ package vellum import ( "io" - "github.com/willf/bitset" + "github.com/bits-and-blooms/bitset" ) // FST is an in-memory representation of a finite state transducer, diff --git a/vendor/github.com/couchbase/vellum/fst_iterator.go b/vendor/github.com/blevesearch/vellum/fst_iterator.go similarity index 100% rename from vendor/github.com/couchbase/vellum/fst_iterator.go rename to vendor/github.com/blevesearch/vellum/fst_iterator.go diff --git a/vendor/github.com/couchbase/vellum/levenshtein/LICENSE b/vendor/github.com/blevesearch/vellum/levenshtein/LICENSE similarity index 100% rename from vendor/github.com/couchbase/vellum/levenshtein/LICENSE rename to vendor/github.com/blevesearch/vellum/levenshtein/LICENSE diff --git a/vendor/github.com/couchbase/vellum/levenshtein/README.md b/vendor/github.com/blevesearch/vellum/levenshtein/README.md similarity index 100% rename from vendor/github.com/couchbase/vellum/levenshtein/README.md rename to vendor/github.com/blevesearch/vellum/levenshtein/README.md diff --git a/vendor/github.com/couchbase/vellum/levenshtein/alphabet.go b/vendor/github.com/blevesearch/vellum/levenshtein/alphabet.go similarity index 100% rename from vendor/github.com/couchbase/vellum/levenshtein/alphabet.go rename to vendor/github.com/blevesearch/vellum/levenshtein/alphabet.go diff --git a/vendor/github.com/couchbase/vellum/levenshtein/dfa.go b/vendor/github.com/blevesearch/vellum/levenshtein/dfa.go similarity index 100% rename from vendor/github.com/couchbase/vellum/levenshtein/dfa.go rename to vendor/github.com/blevesearch/vellum/levenshtein/dfa.go diff --git a/vendor/github.com/couchbase/vellum/levenshtein/levenshtein.go b/vendor/github.com/blevesearch/vellum/levenshtein/levenshtein.go similarity index 100% rename from vendor/github.com/couchbase/vellum/levenshtein/levenshtein.go rename to vendor/github.com/blevesearch/vellum/levenshtein/levenshtein.go diff --git a/vendor/github.com/couchbase/vellum/levenshtein/levenshtein_nfa.go b/vendor/github.com/blevesearch/vellum/levenshtein/levenshtein_nfa.go similarity index 100% rename from vendor/github.com/couchbase/vellum/levenshtein/levenshtein_nfa.go rename to vendor/github.com/blevesearch/vellum/levenshtein/levenshtein_nfa.go diff --git a/vendor/github.com/couchbase/vellum/levenshtein/parametric_dfa.go b/vendor/github.com/blevesearch/vellum/levenshtein/parametric_dfa.go similarity index 100% rename from vendor/github.com/couchbase/vellum/levenshtein/parametric_dfa.go rename to vendor/github.com/blevesearch/vellum/levenshtein/parametric_dfa.go diff --git a/vendor/github.com/couchbase/vellum/merge_iterator.go b/vendor/github.com/blevesearch/vellum/merge_iterator.go similarity index 100% rename from vendor/github.com/couchbase/vellum/merge_iterator.go rename to vendor/github.com/blevesearch/vellum/merge_iterator.go diff --git a/vendor/github.com/couchbase/vellum/pack.go b/vendor/github.com/blevesearch/vellum/pack.go similarity index 100% rename from vendor/github.com/couchbase/vellum/pack.go rename to vendor/github.com/blevesearch/vellum/pack.go diff --git a/vendor/github.com/couchbase/vellum/regexp/compile.go b/vendor/github.com/blevesearch/vellum/regexp/compile.go similarity index 99% rename from vendor/github.com/couchbase/vellum/regexp/compile.go rename to vendor/github.com/blevesearch/vellum/regexp/compile.go index 92284d0a..558ce419 100644 --- a/vendor/github.com/couchbase/vellum/regexp/compile.go +++ b/vendor/github.com/blevesearch/vellum/regexp/compile.go @@ -20,7 +20,7 @@ import ( unicode_utf8 "unicode/utf8" - "github.com/couchbase/vellum/utf8" + "github.com/blevesearch/vellum/utf8" ) type compiler struct { diff --git a/vendor/github.com/couchbase/vellum/regexp/dfa.go b/vendor/github.com/blevesearch/vellum/regexp/dfa.go similarity index 100% rename from vendor/github.com/couchbase/vellum/regexp/dfa.go rename to vendor/github.com/blevesearch/vellum/regexp/dfa.go diff --git a/vendor/github.com/couchbase/vellum/regexp/inst.go b/vendor/github.com/blevesearch/vellum/regexp/inst.go similarity index 100% rename from vendor/github.com/couchbase/vellum/regexp/inst.go rename to vendor/github.com/blevesearch/vellum/regexp/inst.go diff --git a/vendor/github.com/couchbase/vellum/regexp/regexp.go b/vendor/github.com/blevesearch/vellum/regexp/regexp.go similarity index 100% rename from vendor/github.com/couchbase/vellum/regexp/regexp.go rename to vendor/github.com/blevesearch/vellum/regexp/regexp.go diff --git a/vendor/github.com/couchbase/vellum/regexp/sparse.go b/vendor/github.com/blevesearch/vellum/regexp/sparse.go similarity index 100% rename from vendor/github.com/couchbase/vellum/regexp/sparse.go rename to vendor/github.com/blevesearch/vellum/regexp/sparse.go diff --git a/vendor/github.com/couchbase/vellum/registry.go b/vendor/github.com/blevesearch/vellum/registry.go similarity index 100% rename from vendor/github.com/couchbase/vellum/registry.go rename to vendor/github.com/blevesearch/vellum/registry.go diff --git a/vendor/github.com/couchbase/vellum/transducer.go b/vendor/github.com/blevesearch/vellum/transducer.go similarity index 100% rename from vendor/github.com/couchbase/vellum/transducer.go rename to vendor/github.com/blevesearch/vellum/transducer.go diff --git a/vendor/github.com/couchbase/vellum/utf8/utf8.go b/vendor/github.com/blevesearch/vellum/utf8/utf8.go similarity index 100% rename from vendor/github.com/couchbase/vellum/utf8/utf8.go rename to vendor/github.com/blevesearch/vellum/utf8/utf8.go diff --git a/vendor/github.com/couchbase/vellum/vellum.go b/vendor/github.com/blevesearch/vellum/vellum.go similarity index 100% rename from vendor/github.com/couchbase/vellum/vellum.go rename to vendor/github.com/blevesearch/vellum/vellum.go diff --git a/vendor/github.com/couchbase/vellum/vellum_mmap.go b/vendor/github.com/blevesearch/vellum/vellum_mmap.go similarity index 100% rename from vendor/github.com/couchbase/vellum/vellum_mmap.go rename to vendor/github.com/blevesearch/vellum/vellum_mmap.go diff --git a/vendor/github.com/couchbase/vellum/vellum_nommap.go b/vendor/github.com/blevesearch/vellum/vellum_nommap.go similarity index 100% rename from vendor/github.com/couchbase/vellum/vellum_nommap.go rename to vendor/github.com/blevesearch/vellum/vellum_nommap.go diff --git a/vendor/github.com/couchbase/vellum/writer.go b/vendor/github.com/blevesearch/vellum/writer.go similarity index 100% rename from vendor/github.com/couchbase/vellum/writer.go rename to vendor/github.com/blevesearch/vellum/writer.go diff --git a/vendor/github.com/blevesearch/zap/v11/README.md b/vendor/github.com/blevesearch/zap/v11/README.md deleted file mode 100644 index 0facb669..00000000 --- a/vendor/github.com/blevesearch/zap/v11/README.md +++ /dev/null @@ -1,158 +0,0 @@ -# zap file format - -Advanced ZAP File Format Documentation is [here](zap.md). - -The file is written in the reverse order that we typically access data. This helps us write in one pass since later sections of the file require file offsets of things we've already written. - -Current usage: - -- mmap the entire file -- crc-32 bytes and version are in fixed position at end of the file -- reading remainder of footer could be version specific -- remainder of footer gives us: - - 3 important offsets (docValue , fields index and stored data index) - - 2 important values (number of docs and chunk factor) -- field data is processed once and memoized onto the heap so that we never have to go back to disk for it -- access to stored data by doc number means first navigating to the stored data index, then accessing a fixed position offset into that slice, which gives us the actual address of the data. the first bytes of that section tell us the size of data so that we know where it ends. -- access to all other indexed data follows the following pattern: - - first know the field name -> convert to id - - next navigate to term dictionary for that field - - some operations stop here and do dictionary ops - - next use dictionary to navigate to posting list for a specific term - - walk posting list - - if necessary, walk posting details as we go - - if location info is desired, consult location bitmap to see if it is there - -## stored fields section - -- for each document - - preparation phase: - - produce a slice of metadata bytes and data bytes - - produce these slices in field id order - - field value is appended to the data slice - - metadata slice is varint encoded with the following values for each field value - - field id (uint16) - - field type (byte) - - field value start offset in uncompressed data slice (uint64) - - field value length (uint64) - - field number of array positions (uint64) - - one additional value for each array position (uint64) - - compress the data slice using snappy - - file writing phase: - - remember the start offset for this document - - write out meta data length (varint uint64) - - write out compressed data length (varint uint64) - - write out the metadata bytes - - write out the compressed data bytes - -## stored fields idx - -- for each document - - write start offset (remembered from previous section) of stored data (big endian uint64) - -With this index and a known document number, we have direct access to all the stored field data. - -## posting details (freq/norm) section - -- for each posting list - - produce a slice containing multiple consecutive chunks (each chunk is varint stream) - - produce a slice remembering offsets of where each chunk starts - - preparation phase: - - for each hit in the posting list - - if this hit is in next chunk close out encoding of last chunk and record offset start of next - - encode term frequency (uint64) - - encode norm factor (float32) - - file writing phase: - - remember start position for this posting list details - - write out number of chunks that follow (varint uint64) - - write out length of each chunk (each a varint uint64) - - write out the byte slice containing all the chunk data - -If you know the doc number you're interested in, this format lets you jump to the correct chunk (docNum/chunkFactor) directly and then seek within that chunk until you find it. - -## posting details (location) section - -- for each posting list - - produce a slice containing multiple consecutive chunks (each chunk is varint stream) - - produce a slice remembering offsets of where each chunk starts - - preparation phase: - - for each hit in the posting list - - if this hit is in next chunk close out encoding of last chunk and record offset start of next - - encode field (uint16) - - encode field pos (uint64) - - encode field start (uint64) - - encode field end (uint64) - - encode number of array positions to follow (uint64) - - encode each array position (each uint64) - - file writing phase: - - remember start position for this posting list details - - write out number of chunks that follow (varint uint64) - - write out length of each chunk (each a varint uint64) - - write out the byte slice containing all the chunk data - -If you know the doc number you're interested in, this format lets you jump to the correct chunk (docNum/chunkFactor) directly and then seek within that chunk until you find it. - -## postings list section - -- for each posting list - - preparation phase: - - encode roaring bitmap posting list to bytes (so we know the length) - - file writing phase: - - remember the start position for this posting list - - write freq/norm details offset (remembered from previous, as varint uint64) - - write location details offset (remembered from previous, as varint uint64) - - write length of encoded roaring bitmap - - write the serialized roaring bitmap data - -## dictionary - -- for each field - - preparation phase: - - encode vellum FST with dictionary data pointing to file offset of posting list (remembered from previous) - - file writing phase: - - remember the start position of this persistDictionary - - write length of vellum data (varint uint64) - - write out vellum data - -## fields section - -- for each field - - file writing phase: - - remember start offset for each field - - write dictionary address (remembered from previous) (varint uint64) - - write length of field name (varint uint64) - - write field name bytes - -## fields idx - -- for each field - - file writing phase: - - write big endian uint64 of start offset for each field - -NOTE: currently we don't know or record the length of this fields index. Instead we rely on the fact that we know it immediately precedes a footer of known size. - -## fields DocValue - -- for each field - - preparation phase: - - produce a slice containing multiple consecutive chunks, where each chunk is composed of a meta section followed by compressed columnar field data - - produce a slice remembering the length of each chunk - - file writing phase: - - remember the start position of this first field DocValue offset in the footer - - write out number of chunks that follow (varint uint64) - - write out length of each chunk (each a varint uint64) - - write out the byte slice containing all the chunk data - -NOTE: currently the meta header inside each chunk gives clue to the location offsets and size of the data pertaining to a given docID and any -read operation leverage that meta information to extract the document specific data from the file. - -## footer - -- file writing phase - - write number of docs (big endian uint64) - - write stored field index location (big endian uint64) - - write field index location (big endian uint64) - - write field docValue location (big endian uint64) - - write out chunk factor (big endian uint32) - - write out version (big endian uint32) - - write out file CRC of everything preceding this (big endian uint32) diff --git a/vendor/github.com/blevesearch/zap/v11/build.go b/vendor/github.com/blevesearch/zap/v11/build.go deleted file mode 100644 index bac1edb5..00000000 --- a/vendor/github.com/blevesearch/zap/v11/build.go +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "bufio" - "math" - "os" - - "github.com/couchbase/vellum" -) - -const Version uint32 = 11 - -const Type string = "zap" - -const fieldNotUninverted = math.MaxUint64 - -func (sb *SegmentBase) Persist(path string) error { - return PersistSegmentBase(sb, path) -} - -// PersistSegmentBase persists SegmentBase in the zap file format. -func PersistSegmentBase(sb *SegmentBase, path string) error { - flag := os.O_RDWR | os.O_CREATE - - f, err := os.OpenFile(path, flag, 0600) - if err != nil { - return err - } - - cleanup := func() { - _ = f.Close() - _ = os.Remove(path) - } - - br := bufio.NewWriter(f) - - _, err = br.Write(sb.mem) - if err != nil { - cleanup() - return err - } - - err = persistFooter(sb.numDocs, sb.storedIndexOffset, sb.fieldsIndexOffset, sb.docValueOffset, - sb.chunkFactor, sb.memCRC, br) - if err != nil { - cleanup() - return err - } - - err = br.Flush() - if err != nil { - cleanup() - return err - } - - err = f.Sync() - if err != nil { - cleanup() - return err - } - - err = f.Close() - if err != nil { - cleanup() - return err - } - - return nil -} - -func persistStoredFieldValues(fieldID int, - storedFieldValues [][]byte, stf []byte, spf [][]uint64, - curr int, metaEncode varintEncoder, data []byte) ( - int, []byte, error) { - for i := 0; i < len(storedFieldValues); i++ { - // encode field - _, err := metaEncode(uint64(fieldID)) - if err != nil { - return 0, nil, err - } - // encode type - _, err = metaEncode(uint64(stf[i])) - if err != nil { - return 0, nil, err - } - // encode start offset - _, err = metaEncode(uint64(curr)) - if err != nil { - return 0, nil, err - } - // end len - _, err = metaEncode(uint64(len(storedFieldValues[i]))) - if err != nil { - return 0, nil, err - } - // encode number of array pos - _, err = metaEncode(uint64(len(spf[i]))) - if err != nil { - return 0, nil, err - } - // encode all array positions - for _, pos := range spf[i] { - _, err = metaEncode(pos) - if err != nil { - return 0, nil, err - } - } - - data = append(data, storedFieldValues[i]...) - curr += len(storedFieldValues[i]) - } - - return curr, data, nil -} - -func InitSegmentBase(mem []byte, memCRC uint32, chunkFactor uint32, - fieldsMap map[string]uint16, fieldsInv []string, numDocs uint64, - storedIndexOffset uint64, fieldsIndexOffset uint64, docValueOffset uint64, - dictLocs []uint64) (*SegmentBase, error) { - sb := &SegmentBase{ - mem: mem, - memCRC: memCRC, - chunkFactor: chunkFactor, - fieldsMap: fieldsMap, - fieldsInv: fieldsInv, - numDocs: numDocs, - storedIndexOffset: storedIndexOffset, - fieldsIndexOffset: fieldsIndexOffset, - docValueOffset: docValueOffset, - dictLocs: dictLocs, - fieldDvReaders: make(map[uint16]*docValueReader), - fieldFSTs: make(map[uint16]*vellum.FST), - } - sb.updateSize() - - err := sb.loadDvReaders() - if err != nil { - return nil, err - } - - return sb, nil -} diff --git a/vendor/github.com/blevesearch/zap/v11/count.go b/vendor/github.com/blevesearch/zap/v11/count.go deleted file mode 100644 index 50290f88..00000000 --- a/vendor/github.com/blevesearch/zap/v11/count.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "hash/crc32" - "io" - - "github.com/blevesearch/bleve/index/scorch/segment" -) - -// CountHashWriter is a wrapper around a Writer which counts the number of -// bytes which have been written and computes a crc32 hash -type CountHashWriter struct { - w io.Writer - crc uint32 - n int - s segment.StatsReporter -} - -// NewCountHashWriter returns a CountHashWriter which wraps the provided Writer -func NewCountHashWriter(w io.Writer) *CountHashWriter { - return &CountHashWriter{w: w} -} - -func NewCountHashWriterWithStatsReporter(w io.Writer, s segment.StatsReporter) *CountHashWriter { - return &CountHashWriter{w: w, s: s} -} - -// Write writes the provided bytes to the wrapped writer and counts the bytes -func (c *CountHashWriter) Write(b []byte) (int, error) { - n, err := c.w.Write(b) - c.crc = crc32.Update(c.crc, crc32.IEEETable, b[:n]) - c.n += n - if c.s != nil { - c.s.ReportBytesWritten(uint64(n)) - } - return n, err -} - -// Count returns the number of bytes written -func (c *CountHashWriter) Count() int { - return c.n -} - -// Sum32 returns the CRC-32 hash of the content written to this writer -func (c *CountHashWriter) Sum32() uint32 { - return c.crc -} diff --git a/vendor/github.com/blevesearch/zap/v11/dict.go b/vendor/github.com/blevesearch/zap/v11/dict.go deleted file mode 100644 index ad4a8f8d..00000000 --- a/vendor/github.com/blevesearch/zap/v11/dict.go +++ /dev/null @@ -1,263 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "bytes" - "fmt" - - "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/couchbase/vellum" -) - -// Dictionary is the zap representation of the term dictionary -type Dictionary struct { - sb *SegmentBase - field string - fieldID uint16 - fst *vellum.FST - fstReader *vellum.Reader -} - -// PostingsList returns the postings list for the specified term -func (d *Dictionary) PostingsList(term []byte, except *roaring.Bitmap, - prealloc segment.PostingsList) (segment.PostingsList, error) { - var preallocPL *PostingsList - pl, ok := prealloc.(*PostingsList) - if ok && pl != nil { - preallocPL = pl - } - return d.postingsList(term, except, preallocPL) -} - -func (d *Dictionary) postingsList(term []byte, except *roaring.Bitmap, rv *PostingsList) (*PostingsList, error) { - if d.fstReader == nil { - if rv == nil || rv == emptyPostingsList { - return emptyPostingsList, nil - } - return d.postingsListInit(rv, except), nil - } - - postingsOffset, exists, err := d.fstReader.Get(term) - if err != nil { - return nil, fmt.Errorf("vellum err: %v", err) - } - if !exists { - if rv == nil || rv == emptyPostingsList { - return emptyPostingsList, nil - } - return d.postingsListInit(rv, except), nil - } - - return d.postingsListFromOffset(postingsOffset, except, rv) -} - -func (d *Dictionary) postingsListFromOffset(postingsOffset uint64, except *roaring.Bitmap, rv *PostingsList) (*PostingsList, error) { - rv = d.postingsListInit(rv, except) - - err := rv.read(postingsOffset, d) - if err != nil { - return nil, err - } - - return rv, nil -} - -func (d *Dictionary) postingsListInit(rv *PostingsList, except *roaring.Bitmap) *PostingsList { - if rv == nil || rv == emptyPostingsList { - rv = &PostingsList{} - } else { - postings := rv.postings - if postings != nil { - postings.Clear() - } - - *rv = PostingsList{} // clear the struct - - rv.postings = postings - } - rv.sb = d.sb - rv.except = except - return rv -} - -func (d *Dictionary) Contains(key []byte) (bool, error) { - return d.fst.Contains(key) -} - -// Iterator returns an iterator for this dictionary -func (d *Dictionary) Iterator() segment.DictionaryIterator { - rv := &DictionaryIterator{ - d: d, - } - - if d.fst != nil { - itr, err := d.fst.Iterator(nil, nil) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - } - - return rv -} - -// PrefixIterator returns an iterator which only visits terms having the -// the specified prefix -func (d *Dictionary) PrefixIterator(prefix string) segment.DictionaryIterator { - rv := &DictionaryIterator{ - d: d, - } - - kBeg := []byte(prefix) - kEnd := segment.IncrementBytes(kBeg) - - if d.fst != nil { - itr, err := d.fst.Iterator(kBeg, kEnd) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - } - - return rv -} - -// RangeIterator returns an iterator which only visits terms between the -// start and end terms. NOTE: bleve.index API specifies the end is inclusive. -func (d *Dictionary) RangeIterator(start, end string) segment.DictionaryIterator { - rv := &DictionaryIterator{ - d: d, - } - - // need to increment the end position to be inclusive - var endBytes []byte - if len(end) > 0 { - endBytes = []byte(end) - if endBytes[len(endBytes)-1] < 0xff { - endBytes[len(endBytes)-1]++ - } else { - endBytes = append(endBytes, 0xff) - } - } - - if d.fst != nil { - itr, err := d.fst.Iterator([]byte(start), endBytes) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - } - - return rv -} - -// AutomatonIterator returns an iterator which only visits terms -// having the the vellum automaton and start/end key range -func (d *Dictionary) AutomatonIterator(a vellum.Automaton, - startKeyInclusive, endKeyExclusive []byte) segment.DictionaryIterator { - rv := &DictionaryIterator{ - d: d, - } - - if d.fst != nil { - itr, err := d.fst.Search(a, startKeyInclusive, endKeyExclusive) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - } - - return rv -} - -func (d *Dictionary) OnlyIterator(onlyTerms [][]byte, - includeCount bool) segment.DictionaryIterator { - - rv := &DictionaryIterator{ - d: d, - omitCount: !includeCount, - } - - var buf bytes.Buffer - builder, err := vellum.New(&buf, nil) - if err != nil { - rv.err = err - return rv - } - for _, term := range onlyTerms { - err = builder.Insert(term, 0) - if err != nil { - rv.err = err - return rv - } - } - err = builder.Close() - if err != nil { - rv.err = err - return rv - } - - onlyFST, err := vellum.Load(buf.Bytes()) - if err != nil { - rv.err = err - return rv - } - - itr, err := d.fst.Search(onlyFST, nil, nil) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - - return rv -} - -// DictionaryIterator is an iterator for term dictionary -type DictionaryIterator struct { - d *Dictionary - itr vellum.Iterator - err error - tmp PostingsList - entry index.DictEntry - omitCount bool -} - -// Next returns the next entry in the dictionary -func (i *DictionaryIterator) Next() (*index.DictEntry, error) { - if i.err != nil && i.err != vellum.ErrIteratorDone { - return nil, i.err - } else if i.itr == nil || i.err == vellum.ErrIteratorDone { - return nil, nil - } - term, postingsOffset := i.itr.Current() - i.entry.Term = string(term) - if !i.omitCount { - i.err = i.tmp.read(postingsOffset, i.d) - if i.err != nil { - return nil, i.err - } - i.entry.Count = i.tmp.Count() - } - i.err = i.itr.Next() - return &i.entry, nil -} diff --git a/vendor/github.com/blevesearch/zap/v11/docvalues.go b/vendor/github.com/blevesearch/zap/v11/docvalues.go deleted file mode 100644 index 2566dc6d..00000000 --- a/vendor/github.com/blevesearch/zap/v11/docvalues.go +++ /dev/null @@ -1,307 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "bytes" - "encoding/binary" - "fmt" - "math" - "reflect" - "sort" - - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/blevesearch/bleve/size" - "github.com/golang/snappy" -) - -var reflectStaticSizedocValueReader int - -func init() { - var dvi docValueReader - reflectStaticSizedocValueReader = int(reflect.TypeOf(dvi).Size()) -} - -type docNumTermsVisitor func(docNum uint64, terms []byte) error - -type docVisitState struct { - dvrs map[uint16]*docValueReader - segment *SegmentBase -} - -type docValueReader struct { - field string - curChunkNum uint64 - chunkOffsets []uint64 - dvDataLoc uint64 - curChunkHeader []MetaData - curChunkData []byte // compressed data cache - uncompressed []byte // temp buf for snappy decompression -} - -func (di *docValueReader) size() int { - return reflectStaticSizedocValueReader + size.SizeOfPtr + - len(di.field) + - len(di.chunkOffsets)*size.SizeOfUint64 + - len(di.curChunkHeader)*reflectStaticSizeMetaData + - len(di.curChunkData) -} - -func (di *docValueReader) cloneInto(rv *docValueReader) *docValueReader { - if rv == nil { - rv = &docValueReader{} - } - - rv.field = di.field - rv.curChunkNum = math.MaxUint64 - rv.chunkOffsets = di.chunkOffsets // immutable, so it's sharable - rv.dvDataLoc = di.dvDataLoc - rv.curChunkHeader = rv.curChunkHeader[:0] - rv.curChunkData = nil - rv.uncompressed = rv.uncompressed[:0] - - return rv -} - -func (di *docValueReader) curChunkNumber() uint64 { - return di.curChunkNum -} - -func (s *SegmentBase) loadFieldDocValueReader(field string, - fieldDvLocStart, fieldDvLocEnd uint64) (*docValueReader, error) { - // get the docValue offset for the given fields - if fieldDvLocStart == fieldNotUninverted { - // no docValues found, nothing to do - return nil, nil - } - - // read the number of chunks, and chunk offsets position - var numChunks, chunkOffsetsPosition uint64 - - if fieldDvLocEnd-fieldDvLocStart > 16 { - numChunks = binary.BigEndian.Uint64(s.mem[fieldDvLocEnd-8 : fieldDvLocEnd]) - // read the length of chunk offsets - chunkOffsetsLen := binary.BigEndian.Uint64(s.mem[fieldDvLocEnd-16 : fieldDvLocEnd-8]) - // acquire position of chunk offsets - chunkOffsetsPosition = (fieldDvLocEnd - 16) - chunkOffsetsLen - } else { - return nil, fmt.Errorf("loadFieldDocValueReader: fieldDvLoc too small: %d-%d", fieldDvLocEnd, fieldDvLocStart) - } - - fdvIter := &docValueReader{ - curChunkNum: math.MaxUint64, - field: field, - chunkOffsets: make([]uint64, int(numChunks)), - } - - // read the chunk offsets - var offset uint64 - for i := 0; i < int(numChunks); i++ { - loc, read := binary.Uvarint(s.mem[chunkOffsetsPosition+offset : chunkOffsetsPosition+offset+binary.MaxVarintLen64]) - if read <= 0 { - return nil, fmt.Errorf("corrupted chunk offset during segment load") - } - fdvIter.chunkOffsets[i] = loc - offset += uint64(read) - } - - // set the data offset - fdvIter.dvDataLoc = fieldDvLocStart - - return fdvIter, nil -} - -func (di *docValueReader) loadDvChunk(chunkNumber uint64, s *SegmentBase) error { - // advance to the chunk where the docValues - // reside for the given docNum - destChunkDataLoc, curChunkEnd := di.dvDataLoc, di.dvDataLoc - start, end := readChunkBoundary(int(chunkNumber), di.chunkOffsets) - if start >= end { - di.curChunkHeader = di.curChunkHeader[:0] - di.curChunkData = nil - di.curChunkNum = chunkNumber - di.uncompressed = di.uncompressed[:0] - return nil - } - - destChunkDataLoc += start - curChunkEnd += end - - // read the number of docs reside in the chunk - numDocs, read := binary.Uvarint(s.mem[destChunkDataLoc : destChunkDataLoc+binary.MaxVarintLen64]) - if read <= 0 { - return fmt.Errorf("failed to read the chunk") - } - chunkMetaLoc := destChunkDataLoc + uint64(read) - - offset := uint64(0) - if cap(di.curChunkHeader) < int(numDocs) { - di.curChunkHeader = make([]MetaData, int(numDocs)) - } else { - di.curChunkHeader = di.curChunkHeader[:int(numDocs)] - } - for i := 0; i < int(numDocs); i++ { - di.curChunkHeader[i].DocNum, read = binary.Uvarint(s.mem[chunkMetaLoc+offset : chunkMetaLoc+offset+binary.MaxVarintLen64]) - offset += uint64(read) - di.curChunkHeader[i].DocDvOffset, read = binary.Uvarint(s.mem[chunkMetaLoc+offset : chunkMetaLoc+offset+binary.MaxVarintLen64]) - offset += uint64(read) - } - - compressedDataLoc := chunkMetaLoc + offset - dataLength := curChunkEnd - compressedDataLoc - di.curChunkData = s.mem[compressedDataLoc : compressedDataLoc+dataLength] - di.curChunkNum = chunkNumber - di.uncompressed = di.uncompressed[:0] - return nil -} - -func (di *docValueReader) iterateAllDocValues(s *SegmentBase, visitor docNumTermsVisitor) error { - for i := 0; i < len(di.chunkOffsets); i++ { - err := di.loadDvChunk(uint64(i), s) - if err != nil { - return err - } - if di.curChunkData == nil || len(di.curChunkHeader) == 0 { - continue - } - - // uncompress the already loaded data - uncompressed, err := snappy.Decode(di.uncompressed[:cap(di.uncompressed)], di.curChunkData) - if err != nil { - return err - } - di.uncompressed = uncompressed - - start := uint64(0) - for _, entry := range di.curChunkHeader { - err = visitor(entry.DocNum, uncompressed[start:entry.DocDvOffset]) - if err != nil { - return err - } - - start = entry.DocDvOffset - } - } - - return nil -} - -func (di *docValueReader) visitDocValues(docNum uint64, - visitor index.DocumentFieldTermVisitor) error { - // binary search the term locations for the docNum - start, end := di.getDocValueLocs(docNum) - if start == math.MaxUint64 || end == math.MaxUint64 || start == end { - return nil - } - - var uncompressed []byte - var err error - // use the uncompressed copy if available - if len(di.uncompressed) > 0 { - uncompressed = di.uncompressed - } else { - // uncompress the already loaded data - uncompressed, err = snappy.Decode(di.uncompressed[:cap(di.uncompressed)], di.curChunkData) - if err != nil { - return err - } - di.uncompressed = uncompressed - } - - // pick the terms for the given docNum - uncompressed = uncompressed[start:end] - for { - i := bytes.Index(uncompressed, termSeparatorSplitSlice) - if i < 0 { - break - } - - visitor(di.field, uncompressed[0:i]) - uncompressed = uncompressed[i+1:] - } - - return nil -} - -func (di *docValueReader) getDocValueLocs(docNum uint64) (uint64, uint64) { - i := sort.Search(len(di.curChunkHeader), func(i int) bool { - return di.curChunkHeader[i].DocNum >= docNum - }) - if i < len(di.curChunkHeader) && di.curChunkHeader[i].DocNum == docNum { - return ReadDocValueBoundary(i, di.curChunkHeader) - } - return math.MaxUint64, math.MaxUint64 -} - -// VisitDocumentFieldTerms is an implementation of the -// DocumentFieldTermVisitable interface -func (s *SegmentBase) VisitDocumentFieldTerms(localDocNum uint64, fields []string, - visitor index.DocumentFieldTermVisitor, dvsIn segment.DocVisitState) ( - segment.DocVisitState, error) { - dvs, ok := dvsIn.(*docVisitState) - if !ok || dvs == nil { - dvs = &docVisitState{} - } else { - if dvs.segment != s { - dvs.segment = s - dvs.dvrs = nil - } - } - - var fieldIDPlus1 uint16 - if dvs.dvrs == nil { - dvs.dvrs = make(map[uint16]*docValueReader, len(fields)) - for _, field := range fields { - if fieldIDPlus1, ok = s.fieldsMap[field]; !ok { - continue - } - fieldID := fieldIDPlus1 - 1 - if dvIter, exists := s.fieldDvReaders[fieldID]; exists && - dvIter != nil { - dvs.dvrs[fieldID] = dvIter.cloneInto(dvs.dvrs[fieldID]) - } - } - } - - // find the chunkNumber where the docValues are stored - docInChunk := localDocNum / uint64(s.chunkFactor) - var dvr *docValueReader - for _, field := range fields { - if fieldIDPlus1, ok = s.fieldsMap[field]; !ok { - continue - } - fieldID := fieldIDPlus1 - 1 - if dvr, ok = dvs.dvrs[fieldID]; ok && dvr != nil { - // check if the chunk is already loaded - if docInChunk != dvr.curChunkNumber() { - err := dvr.loadDvChunk(docInChunk, s) - if err != nil { - return dvs, err - } - } - - _ = dvr.visitDocValues(localDocNum, visitor) - } - } - return dvs, nil -} - -// VisitableDocValueFields returns the list of fields with -// persisted doc value terms ready to be visitable using the -// VisitDocumentFieldTerms method. -func (s *SegmentBase) VisitableDocValueFields() ([]string, error) { - return s.fieldDvNames, nil -} diff --git a/vendor/github.com/blevesearch/zap/v11/enumerator.go b/vendor/github.com/blevesearch/zap/v11/enumerator.go deleted file mode 100644 index cd6ff73c..00000000 --- a/vendor/github.com/blevesearch/zap/v11/enumerator.go +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright (c) 2018 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "bytes" - - "github.com/couchbase/vellum" -) - -// enumerator provides an ordered traversal of multiple vellum -// iterators. Like JOIN of iterators, the enumerator produces a -// sequence of (key, iteratorIndex, value) tuples, sorted by key ASC, -// then iteratorIndex ASC, where the same key might be seen or -// repeated across multiple child iterators. -type enumerator struct { - itrs []vellum.Iterator - currKs [][]byte - currVs []uint64 - - lowK []byte - lowIdxs []int - lowCurr int -} - -// newEnumerator returns a new enumerator over the vellum Iterators -func newEnumerator(itrs []vellum.Iterator) (*enumerator, error) { - rv := &enumerator{ - itrs: itrs, - currKs: make([][]byte, len(itrs)), - currVs: make([]uint64, len(itrs)), - lowIdxs: make([]int, 0, len(itrs)), - } - for i, itr := range rv.itrs { - rv.currKs[i], rv.currVs[i] = itr.Current() - } - rv.updateMatches(false) - if rv.lowK == nil && len(rv.lowIdxs) == 0 { - return rv, vellum.ErrIteratorDone - } - return rv, nil -} - -// updateMatches maintains the low key matches based on the currKs -func (m *enumerator) updateMatches(skipEmptyKey bool) { - m.lowK = nil - m.lowIdxs = m.lowIdxs[:0] - m.lowCurr = 0 - - for i, key := range m.currKs { - if (key == nil && m.currVs[i] == 0) || // in case of empty iterator - (len(key) == 0 && skipEmptyKey) { // skip empty keys - continue - } - - cmp := bytes.Compare(key, m.lowK) - if cmp < 0 || len(m.lowIdxs) == 0 { - // reached a new low - m.lowK = key - m.lowIdxs = m.lowIdxs[:0] - m.lowIdxs = append(m.lowIdxs, i) - } else if cmp == 0 { - m.lowIdxs = append(m.lowIdxs, i) - } - } -} - -// Current returns the enumerator's current key, iterator-index, and -// value. If the enumerator is not pointing at a valid value (because -// Next returned an error previously), Current will return nil,0,0. -func (m *enumerator) Current() ([]byte, int, uint64) { - var i int - var v uint64 - if m.lowCurr < len(m.lowIdxs) { - i = m.lowIdxs[m.lowCurr] - v = m.currVs[i] - } - return m.lowK, i, v -} - -// Next advances the enumerator to the next key/iterator/value result, -// else vellum.ErrIteratorDone is returned. -func (m *enumerator) Next() error { - m.lowCurr += 1 - if m.lowCurr >= len(m.lowIdxs) { - // move all the current low iterators forwards - for _, vi := range m.lowIdxs { - err := m.itrs[vi].Next() - if err != nil && err != vellum.ErrIteratorDone { - return err - } - m.currKs[vi], m.currVs[vi] = m.itrs[vi].Current() - } - // can skip any empty keys encountered at this point - m.updateMatches(true) - } - if m.lowK == nil && len(m.lowIdxs) == 0 { - return vellum.ErrIteratorDone - } - return nil -} - -// Close all the underlying Iterators. The first error, if any, will -// be returned. -func (m *enumerator) Close() error { - var rv error - for _, itr := range m.itrs { - err := itr.Close() - if rv == nil { - rv = err - } - } - return rv -} diff --git a/vendor/github.com/blevesearch/zap/v11/merge.go b/vendor/github.com/blevesearch/zap/v11/merge.go deleted file mode 100644 index 0d3f5458..00000000 --- a/vendor/github.com/blevesearch/zap/v11/merge.go +++ /dev/null @@ -1,860 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "bufio" - "bytes" - "encoding/binary" - "fmt" - "math" - "os" - "sort" - - "github.com/RoaringBitmap/roaring" - seg "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/couchbase/vellum" - "github.com/golang/snappy" -) - -var DefaultFileMergerBufferSize = 1024 * 1024 - -const docDropped = math.MaxUint64 // sentinel docNum to represent a deleted doc - -// Merge takes a slice of segments and bit masks describing which -// documents may be dropped, and creates a new segment containing the -// remaining data. This new segment is built at the specified path. -func (*ZapPlugin) Merge(segments []seg.Segment, drops []*roaring.Bitmap, path string, - closeCh chan struct{}, s seg.StatsReporter) ( - [][]uint64, uint64, error) { - - segmentBases := make([]*SegmentBase, len(segments)) - for segmenti, segment := range segments { - switch segmentx := segment.(type) { - case *Segment: - segmentBases[segmenti] = &segmentx.SegmentBase - case *SegmentBase: - segmentBases[segmenti] = segmentx - default: - panic(fmt.Sprintf("oops, unexpected segment type: %T", segment)) - } - } - return mergeSegmentBases(segmentBases, drops, path, defaultChunkFactor, closeCh, s) -} - -func mergeSegmentBases(segmentBases []*SegmentBase, drops []*roaring.Bitmap, path string, - chunkFactor uint32, closeCh chan struct{}, s seg.StatsReporter) ( - [][]uint64, uint64, error) { - flag := os.O_RDWR | os.O_CREATE - - f, err := os.OpenFile(path, flag, 0600) - if err != nil { - return nil, 0, err - } - - cleanup := func() { - _ = f.Close() - _ = os.Remove(path) - } - - // buffer the output - br := bufio.NewWriterSize(f, DefaultFileMergerBufferSize) - - // wrap it for counting (tracking offsets) - cr := NewCountHashWriterWithStatsReporter(br, s) - - newDocNums, numDocs, storedIndexOffset, fieldsIndexOffset, docValueOffset, _, _, _, err := - MergeToWriter(segmentBases, drops, chunkFactor, cr, closeCh) - if err != nil { - cleanup() - return nil, 0, err - } - - err = persistFooter(numDocs, storedIndexOffset, fieldsIndexOffset, - docValueOffset, chunkFactor, cr.Sum32(), cr) - if err != nil { - cleanup() - return nil, 0, err - } - - err = br.Flush() - if err != nil { - cleanup() - return nil, 0, err - } - - err = f.Sync() - if err != nil { - cleanup() - return nil, 0, err - } - - err = f.Close() - if err != nil { - cleanup() - return nil, 0, err - } - - return newDocNums, uint64(cr.Count()), nil -} - -func MergeToWriter(segments []*SegmentBase, drops []*roaring.Bitmap, - chunkFactor uint32, cr *CountHashWriter, closeCh chan struct{}) ( - newDocNums [][]uint64, - numDocs, storedIndexOffset, fieldsIndexOffset, docValueOffset uint64, - dictLocs []uint64, fieldsInv []string, fieldsMap map[string]uint16, - err error) { - docValueOffset = uint64(fieldNotUninverted) - - var fieldsSame bool - fieldsSame, fieldsInv = mergeFields(segments) - fieldsMap = mapFields(fieldsInv) - - numDocs = computeNewDocCount(segments, drops) - - if isClosed(closeCh) { - return nil, 0, 0, 0, 0, nil, nil, nil, seg.ErrClosed - } - - if numDocs > 0 { - storedIndexOffset, newDocNums, err = mergeStoredAndRemap(segments, drops, - fieldsMap, fieldsInv, fieldsSame, numDocs, cr, closeCh) - if err != nil { - return nil, 0, 0, 0, 0, nil, nil, nil, err - } - - dictLocs, docValueOffset, err = persistMergedRest(segments, drops, - fieldsInv, fieldsMap, fieldsSame, - newDocNums, numDocs, chunkFactor, cr, closeCh) - if err != nil { - return nil, 0, 0, 0, 0, nil, nil, nil, err - } - } else { - dictLocs = make([]uint64, len(fieldsInv)) - } - - fieldsIndexOffset, err = persistFields(fieldsInv, cr, dictLocs) - if err != nil { - return nil, 0, 0, 0, 0, nil, nil, nil, err - } - - return newDocNums, numDocs, storedIndexOffset, fieldsIndexOffset, docValueOffset, dictLocs, fieldsInv, fieldsMap, nil -} - -// mapFields takes the fieldsInv list and returns a map of fieldName -// to fieldID+1 -func mapFields(fields []string) map[string]uint16 { - rv := make(map[string]uint16, len(fields)) - for i, fieldName := range fields { - rv[fieldName] = uint16(i) + 1 - } - return rv -} - -// computeNewDocCount determines how many documents will be in the newly -// merged segment when obsoleted docs are dropped -func computeNewDocCount(segments []*SegmentBase, drops []*roaring.Bitmap) uint64 { - var newDocCount uint64 - for segI, segment := range segments { - newDocCount += segment.numDocs - if drops[segI] != nil { - newDocCount -= drops[segI].GetCardinality() - } - } - return newDocCount -} - -func persistMergedRest(segments []*SegmentBase, dropsIn []*roaring.Bitmap, - fieldsInv []string, fieldsMap map[string]uint16, fieldsSame bool, - newDocNumsIn [][]uint64, newSegDocCount uint64, chunkFactor uint32, - w *CountHashWriter, closeCh chan struct{}) ([]uint64, uint64, error) { - - var bufMaxVarintLen64 []byte = make([]byte, binary.MaxVarintLen64) - var bufLoc []uint64 - - var postings *PostingsList - var postItr *PostingsIterator - - rv := make([]uint64, len(fieldsInv)) - fieldDvLocsStart := make([]uint64, len(fieldsInv)) - fieldDvLocsEnd := make([]uint64, len(fieldsInv)) - - tfEncoder := newChunkedIntCoder(uint64(chunkFactor), newSegDocCount-1) - locEncoder := newChunkedIntCoder(uint64(chunkFactor), newSegDocCount-1) - - var vellumBuf bytes.Buffer - newVellum, err := vellum.New(&vellumBuf, nil) - if err != nil { - return nil, 0, err - } - - newRoaring := roaring.NewBitmap() - - // for each field - for fieldID, fieldName := range fieldsInv { - - // collect FST iterators from all active segments for this field - var newDocNums [][]uint64 - var drops []*roaring.Bitmap - var dicts []*Dictionary - var itrs []vellum.Iterator - - var segmentsInFocus []*SegmentBase - - for segmentI, segment := range segments { - - // check for the closure in meantime - if isClosed(closeCh) { - return nil, 0, seg.ErrClosed - } - - dict, err2 := segment.dictionary(fieldName) - if err2 != nil { - return nil, 0, err2 - } - if dict != nil && dict.fst != nil { - itr, err2 := dict.fst.Iterator(nil, nil) - if err2 != nil && err2 != vellum.ErrIteratorDone { - return nil, 0, err2 - } - if itr != nil { - newDocNums = append(newDocNums, newDocNumsIn[segmentI]) - if dropsIn[segmentI] != nil && !dropsIn[segmentI].IsEmpty() { - drops = append(drops, dropsIn[segmentI]) - } else { - drops = append(drops, nil) - } - dicts = append(dicts, dict) - itrs = append(itrs, itr) - segmentsInFocus = append(segmentsInFocus, segment) - } - } - } - - var prevTerm []byte - - newRoaring.Clear() - - var lastDocNum, lastFreq, lastNorm uint64 - - // determines whether to use "1-hit" encoding optimization - // when a term appears in only 1 doc, with no loc info, - // has freq of 1, and the docNum fits into 31-bits - use1HitEncoding := func(termCardinality uint64) (bool, uint64, uint64) { - if termCardinality == uint64(1) && locEncoder.FinalSize() <= 0 { - docNum := uint64(newRoaring.Minimum()) - if under32Bits(docNum) && docNum == lastDocNum && lastFreq == 1 { - return true, docNum, lastNorm - } - } - return false, 0, 0 - } - - finishTerm := func(term []byte) error { - tfEncoder.Close() - locEncoder.Close() - - postingsOffset, err := writePostings(newRoaring, - tfEncoder, locEncoder, use1HitEncoding, w, bufMaxVarintLen64) - if err != nil { - return err - } - - if postingsOffset > 0 { - err = newVellum.Insert(term, postingsOffset) - if err != nil { - return err - } - } - - newRoaring.Clear() - - tfEncoder.Reset() - locEncoder.Reset() - - lastDocNum = 0 - lastFreq = 0 - lastNorm = 0 - - return nil - } - - enumerator, err := newEnumerator(itrs) - - for err == nil { - term, itrI, postingsOffset := enumerator.Current() - - if !bytes.Equal(prevTerm, term) { - // check for the closure in meantime - if isClosed(closeCh) { - return nil, 0, seg.ErrClosed - } - - // if the term changed, write out the info collected - // for the previous term - err = finishTerm(prevTerm) - if err != nil { - return nil, 0, err - } - } - - postings, err = dicts[itrI].postingsListFromOffset( - postingsOffset, drops[itrI], postings) - if err != nil { - return nil, 0, err - } - - postItr = postings.iterator(true, true, true, postItr) - - if fieldsSame { - // can optimize by copying freq/norm/loc bytes directly - lastDocNum, lastFreq, lastNorm, err = mergeTermFreqNormLocsByCopying( - term, postItr, newDocNums[itrI], newRoaring, - tfEncoder, locEncoder) - } else { - lastDocNum, lastFreq, lastNorm, bufLoc, err = mergeTermFreqNormLocs( - fieldsMap, term, postItr, newDocNums[itrI], newRoaring, - tfEncoder, locEncoder, bufLoc) - } - if err != nil { - return nil, 0, err - } - - prevTerm = prevTerm[:0] // copy to prevTerm in case Next() reuses term mem - prevTerm = append(prevTerm, term...) - - err = enumerator.Next() - } - if err != vellum.ErrIteratorDone { - return nil, 0, err - } - - err = finishTerm(prevTerm) - if err != nil { - return nil, 0, err - } - - dictOffset := uint64(w.Count()) - - err = newVellum.Close() - if err != nil { - return nil, 0, err - } - vellumData := vellumBuf.Bytes() - - // write out the length of the vellum data - n := binary.PutUvarint(bufMaxVarintLen64, uint64(len(vellumData))) - _, err = w.Write(bufMaxVarintLen64[:n]) - if err != nil { - return nil, 0, err - } - - // write this vellum to disk - _, err = w.Write(vellumData) - if err != nil { - return nil, 0, err - } - - rv[fieldID] = dictOffset - - // get the field doc value offset (start) - fieldDvLocsStart[fieldID] = uint64(w.Count()) - - // update the field doc values - fdvEncoder := newChunkedContentCoder(uint64(chunkFactor), newSegDocCount-1, w, true) - - fdvReadersAvailable := false - var dvIterClone *docValueReader - for segmentI, segment := range segmentsInFocus { - // check for the closure in meantime - if isClosed(closeCh) { - return nil, 0, seg.ErrClosed - } - - fieldIDPlus1 := uint16(segment.fieldsMap[fieldName]) - if dvIter, exists := segment.fieldDvReaders[fieldIDPlus1-1]; exists && - dvIter != nil { - fdvReadersAvailable = true - dvIterClone = dvIter.cloneInto(dvIterClone) - err = dvIterClone.iterateAllDocValues(segment, func(docNum uint64, terms []byte) error { - if newDocNums[segmentI][docNum] == docDropped { - return nil - } - err := fdvEncoder.Add(newDocNums[segmentI][docNum], terms) - if err != nil { - return err - } - return nil - }) - if err != nil { - return nil, 0, err - } - } - } - - if fdvReadersAvailable { - err = fdvEncoder.Close() - if err != nil { - return nil, 0, err - } - - // persist the doc value details for this field - _, err = fdvEncoder.Write() - if err != nil { - return nil, 0, err - } - - // get the field doc value offset (end) - fieldDvLocsEnd[fieldID] = uint64(w.Count()) - } else { - fieldDvLocsStart[fieldID] = fieldNotUninverted - fieldDvLocsEnd[fieldID] = fieldNotUninverted - } - - // reset vellum buffer and vellum builder - vellumBuf.Reset() - err = newVellum.Reset(&vellumBuf) - if err != nil { - return nil, 0, err - } - } - - fieldDvLocsOffset := uint64(w.Count()) - - buf := bufMaxVarintLen64 - for i := 0; i < len(fieldDvLocsStart); i++ { - n := binary.PutUvarint(buf, fieldDvLocsStart[i]) - _, err := w.Write(buf[:n]) - if err != nil { - return nil, 0, err - } - n = binary.PutUvarint(buf, fieldDvLocsEnd[i]) - _, err = w.Write(buf[:n]) - if err != nil { - return nil, 0, err - } - } - - return rv, fieldDvLocsOffset, nil -} - -func mergeTermFreqNormLocs(fieldsMap map[string]uint16, term []byte, postItr *PostingsIterator, - newDocNums []uint64, newRoaring *roaring.Bitmap, - tfEncoder *chunkedIntCoder, locEncoder *chunkedIntCoder, bufLoc []uint64) ( - lastDocNum uint64, lastFreq uint64, lastNorm uint64, bufLocOut []uint64, err error) { - next, err := postItr.Next() - for next != nil && err == nil { - hitNewDocNum := newDocNums[next.Number()] - if hitNewDocNum == docDropped { - return 0, 0, 0, nil, fmt.Errorf("see hit with dropped docNum") - } - - newRoaring.Add(uint32(hitNewDocNum)) - - nextFreq := next.Frequency() - nextNorm := uint64(math.Float32bits(float32(next.Norm()))) - - locs := next.Locations() - - err = tfEncoder.Add(hitNewDocNum, - encodeFreqHasLocs(nextFreq, len(locs) > 0), nextNorm) - if err != nil { - return 0, 0, 0, nil, err - } - - if len(locs) > 0 { - numBytesLocs := 0 - for _, loc := range locs { - ap := loc.ArrayPositions() - numBytesLocs += totalUvarintBytes(uint64(fieldsMap[loc.Field()]-1), - loc.Pos(), loc.Start(), loc.End(), uint64(len(ap)), ap) - } - - err = locEncoder.Add(hitNewDocNum, uint64(numBytesLocs)) - if err != nil { - return 0, 0, 0, nil, err - } - - for _, loc := range locs { - ap := loc.ArrayPositions() - if cap(bufLoc) < 5+len(ap) { - bufLoc = make([]uint64, 0, 5+len(ap)) - } - args := bufLoc[0:5] - args[0] = uint64(fieldsMap[loc.Field()] - 1) - args[1] = loc.Pos() - args[2] = loc.Start() - args[3] = loc.End() - args[4] = uint64(len(ap)) - args = append(args, ap...) - err = locEncoder.Add(hitNewDocNum, args...) - if err != nil { - return 0, 0, 0, nil, err - } - } - } - - lastDocNum = hitNewDocNum - lastFreq = nextFreq - lastNorm = nextNorm - - next, err = postItr.Next() - } - - return lastDocNum, lastFreq, lastNorm, bufLoc, err -} - -func mergeTermFreqNormLocsByCopying(term []byte, postItr *PostingsIterator, - newDocNums []uint64, newRoaring *roaring.Bitmap, - tfEncoder *chunkedIntCoder, locEncoder *chunkedIntCoder) ( - lastDocNum uint64, lastFreq uint64, lastNorm uint64, err error) { - nextDocNum, nextFreq, nextNorm, nextFreqNormBytes, nextLocBytes, err := - postItr.nextBytes() - for err == nil && len(nextFreqNormBytes) > 0 { - hitNewDocNum := newDocNums[nextDocNum] - if hitNewDocNum == docDropped { - return 0, 0, 0, fmt.Errorf("see hit with dropped doc num") - } - - newRoaring.Add(uint32(hitNewDocNum)) - err = tfEncoder.AddBytes(hitNewDocNum, nextFreqNormBytes) - if err != nil { - return 0, 0, 0, err - } - - if len(nextLocBytes) > 0 { - err = locEncoder.AddBytes(hitNewDocNum, nextLocBytes) - if err != nil { - return 0, 0, 0, err - } - } - - lastDocNum = hitNewDocNum - lastFreq = nextFreq - lastNorm = nextNorm - - nextDocNum, nextFreq, nextNorm, nextFreqNormBytes, nextLocBytes, err = - postItr.nextBytes() - } - - return lastDocNum, lastFreq, lastNorm, err -} - -func writePostings(postings *roaring.Bitmap, tfEncoder, locEncoder *chunkedIntCoder, - use1HitEncoding func(uint64) (bool, uint64, uint64), - w *CountHashWriter, bufMaxVarintLen64 []byte) ( - offset uint64, err error) { - termCardinality := postings.GetCardinality() - if termCardinality <= 0 { - return 0, nil - } - - if use1HitEncoding != nil { - encodeAs1Hit, docNum1Hit, normBits1Hit := use1HitEncoding(termCardinality) - if encodeAs1Hit { - return FSTValEncode1Hit(docNum1Hit, normBits1Hit), nil - } - } - - tfOffset := uint64(w.Count()) - _, err = tfEncoder.Write(w) - if err != nil { - return 0, err - } - - locOffset := uint64(w.Count()) - _, err = locEncoder.Write(w) - if err != nil { - return 0, err - } - - postingsOffset := uint64(w.Count()) - - n := binary.PutUvarint(bufMaxVarintLen64, tfOffset) - _, err = w.Write(bufMaxVarintLen64[:n]) - if err != nil { - return 0, err - } - - n = binary.PutUvarint(bufMaxVarintLen64, locOffset) - _, err = w.Write(bufMaxVarintLen64[:n]) - if err != nil { - return 0, err - } - - _, err = writeRoaringWithLen(postings, w, bufMaxVarintLen64) - if err != nil { - return 0, err - } - - return postingsOffset, nil -} - -type varintEncoder func(uint64) (int, error) - -func mergeStoredAndRemap(segments []*SegmentBase, drops []*roaring.Bitmap, - fieldsMap map[string]uint16, fieldsInv []string, fieldsSame bool, newSegDocCount uint64, - w *CountHashWriter, closeCh chan struct{}) (uint64, [][]uint64, error) { - var rv [][]uint64 // The remapped or newDocNums for each segment. - - var newDocNum uint64 - - var curr int - var data, compressed []byte - var metaBuf bytes.Buffer - varBuf := make([]byte, binary.MaxVarintLen64) - metaEncode := func(val uint64) (int, error) { - wb := binary.PutUvarint(varBuf, val) - return metaBuf.Write(varBuf[:wb]) - } - - vals := make([][][]byte, len(fieldsInv)) - typs := make([][]byte, len(fieldsInv)) - poss := make([][][]uint64, len(fieldsInv)) - - var posBuf []uint64 - - docNumOffsets := make([]uint64, newSegDocCount) - - vdc := visitDocumentCtxPool.Get().(*visitDocumentCtx) - defer visitDocumentCtxPool.Put(vdc) - - // for each segment - for segI, segment := range segments { - // check for the closure in meantime - if isClosed(closeCh) { - return 0, nil, seg.ErrClosed - } - - segNewDocNums := make([]uint64, segment.numDocs) - - dropsI := drops[segI] - - // optimize when the field mapping is the same across all - // segments and there are no deletions, via byte-copying - // of stored docs bytes directly to the writer - if fieldsSame && (dropsI == nil || dropsI.GetCardinality() == 0) { - err := segment.copyStoredDocs(newDocNum, docNumOffsets, w) - if err != nil { - return 0, nil, err - } - - for i := uint64(0); i < segment.numDocs; i++ { - segNewDocNums[i] = newDocNum - newDocNum++ - } - rv = append(rv, segNewDocNums) - - continue - } - - // for each doc num - for docNum := uint64(0); docNum < segment.numDocs; docNum++ { - // TODO: roaring's API limits docNums to 32-bits? - if dropsI != nil && dropsI.Contains(uint32(docNum)) { - segNewDocNums[docNum] = docDropped - continue - } - - segNewDocNums[docNum] = newDocNum - - curr = 0 - metaBuf.Reset() - data = data[:0] - - posTemp := posBuf - - // collect all the data - for i := 0; i < len(fieldsInv); i++ { - vals[i] = vals[i][:0] - typs[i] = typs[i][:0] - poss[i] = poss[i][:0] - } - err := segment.visitDocument(vdc, docNum, func(field string, typ byte, value []byte, pos []uint64) bool { - fieldID := int(fieldsMap[field]) - 1 - vals[fieldID] = append(vals[fieldID], value) - typs[fieldID] = append(typs[fieldID], typ) - - // copy array positions to preserve them beyond the scope of this callback - var curPos []uint64 - if len(pos) > 0 { - if cap(posTemp) < len(pos) { - posBuf = make([]uint64, len(pos)*len(fieldsInv)) - posTemp = posBuf - } - curPos = posTemp[0:len(pos)] - copy(curPos, pos) - posTemp = posTemp[len(pos):] - } - poss[fieldID] = append(poss[fieldID], curPos) - - return true - }) - if err != nil { - return 0, nil, err - } - - // _id field special case optimizes ExternalID() lookups - idFieldVal := vals[uint16(0)][0] - _, err = metaEncode(uint64(len(idFieldVal))) - if err != nil { - return 0, nil, err - } - - // now walk the non-"_id" fields in order - for fieldID := 1; fieldID < len(fieldsInv); fieldID++ { - storedFieldValues := vals[fieldID] - - stf := typs[fieldID] - spf := poss[fieldID] - - var err2 error - curr, data, err2 = persistStoredFieldValues(fieldID, - storedFieldValues, stf, spf, curr, metaEncode, data) - if err2 != nil { - return 0, nil, err2 - } - } - - metaBytes := metaBuf.Bytes() - - compressed = snappy.Encode(compressed[:cap(compressed)], data) - - // record where we're about to start writing - docNumOffsets[newDocNum] = uint64(w.Count()) - - // write out the meta len and compressed data len - _, err = writeUvarints(w, - uint64(len(metaBytes)), - uint64(len(idFieldVal)+len(compressed))) - if err != nil { - return 0, nil, err - } - // now write the meta - _, err = w.Write(metaBytes) - if err != nil { - return 0, nil, err - } - // now write the _id field val (counted as part of the 'compressed' data) - _, err = w.Write(idFieldVal) - if err != nil { - return 0, nil, err - } - // now write the compressed data - _, err = w.Write(compressed) - if err != nil { - return 0, nil, err - } - - newDocNum++ - } - - rv = append(rv, segNewDocNums) - } - - // return value is the start of the stored index - storedIndexOffset := uint64(w.Count()) - - // now write out the stored doc index - for _, docNumOffset := range docNumOffsets { - err := binary.Write(w, binary.BigEndian, docNumOffset) - if err != nil { - return 0, nil, err - } - } - - return storedIndexOffset, rv, nil -} - -// copyStoredDocs writes out a segment's stored doc info, optimized by -// using a single Write() call for the entire set of bytes. The -// newDocNumOffsets is filled with the new offsets for each doc. -func (s *SegmentBase) copyStoredDocs(newDocNum uint64, newDocNumOffsets []uint64, - w *CountHashWriter) error { - if s.numDocs <= 0 { - return nil - } - - indexOffset0, storedOffset0, _, _, _ := - s.getDocStoredOffsets(0) // the segment's first doc - - indexOffsetN, storedOffsetN, readN, metaLenN, dataLenN := - s.getDocStoredOffsets(s.numDocs - 1) // the segment's last doc - - storedOffset0New := uint64(w.Count()) - - storedBytes := s.mem[storedOffset0 : storedOffsetN+readN+metaLenN+dataLenN] - _, err := w.Write(storedBytes) - if err != nil { - return err - } - - // remap the storedOffset's for the docs into new offsets relative - // to storedOffset0New, filling the given docNumOffsetsOut array - for indexOffset := indexOffset0; indexOffset <= indexOffsetN; indexOffset += 8 { - storedOffset := binary.BigEndian.Uint64(s.mem[indexOffset : indexOffset+8]) - storedOffsetNew := storedOffset - storedOffset0 + storedOffset0New - newDocNumOffsets[newDocNum] = storedOffsetNew - newDocNum += 1 - } - - return nil -} - -// mergeFields builds a unified list of fields used across all the -// input segments, and computes whether the fields are the same across -// segments (which depends on fields to be sorted in the same way -// across segments) -func mergeFields(segments []*SegmentBase) (bool, []string) { - fieldsSame := true - - var segment0Fields []string - if len(segments) > 0 { - segment0Fields = segments[0].Fields() - } - - fieldsExist := map[string]struct{}{} - for _, segment := range segments { - fields := segment.Fields() - for fieldi, field := range fields { - fieldsExist[field] = struct{}{} - if len(segment0Fields) != len(fields) || segment0Fields[fieldi] != field { - fieldsSame = false - } - } - } - - rv := make([]string, 0, len(fieldsExist)) - // ensure _id stays first - rv = append(rv, "_id") - for k := range fieldsExist { - if k != "_id" { - rv = append(rv, k) - } - } - - sort.Strings(rv[1:]) // leave _id as first - - return fieldsSame, rv -} - -func isClosed(closeCh chan struct{}) bool { - select { - case <-closeCh: - return true - default: - return false - } -} diff --git a/vendor/github.com/blevesearch/zap/v11/new.go b/vendor/github.com/blevesearch/zap/v11/new.go deleted file mode 100644 index 6c75f2fd..00000000 --- a/vendor/github.com/blevesearch/zap/v11/new.go +++ /dev/null @@ -1,847 +0,0 @@ -// Copyright (c) 2018 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "bytes" - "encoding/binary" - "math" - "sort" - "sync" - - "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/couchbase/vellum" - "github.com/golang/snappy" -) - -var NewSegmentBufferNumResultsBump int = 100 -var NewSegmentBufferNumResultsFactor float64 = 1.0 -var NewSegmentBufferAvgBytesPerDocFactor float64 = 1.0 - -// ValidateDocFields can be set by applications to perform additional checks -// on fields in a document being added to a new segment, by default it does -// nothing. -// This API is experimental and may be removed at any time. -var ValidateDocFields = func(field document.Field) error { - return nil -} - -var defaultChunkFactor uint32 = 1024 - -// AnalysisResultsToSegmentBase produces an in-memory zap-encoded -// SegmentBase from analysis results -func (z *ZapPlugin) New(results []*index.AnalysisResult) ( - segment.Segment, uint64, error) { - return z.newWithChunkFactor(results, defaultChunkFactor) -} - -func (*ZapPlugin) newWithChunkFactor(results []*index.AnalysisResult, - chunkFactor uint32) (segment.Segment, uint64, error) { - s := interimPool.Get().(*interim) - - var br bytes.Buffer - if s.lastNumDocs > 0 { - // use previous results to initialize the buf with an estimate - // size, but note that the interim instance comes from a - // global interimPool, so multiple scorch instances indexing - // different docs can lead to low quality estimates - estimateAvgBytesPerDoc := int(float64(s.lastOutSize/s.lastNumDocs) * - NewSegmentBufferNumResultsFactor) - estimateNumResults := int(float64(len(results)+NewSegmentBufferNumResultsBump) * - NewSegmentBufferAvgBytesPerDocFactor) - br.Grow(estimateAvgBytesPerDoc * estimateNumResults) - } - - s.results = results - s.chunkFactor = chunkFactor - s.w = NewCountHashWriter(&br) - - storedIndexOffset, fieldsIndexOffset, fdvIndexOffset, dictOffsets, - err := s.convert() - if err != nil { - return nil, uint64(0), err - } - - sb, err := InitSegmentBase(br.Bytes(), s.w.Sum32(), chunkFactor, - s.FieldsMap, s.FieldsInv, uint64(len(results)), - storedIndexOffset, fieldsIndexOffset, fdvIndexOffset, dictOffsets) - - if err == nil && s.reset() == nil { - s.lastNumDocs = len(results) - s.lastOutSize = len(br.Bytes()) - interimPool.Put(s) - } - - return sb, uint64(len(br.Bytes())), err -} - -var interimPool = sync.Pool{New: func() interface{} { return &interim{} }} - -// interim holds temporary working data used while converting from -// analysis results to a zap-encoded segment -type interim struct { - results []*index.AnalysisResult - - chunkFactor uint32 - - w *CountHashWriter - - // FieldsMap adds 1 to field id to avoid zero value issues - // name -> field id + 1 - FieldsMap map[string]uint16 - - // FieldsInv is the inverse of FieldsMap - // field id -> name - FieldsInv []string - - // Term dictionaries for each field - // field id -> term -> postings list id + 1 - Dicts []map[string]uint64 - - // Terms for each field, where terms are sorted ascending - // field id -> []term - DictKeys [][]string - - // Fields whose IncludeDocValues is true - // field id -> bool - IncludeDocValues []bool - - // postings id -> bitmap of docNums - Postings []*roaring.Bitmap - - // postings id -> freq/norm's, one for each docNum in postings - FreqNorms [][]interimFreqNorm - freqNormsBacking []interimFreqNorm - - // postings id -> locs, one for each freq - Locs [][]interimLoc - locsBacking []interimLoc - - numTermsPerPostingsList []int // key is postings list id - numLocsPerPostingsList []int // key is postings list id - - builder *vellum.Builder - builderBuf bytes.Buffer - - metaBuf bytes.Buffer - - tmp0 []byte - tmp1 []byte - - lastNumDocs int - lastOutSize int -} - -func (s *interim) reset() (err error) { - s.results = nil - s.chunkFactor = 0 - s.w = nil - s.FieldsMap = nil - s.FieldsInv = nil - for i := range s.Dicts { - s.Dicts[i] = nil - } - s.Dicts = s.Dicts[:0] - for i := range s.DictKeys { - s.DictKeys[i] = s.DictKeys[i][:0] - } - s.DictKeys = s.DictKeys[:0] - for i := range s.IncludeDocValues { - s.IncludeDocValues[i] = false - } - s.IncludeDocValues = s.IncludeDocValues[:0] - for _, idn := range s.Postings { - idn.Clear() - } - s.Postings = s.Postings[:0] - s.FreqNorms = s.FreqNorms[:0] - for i := range s.freqNormsBacking { - s.freqNormsBacking[i] = interimFreqNorm{} - } - s.freqNormsBacking = s.freqNormsBacking[:0] - s.Locs = s.Locs[:0] - for i := range s.locsBacking { - s.locsBacking[i] = interimLoc{} - } - s.locsBacking = s.locsBacking[:0] - s.numTermsPerPostingsList = s.numTermsPerPostingsList[:0] - s.numLocsPerPostingsList = s.numLocsPerPostingsList[:0] - s.builderBuf.Reset() - if s.builder != nil { - err = s.builder.Reset(&s.builderBuf) - } - s.metaBuf.Reset() - s.tmp0 = s.tmp0[:0] - s.tmp1 = s.tmp1[:0] - s.lastNumDocs = 0 - s.lastOutSize = 0 - - return err -} - -func (s *interim) grabBuf(size int) []byte { - buf := s.tmp0 - if cap(buf) < size { - buf = make([]byte, size) - s.tmp0 = buf - } - return buf[0:size] -} - -type interimStoredField struct { - vals [][]byte - typs []byte - arrayposs [][]uint64 // array positions -} - -type interimFreqNorm struct { - freq uint64 - norm float32 - numLocs int -} - -type interimLoc struct { - fieldID uint16 - pos uint64 - start uint64 - end uint64 - arrayposs []uint64 -} - -func (s *interim) convert() (uint64, uint64, uint64, []uint64, error) { - s.FieldsMap = map[string]uint16{} - - s.getOrDefineField("_id") // _id field is fieldID 0 - - for _, result := range s.results { - for _, field := range result.Document.CompositeFields { - s.getOrDefineField(field.Name()) - } - for _, field := range result.Document.Fields { - s.getOrDefineField(field.Name()) - } - } - - sort.Strings(s.FieldsInv[1:]) // keep _id as first field - - for fieldID, fieldName := range s.FieldsInv { - s.FieldsMap[fieldName] = uint16(fieldID + 1) - } - - if cap(s.IncludeDocValues) >= len(s.FieldsInv) { - s.IncludeDocValues = s.IncludeDocValues[:len(s.FieldsInv)] - } else { - s.IncludeDocValues = make([]bool, len(s.FieldsInv)) - } - - s.prepareDicts() - - for _, dict := range s.DictKeys { - sort.Strings(dict) - } - - s.processDocuments() - - storedIndexOffset, err := s.writeStoredFields() - if err != nil { - return 0, 0, 0, nil, err - } - - var fdvIndexOffset uint64 - var dictOffsets []uint64 - - if len(s.results) > 0 { - fdvIndexOffset, dictOffsets, err = s.writeDicts() - if err != nil { - return 0, 0, 0, nil, err - } - } else { - dictOffsets = make([]uint64, len(s.FieldsInv)) - } - - fieldsIndexOffset, err := persistFields(s.FieldsInv, s.w, dictOffsets) - if err != nil { - return 0, 0, 0, nil, err - } - - return storedIndexOffset, fieldsIndexOffset, fdvIndexOffset, dictOffsets, nil -} - -func (s *interim) getOrDefineField(fieldName string) int { - fieldIDPlus1, exists := s.FieldsMap[fieldName] - if !exists { - fieldIDPlus1 = uint16(len(s.FieldsInv) + 1) - s.FieldsMap[fieldName] = fieldIDPlus1 - s.FieldsInv = append(s.FieldsInv, fieldName) - - s.Dicts = append(s.Dicts, make(map[string]uint64)) - - n := len(s.DictKeys) - if n < cap(s.DictKeys) { - s.DictKeys = s.DictKeys[:n+1] - s.DictKeys[n] = s.DictKeys[n][:0] - } else { - s.DictKeys = append(s.DictKeys, []string(nil)) - } - } - - return int(fieldIDPlus1 - 1) -} - -// fill Dicts and DictKeys from analysis results -func (s *interim) prepareDicts() { - var pidNext int - - var totTFs int - var totLocs int - - visitField := func(fieldID uint16, tfs analysis.TokenFrequencies) { - dict := s.Dicts[fieldID] - dictKeys := s.DictKeys[fieldID] - - for term, tf := range tfs { - pidPlus1, exists := dict[term] - if !exists { - pidNext++ - pidPlus1 = uint64(pidNext) - - dict[term] = pidPlus1 - dictKeys = append(dictKeys, term) - - s.numTermsPerPostingsList = append(s.numTermsPerPostingsList, 0) - s.numLocsPerPostingsList = append(s.numLocsPerPostingsList, 0) - } - - pid := pidPlus1 - 1 - - s.numTermsPerPostingsList[pid] += 1 - s.numLocsPerPostingsList[pid] += len(tf.Locations) - - totLocs += len(tf.Locations) - } - - totTFs += len(tfs) - - s.DictKeys[fieldID] = dictKeys - } - - for _, result := range s.results { - // walk each composite field - for _, field := range result.Document.CompositeFields { - fieldID := uint16(s.getOrDefineField(field.Name())) - _, tf := field.Analyze() - visitField(fieldID, tf) - } - - // walk each field - for i, field := range result.Document.Fields { - fieldID := uint16(s.getOrDefineField(field.Name())) - tf := result.Analyzed[i] - visitField(fieldID, tf) - } - } - - numPostingsLists := pidNext - - if cap(s.Postings) >= numPostingsLists { - s.Postings = s.Postings[:numPostingsLists] - } else { - postings := make([]*roaring.Bitmap, numPostingsLists) - copy(postings, s.Postings[:cap(s.Postings)]) - for i := 0; i < numPostingsLists; i++ { - if postings[i] == nil { - postings[i] = roaring.New() - } - } - s.Postings = postings - } - - if cap(s.FreqNorms) >= numPostingsLists { - s.FreqNorms = s.FreqNorms[:numPostingsLists] - } else { - s.FreqNorms = make([][]interimFreqNorm, numPostingsLists) - } - - if cap(s.freqNormsBacking) >= totTFs { - s.freqNormsBacking = s.freqNormsBacking[:totTFs] - } else { - s.freqNormsBacking = make([]interimFreqNorm, totTFs) - } - - freqNormsBacking := s.freqNormsBacking - for pid, numTerms := range s.numTermsPerPostingsList { - s.FreqNorms[pid] = freqNormsBacking[0:0] - freqNormsBacking = freqNormsBacking[numTerms:] - } - - if cap(s.Locs) >= numPostingsLists { - s.Locs = s.Locs[:numPostingsLists] - } else { - s.Locs = make([][]interimLoc, numPostingsLists) - } - - if cap(s.locsBacking) >= totLocs { - s.locsBacking = s.locsBacking[:totLocs] - } else { - s.locsBacking = make([]interimLoc, totLocs) - } - - locsBacking := s.locsBacking - for pid, numLocs := range s.numLocsPerPostingsList { - s.Locs[pid] = locsBacking[0:0] - locsBacking = locsBacking[numLocs:] - } -} - -func (s *interim) processDocuments() { - numFields := len(s.FieldsInv) - reuseFieldLens := make([]int, numFields) - reuseFieldTFs := make([]analysis.TokenFrequencies, numFields) - - for docNum, result := range s.results { - for i := 0; i < numFields; i++ { // clear these for reuse - reuseFieldLens[i] = 0 - reuseFieldTFs[i] = nil - } - - s.processDocument(uint64(docNum), result, - reuseFieldLens, reuseFieldTFs) - } -} - -func (s *interim) processDocument(docNum uint64, - result *index.AnalysisResult, - fieldLens []int, fieldTFs []analysis.TokenFrequencies) { - visitField := func(fieldID uint16, fieldName string, - ln int, tf analysis.TokenFrequencies) { - fieldLens[fieldID] += ln - - existingFreqs := fieldTFs[fieldID] - if existingFreqs != nil { - existingFreqs.MergeAll(fieldName, tf) - } else { - fieldTFs[fieldID] = tf - } - } - - // walk each composite field - for _, field := range result.Document.CompositeFields { - fieldID := uint16(s.getOrDefineField(field.Name())) - ln, tf := field.Analyze() - visitField(fieldID, field.Name(), ln, tf) - } - - // walk each field - for i, field := range result.Document.Fields { - fieldID := uint16(s.getOrDefineField(field.Name())) - ln := result.Length[i] - tf := result.Analyzed[i] - visitField(fieldID, field.Name(), ln, tf) - } - - // now that it's been rolled up into fieldTFs, walk that - for fieldID, tfs := range fieldTFs { - dict := s.Dicts[fieldID] - norm := float32(1.0 / math.Sqrt(float64(fieldLens[fieldID]))) - - for term, tf := range tfs { - pid := dict[term] - 1 - bs := s.Postings[pid] - bs.Add(uint32(docNum)) - - s.FreqNorms[pid] = append(s.FreqNorms[pid], - interimFreqNorm{ - freq: uint64(tf.Frequency()), - norm: norm, - numLocs: len(tf.Locations), - }) - - if len(tf.Locations) > 0 { - locs := s.Locs[pid] - - for _, loc := range tf.Locations { - var locf = uint16(fieldID) - if loc.Field != "" { - locf = uint16(s.getOrDefineField(loc.Field)) - } - var arrayposs []uint64 - if len(loc.ArrayPositions) > 0 { - arrayposs = loc.ArrayPositions - } - locs = append(locs, interimLoc{ - fieldID: locf, - pos: uint64(loc.Position), - start: uint64(loc.Start), - end: uint64(loc.End), - arrayposs: arrayposs, - }) - } - - s.Locs[pid] = locs - } - } - } -} - -func (s *interim) writeStoredFields() ( - storedIndexOffset uint64, err error) { - varBuf := make([]byte, binary.MaxVarintLen64) - metaEncode := func(val uint64) (int, error) { - wb := binary.PutUvarint(varBuf, val) - return s.metaBuf.Write(varBuf[:wb]) - } - - data, compressed := s.tmp0[:0], s.tmp1[:0] - defer func() { s.tmp0, s.tmp1 = data, compressed }() - - // keyed by docNum - docStoredOffsets := make([]uint64, len(s.results)) - - // keyed by fieldID, for the current doc in the loop - docStoredFields := map[uint16]interimStoredField{} - - for docNum, result := range s.results { - for fieldID := range docStoredFields { // reset for next doc - delete(docStoredFields, fieldID) - } - - for _, field := range result.Document.Fields { - fieldID := uint16(s.getOrDefineField(field.Name())) - - opts := field.Options() - - if opts.IsStored() { - isf := docStoredFields[fieldID] - isf.vals = append(isf.vals, field.Value()) - isf.typs = append(isf.typs, encodeFieldType(field)) - isf.arrayposs = append(isf.arrayposs, field.ArrayPositions()) - docStoredFields[fieldID] = isf - } - - if opts.IncludeDocValues() { - s.IncludeDocValues[fieldID] = true - } - - err := ValidateDocFields(field) - if err != nil { - return 0, err - } - } - - var curr int - - s.metaBuf.Reset() - data = data[:0] - - // _id field special case optimizes ExternalID() lookups - idFieldVal := docStoredFields[uint16(0)].vals[0] - _, err = metaEncode(uint64(len(idFieldVal))) - if err != nil { - return 0, err - } - - // handle non-"_id" fields - for fieldID := 1; fieldID < len(s.FieldsInv); fieldID++ { - isf, exists := docStoredFields[uint16(fieldID)] - if exists { - curr, data, err = persistStoredFieldValues( - fieldID, isf.vals, isf.typs, isf.arrayposs, - curr, metaEncode, data) - if err != nil { - return 0, err - } - } - } - - metaBytes := s.metaBuf.Bytes() - - compressed = snappy.Encode(compressed[:cap(compressed)], data) - - docStoredOffsets[docNum] = uint64(s.w.Count()) - - _, err := writeUvarints(s.w, - uint64(len(metaBytes)), - uint64(len(idFieldVal)+len(compressed))) - if err != nil { - return 0, err - } - - _, err = s.w.Write(metaBytes) - if err != nil { - return 0, err - } - - _, err = s.w.Write(idFieldVal) - if err != nil { - return 0, err - } - - _, err = s.w.Write(compressed) - if err != nil { - return 0, err - } - } - - storedIndexOffset = uint64(s.w.Count()) - - for _, docStoredOffset := range docStoredOffsets { - err = binary.Write(s.w, binary.BigEndian, docStoredOffset) - if err != nil { - return 0, err - } - } - - return storedIndexOffset, nil -} - -func (s *interim) writeDicts() (fdvIndexOffset uint64, dictOffsets []uint64, err error) { - dictOffsets = make([]uint64, len(s.FieldsInv)) - - fdvOffsetsStart := make([]uint64, len(s.FieldsInv)) - fdvOffsetsEnd := make([]uint64, len(s.FieldsInv)) - - buf := s.grabBuf(binary.MaxVarintLen64) - - tfEncoder := newChunkedIntCoder(uint64(s.chunkFactor), uint64(len(s.results)-1)) - locEncoder := newChunkedIntCoder(uint64(s.chunkFactor), uint64(len(s.results)-1)) - fdvEncoder := newChunkedContentCoder(uint64(s.chunkFactor), uint64(len(s.results)-1), s.w, false) - - var docTermMap [][]byte - - if s.builder == nil { - s.builder, err = vellum.New(&s.builderBuf, nil) - if err != nil { - return 0, nil, err - } - } - - for fieldID, terms := range s.DictKeys { - if cap(docTermMap) < len(s.results) { - docTermMap = make([][]byte, len(s.results)) - } else { - docTermMap = docTermMap[0:len(s.results)] - for docNum := range docTermMap { // reset the docTermMap - docTermMap[docNum] = docTermMap[docNum][:0] - } - } - - dict := s.Dicts[fieldID] - - for _, term := range terms { // terms are already sorted - pid := dict[term] - 1 - - postingsBS := s.Postings[pid] - - freqNorms := s.FreqNorms[pid] - freqNormOffset := 0 - - locs := s.Locs[pid] - locOffset := 0 - - postingsItr := postingsBS.Iterator() - for postingsItr.HasNext() { - docNum := uint64(postingsItr.Next()) - - freqNorm := freqNorms[freqNormOffset] - - err = tfEncoder.Add(docNum, - encodeFreqHasLocs(freqNorm.freq, freqNorm.numLocs > 0), - uint64(math.Float32bits(freqNorm.norm))) - if err != nil { - return 0, nil, err - } - - if freqNorm.numLocs > 0 { - numBytesLocs := 0 - for _, loc := range locs[locOffset : locOffset+freqNorm.numLocs] { - numBytesLocs += totalUvarintBytes( - uint64(loc.fieldID), loc.pos, loc.start, loc.end, - uint64(len(loc.arrayposs)), loc.arrayposs) - } - - err = locEncoder.Add(docNum, uint64(numBytesLocs)) - if err != nil { - return 0, nil, err - } - - for _, loc := range locs[locOffset : locOffset+freqNorm.numLocs] { - err = locEncoder.Add(docNum, - uint64(loc.fieldID), loc.pos, loc.start, loc.end, - uint64(len(loc.arrayposs))) - if err != nil { - return 0, nil, err - } - - err = locEncoder.Add(docNum, loc.arrayposs...) - if err != nil { - return 0, nil, err - } - } - - locOffset += freqNorm.numLocs - } - - freqNormOffset++ - - docTermMap[docNum] = append( - append(docTermMap[docNum], term...), - termSeparator) - } - - tfEncoder.Close() - locEncoder.Close() - - postingsOffset, err := - writePostings(postingsBS, tfEncoder, locEncoder, nil, s.w, buf) - if err != nil { - return 0, nil, err - } - - if postingsOffset > uint64(0) { - err = s.builder.Insert([]byte(term), postingsOffset) - if err != nil { - return 0, nil, err - } - } - - tfEncoder.Reset() - locEncoder.Reset() - } - - err = s.builder.Close() - if err != nil { - return 0, nil, err - } - - // record where this dictionary starts - dictOffsets[fieldID] = uint64(s.w.Count()) - - vellumData := s.builderBuf.Bytes() - - // write out the length of the vellum data - n := binary.PutUvarint(buf, uint64(len(vellumData))) - _, err = s.w.Write(buf[:n]) - if err != nil { - return 0, nil, err - } - - // write this vellum to disk - _, err = s.w.Write(vellumData) - if err != nil { - return 0, nil, err - } - - // reset vellum for reuse - s.builderBuf.Reset() - - err = s.builder.Reset(&s.builderBuf) - if err != nil { - return 0, nil, err - } - - // write the field doc values - if s.IncludeDocValues[fieldID] { - for docNum, docTerms := range docTermMap { - if len(docTerms) > 0 { - err = fdvEncoder.Add(uint64(docNum), docTerms) - if err != nil { - return 0, nil, err - } - } - } - err = fdvEncoder.Close() - if err != nil { - return 0, nil, err - } - - fdvOffsetsStart[fieldID] = uint64(s.w.Count()) - - _, err = fdvEncoder.Write() - if err != nil { - return 0, nil, err - } - - fdvOffsetsEnd[fieldID] = uint64(s.w.Count()) - - fdvEncoder.Reset() - } else { - fdvOffsetsStart[fieldID] = fieldNotUninverted - fdvOffsetsEnd[fieldID] = fieldNotUninverted - } - } - - fdvIndexOffset = uint64(s.w.Count()) - - for i := 0; i < len(fdvOffsetsStart); i++ { - n := binary.PutUvarint(buf, fdvOffsetsStart[i]) - _, err := s.w.Write(buf[:n]) - if err != nil { - return 0, nil, err - } - n = binary.PutUvarint(buf, fdvOffsetsEnd[i]) - _, err = s.w.Write(buf[:n]) - if err != nil { - return 0, nil, err - } - } - - return fdvIndexOffset, dictOffsets, nil -} - -func encodeFieldType(f document.Field) byte { - fieldType := byte('x') - switch f.(type) { - case *document.TextField: - fieldType = 't' - case *document.NumericField: - fieldType = 'n' - case *document.DateTimeField: - fieldType = 'd' - case *document.BooleanField: - fieldType = 'b' - case *document.GeoPointField: - fieldType = 'g' - case *document.CompositeField: - fieldType = 'c' - } - return fieldType -} - -// returns the total # of bytes needed to encode the given uint64's -// into binary.PutUVarint() encoding -func totalUvarintBytes(a, b, c, d, e uint64, more []uint64) (n int) { - n = numUvarintBytes(a) - n += numUvarintBytes(b) - n += numUvarintBytes(c) - n += numUvarintBytes(d) - n += numUvarintBytes(e) - for _, v := range more { - n += numUvarintBytes(v) - } - return n -} - -// returns # of bytes needed to encode x in binary.PutUvarint() encoding -func numUvarintBytes(x uint64) (n int) { - for x >= 0x80 { - x >>= 7 - n++ - } - return n + 1 -} diff --git a/vendor/github.com/blevesearch/zap/v11/plugin.go b/vendor/github.com/blevesearch/zap/v11/plugin.go deleted file mode 100644 index 38a0638d..00000000 --- a/vendor/github.com/blevesearch/zap/v11/plugin.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (c) 2020 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "github.com/blevesearch/bleve/index/scorch/segment" -) - -// ZapPlugin implements the Plugin interface of -// the blevesearch/bleve/index/scorch/segment pkg -type ZapPlugin struct{} - -func (*ZapPlugin) Type() string { - return Type -} - -func (*ZapPlugin) Version() uint32 { - return Version -} - -// Plugin returns an instance segment.Plugin for use -// by the Scorch indexing scheme -func Plugin() segment.Plugin { - return &ZapPlugin{} -} diff --git a/vendor/github.com/blevesearch/zap/v11/posting.go b/vendor/github.com/blevesearch/zap/v11/posting.go deleted file mode 100644 index 619dc4c3..00000000 --- a/vendor/github.com/blevesearch/zap/v11/posting.go +++ /dev/null @@ -1,910 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "encoding/binary" - "fmt" - "math" - "reflect" - - "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/blevesearch/bleve/size" -) - -var reflectStaticSizePostingsList int -var reflectStaticSizePostingsIterator int -var reflectStaticSizePosting int -var reflectStaticSizeLocation int - -func init() { - var pl PostingsList - reflectStaticSizePostingsList = int(reflect.TypeOf(pl).Size()) - var pi PostingsIterator - reflectStaticSizePostingsIterator = int(reflect.TypeOf(pi).Size()) - var p Posting - reflectStaticSizePosting = int(reflect.TypeOf(p).Size()) - var l Location - reflectStaticSizeLocation = int(reflect.TypeOf(l).Size()) -} - -// FST or vellum value (uint64) encoding is determined by the top two -// highest-order or most significant bits... -// -// encoding : MSB -// name : 63 62 61...to...bit #0 (LSB) -// ----------+---+---+--------------------------------------------------- -// general : 0 | 0 | 62-bits of postingsOffset. -// ~ : 0 | 1 | reserved for future. -// 1-hit : 1 | 0 | 31-bits of positive float31 norm | 31-bits docNum. -// ~ : 1 | 1 | reserved for future. -// -// Encoding "general" is able to handle all cases, where the -// postingsOffset points to more information about the postings for -// the term. -// -// Encoding "1-hit" is used to optimize a commonly seen case when a -// term has only a single hit. For example, a term in the _id field -// will have only 1 hit. The "1-hit" encoding is used for a term -// in a field when... -// -// - term vector info is disabled for that field; -// - and, the term appears in only a single doc for that field; -// - and, the term's freq is exactly 1 in that single doc for that field; -// - and, the docNum must fit into 31-bits; -// -// Otherwise, the "general" encoding is used instead. -// -// In the "1-hit" encoding, the field in that single doc may have -// other terms, which is supported in the "1-hit" encoding by the -// positive float31 norm. - -const FSTValEncodingMask = uint64(0xc000000000000000) -const FSTValEncodingGeneral = uint64(0x0000000000000000) -const FSTValEncoding1Hit = uint64(0x8000000000000000) - -func FSTValEncode1Hit(docNum uint64, normBits uint64) uint64 { - return FSTValEncoding1Hit | ((mask31Bits & normBits) << 31) | (mask31Bits & docNum) -} - -func FSTValDecode1Hit(v uint64) (docNum uint64, normBits uint64) { - return (mask31Bits & v), (mask31Bits & (v >> 31)) -} - -const mask31Bits = uint64(0x000000007fffffff) - -func under32Bits(x uint64) bool { - return x <= mask31Bits -} - -const DocNum1HitFinished = math.MaxUint64 - -var NormBits1Hit = uint64(math.Float32bits(float32(1))) - -// PostingsList is an in-memory representation of a postings list -type PostingsList struct { - sb *SegmentBase - postingsOffset uint64 - freqOffset uint64 - locOffset uint64 - postings *roaring.Bitmap - except *roaring.Bitmap - - // when normBits1Hit != 0, then this postings list came from a - // 1-hit encoding, and only the docNum1Hit & normBits1Hit apply - docNum1Hit uint64 - normBits1Hit uint64 -} - -// represents an immutable, empty postings list -var emptyPostingsList = &PostingsList{} - -func (p *PostingsList) Size() int { - sizeInBytes := reflectStaticSizePostingsList + size.SizeOfPtr - - if p.except != nil { - sizeInBytes += int(p.except.GetSizeInBytes()) - } - - return sizeInBytes -} - -func (p *PostingsList) OrInto(receiver *roaring.Bitmap) { - if p.normBits1Hit != 0 { - receiver.Add(uint32(p.docNum1Hit)) - return - } - - if p.postings != nil { - receiver.Or(p.postings) - } -} - -// Iterator returns an iterator for this postings list -func (p *PostingsList) Iterator(includeFreq, includeNorm, includeLocs bool, - prealloc segment.PostingsIterator) segment.PostingsIterator { - if p.normBits1Hit == 0 && p.postings == nil { - return emptyPostingsIterator - } - - var preallocPI *PostingsIterator - pi, ok := prealloc.(*PostingsIterator) - if ok && pi != nil { - preallocPI = pi - } - if preallocPI == emptyPostingsIterator { - preallocPI = nil - } - - return p.iterator(includeFreq, includeNorm, includeLocs, preallocPI) -} - -func (p *PostingsList) iterator(includeFreq, includeNorm, includeLocs bool, - rv *PostingsIterator) *PostingsIterator { - if rv == nil { - rv = &PostingsIterator{} - } else { - freqNormReader := rv.freqNormReader - if freqNormReader != nil { - freqNormReader.Reset([]byte(nil)) - } - - locReader := rv.locReader - if locReader != nil { - locReader.Reset([]byte(nil)) - } - - freqChunkOffsets := rv.freqChunkOffsets[:0] - locChunkOffsets := rv.locChunkOffsets[:0] - - nextLocs := rv.nextLocs[:0] - nextSegmentLocs := rv.nextSegmentLocs[:0] - - buf := rv.buf - - *rv = PostingsIterator{} // clear the struct - - rv.freqNormReader = freqNormReader - rv.locReader = locReader - - rv.freqChunkOffsets = freqChunkOffsets - rv.locChunkOffsets = locChunkOffsets - - rv.nextLocs = nextLocs - rv.nextSegmentLocs = nextSegmentLocs - - rv.buf = buf - } - - rv.postings = p - rv.includeFreqNorm = includeFreq || includeNorm || includeLocs - rv.includeLocs = includeLocs - - if p.normBits1Hit != 0 { - // "1-hit" encoding - rv.docNum1Hit = p.docNum1Hit - rv.normBits1Hit = p.normBits1Hit - - if p.except != nil && p.except.Contains(uint32(rv.docNum1Hit)) { - rv.docNum1Hit = DocNum1HitFinished - } - - return rv - } - - // "general" encoding, check if empty - if p.postings == nil { - return rv - } - - var n uint64 - var read int - - // prepare the freq chunk details - if rv.includeFreqNorm { - var numFreqChunks uint64 - numFreqChunks, read = binary.Uvarint(p.sb.mem[p.freqOffset+n : p.freqOffset+n+binary.MaxVarintLen64]) - n += uint64(read) - if cap(rv.freqChunkOffsets) >= int(numFreqChunks) { - rv.freqChunkOffsets = rv.freqChunkOffsets[:int(numFreqChunks)] - } else { - rv.freqChunkOffsets = make([]uint64, int(numFreqChunks)) - } - for i := 0; i < int(numFreqChunks); i++ { - rv.freqChunkOffsets[i], read = binary.Uvarint(p.sb.mem[p.freqOffset+n : p.freqOffset+n+binary.MaxVarintLen64]) - n += uint64(read) - } - rv.freqChunkStart = p.freqOffset + n - } - - // prepare the loc chunk details - if rv.includeLocs { - n = 0 - var numLocChunks uint64 - numLocChunks, read = binary.Uvarint(p.sb.mem[p.locOffset+n : p.locOffset+n+binary.MaxVarintLen64]) - n += uint64(read) - if cap(rv.locChunkOffsets) >= int(numLocChunks) { - rv.locChunkOffsets = rv.locChunkOffsets[:int(numLocChunks)] - } else { - rv.locChunkOffsets = make([]uint64, int(numLocChunks)) - } - for i := 0; i < int(numLocChunks); i++ { - rv.locChunkOffsets[i], read = binary.Uvarint(p.sb.mem[p.locOffset+n : p.locOffset+n+binary.MaxVarintLen64]) - n += uint64(read) - } - rv.locChunkStart = p.locOffset + n - } - - rv.all = p.postings.Iterator() - if p.except != nil { - rv.ActualBM = roaring.AndNot(p.postings, p.except) - rv.Actual = rv.ActualBM.Iterator() - } else { - rv.ActualBM = p.postings - rv.Actual = rv.all // Optimize to use same iterator for all & Actual. - } - - return rv -} - -// Count returns the number of items on this postings list -func (p *PostingsList) Count() uint64 { - var n, e uint64 - if p.normBits1Hit != 0 { - n = 1 - if p.except != nil && p.except.Contains(uint32(p.docNum1Hit)) { - e = 1 - } - } else if p.postings != nil { - n = p.postings.GetCardinality() - if p.except != nil { - e = p.postings.AndCardinality(p.except) - } - } - return n - e -} - -func (rv *PostingsList) read(postingsOffset uint64, d *Dictionary) error { - rv.postingsOffset = postingsOffset - - // handle "1-hit" encoding special case - if rv.postingsOffset&FSTValEncodingMask == FSTValEncoding1Hit { - return rv.init1Hit(postingsOffset) - } - - // read the location of the freq/norm details - var n uint64 - var read int - - rv.freqOffset, read = binary.Uvarint(d.sb.mem[postingsOffset+n : postingsOffset+binary.MaxVarintLen64]) - n += uint64(read) - - rv.locOffset, read = binary.Uvarint(d.sb.mem[postingsOffset+n : postingsOffset+n+binary.MaxVarintLen64]) - n += uint64(read) - - var postingsLen uint64 - postingsLen, read = binary.Uvarint(d.sb.mem[postingsOffset+n : postingsOffset+n+binary.MaxVarintLen64]) - n += uint64(read) - - roaringBytes := d.sb.mem[postingsOffset+n : postingsOffset+n+postingsLen] - - if rv.postings == nil { - rv.postings = roaring.NewBitmap() - } - _, err := rv.postings.FromBuffer(roaringBytes) - if err != nil { - return fmt.Errorf("error loading roaring bitmap: %v", err) - } - - return nil -} - -func (rv *PostingsList) init1Hit(fstVal uint64) error { - docNum, normBits := FSTValDecode1Hit(fstVal) - - rv.docNum1Hit = docNum - rv.normBits1Hit = normBits - - return nil -} - -// PostingsIterator provides a way to iterate through the postings list -type PostingsIterator struct { - postings *PostingsList - all roaring.IntPeekable - Actual roaring.IntPeekable - ActualBM *roaring.Bitmap - - currChunk uint32 - currChunkFreqNorm []byte - currChunkLoc []byte - - freqNormReader *segment.MemUvarintReader - locReader *segment.MemUvarintReader - - freqChunkOffsets []uint64 - freqChunkStart uint64 - - locChunkOffsets []uint64 - locChunkStart uint64 - - next Posting // reused across Next() calls - nextLocs []Location // reused across Next() calls - nextSegmentLocs []segment.Location // reused across Next() calls - - docNum1Hit uint64 - normBits1Hit uint64 - - buf []byte - - includeFreqNorm bool - includeLocs bool -} - -var emptyPostingsIterator = &PostingsIterator{} - -func (i *PostingsIterator) Size() int { - sizeInBytes := reflectStaticSizePostingsIterator + size.SizeOfPtr + - len(i.currChunkFreqNorm) + - len(i.currChunkLoc) + - len(i.freqChunkOffsets)*size.SizeOfUint64 + - len(i.locChunkOffsets)*size.SizeOfUint64 + - i.next.Size() - - for _, entry := range i.nextLocs { - sizeInBytes += entry.Size() - } - - return sizeInBytes -} - -func (i *PostingsIterator) loadChunk(chunk int) error { - if i.includeFreqNorm { - if chunk >= len(i.freqChunkOffsets) { - return fmt.Errorf("tried to load freq chunk that doesn't exist %d/(%d)", - chunk, len(i.freqChunkOffsets)) - } - - end, start := i.freqChunkStart, i.freqChunkStart - s, e := readChunkBoundary(chunk, i.freqChunkOffsets) - start += s - end += e - i.currChunkFreqNorm = i.postings.sb.mem[start:end] - if i.freqNormReader == nil { - i.freqNormReader = segment.NewMemUvarintReader(i.currChunkFreqNorm) - } else { - i.freqNormReader.Reset(i.currChunkFreqNorm) - } - } - - if i.includeLocs { - if chunk >= len(i.locChunkOffsets) { - return fmt.Errorf("tried to load loc chunk that doesn't exist %d/(%d)", - chunk, len(i.locChunkOffsets)) - } - - end, start := i.locChunkStart, i.locChunkStart - s, e := readChunkBoundary(chunk, i.locChunkOffsets) - start += s - end += e - i.currChunkLoc = i.postings.sb.mem[start:end] - if i.locReader == nil { - i.locReader = segment.NewMemUvarintReader(i.currChunkLoc) - } else { - i.locReader.Reset(i.currChunkLoc) - } - } - - i.currChunk = uint32(chunk) - return nil -} - -func (i *PostingsIterator) readFreqNormHasLocs() (uint64, uint64, bool, error) { - if i.normBits1Hit != 0 { - return 1, i.normBits1Hit, false, nil - } - - freqHasLocs, err := i.freqNormReader.ReadUvarint() - if err != nil { - return 0, 0, false, fmt.Errorf("error reading frequency: %v", err) - } - - freq, hasLocs := decodeFreqHasLocs(freqHasLocs) - - normBits, err := i.freqNormReader.ReadUvarint() - if err != nil { - return 0, 0, false, fmt.Errorf("error reading norm: %v", err) - } - - return freq, normBits, hasLocs, nil -} - -func (i *PostingsIterator) skipFreqNormReadHasLocs() (bool, error) { - if i.normBits1Hit != 0 { - return false, nil - } - - freqHasLocs, err := i.freqNormReader.ReadUvarint() - if err != nil { - return false, fmt.Errorf("error reading freqHasLocs: %v", err) - } - - i.freqNormReader.SkipUvarint() // Skip normBits. - - return freqHasLocs&0x01 != 0, nil // See decodeFreqHasLocs() / hasLocs. -} - -func encodeFreqHasLocs(freq uint64, hasLocs bool) uint64 { - rv := freq << 1 - if hasLocs { - rv = rv | 0x01 // 0'th LSB encodes whether there are locations - } - return rv -} - -func decodeFreqHasLocs(freqHasLocs uint64) (uint64, bool) { - freq := freqHasLocs >> 1 - hasLocs := freqHasLocs&0x01 != 0 - return freq, hasLocs -} - -// readLocation processes all the integers on the stream representing a single -// location. -func (i *PostingsIterator) readLocation(l *Location) error { - // read off field - fieldID, err := i.locReader.ReadUvarint() - if err != nil { - return fmt.Errorf("error reading location field: %v", err) - } - // read off pos - pos, err := i.locReader.ReadUvarint() - if err != nil { - return fmt.Errorf("error reading location pos: %v", err) - } - // read off start - start, err := i.locReader.ReadUvarint() - if err != nil { - return fmt.Errorf("error reading location start: %v", err) - } - // read off end - end, err := i.locReader.ReadUvarint() - if err != nil { - return fmt.Errorf("error reading location end: %v", err) - } - // read off num array pos - numArrayPos, err := i.locReader.ReadUvarint() - if err != nil { - return fmt.Errorf("error reading location num array pos: %v", err) - } - - l.field = i.postings.sb.fieldsInv[fieldID] - l.pos = pos - l.start = start - l.end = end - - if cap(l.ap) < int(numArrayPos) { - l.ap = make([]uint64, int(numArrayPos)) - } else { - l.ap = l.ap[:int(numArrayPos)] - } - - // read off array positions - for k := 0; k < int(numArrayPos); k++ { - ap, err := i.locReader.ReadUvarint() - if err != nil { - return fmt.Errorf("error reading array position: %v", err) - } - - l.ap[k] = ap - } - - return nil -} - -// Next returns the next posting on the postings list, or nil at the end -func (i *PostingsIterator) Next() (segment.Posting, error) { - return i.nextAtOrAfter(0) -} - -// Advance returns the posting at the specified docNum or it is not present -// the next posting, or if the end is reached, nil -func (i *PostingsIterator) Advance(docNum uint64) (segment.Posting, error) { - return i.nextAtOrAfter(docNum) -} - -// Next returns the next posting on the postings list, or nil at the end -func (i *PostingsIterator) nextAtOrAfter(atOrAfter uint64) (segment.Posting, error) { - docNum, exists, err := i.nextDocNumAtOrAfter(atOrAfter) - if err != nil || !exists { - return nil, err - } - - i.next = Posting{} // clear the struct - rv := &i.next - rv.docNum = docNum - - if !i.includeFreqNorm { - return rv, nil - } - - var normBits uint64 - var hasLocs bool - - rv.freq, normBits, hasLocs, err = i.readFreqNormHasLocs() - if err != nil { - return nil, err - } - - rv.norm = math.Float32frombits(uint32(normBits)) - - if i.includeLocs && hasLocs { - // prepare locations into reused slices, where we assume - // rv.freq >= "number of locs", since in a composite field, - // some component fields might have their IncludeTermVector - // flags disabled while other component fields are enabled - if cap(i.nextLocs) >= int(rv.freq) { - i.nextLocs = i.nextLocs[0:rv.freq] - } else { - i.nextLocs = make([]Location, rv.freq, rv.freq*2) - } - if cap(i.nextSegmentLocs) < int(rv.freq) { - i.nextSegmentLocs = make([]segment.Location, rv.freq, rv.freq*2) - } - rv.locs = i.nextSegmentLocs[:0] - - numLocsBytes, err := i.locReader.ReadUvarint() - if err != nil { - return nil, fmt.Errorf("error reading location numLocsBytes: %v", err) - } - - j := 0 - startBytesRemaining := i.locReader.Len() // # bytes remaining in the locReader - for startBytesRemaining-i.locReader.Len() < int(numLocsBytes) { - err := i.readLocation(&i.nextLocs[j]) - if err != nil { - return nil, err - } - rv.locs = append(rv.locs, &i.nextLocs[j]) - j++ - } - } - - return rv, nil -} - -var freqHasLocs1Hit = encodeFreqHasLocs(1, false) - -// nextBytes returns the docNum and the encoded freq & loc bytes for -// the next posting -func (i *PostingsIterator) nextBytes() ( - docNumOut uint64, freq uint64, normBits uint64, - bytesFreqNorm []byte, bytesLoc []byte, err error) { - docNum, exists, err := i.nextDocNumAtOrAfter(0) - if err != nil || !exists { - return 0, 0, 0, nil, nil, err - } - - if i.normBits1Hit != 0 { - if i.buf == nil { - i.buf = make([]byte, binary.MaxVarintLen64*2) - } - n := binary.PutUvarint(i.buf, freqHasLocs1Hit) - n += binary.PutUvarint(i.buf[n:], i.normBits1Hit) - return docNum, uint64(1), i.normBits1Hit, i.buf[:n], nil, nil - } - - startFreqNorm := len(i.currChunkFreqNorm) - i.freqNormReader.Len() - - var hasLocs bool - - freq, normBits, hasLocs, err = i.readFreqNormHasLocs() - if err != nil { - return 0, 0, 0, nil, nil, err - } - - endFreqNorm := len(i.currChunkFreqNorm) - i.freqNormReader.Len() - bytesFreqNorm = i.currChunkFreqNorm[startFreqNorm:endFreqNorm] - - if hasLocs { - startLoc := len(i.currChunkLoc) - i.locReader.Len() - - numLocsBytes, err := i.locReader.ReadUvarint() - if err != nil { - return 0, 0, 0, nil, nil, - fmt.Errorf("error reading location nextBytes numLocs: %v", err) - } - - // skip over all the location bytes - i.locReader.SkipBytes(int(numLocsBytes)) - - endLoc := len(i.currChunkLoc) - i.locReader.Len() - bytesLoc = i.currChunkLoc[startLoc:endLoc] - } - - return docNum, freq, normBits, bytesFreqNorm, bytesLoc, nil -} - -// nextDocNum returns the next docNum on the postings list, and also -// sets up the currChunk / loc related fields of the iterator. -func (i *PostingsIterator) nextDocNumAtOrAfter(atOrAfter uint64) (uint64, bool, error) { - if i.normBits1Hit != 0 { - if i.docNum1Hit == DocNum1HitFinished { - return 0, false, nil - } - if i.docNum1Hit < atOrAfter { - // advanced past our 1-hit - i.docNum1Hit = DocNum1HitFinished // consume our 1-hit docNum - return 0, false, nil - } - docNum := i.docNum1Hit - i.docNum1Hit = DocNum1HitFinished // consume our 1-hit docNum - return docNum, true, nil - } - - if i.Actual == nil || !i.Actual.HasNext() { - return 0, false, nil - } - - if i.postings == nil || i.postings.postings == i.ActualBM { - return i.nextDocNumAtOrAfterClean(atOrAfter) - } - - i.Actual.AdvanceIfNeeded(uint32(atOrAfter)) - - if !i.Actual.HasNext() { - // couldn't find anything - return 0, false, nil - } - - n := i.Actual.Next() - allN := i.all.Next() - - nChunk := n / i.postings.sb.chunkFactor - - // when allN becomes >= to here, then allN is in the same chunk as nChunk. - allNReachesNChunk := nChunk * i.postings.sb.chunkFactor - - // n is the next actual hit (excluding some postings), and - // allN is the next hit in the full postings, and - // if they don't match, move 'all' forwards until they do - for allN != n { - // we've reached same chunk, so move the freq/norm/loc decoders forward - if i.includeFreqNorm && allN >= allNReachesNChunk { - err := i.currChunkNext(nChunk) - if err != nil { - return 0, false, err - } - } - - allN = i.all.Next() - } - - if i.includeFreqNorm && (i.currChunk != nChunk || i.currChunkFreqNorm == nil) { - err := i.loadChunk(int(nChunk)) - if err != nil { - return 0, false, fmt.Errorf("error loading chunk: %v", err) - } - } - - return uint64(n), true, nil -} - -// optimization when the postings list is "clean" (e.g., no updates & -// no deletions) where the all bitmap is the same as the actual bitmap -func (i *PostingsIterator) nextDocNumAtOrAfterClean( - atOrAfter uint64) (uint64, bool, error) { - - if !i.includeFreqNorm { - i.Actual.AdvanceIfNeeded(uint32(atOrAfter)) - - if !i.Actual.HasNext() { - return 0, false, nil // couldn't find anything - } - - return uint64(i.Actual.Next()), true, nil - } - - // freq-norm's needed, so maintain freq-norm chunk reader - sameChunkNexts := 0 // # of times we called Next() in the same chunk - n := i.Actual.Next() - nChunk := n / i.postings.sb.chunkFactor - - for uint64(n) < atOrAfter && i.Actual.HasNext() { - n = i.Actual.Next() - - nChunkPrev := nChunk - nChunk = n / i.postings.sb.chunkFactor - - if nChunk != nChunkPrev { - sameChunkNexts = 0 - } else { - sameChunkNexts += 1 - } - } - - if uint64(n) < atOrAfter { - // couldn't find anything - return 0, false, nil - } - - for j := 0; j < sameChunkNexts; j++ { - err := i.currChunkNext(nChunk) - if err != nil { - return 0, false, fmt.Errorf("error optimized currChunkNext: %v", err) - } - } - - if i.currChunk != nChunk || i.currChunkFreqNorm == nil { - err := i.loadChunk(int(nChunk)) - if err != nil { - return 0, false, fmt.Errorf("error loading chunk: %v", err) - } - } - - return uint64(n), true, nil -} - -func (i *PostingsIterator) currChunkNext(nChunk uint32) error { - if i.currChunk != nChunk || i.currChunkFreqNorm == nil { - err := i.loadChunk(int(nChunk)) - if err != nil { - return fmt.Errorf("error loading chunk: %v", err) - } - } - - // read off freq/offsets even though we don't care about them - hasLocs, err := i.skipFreqNormReadHasLocs() - if err != nil { - return err - } - - if i.includeLocs && hasLocs { - numLocsBytes, err := i.locReader.ReadUvarint() - if err != nil { - return fmt.Errorf("error reading location numLocsBytes: %v", err) - } - - // skip over all the location bytes - i.locReader.SkipBytes(int(numLocsBytes)) - } - - return nil -} - -// DocNum1Hit returns the docNum and true if this is "1-hit" optimized -// and the docNum is available. -func (p *PostingsIterator) DocNum1Hit() (uint64, bool) { - if p.normBits1Hit != 0 && p.docNum1Hit != DocNum1HitFinished { - return p.docNum1Hit, true - } - return 0, false -} - -// ActualBitmap returns the underlying actual bitmap -// which can be used up the stack for optimizations -func (p *PostingsIterator) ActualBitmap() *roaring.Bitmap { - return p.ActualBM -} - -// ReplaceActual replaces the ActualBM with the provided -// bitmap -func (p *PostingsIterator) ReplaceActual(abm *roaring.Bitmap) { - p.ActualBM = abm - p.Actual = abm.Iterator() -} - -// PostingsIteratorFromBitmap constructs a PostingsIterator given an -// "actual" bitmap. -func PostingsIteratorFromBitmap(bm *roaring.Bitmap, - includeFreqNorm, includeLocs bool) (segment.PostingsIterator, error) { - return &PostingsIterator{ - ActualBM: bm, - Actual: bm.Iterator(), - includeFreqNorm: includeFreqNorm, - includeLocs: includeLocs, - }, nil -} - -// PostingsIteratorFrom1Hit constructs a PostingsIterator given a -// 1-hit docNum. -func PostingsIteratorFrom1Hit(docNum1Hit uint64, - includeFreqNorm, includeLocs bool) (segment.PostingsIterator, error) { - return &PostingsIterator{ - docNum1Hit: docNum1Hit, - normBits1Hit: NormBits1Hit, - includeFreqNorm: includeFreqNorm, - includeLocs: includeLocs, - }, nil -} - -// Posting is a single entry in a postings list -type Posting struct { - docNum uint64 - freq uint64 - norm float32 - locs []segment.Location -} - -func (p *Posting) Size() int { - sizeInBytes := reflectStaticSizePosting - - for _, entry := range p.locs { - sizeInBytes += entry.Size() - } - - return sizeInBytes -} - -// Number returns the document number of this posting in this segment -func (p *Posting) Number() uint64 { - return p.docNum -} - -// Frequency returns the frequencies of occurrence of this term in this doc/field -func (p *Posting) Frequency() uint64 { - return p.freq -} - -// Norm returns the normalization factor for this posting -func (p *Posting) Norm() float64 { - return float64(p.norm) -} - -// Locations returns the location information for each occurrence -func (p *Posting) Locations() []segment.Location { - return p.locs -} - -// Location represents the location of a single occurrence -type Location struct { - field string - pos uint64 - start uint64 - end uint64 - ap []uint64 -} - -func (l *Location) Size() int { - return reflectStaticSizeLocation + - len(l.field) + - len(l.ap)*size.SizeOfUint64 -} - -// Field returns the name of the field (useful in composite fields to know -// which original field the value came from) -func (l *Location) Field() string { - return l.field -} - -// Start returns the start byte offset of this occurrence -func (l *Location) Start() uint64 { - return l.start -} - -// End returns the end byte offset of this occurrence -func (l *Location) End() uint64 { - return l.end -} - -// Pos returns the 1-based phrase position of this occurrence -func (l *Location) Pos() uint64 { - return l.pos -} - -// ArrayPositions returns the array position vector associated with this occurrence -func (l *Location) ArrayPositions() []uint64 { - return l.ap -} diff --git a/vendor/github.com/blevesearch/zap/v11/segment.go b/vendor/github.com/blevesearch/zap/v11/segment.go deleted file mode 100644 index 517b6afb..00000000 --- a/vendor/github.com/blevesearch/zap/v11/segment.go +++ /dev/null @@ -1,572 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "bytes" - "encoding/binary" - "fmt" - "io" - "os" - "sync" - "unsafe" - - "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/blevesearch/bleve/size" - "github.com/couchbase/vellum" - mmap "github.com/blevesearch/mmap-go" - "github.com/golang/snappy" -) - -var reflectStaticSizeSegmentBase int - -func init() { - var sb SegmentBase - reflectStaticSizeSegmentBase = int(unsafe.Sizeof(sb)) -} - -// Open returns a zap impl of a segment -func (*ZapPlugin) Open(path string) (segment.Segment, error) { - f, err := os.Open(path) - if err != nil { - return nil, err - } - mm, err := mmap.Map(f, mmap.RDONLY, 0) - if err != nil { - // mmap failed, try to close the file - _ = f.Close() - return nil, err - } - - rv := &Segment{ - SegmentBase: SegmentBase{ - mem: mm[0 : len(mm)-FooterSize], - fieldsMap: make(map[string]uint16), - fieldDvReaders: make(map[uint16]*docValueReader), - fieldFSTs: make(map[uint16]*vellum.FST), - }, - f: f, - mm: mm, - path: path, - refs: 1, - } - rv.SegmentBase.updateSize() - - err = rv.loadConfig() - if err != nil { - _ = rv.Close() - return nil, err - } - - err = rv.loadFields() - if err != nil { - _ = rv.Close() - return nil, err - } - - err = rv.loadDvReaders() - if err != nil { - _ = rv.Close() - return nil, err - } - - return rv, nil -} - -// SegmentBase is a memory only, read-only implementation of the -// segment.Segment interface, using zap's data representation. -type SegmentBase struct { - mem []byte - memCRC uint32 - chunkFactor uint32 - fieldsMap map[string]uint16 // fieldName -> fieldID+1 - fieldsInv []string // fieldID -> fieldName - numDocs uint64 - storedIndexOffset uint64 - fieldsIndexOffset uint64 - docValueOffset uint64 - dictLocs []uint64 - fieldDvReaders map[uint16]*docValueReader // naive chunk cache per field - fieldDvNames []string // field names cached in fieldDvReaders - size uint64 - - m sync.Mutex - fieldFSTs map[uint16]*vellum.FST -} - -func (sb *SegmentBase) Size() int { - return int(sb.size) -} - -func (sb *SegmentBase) updateSize() { - sizeInBytes := reflectStaticSizeSegmentBase + - cap(sb.mem) - - // fieldsMap - for k := range sb.fieldsMap { - sizeInBytes += (len(k) + size.SizeOfString) + size.SizeOfUint16 - } - - // fieldsInv, dictLocs - for _, entry := range sb.fieldsInv { - sizeInBytes += len(entry) + size.SizeOfString - } - sizeInBytes += len(sb.dictLocs) * size.SizeOfUint64 - - // fieldDvReaders - for _, v := range sb.fieldDvReaders { - sizeInBytes += size.SizeOfUint16 + size.SizeOfPtr - if v != nil { - sizeInBytes += v.size() - } - } - - sb.size = uint64(sizeInBytes) -} - -func (sb *SegmentBase) AddRef() {} -func (sb *SegmentBase) DecRef() (err error) { return nil } -func (sb *SegmentBase) Close() (err error) { return nil } - -// Segment implements a persisted segment.Segment interface, by -// embedding an mmap()'ed SegmentBase. -type Segment struct { - SegmentBase - - f *os.File - mm mmap.MMap - path string - version uint32 - crc uint32 - - m sync.Mutex // Protects the fields that follow. - refs int64 -} - -func (s *Segment) Size() int { - // 8 /* size of file pointer */ - // 4 /* size of version -> uint32 */ - // 4 /* size of crc -> uint32 */ - sizeOfUints := 16 - - sizeInBytes := (len(s.path) + size.SizeOfString) + sizeOfUints - - // mutex, refs -> int64 - sizeInBytes += 16 - - // do not include the mmap'ed part - return sizeInBytes + s.SegmentBase.Size() - cap(s.mem) -} - -func (s *Segment) AddRef() { - s.m.Lock() - s.refs++ - s.m.Unlock() -} - -func (s *Segment) DecRef() (err error) { - s.m.Lock() - s.refs-- - if s.refs == 0 { - err = s.closeActual() - } - s.m.Unlock() - return err -} - -func (s *Segment) loadConfig() error { - crcOffset := len(s.mm) - 4 - s.crc = binary.BigEndian.Uint32(s.mm[crcOffset : crcOffset+4]) - - verOffset := crcOffset - 4 - s.version = binary.BigEndian.Uint32(s.mm[verOffset : verOffset+4]) - if s.version != Version { - return fmt.Errorf("unsupported version %d", s.version) - } - - chunkOffset := verOffset - 4 - s.chunkFactor = binary.BigEndian.Uint32(s.mm[chunkOffset : chunkOffset+4]) - - docValueOffset := chunkOffset - 8 - s.docValueOffset = binary.BigEndian.Uint64(s.mm[docValueOffset : docValueOffset+8]) - - fieldsIndexOffset := docValueOffset - 8 - s.fieldsIndexOffset = binary.BigEndian.Uint64(s.mm[fieldsIndexOffset : fieldsIndexOffset+8]) - - storedIndexOffset := fieldsIndexOffset - 8 - s.storedIndexOffset = binary.BigEndian.Uint64(s.mm[storedIndexOffset : storedIndexOffset+8]) - - numDocsOffset := storedIndexOffset - 8 - s.numDocs = binary.BigEndian.Uint64(s.mm[numDocsOffset : numDocsOffset+8]) - return nil -} - -func (s *SegmentBase) loadFields() error { - // NOTE for now we assume the fields index immediately precedes - // the footer, and if this changes, need to adjust accordingly (or - // store explicit length), where s.mem was sliced from s.mm in Open(). - fieldsIndexEnd := uint64(len(s.mem)) - - // iterate through fields index - var fieldID uint64 - for s.fieldsIndexOffset+(8*fieldID) < fieldsIndexEnd { - addr := binary.BigEndian.Uint64(s.mem[s.fieldsIndexOffset+(8*fieldID) : s.fieldsIndexOffset+(8*fieldID)+8]) - - dictLoc, read := binary.Uvarint(s.mem[addr:fieldsIndexEnd]) - n := uint64(read) - s.dictLocs = append(s.dictLocs, dictLoc) - - var nameLen uint64 - nameLen, read = binary.Uvarint(s.mem[addr+n : fieldsIndexEnd]) - n += uint64(read) - - name := string(s.mem[addr+n : addr+n+nameLen]) - s.fieldsInv = append(s.fieldsInv, name) - s.fieldsMap[name] = uint16(fieldID + 1) - - fieldID++ - } - return nil -} - -// Dictionary returns the term dictionary for the specified field -func (s *SegmentBase) Dictionary(field string) (segment.TermDictionary, error) { - dict, err := s.dictionary(field) - if err == nil && dict == nil { - return &segment.EmptyDictionary{}, nil - } - return dict, err -} - -func (sb *SegmentBase) dictionary(field string) (rv *Dictionary, err error) { - fieldIDPlus1 := sb.fieldsMap[field] - if fieldIDPlus1 > 0 { - rv = &Dictionary{ - sb: sb, - field: field, - fieldID: fieldIDPlus1 - 1, - } - - dictStart := sb.dictLocs[rv.fieldID] - if dictStart > 0 { - var ok bool - sb.m.Lock() - if rv.fst, ok = sb.fieldFSTs[rv.fieldID]; !ok { - // read the length of the vellum data - vellumLen, read := binary.Uvarint(sb.mem[dictStart : dictStart+binary.MaxVarintLen64]) - fstBytes := sb.mem[dictStart+uint64(read) : dictStart+uint64(read)+vellumLen] - rv.fst, err = vellum.Load(fstBytes) - if err != nil { - sb.m.Unlock() - return nil, fmt.Errorf("dictionary field %s vellum err: %v", field, err) - } - - sb.fieldFSTs[rv.fieldID] = rv.fst - } - - sb.m.Unlock() - rv.fstReader, err = rv.fst.Reader() - if err != nil { - return nil, fmt.Errorf("dictionary field %s vellum reader err: %v", field, err) - } - - } - } - - return rv, nil -} - -// visitDocumentCtx holds data structures that are reusable across -// multiple VisitDocument() calls to avoid memory allocations -type visitDocumentCtx struct { - buf []byte - reader bytes.Reader - arrayPos []uint64 -} - -var visitDocumentCtxPool = sync.Pool{ - New: func() interface{} { - reuse := &visitDocumentCtx{} - return reuse - }, -} - -// VisitDocument invokes the DocFieldValueVistor for each stored field -// for the specified doc number -func (s *SegmentBase) VisitDocument(num uint64, visitor segment.DocumentFieldValueVisitor) error { - vdc := visitDocumentCtxPool.Get().(*visitDocumentCtx) - defer visitDocumentCtxPool.Put(vdc) - return s.visitDocument(vdc, num, visitor) -} - -func (s *SegmentBase) visitDocument(vdc *visitDocumentCtx, num uint64, - visitor segment.DocumentFieldValueVisitor) error { - // first make sure this is a valid number in this segment - if num < s.numDocs { - meta, compressed := s.getDocStoredMetaAndCompressed(num) - - vdc.reader.Reset(meta) - - // handle _id field special case - idFieldValLen, err := binary.ReadUvarint(&vdc.reader) - if err != nil { - return err - } - idFieldVal := compressed[:idFieldValLen] - - keepGoing := visitor("_id", byte('t'), idFieldVal, nil) - if !keepGoing { - visitDocumentCtxPool.Put(vdc) - return nil - } - - // handle non-"_id" fields - compressed = compressed[idFieldValLen:] - - uncompressed, err := snappy.Decode(vdc.buf[:cap(vdc.buf)], compressed) - if err != nil { - return err - } - - for keepGoing { - field, err := binary.ReadUvarint(&vdc.reader) - if err == io.EOF { - break - } - if err != nil { - return err - } - typ, err := binary.ReadUvarint(&vdc.reader) - if err != nil { - return err - } - offset, err := binary.ReadUvarint(&vdc.reader) - if err != nil { - return err - } - l, err := binary.ReadUvarint(&vdc.reader) - if err != nil { - return err - } - numap, err := binary.ReadUvarint(&vdc.reader) - if err != nil { - return err - } - var arrayPos []uint64 - if numap > 0 { - if cap(vdc.arrayPos) < int(numap) { - vdc.arrayPos = make([]uint64, numap) - } - arrayPos = vdc.arrayPos[:numap] - for i := 0; i < int(numap); i++ { - ap, err := binary.ReadUvarint(&vdc.reader) - if err != nil { - return err - } - arrayPos[i] = ap - } - } - - value := uncompressed[offset : offset+l] - keepGoing = visitor(s.fieldsInv[field], byte(typ), value, arrayPos) - } - - vdc.buf = uncompressed - } - return nil -} - -// DocID returns the value of the _id field for the given docNum -func (s *SegmentBase) DocID(num uint64) ([]byte, error) { - if num >= s.numDocs { - return nil, nil - } - - vdc := visitDocumentCtxPool.Get().(*visitDocumentCtx) - - meta, compressed := s.getDocStoredMetaAndCompressed(num) - - vdc.reader.Reset(meta) - - // handle _id field special case - idFieldValLen, err := binary.ReadUvarint(&vdc.reader) - if err != nil { - return nil, err - } - idFieldVal := compressed[:idFieldValLen] - - visitDocumentCtxPool.Put(vdc) - - return idFieldVal, nil -} - -// Count returns the number of documents in this segment. -func (s *SegmentBase) Count() uint64 { - return s.numDocs -} - -// DocNumbers returns a bitset corresponding to the doc numbers of all the -// provided _id strings -func (s *SegmentBase) DocNumbers(ids []string) (*roaring.Bitmap, error) { - rv := roaring.New() - - if len(s.fieldsMap) > 0 { - idDict, err := s.dictionary("_id") - if err != nil { - return nil, err - } - - postingsList := emptyPostingsList - - sMax, err := idDict.fst.GetMaxKey() - if err != nil { - return nil, err - } - sMaxStr := string(sMax) - filteredIds := make([]string, 0, len(ids)) - for _, id := range ids { - if id <= sMaxStr { - filteredIds = append(filteredIds, id) - } - } - - for _, id := range filteredIds { - postingsList, err = idDict.postingsList([]byte(id), nil, postingsList) - if err != nil { - return nil, err - } - postingsList.OrInto(rv) - } - } - - return rv, nil -} - -// Fields returns the field names used in this segment -func (s *SegmentBase) Fields() []string { - return s.fieldsInv -} - -// Path returns the path of this segment on disk -func (s *Segment) Path() string { - return s.path -} - -// Close releases all resources associated with this segment -func (s *Segment) Close() (err error) { - return s.DecRef() -} - -func (s *Segment) closeActual() (err error) { - if s.mm != nil { - err = s.mm.Unmap() - } - // try to close file even if unmap failed - if s.f != nil { - err2 := s.f.Close() - if err == nil { - // try to return first error - err = err2 - } - } - return -} - -// some helpers i started adding for the command-line utility - -// Data returns the underlying mmaped data slice -func (s *Segment) Data() []byte { - return s.mm -} - -// CRC returns the CRC value stored in the file footer -func (s *Segment) CRC() uint32 { - return s.crc -} - -// Version returns the file version in the file footer -func (s *Segment) Version() uint32 { - return s.version -} - -// ChunkFactor returns the chunk factor in the file footer -func (s *Segment) ChunkFactor() uint32 { - return s.chunkFactor -} - -// FieldsIndexOffset returns the fields index offset in the file footer -func (s *Segment) FieldsIndexOffset() uint64 { - return s.fieldsIndexOffset -} - -// StoredIndexOffset returns the stored value index offset in the file footer -func (s *Segment) StoredIndexOffset() uint64 { - return s.storedIndexOffset -} - -// DocValueOffset returns the docValue offset in the file footer -func (s *Segment) DocValueOffset() uint64 { - return s.docValueOffset -} - -// NumDocs returns the number of documents in the file footer -func (s *Segment) NumDocs() uint64 { - return s.numDocs -} - -// DictAddr is a helper function to compute the file offset where the -// dictionary is stored for the specified field. -func (s *Segment) DictAddr(field string) (uint64, error) { - fieldIDPlus1, ok := s.fieldsMap[field] - if !ok { - return 0, fmt.Errorf("no such field '%s'", field) - } - - return s.dictLocs[fieldIDPlus1-1], nil -} - -func (s *SegmentBase) loadDvReaders() error { - if s.docValueOffset == fieldNotUninverted || s.numDocs == 0 { - return nil - } - - var read uint64 - for fieldID, field := range s.fieldsInv { - var fieldLocStart, fieldLocEnd uint64 - var n int - fieldLocStart, n = binary.Uvarint(s.mem[s.docValueOffset+read : s.docValueOffset+read+binary.MaxVarintLen64]) - if n <= 0 { - return fmt.Errorf("loadDvReaders: failed to read the docvalue offset start for field %d", fieldID) - } - read += uint64(n) - fieldLocEnd, n = binary.Uvarint(s.mem[s.docValueOffset+read : s.docValueOffset+read+binary.MaxVarintLen64]) - if n <= 0 { - return fmt.Errorf("loadDvReaders: failed to read the docvalue offset end for field %d", fieldID) - } - read += uint64(n) - - fieldDvReader, err := s.loadFieldDocValueReader(field, fieldLocStart, fieldLocEnd) - if err != nil { - return err - } - if fieldDvReader != nil { - s.fieldDvReaders[uint16(fieldID)] = fieldDvReader - s.fieldDvNames = append(s.fieldDvNames, field) - } - } - - return nil -} diff --git a/vendor/github.com/blevesearch/zap/v12/README.md b/vendor/github.com/blevesearch/zap/v12/README.md deleted file mode 100644 index 0facb669..00000000 --- a/vendor/github.com/blevesearch/zap/v12/README.md +++ /dev/null @@ -1,158 +0,0 @@ -# zap file format - -Advanced ZAP File Format Documentation is [here](zap.md). - -The file is written in the reverse order that we typically access data. This helps us write in one pass since later sections of the file require file offsets of things we've already written. - -Current usage: - -- mmap the entire file -- crc-32 bytes and version are in fixed position at end of the file -- reading remainder of footer could be version specific -- remainder of footer gives us: - - 3 important offsets (docValue , fields index and stored data index) - - 2 important values (number of docs and chunk factor) -- field data is processed once and memoized onto the heap so that we never have to go back to disk for it -- access to stored data by doc number means first navigating to the stored data index, then accessing a fixed position offset into that slice, which gives us the actual address of the data. the first bytes of that section tell us the size of data so that we know where it ends. -- access to all other indexed data follows the following pattern: - - first know the field name -> convert to id - - next navigate to term dictionary for that field - - some operations stop here and do dictionary ops - - next use dictionary to navigate to posting list for a specific term - - walk posting list - - if necessary, walk posting details as we go - - if location info is desired, consult location bitmap to see if it is there - -## stored fields section - -- for each document - - preparation phase: - - produce a slice of metadata bytes and data bytes - - produce these slices in field id order - - field value is appended to the data slice - - metadata slice is varint encoded with the following values for each field value - - field id (uint16) - - field type (byte) - - field value start offset in uncompressed data slice (uint64) - - field value length (uint64) - - field number of array positions (uint64) - - one additional value for each array position (uint64) - - compress the data slice using snappy - - file writing phase: - - remember the start offset for this document - - write out meta data length (varint uint64) - - write out compressed data length (varint uint64) - - write out the metadata bytes - - write out the compressed data bytes - -## stored fields idx - -- for each document - - write start offset (remembered from previous section) of stored data (big endian uint64) - -With this index and a known document number, we have direct access to all the stored field data. - -## posting details (freq/norm) section - -- for each posting list - - produce a slice containing multiple consecutive chunks (each chunk is varint stream) - - produce a slice remembering offsets of where each chunk starts - - preparation phase: - - for each hit in the posting list - - if this hit is in next chunk close out encoding of last chunk and record offset start of next - - encode term frequency (uint64) - - encode norm factor (float32) - - file writing phase: - - remember start position for this posting list details - - write out number of chunks that follow (varint uint64) - - write out length of each chunk (each a varint uint64) - - write out the byte slice containing all the chunk data - -If you know the doc number you're interested in, this format lets you jump to the correct chunk (docNum/chunkFactor) directly and then seek within that chunk until you find it. - -## posting details (location) section - -- for each posting list - - produce a slice containing multiple consecutive chunks (each chunk is varint stream) - - produce a slice remembering offsets of where each chunk starts - - preparation phase: - - for each hit in the posting list - - if this hit is in next chunk close out encoding of last chunk and record offset start of next - - encode field (uint16) - - encode field pos (uint64) - - encode field start (uint64) - - encode field end (uint64) - - encode number of array positions to follow (uint64) - - encode each array position (each uint64) - - file writing phase: - - remember start position for this posting list details - - write out number of chunks that follow (varint uint64) - - write out length of each chunk (each a varint uint64) - - write out the byte slice containing all the chunk data - -If you know the doc number you're interested in, this format lets you jump to the correct chunk (docNum/chunkFactor) directly and then seek within that chunk until you find it. - -## postings list section - -- for each posting list - - preparation phase: - - encode roaring bitmap posting list to bytes (so we know the length) - - file writing phase: - - remember the start position for this posting list - - write freq/norm details offset (remembered from previous, as varint uint64) - - write location details offset (remembered from previous, as varint uint64) - - write length of encoded roaring bitmap - - write the serialized roaring bitmap data - -## dictionary - -- for each field - - preparation phase: - - encode vellum FST with dictionary data pointing to file offset of posting list (remembered from previous) - - file writing phase: - - remember the start position of this persistDictionary - - write length of vellum data (varint uint64) - - write out vellum data - -## fields section - -- for each field - - file writing phase: - - remember start offset for each field - - write dictionary address (remembered from previous) (varint uint64) - - write length of field name (varint uint64) - - write field name bytes - -## fields idx - -- for each field - - file writing phase: - - write big endian uint64 of start offset for each field - -NOTE: currently we don't know or record the length of this fields index. Instead we rely on the fact that we know it immediately precedes a footer of known size. - -## fields DocValue - -- for each field - - preparation phase: - - produce a slice containing multiple consecutive chunks, where each chunk is composed of a meta section followed by compressed columnar field data - - produce a slice remembering the length of each chunk - - file writing phase: - - remember the start position of this first field DocValue offset in the footer - - write out number of chunks that follow (varint uint64) - - write out length of each chunk (each a varint uint64) - - write out the byte slice containing all the chunk data - -NOTE: currently the meta header inside each chunk gives clue to the location offsets and size of the data pertaining to a given docID and any -read operation leverage that meta information to extract the document specific data from the file. - -## footer - -- file writing phase - - write number of docs (big endian uint64) - - write stored field index location (big endian uint64) - - write field index location (big endian uint64) - - write field docValue location (big endian uint64) - - write out chunk factor (big endian uint32) - - write out version (big endian uint32) - - write out file CRC of everything preceding this (big endian uint32) diff --git a/vendor/github.com/blevesearch/zap/v12/build.go b/vendor/github.com/blevesearch/zap/v12/build.go deleted file mode 100644 index 467e5e00..00000000 --- a/vendor/github.com/blevesearch/zap/v12/build.go +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "bufio" - "math" - "os" - - "github.com/couchbase/vellum" -) - -const Version uint32 = 12 - -const Type string = "zap" - -const fieldNotUninverted = math.MaxUint64 - -func (sb *SegmentBase) Persist(path string) error { - return PersistSegmentBase(sb, path) -} - -// PersistSegmentBase persists SegmentBase in the zap file format. -func PersistSegmentBase(sb *SegmentBase, path string) error { - flag := os.O_RDWR | os.O_CREATE - - f, err := os.OpenFile(path, flag, 0600) - if err != nil { - return err - } - - cleanup := func() { - _ = f.Close() - _ = os.Remove(path) - } - - br := bufio.NewWriter(f) - - _, err = br.Write(sb.mem) - if err != nil { - cleanup() - return err - } - - err = persistFooter(sb.numDocs, sb.storedIndexOffset, sb.fieldsIndexOffset, sb.docValueOffset, - sb.chunkMode, sb.memCRC, br) - if err != nil { - cleanup() - return err - } - - err = br.Flush() - if err != nil { - cleanup() - return err - } - - err = f.Sync() - if err != nil { - cleanup() - return err - } - - err = f.Close() - if err != nil { - cleanup() - return err - } - - return nil -} - -func persistStoredFieldValues(fieldID int, - storedFieldValues [][]byte, stf []byte, spf [][]uint64, - curr int, metaEncode varintEncoder, data []byte) ( - int, []byte, error) { - for i := 0; i < len(storedFieldValues); i++ { - // encode field - _, err := metaEncode(uint64(fieldID)) - if err != nil { - return 0, nil, err - } - // encode type - _, err = metaEncode(uint64(stf[i])) - if err != nil { - return 0, nil, err - } - // encode start offset - _, err = metaEncode(uint64(curr)) - if err != nil { - return 0, nil, err - } - // end len - _, err = metaEncode(uint64(len(storedFieldValues[i]))) - if err != nil { - return 0, nil, err - } - // encode number of array pos - _, err = metaEncode(uint64(len(spf[i]))) - if err != nil { - return 0, nil, err - } - // encode all array positions - for _, pos := range spf[i] { - _, err = metaEncode(pos) - if err != nil { - return 0, nil, err - } - } - - data = append(data, storedFieldValues[i]...) - curr += len(storedFieldValues[i]) - } - - return curr, data, nil -} - -func InitSegmentBase(mem []byte, memCRC uint32, chunkMode uint32, - fieldsMap map[string]uint16, fieldsInv []string, numDocs uint64, - storedIndexOffset uint64, fieldsIndexOffset uint64, docValueOffset uint64, - dictLocs []uint64) (*SegmentBase, error) { - sb := &SegmentBase{ - mem: mem, - memCRC: memCRC, - chunkMode: chunkMode, - fieldsMap: fieldsMap, - fieldsInv: fieldsInv, - numDocs: numDocs, - storedIndexOffset: storedIndexOffset, - fieldsIndexOffset: fieldsIndexOffset, - docValueOffset: docValueOffset, - dictLocs: dictLocs, - fieldDvReaders: make(map[uint16]*docValueReader), - fieldFSTs: make(map[uint16]*vellum.FST), - } - sb.updateSize() - - err := sb.loadDvReaders() - if err != nil { - return nil, err - } - - return sb, nil -} diff --git a/vendor/github.com/blevesearch/zap/v12/count.go b/vendor/github.com/blevesearch/zap/v12/count.go deleted file mode 100644 index 50290f88..00000000 --- a/vendor/github.com/blevesearch/zap/v12/count.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "hash/crc32" - "io" - - "github.com/blevesearch/bleve/index/scorch/segment" -) - -// CountHashWriter is a wrapper around a Writer which counts the number of -// bytes which have been written and computes a crc32 hash -type CountHashWriter struct { - w io.Writer - crc uint32 - n int - s segment.StatsReporter -} - -// NewCountHashWriter returns a CountHashWriter which wraps the provided Writer -func NewCountHashWriter(w io.Writer) *CountHashWriter { - return &CountHashWriter{w: w} -} - -func NewCountHashWriterWithStatsReporter(w io.Writer, s segment.StatsReporter) *CountHashWriter { - return &CountHashWriter{w: w, s: s} -} - -// Write writes the provided bytes to the wrapped writer and counts the bytes -func (c *CountHashWriter) Write(b []byte) (int, error) { - n, err := c.w.Write(b) - c.crc = crc32.Update(c.crc, crc32.IEEETable, b[:n]) - c.n += n - if c.s != nil { - c.s.ReportBytesWritten(uint64(n)) - } - return n, err -} - -// Count returns the number of bytes written -func (c *CountHashWriter) Count() int { - return c.n -} - -// Sum32 returns the CRC-32 hash of the content written to this writer -func (c *CountHashWriter) Sum32() uint32 { - return c.crc -} diff --git a/vendor/github.com/blevesearch/zap/v12/dict.go b/vendor/github.com/blevesearch/zap/v12/dict.go deleted file mode 100644 index ad4a8f8d..00000000 --- a/vendor/github.com/blevesearch/zap/v12/dict.go +++ /dev/null @@ -1,263 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "bytes" - "fmt" - - "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/couchbase/vellum" -) - -// Dictionary is the zap representation of the term dictionary -type Dictionary struct { - sb *SegmentBase - field string - fieldID uint16 - fst *vellum.FST - fstReader *vellum.Reader -} - -// PostingsList returns the postings list for the specified term -func (d *Dictionary) PostingsList(term []byte, except *roaring.Bitmap, - prealloc segment.PostingsList) (segment.PostingsList, error) { - var preallocPL *PostingsList - pl, ok := prealloc.(*PostingsList) - if ok && pl != nil { - preallocPL = pl - } - return d.postingsList(term, except, preallocPL) -} - -func (d *Dictionary) postingsList(term []byte, except *roaring.Bitmap, rv *PostingsList) (*PostingsList, error) { - if d.fstReader == nil { - if rv == nil || rv == emptyPostingsList { - return emptyPostingsList, nil - } - return d.postingsListInit(rv, except), nil - } - - postingsOffset, exists, err := d.fstReader.Get(term) - if err != nil { - return nil, fmt.Errorf("vellum err: %v", err) - } - if !exists { - if rv == nil || rv == emptyPostingsList { - return emptyPostingsList, nil - } - return d.postingsListInit(rv, except), nil - } - - return d.postingsListFromOffset(postingsOffset, except, rv) -} - -func (d *Dictionary) postingsListFromOffset(postingsOffset uint64, except *roaring.Bitmap, rv *PostingsList) (*PostingsList, error) { - rv = d.postingsListInit(rv, except) - - err := rv.read(postingsOffset, d) - if err != nil { - return nil, err - } - - return rv, nil -} - -func (d *Dictionary) postingsListInit(rv *PostingsList, except *roaring.Bitmap) *PostingsList { - if rv == nil || rv == emptyPostingsList { - rv = &PostingsList{} - } else { - postings := rv.postings - if postings != nil { - postings.Clear() - } - - *rv = PostingsList{} // clear the struct - - rv.postings = postings - } - rv.sb = d.sb - rv.except = except - return rv -} - -func (d *Dictionary) Contains(key []byte) (bool, error) { - return d.fst.Contains(key) -} - -// Iterator returns an iterator for this dictionary -func (d *Dictionary) Iterator() segment.DictionaryIterator { - rv := &DictionaryIterator{ - d: d, - } - - if d.fst != nil { - itr, err := d.fst.Iterator(nil, nil) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - } - - return rv -} - -// PrefixIterator returns an iterator which only visits terms having the -// the specified prefix -func (d *Dictionary) PrefixIterator(prefix string) segment.DictionaryIterator { - rv := &DictionaryIterator{ - d: d, - } - - kBeg := []byte(prefix) - kEnd := segment.IncrementBytes(kBeg) - - if d.fst != nil { - itr, err := d.fst.Iterator(kBeg, kEnd) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - } - - return rv -} - -// RangeIterator returns an iterator which only visits terms between the -// start and end terms. NOTE: bleve.index API specifies the end is inclusive. -func (d *Dictionary) RangeIterator(start, end string) segment.DictionaryIterator { - rv := &DictionaryIterator{ - d: d, - } - - // need to increment the end position to be inclusive - var endBytes []byte - if len(end) > 0 { - endBytes = []byte(end) - if endBytes[len(endBytes)-1] < 0xff { - endBytes[len(endBytes)-1]++ - } else { - endBytes = append(endBytes, 0xff) - } - } - - if d.fst != nil { - itr, err := d.fst.Iterator([]byte(start), endBytes) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - } - - return rv -} - -// AutomatonIterator returns an iterator which only visits terms -// having the the vellum automaton and start/end key range -func (d *Dictionary) AutomatonIterator(a vellum.Automaton, - startKeyInclusive, endKeyExclusive []byte) segment.DictionaryIterator { - rv := &DictionaryIterator{ - d: d, - } - - if d.fst != nil { - itr, err := d.fst.Search(a, startKeyInclusive, endKeyExclusive) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - } - - return rv -} - -func (d *Dictionary) OnlyIterator(onlyTerms [][]byte, - includeCount bool) segment.DictionaryIterator { - - rv := &DictionaryIterator{ - d: d, - omitCount: !includeCount, - } - - var buf bytes.Buffer - builder, err := vellum.New(&buf, nil) - if err != nil { - rv.err = err - return rv - } - for _, term := range onlyTerms { - err = builder.Insert(term, 0) - if err != nil { - rv.err = err - return rv - } - } - err = builder.Close() - if err != nil { - rv.err = err - return rv - } - - onlyFST, err := vellum.Load(buf.Bytes()) - if err != nil { - rv.err = err - return rv - } - - itr, err := d.fst.Search(onlyFST, nil, nil) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - - return rv -} - -// DictionaryIterator is an iterator for term dictionary -type DictionaryIterator struct { - d *Dictionary - itr vellum.Iterator - err error - tmp PostingsList - entry index.DictEntry - omitCount bool -} - -// Next returns the next entry in the dictionary -func (i *DictionaryIterator) Next() (*index.DictEntry, error) { - if i.err != nil && i.err != vellum.ErrIteratorDone { - return nil, i.err - } else if i.itr == nil || i.err == vellum.ErrIteratorDone { - return nil, nil - } - term, postingsOffset := i.itr.Current() - i.entry.Term = string(term) - if !i.omitCount { - i.err = i.tmp.read(postingsOffset, i.d) - if i.err != nil { - return nil, i.err - } - i.entry.Count = i.tmp.Count() - } - i.err = i.itr.Next() - return &i.entry, nil -} diff --git a/vendor/github.com/blevesearch/zap/v12/docvalues.go b/vendor/github.com/blevesearch/zap/v12/docvalues.go deleted file mode 100644 index 793797bd..00000000 --- a/vendor/github.com/blevesearch/zap/v12/docvalues.go +++ /dev/null @@ -1,312 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "bytes" - "encoding/binary" - "fmt" - "math" - "reflect" - "sort" - - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/blevesearch/bleve/size" - "github.com/golang/snappy" -) - -var reflectStaticSizedocValueReader int - -func init() { - var dvi docValueReader - reflectStaticSizedocValueReader = int(reflect.TypeOf(dvi).Size()) -} - -type docNumTermsVisitor func(docNum uint64, terms []byte) error - -type docVisitState struct { - dvrs map[uint16]*docValueReader - segment *SegmentBase -} - -type docValueReader struct { - field string - curChunkNum uint64 - chunkOffsets []uint64 - dvDataLoc uint64 - curChunkHeader []MetaData - curChunkData []byte // compressed data cache - uncompressed []byte // temp buf for snappy decompression -} - -func (di *docValueReader) size() int { - return reflectStaticSizedocValueReader + size.SizeOfPtr + - len(di.field) + - len(di.chunkOffsets)*size.SizeOfUint64 + - len(di.curChunkHeader)*reflectStaticSizeMetaData + - len(di.curChunkData) -} - -func (di *docValueReader) cloneInto(rv *docValueReader) *docValueReader { - if rv == nil { - rv = &docValueReader{} - } - - rv.field = di.field - rv.curChunkNum = math.MaxUint64 - rv.chunkOffsets = di.chunkOffsets // immutable, so it's sharable - rv.dvDataLoc = di.dvDataLoc - rv.curChunkHeader = rv.curChunkHeader[:0] - rv.curChunkData = nil - rv.uncompressed = rv.uncompressed[:0] - - return rv -} - -func (di *docValueReader) curChunkNumber() uint64 { - return di.curChunkNum -} - -func (s *SegmentBase) loadFieldDocValueReader(field string, - fieldDvLocStart, fieldDvLocEnd uint64) (*docValueReader, error) { - // get the docValue offset for the given fields - if fieldDvLocStart == fieldNotUninverted { - // no docValues found, nothing to do - return nil, nil - } - - // read the number of chunks, and chunk offsets position - var numChunks, chunkOffsetsPosition uint64 - - if fieldDvLocEnd-fieldDvLocStart > 16 { - numChunks = binary.BigEndian.Uint64(s.mem[fieldDvLocEnd-8 : fieldDvLocEnd]) - // read the length of chunk offsets - chunkOffsetsLen := binary.BigEndian.Uint64(s.mem[fieldDvLocEnd-16 : fieldDvLocEnd-8]) - // acquire position of chunk offsets - chunkOffsetsPosition = (fieldDvLocEnd - 16) - chunkOffsetsLen - } else { - return nil, fmt.Errorf("loadFieldDocValueReader: fieldDvLoc too small: %d-%d", fieldDvLocEnd, fieldDvLocStart) - } - - fdvIter := &docValueReader{ - curChunkNum: math.MaxUint64, - field: field, - chunkOffsets: make([]uint64, int(numChunks)), - } - - // read the chunk offsets - var offset uint64 - for i := 0; i < int(numChunks); i++ { - loc, read := binary.Uvarint(s.mem[chunkOffsetsPosition+offset : chunkOffsetsPosition+offset+binary.MaxVarintLen64]) - if read <= 0 { - return nil, fmt.Errorf("corrupted chunk offset during segment load") - } - fdvIter.chunkOffsets[i] = loc - offset += uint64(read) - } - - // set the data offset - fdvIter.dvDataLoc = fieldDvLocStart - - return fdvIter, nil -} - -func (di *docValueReader) loadDvChunk(chunkNumber uint64, s *SegmentBase) error { - // advance to the chunk where the docValues - // reside for the given docNum - destChunkDataLoc, curChunkEnd := di.dvDataLoc, di.dvDataLoc - start, end := readChunkBoundary(int(chunkNumber), di.chunkOffsets) - if start >= end { - di.curChunkHeader = di.curChunkHeader[:0] - di.curChunkData = nil - di.curChunkNum = chunkNumber - di.uncompressed = di.uncompressed[:0] - return nil - } - - destChunkDataLoc += start - curChunkEnd += end - - // read the number of docs reside in the chunk - numDocs, read := binary.Uvarint(s.mem[destChunkDataLoc : destChunkDataLoc+binary.MaxVarintLen64]) - if read <= 0 { - return fmt.Errorf("failed to read the chunk") - } - chunkMetaLoc := destChunkDataLoc + uint64(read) - - offset := uint64(0) - if cap(di.curChunkHeader) < int(numDocs) { - di.curChunkHeader = make([]MetaData, int(numDocs)) - } else { - di.curChunkHeader = di.curChunkHeader[:int(numDocs)] - } - for i := 0; i < int(numDocs); i++ { - di.curChunkHeader[i].DocNum, read = binary.Uvarint(s.mem[chunkMetaLoc+offset : chunkMetaLoc+offset+binary.MaxVarintLen64]) - offset += uint64(read) - di.curChunkHeader[i].DocDvOffset, read = binary.Uvarint(s.mem[chunkMetaLoc+offset : chunkMetaLoc+offset+binary.MaxVarintLen64]) - offset += uint64(read) - } - - compressedDataLoc := chunkMetaLoc + offset - dataLength := curChunkEnd - compressedDataLoc - di.curChunkData = s.mem[compressedDataLoc : compressedDataLoc+dataLength] - di.curChunkNum = chunkNumber - di.uncompressed = di.uncompressed[:0] - return nil -} - -func (di *docValueReader) iterateAllDocValues(s *SegmentBase, visitor docNumTermsVisitor) error { - for i := 0; i < len(di.chunkOffsets); i++ { - err := di.loadDvChunk(uint64(i), s) - if err != nil { - return err - } - if di.curChunkData == nil || len(di.curChunkHeader) == 0 { - continue - } - - // uncompress the already loaded data - uncompressed, err := snappy.Decode(di.uncompressed[:cap(di.uncompressed)], di.curChunkData) - if err != nil { - return err - } - di.uncompressed = uncompressed - - start := uint64(0) - for _, entry := range di.curChunkHeader { - err = visitor(entry.DocNum, uncompressed[start:entry.DocDvOffset]) - if err != nil { - return err - } - - start = entry.DocDvOffset - } - } - - return nil -} - -func (di *docValueReader) visitDocValues(docNum uint64, - visitor index.DocumentFieldTermVisitor) error { - // binary search the term locations for the docNum - start, end := di.getDocValueLocs(docNum) - if start == math.MaxUint64 || end == math.MaxUint64 || start == end { - return nil - } - - var uncompressed []byte - var err error - // use the uncompressed copy if available - if len(di.uncompressed) > 0 { - uncompressed = di.uncompressed - } else { - // uncompress the already loaded data - uncompressed, err = snappy.Decode(di.uncompressed[:cap(di.uncompressed)], di.curChunkData) - if err != nil { - return err - } - di.uncompressed = uncompressed - } - - // pick the terms for the given docNum - uncompressed = uncompressed[start:end] - for { - i := bytes.Index(uncompressed, termSeparatorSplitSlice) - if i < 0 { - break - } - - visitor(di.field, uncompressed[0:i]) - uncompressed = uncompressed[i+1:] - } - - return nil -} - -func (di *docValueReader) getDocValueLocs(docNum uint64) (uint64, uint64) { - i := sort.Search(len(di.curChunkHeader), func(i int) bool { - return di.curChunkHeader[i].DocNum >= docNum - }) - if i < len(di.curChunkHeader) && di.curChunkHeader[i].DocNum == docNum { - return ReadDocValueBoundary(i, di.curChunkHeader) - } - return math.MaxUint64, math.MaxUint64 -} - -// VisitDocumentFieldTerms is an implementation of the -// DocumentFieldTermVisitable interface -func (s *SegmentBase) VisitDocumentFieldTerms(localDocNum uint64, fields []string, - visitor index.DocumentFieldTermVisitor, dvsIn segment.DocVisitState) ( - segment.DocVisitState, error) { - dvs, ok := dvsIn.(*docVisitState) - if !ok || dvs == nil { - dvs = &docVisitState{} - } else { - if dvs.segment != s { - dvs.segment = s - dvs.dvrs = nil - } - } - - var fieldIDPlus1 uint16 - if dvs.dvrs == nil { - dvs.dvrs = make(map[uint16]*docValueReader, len(fields)) - for _, field := range fields { - if fieldIDPlus1, ok = s.fieldsMap[field]; !ok { - continue - } - fieldID := fieldIDPlus1 - 1 - if dvIter, exists := s.fieldDvReaders[fieldID]; exists && - dvIter != nil { - dvs.dvrs[fieldID] = dvIter.cloneInto(dvs.dvrs[fieldID]) - } - } - } - - // find the chunkNumber where the docValues are stored - // NOTE: doc values continue to use legacy chunk mode - chunkFactor, err := getChunkSize(LegacyChunkMode, 0, 0) - if err != nil { - return nil, err - } - docInChunk := localDocNum / chunkFactor - var dvr *docValueReader - for _, field := range fields { - if fieldIDPlus1, ok = s.fieldsMap[field]; !ok { - continue - } - fieldID := fieldIDPlus1 - 1 - if dvr, ok = dvs.dvrs[fieldID]; ok && dvr != nil { - // check if the chunk is already loaded - if docInChunk != dvr.curChunkNumber() { - err := dvr.loadDvChunk(docInChunk, s) - if err != nil { - return dvs, err - } - } - - _ = dvr.visitDocValues(localDocNum, visitor) - } - } - return dvs, nil -} - -// VisitableDocValueFields returns the list of fields with -// persisted doc value terms ready to be visitable using the -// VisitDocumentFieldTerms method. -func (s *SegmentBase) VisitableDocValueFields() ([]string, error) { - return s.fieldDvNames, nil -} diff --git a/vendor/github.com/blevesearch/zap/v12/enumerator.go b/vendor/github.com/blevesearch/zap/v12/enumerator.go deleted file mode 100644 index bc5b7e62..00000000 --- a/vendor/github.com/blevesearch/zap/v12/enumerator.go +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright (c) 2018 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "bytes" - - "github.com/couchbase/vellum" -) - -// enumerator provides an ordered traversal of multiple vellum -// iterators. Like JOIN of iterators, the enumerator produces a -// sequence of (key, iteratorIndex, value) tuples, sorted by key ASC, -// then iteratorIndex ASC, where the same key might be seen or -// repeated across multiple child iterators. -type enumerator struct { - itrs []vellum.Iterator - currKs [][]byte - currVs []uint64 - - lowK []byte - lowIdxs []int - lowCurr int -} - -// newEnumerator returns a new enumerator over the vellum Iterators -func newEnumerator(itrs []vellum.Iterator) (*enumerator, error) { - rv := &enumerator{ - itrs: itrs, - currKs: make([][]byte, len(itrs)), - currVs: make([]uint64, len(itrs)), - lowIdxs: make([]int, 0, len(itrs)), - } - for i, itr := range rv.itrs { - rv.currKs[i], rv.currVs[i] = itr.Current() - } - rv.updateMatches(false) - if rv.lowK == nil && len(rv.lowIdxs) == 0 { - return rv, vellum.ErrIteratorDone - } - return rv, nil -} - -// updateMatches maintains the low key matches based on the currKs -func (m *enumerator) updateMatches(skipEmptyKey bool) { - m.lowK = nil - m.lowIdxs = m.lowIdxs[:0] - m.lowCurr = 0 - - for i, key := range m.currKs { - if (key == nil && m.currVs[i] == 0) || // in case of empty iterator - (len(key) == 0 && skipEmptyKey) { // skip empty keys - continue - } - - cmp := bytes.Compare(key, m.lowK) - if cmp < 0 || len(m.lowIdxs) == 0 { - // reached a new low - m.lowK = key - m.lowIdxs = m.lowIdxs[:0] - m.lowIdxs = append(m.lowIdxs, i) - } else if cmp == 0 { - m.lowIdxs = append(m.lowIdxs, i) - } - } -} - -// Current returns the enumerator's current key, iterator-index, and -// value. If the enumerator is not pointing at a valid value (because -// Next returned an error previously), Current will return nil,0,0. -func (m *enumerator) Current() ([]byte, int, uint64) { - var i int - var v uint64 - if m.lowCurr < len(m.lowIdxs) { - i = m.lowIdxs[m.lowCurr] - v = m.currVs[i] - } - return m.lowK, i, v -} - -// GetLowIdxsAndValues will return all of the iterator indices -// which point to the current key, and their corresponding -// values. This can be used by advanced caller which may need -// to peek into these other sets of data before processing. -func (m *enumerator) GetLowIdxsAndValues() ([]int, []uint64) { - values := make([]uint64, 0, len(m.lowIdxs)) - for _, idx := range m.lowIdxs { - values = append(values, m.currVs[idx]) - } - return m.lowIdxs, values -} - -// Next advances the enumerator to the next key/iterator/value result, -// else vellum.ErrIteratorDone is returned. -func (m *enumerator) Next() error { - m.lowCurr += 1 - if m.lowCurr >= len(m.lowIdxs) { - // move all the current low iterators forwards - for _, vi := range m.lowIdxs { - err := m.itrs[vi].Next() - if err != nil && err != vellum.ErrIteratorDone { - return err - } - m.currKs[vi], m.currVs[vi] = m.itrs[vi].Current() - } - // can skip any empty keys encountered at this point - m.updateMatches(true) - } - if m.lowK == nil && len(m.lowIdxs) == 0 { - return vellum.ErrIteratorDone - } - return nil -} - -// Close all the underlying Iterators. The first error, if any, will -// be returned. -func (m *enumerator) Close() error { - var rv error - for _, itr := range m.itrs { - err := itr.Close() - if rv == nil { - rv = err - } - } - return rv -} diff --git a/vendor/github.com/blevesearch/zap/v12/intDecoder.go b/vendor/github.com/blevesearch/zap/v12/intDecoder.go deleted file mode 100644 index 4cd008ff..00000000 --- a/vendor/github.com/blevesearch/zap/v12/intDecoder.go +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright (c) 2019 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "encoding/binary" - "fmt" - - "github.com/blevesearch/bleve/index/scorch/segment" -) - -type chunkedIntDecoder struct { - startOffset uint64 - dataStartOffset uint64 - chunkOffsets []uint64 - curChunkBytes []byte - data []byte - r *segment.MemUvarintReader -} - -func newChunkedIntDecoder(buf []byte, offset uint64) *chunkedIntDecoder { - rv := &chunkedIntDecoder{startOffset: offset, data: buf} - var n, numChunks uint64 - var read int - if offset == termNotEncoded { - numChunks = 0 - } else { - numChunks, read = binary.Uvarint(buf[offset+n : offset+n+binary.MaxVarintLen64]) - } - - n += uint64(read) - if cap(rv.chunkOffsets) >= int(numChunks) { - rv.chunkOffsets = rv.chunkOffsets[:int(numChunks)] - } else { - rv.chunkOffsets = make([]uint64, int(numChunks)) - } - for i := 0; i < int(numChunks); i++ { - rv.chunkOffsets[i], read = binary.Uvarint(buf[offset+n : offset+n+binary.MaxVarintLen64]) - n += uint64(read) - } - rv.dataStartOffset = offset + n - return rv -} - -func (d *chunkedIntDecoder) loadChunk(chunk int) error { - if d.startOffset == termNotEncoded { - d.r = segment.NewMemUvarintReader([]byte(nil)) - return nil - } - - if chunk >= len(d.chunkOffsets) { - return fmt.Errorf("tried to load freq chunk that doesn't exist %d/(%d)", - chunk, len(d.chunkOffsets)) - } - - end, start := d.dataStartOffset, d.dataStartOffset - s, e := readChunkBoundary(chunk, d.chunkOffsets) - start += s - end += e - d.curChunkBytes = d.data[start:end] - if d.r == nil { - d.r = segment.NewMemUvarintReader(d.curChunkBytes) - } else { - d.r.Reset(d.curChunkBytes) - } - - return nil -} - -func (d *chunkedIntDecoder) reset() { - d.startOffset = 0 - d.dataStartOffset = 0 - d.chunkOffsets = d.chunkOffsets[:0] - d.curChunkBytes = d.curChunkBytes[:0] - d.data = d.data[:0] - if d.r != nil { - d.r.Reset([]byte(nil)) - } -} - -func (d *chunkedIntDecoder) isNil() bool { - return d.curChunkBytes == nil -} - -func (d *chunkedIntDecoder) readUvarint() (uint64, error) { - return d.r.ReadUvarint() -} - -func (d *chunkedIntDecoder) SkipUvarint() { - d.r.SkipUvarint() -} - -func (d *chunkedIntDecoder) SkipBytes(count int) { - d.r.SkipBytes(count) -} - -func (d *chunkedIntDecoder) Len() int { - return d.r.Len() -} diff --git a/vendor/github.com/blevesearch/zap/v12/merge.go b/vendor/github.com/blevesearch/zap/v12/merge.go deleted file mode 100644 index 805100fb..00000000 --- a/vendor/github.com/blevesearch/zap/v12/merge.go +++ /dev/null @@ -1,847 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "bufio" - "bytes" - "encoding/binary" - "fmt" - "math" - "os" - "sort" - - "github.com/RoaringBitmap/roaring" - seg "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/couchbase/vellum" - "github.com/golang/snappy" -) - -var DefaultFileMergerBufferSize = 1024 * 1024 - -const docDropped = math.MaxUint64 // sentinel docNum to represent a deleted doc - -// Merge takes a slice of segments and bit masks describing which -// documents may be dropped, and creates a new segment containing the -// remaining data. This new segment is built at the specified path. -func (*ZapPlugin) Merge(segments []seg.Segment, drops []*roaring.Bitmap, path string, - closeCh chan struct{}, s seg.StatsReporter) ( - [][]uint64, uint64, error) { - - segmentBases := make([]*SegmentBase, len(segments)) - for segmenti, segment := range segments { - switch segmentx := segment.(type) { - case *Segment: - segmentBases[segmenti] = &segmentx.SegmentBase - case *SegmentBase: - segmentBases[segmenti] = segmentx - default: - panic(fmt.Sprintf("oops, unexpected segment type: %T", segment)) - } - } - return mergeSegmentBases(segmentBases, drops, path, DefaultChunkMode, closeCh, s) -} - -func mergeSegmentBases(segmentBases []*SegmentBase, drops []*roaring.Bitmap, path string, - chunkMode uint32, closeCh chan struct{}, s seg.StatsReporter) ( - [][]uint64, uint64, error) { - flag := os.O_RDWR | os.O_CREATE - - f, err := os.OpenFile(path, flag, 0600) - if err != nil { - return nil, 0, err - } - - cleanup := func() { - _ = f.Close() - _ = os.Remove(path) - } - - // buffer the output - br := bufio.NewWriterSize(f, DefaultFileMergerBufferSize) - - // wrap it for counting (tracking offsets) - cr := NewCountHashWriterWithStatsReporter(br, s) - - newDocNums, numDocs, storedIndexOffset, fieldsIndexOffset, docValueOffset, _, _, _, err := - MergeToWriter(segmentBases, drops, chunkMode, cr, closeCh) - if err != nil { - cleanup() - return nil, 0, err - } - - err = persistFooter(numDocs, storedIndexOffset, fieldsIndexOffset, - docValueOffset, chunkMode, cr.Sum32(), cr) - if err != nil { - cleanup() - return nil, 0, err - } - - err = br.Flush() - if err != nil { - cleanup() - return nil, 0, err - } - - err = f.Sync() - if err != nil { - cleanup() - return nil, 0, err - } - - err = f.Close() - if err != nil { - cleanup() - return nil, 0, err - } - - return newDocNums, uint64(cr.Count()), nil -} - -func MergeToWriter(segments []*SegmentBase, drops []*roaring.Bitmap, - chunkMode uint32, cr *CountHashWriter, closeCh chan struct{}) ( - newDocNums [][]uint64, - numDocs, storedIndexOffset, fieldsIndexOffset, docValueOffset uint64, - dictLocs []uint64, fieldsInv []string, fieldsMap map[string]uint16, - err error) { - docValueOffset = uint64(fieldNotUninverted) - - var fieldsSame bool - fieldsSame, fieldsInv = mergeFields(segments) - fieldsMap = mapFields(fieldsInv) - - numDocs = computeNewDocCount(segments, drops) - - if isClosed(closeCh) { - return nil, 0, 0, 0, 0, nil, nil, nil, seg.ErrClosed - } - - if numDocs > 0 { - storedIndexOffset, newDocNums, err = mergeStoredAndRemap(segments, drops, - fieldsMap, fieldsInv, fieldsSame, numDocs, cr, closeCh) - if err != nil { - return nil, 0, 0, 0, 0, nil, nil, nil, err - } - - dictLocs, docValueOffset, err = persistMergedRest(segments, drops, - fieldsInv, fieldsMap, fieldsSame, - newDocNums, numDocs, chunkMode, cr, closeCh) - if err != nil { - return nil, 0, 0, 0, 0, nil, nil, nil, err - } - } else { - dictLocs = make([]uint64, len(fieldsInv)) - } - - fieldsIndexOffset, err = persistFields(fieldsInv, cr, dictLocs) - if err != nil { - return nil, 0, 0, 0, 0, nil, nil, nil, err - } - - return newDocNums, numDocs, storedIndexOffset, fieldsIndexOffset, docValueOffset, dictLocs, fieldsInv, fieldsMap, nil -} - -// mapFields takes the fieldsInv list and returns a map of fieldName -// to fieldID+1 -func mapFields(fields []string) map[string]uint16 { - rv := make(map[string]uint16, len(fields)) - for i, fieldName := range fields { - rv[fieldName] = uint16(i) + 1 - } - return rv -} - -// computeNewDocCount determines how many documents will be in the newly -// merged segment when obsoleted docs are dropped -func computeNewDocCount(segments []*SegmentBase, drops []*roaring.Bitmap) uint64 { - var newDocCount uint64 - for segI, segment := range segments { - newDocCount += segment.numDocs - if drops[segI] != nil { - newDocCount -= drops[segI].GetCardinality() - } - } - return newDocCount -} - -func persistMergedRest(segments []*SegmentBase, dropsIn []*roaring.Bitmap, - fieldsInv []string, fieldsMap map[string]uint16, fieldsSame bool, - newDocNumsIn [][]uint64, newSegDocCount uint64, chunkMode uint32, - w *CountHashWriter, closeCh chan struct{}) ([]uint64, uint64, error) { - - var bufMaxVarintLen64 []byte = make([]byte, binary.MaxVarintLen64) - var bufLoc []uint64 - - var postings *PostingsList - var postItr *PostingsIterator - - rv := make([]uint64, len(fieldsInv)) - fieldDvLocsStart := make([]uint64, len(fieldsInv)) - fieldDvLocsEnd := make([]uint64, len(fieldsInv)) - - // these int coders are initialized with chunk size 1024 - // however this will be reset to the correct chunk size - // while processing each individual field-term section - tfEncoder := newChunkedIntCoder(1024, newSegDocCount-1) - locEncoder := newChunkedIntCoder(1024, newSegDocCount-1) - - var vellumBuf bytes.Buffer - newVellum, err := vellum.New(&vellumBuf, nil) - if err != nil { - return nil, 0, err - } - - newRoaring := roaring.NewBitmap() - - // for each field - for fieldID, fieldName := range fieldsInv { - - // collect FST iterators from all active segments for this field - var newDocNums [][]uint64 - var drops []*roaring.Bitmap - var dicts []*Dictionary - var itrs []vellum.Iterator - - var segmentsInFocus []*SegmentBase - - for segmentI, segment := range segments { - - // check for the closure in meantime - if isClosed(closeCh) { - return nil, 0, seg.ErrClosed - } - - dict, err2 := segment.dictionary(fieldName) - if err2 != nil { - return nil, 0, err2 - } - if dict != nil && dict.fst != nil { - itr, err2 := dict.fst.Iterator(nil, nil) - if err2 != nil && err2 != vellum.ErrIteratorDone { - return nil, 0, err2 - } - if itr != nil { - newDocNums = append(newDocNums, newDocNumsIn[segmentI]) - if dropsIn[segmentI] != nil && !dropsIn[segmentI].IsEmpty() { - drops = append(drops, dropsIn[segmentI]) - } else { - drops = append(drops, nil) - } - dicts = append(dicts, dict) - itrs = append(itrs, itr) - segmentsInFocus = append(segmentsInFocus, segment) - } - } - } - - var prevTerm []byte - - newRoaring.Clear() - - var lastDocNum, lastFreq, lastNorm uint64 - - // determines whether to use "1-hit" encoding optimization - // when a term appears in only 1 doc, with no loc info, - // has freq of 1, and the docNum fits into 31-bits - use1HitEncoding := func(termCardinality uint64) (bool, uint64, uint64) { - if termCardinality == uint64(1) && locEncoder.FinalSize() <= 0 { - docNum := uint64(newRoaring.Minimum()) - if under32Bits(docNum) && docNum == lastDocNum && lastFreq == 1 { - return true, docNum, lastNorm - } - } - return false, 0, 0 - } - - finishTerm := func(term []byte) error { - tfEncoder.Close() - locEncoder.Close() - - postingsOffset, err := writePostings(newRoaring, - tfEncoder, locEncoder, use1HitEncoding, w, bufMaxVarintLen64) - if err != nil { - return err - } - - if postingsOffset > 0 { - err = newVellum.Insert(term, postingsOffset) - if err != nil { - return err - } - } - - newRoaring.Clear() - - tfEncoder.Reset() - locEncoder.Reset() - - lastDocNum = 0 - lastFreq = 0 - lastNorm = 0 - - return nil - } - - enumerator, err := newEnumerator(itrs) - - for err == nil { - term, itrI, postingsOffset := enumerator.Current() - - if !bytes.Equal(prevTerm, term) { - // check for the closure in meantime - if isClosed(closeCh) { - return nil, 0, seg.ErrClosed - } - - // if the term changed, write out the info collected - // for the previous term - err = finishTerm(prevTerm) - if err != nil { - return nil, 0, err - } - } - if !bytes.Equal(prevTerm, term) || prevTerm == nil { - // compute cardinality of field-term in new seg - var newCard uint64 - lowItrIdxs, lowItrVals := enumerator.GetLowIdxsAndValues() - for i, idx := range lowItrIdxs { - pl, err := dicts[idx].postingsListFromOffset(lowItrVals[i], drops[idx], nil) - if err != nil { - return nil, 0, err - } - newCard += pl.Count() - } - // compute correct chunk size with this - chunkSize, err := getChunkSize(chunkMode, newCard, newSegDocCount) - if err != nil { - return nil, 0, err - } - // update encoders chunk - tfEncoder.SetChunkSize(chunkSize, newSegDocCount-1) - locEncoder.SetChunkSize(chunkSize, newSegDocCount-1) - } - - postings, err = dicts[itrI].postingsListFromOffset( - postingsOffset, drops[itrI], postings) - if err != nil { - return nil, 0, err - } - - postItr = postings.iterator(true, true, true, postItr) - - // can no longer optimize by copying, since chunk factor could have changed - lastDocNum, lastFreq, lastNorm, bufLoc, err = mergeTermFreqNormLocs( - fieldsMap, term, postItr, newDocNums[itrI], newRoaring, - tfEncoder, locEncoder, bufLoc) - - if err != nil { - return nil, 0, err - } - - prevTerm = prevTerm[:0] // copy to prevTerm in case Next() reuses term mem - prevTerm = append(prevTerm, term...) - - err = enumerator.Next() - } - if err != vellum.ErrIteratorDone { - return nil, 0, err - } - - err = finishTerm(prevTerm) - if err != nil { - return nil, 0, err - } - - dictOffset := uint64(w.Count()) - - err = newVellum.Close() - if err != nil { - return nil, 0, err - } - vellumData := vellumBuf.Bytes() - - // write out the length of the vellum data - n := binary.PutUvarint(bufMaxVarintLen64, uint64(len(vellumData))) - _, err = w.Write(bufMaxVarintLen64[:n]) - if err != nil { - return nil, 0, err - } - - // write this vellum to disk - _, err = w.Write(vellumData) - if err != nil { - return nil, 0, err - } - - rv[fieldID] = dictOffset - - // get the field doc value offset (start) - fieldDvLocsStart[fieldID] = uint64(w.Count()) - - // update the field doc values - // NOTE: doc values continue to use legacy chunk mode - chunkSize, err := getChunkSize(LegacyChunkMode, 0, 0) - if err != nil { - return nil, 0, err - } - fdvEncoder := newChunkedContentCoder(chunkSize, newSegDocCount-1, w, true) - - fdvReadersAvailable := false - var dvIterClone *docValueReader - for segmentI, segment := range segmentsInFocus { - // check for the closure in meantime - if isClosed(closeCh) { - return nil, 0, seg.ErrClosed - } - - fieldIDPlus1 := uint16(segment.fieldsMap[fieldName]) - if dvIter, exists := segment.fieldDvReaders[fieldIDPlus1-1]; exists && - dvIter != nil { - fdvReadersAvailable = true - dvIterClone = dvIter.cloneInto(dvIterClone) - err = dvIterClone.iterateAllDocValues(segment, func(docNum uint64, terms []byte) error { - if newDocNums[segmentI][docNum] == docDropped { - return nil - } - err := fdvEncoder.Add(newDocNums[segmentI][docNum], terms) - if err != nil { - return err - } - return nil - }) - if err != nil { - return nil, 0, err - } - } - } - - if fdvReadersAvailable { - err = fdvEncoder.Close() - if err != nil { - return nil, 0, err - } - - // persist the doc value details for this field - _, err = fdvEncoder.Write() - if err != nil { - return nil, 0, err - } - - // get the field doc value offset (end) - fieldDvLocsEnd[fieldID] = uint64(w.Count()) - } else { - fieldDvLocsStart[fieldID] = fieldNotUninverted - fieldDvLocsEnd[fieldID] = fieldNotUninverted - } - - // reset vellum buffer and vellum builder - vellumBuf.Reset() - err = newVellum.Reset(&vellumBuf) - if err != nil { - return nil, 0, err - } - } - - fieldDvLocsOffset := uint64(w.Count()) - - buf := bufMaxVarintLen64 - for i := 0; i < len(fieldDvLocsStart); i++ { - n := binary.PutUvarint(buf, fieldDvLocsStart[i]) - _, err := w.Write(buf[:n]) - if err != nil { - return nil, 0, err - } - n = binary.PutUvarint(buf, fieldDvLocsEnd[i]) - _, err = w.Write(buf[:n]) - if err != nil { - return nil, 0, err - } - } - - return rv, fieldDvLocsOffset, nil -} - -func mergeTermFreqNormLocs(fieldsMap map[string]uint16, term []byte, postItr *PostingsIterator, - newDocNums []uint64, newRoaring *roaring.Bitmap, - tfEncoder *chunkedIntCoder, locEncoder *chunkedIntCoder, bufLoc []uint64) ( - lastDocNum uint64, lastFreq uint64, lastNorm uint64, bufLocOut []uint64, err error) { - next, err := postItr.Next() - for next != nil && err == nil { - hitNewDocNum := newDocNums[next.Number()] - if hitNewDocNum == docDropped { - return 0, 0, 0, nil, fmt.Errorf("see hit with dropped docNum") - } - - newRoaring.Add(uint32(hitNewDocNum)) - - nextFreq := next.Frequency() - nextNorm := uint64(math.Float32bits(float32(next.Norm()))) - - locs := next.Locations() - - err = tfEncoder.Add(hitNewDocNum, - encodeFreqHasLocs(nextFreq, len(locs) > 0), nextNorm) - if err != nil { - return 0, 0, 0, nil, err - } - - if len(locs) > 0 { - numBytesLocs := 0 - for _, loc := range locs { - ap := loc.ArrayPositions() - numBytesLocs += totalUvarintBytes(uint64(fieldsMap[loc.Field()]-1), - loc.Pos(), loc.Start(), loc.End(), uint64(len(ap)), ap) - } - - err = locEncoder.Add(hitNewDocNum, uint64(numBytesLocs)) - if err != nil { - return 0, 0, 0, nil, err - } - - for _, loc := range locs { - ap := loc.ArrayPositions() - if cap(bufLoc) < 5+len(ap) { - bufLoc = make([]uint64, 0, 5+len(ap)) - } - args := bufLoc[0:5] - args[0] = uint64(fieldsMap[loc.Field()] - 1) - args[1] = loc.Pos() - args[2] = loc.Start() - args[3] = loc.End() - args[4] = uint64(len(ap)) - args = append(args, ap...) - err = locEncoder.Add(hitNewDocNum, args...) - if err != nil { - return 0, 0, 0, nil, err - } - } - } - - lastDocNum = hitNewDocNum - lastFreq = nextFreq - lastNorm = nextNorm - - next, err = postItr.Next() - } - - return lastDocNum, lastFreq, lastNorm, bufLoc, err -} - -func writePostings(postings *roaring.Bitmap, tfEncoder, locEncoder *chunkedIntCoder, - use1HitEncoding func(uint64) (bool, uint64, uint64), - w *CountHashWriter, bufMaxVarintLen64 []byte) ( - offset uint64, err error) { - termCardinality := postings.GetCardinality() - if termCardinality <= 0 { - return 0, nil - } - - if use1HitEncoding != nil { - encodeAs1Hit, docNum1Hit, normBits1Hit := use1HitEncoding(termCardinality) - if encodeAs1Hit { - return FSTValEncode1Hit(docNum1Hit, normBits1Hit), nil - } - } - - var tfOffset uint64 - tfOffset, _, err = tfEncoder.writeAt(w) - if err != nil { - return 0, err - } - - var locOffset uint64 - locOffset, _, err = locEncoder.writeAt(w) - if err != nil { - return 0, err - } - - postingsOffset := uint64(w.Count()) - - n := binary.PutUvarint(bufMaxVarintLen64, tfOffset) - _, err = w.Write(bufMaxVarintLen64[:n]) - if err != nil { - return 0, err - } - - n = binary.PutUvarint(bufMaxVarintLen64, locOffset) - _, err = w.Write(bufMaxVarintLen64[:n]) - if err != nil { - return 0, err - } - - _, err = writeRoaringWithLen(postings, w, bufMaxVarintLen64) - if err != nil { - return 0, err - } - - return postingsOffset, nil -} - -type varintEncoder func(uint64) (int, error) - -func mergeStoredAndRemap(segments []*SegmentBase, drops []*roaring.Bitmap, - fieldsMap map[string]uint16, fieldsInv []string, fieldsSame bool, newSegDocCount uint64, - w *CountHashWriter, closeCh chan struct{}) (uint64, [][]uint64, error) { - var rv [][]uint64 // The remapped or newDocNums for each segment. - - var newDocNum uint64 - - var curr int - var data, compressed []byte - var metaBuf bytes.Buffer - varBuf := make([]byte, binary.MaxVarintLen64) - metaEncode := func(val uint64) (int, error) { - wb := binary.PutUvarint(varBuf, val) - return metaBuf.Write(varBuf[:wb]) - } - - vals := make([][][]byte, len(fieldsInv)) - typs := make([][]byte, len(fieldsInv)) - poss := make([][][]uint64, len(fieldsInv)) - - var posBuf []uint64 - - docNumOffsets := make([]uint64, newSegDocCount) - - vdc := visitDocumentCtxPool.Get().(*visitDocumentCtx) - defer visitDocumentCtxPool.Put(vdc) - - // for each segment - for segI, segment := range segments { - // check for the closure in meantime - if isClosed(closeCh) { - return 0, nil, seg.ErrClosed - } - - segNewDocNums := make([]uint64, segment.numDocs) - - dropsI := drops[segI] - - // optimize when the field mapping is the same across all - // segments and there are no deletions, via byte-copying - // of stored docs bytes directly to the writer - if fieldsSame && (dropsI == nil || dropsI.GetCardinality() == 0) { - err := segment.copyStoredDocs(newDocNum, docNumOffsets, w) - if err != nil { - return 0, nil, err - } - - for i := uint64(0); i < segment.numDocs; i++ { - segNewDocNums[i] = newDocNum - newDocNum++ - } - rv = append(rv, segNewDocNums) - - continue - } - - // for each doc num - for docNum := uint64(0); docNum < segment.numDocs; docNum++ { - // TODO: roaring's API limits docNums to 32-bits? - if dropsI != nil && dropsI.Contains(uint32(docNum)) { - segNewDocNums[docNum] = docDropped - continue - } - - segNewDocNums[docNum] = newDocNum - - curr = 0 - metaBuf.Reset() - data = data[:0] - - posTemp := posBuf - - // collect all the data - for i := 0; i < len(fieldsInv); i++ { - vals[i] = vals[i][:0] - typs[i] = typs[i][:0] - poss[i] = poss[i][:0] - } - err := segment.visitDocument(vdc, docNum, func(field string, typ byte, value []byte, pos []uint64) bool { - fieldID := int(fieldsMap[field]) - 1 - vals[fieldID] = append(vals[fieldID], value) - typs[fieldID] = append(typs[fieldID], typ) - - // copy array positions to preserve them beyond the scope of this callback - var curPos []uint64 - if len(pos) > 0 { - if cap(posTemp) < len(pos) { - posBuf = make([]uint64, len(pos)*len(fieldsInv)) - posTemp = posBuf - } - curPos = posTemp[0:len(pos)] - copy(curPos, pos) - posTemp = posTemp[len(pos):] - } - poss[fieldID] = append(poss[fieldID], curPos) - - return true - }) - if err != nil { - return 0, nil, err - } - - // _id field special case optimizes ExternalID() lookups - idFieldVal := vals[uint16(0)][0] - _, err = metaEncode(uint64(len(idFieldVal))) - if err != nil { - return 0, nil, err - } - - // now walk the non-"_id" fields in order - for fieldID := 1; fieldID < len(fieldsInv); fieldID++ { - storedFieldValues := vals[fieldID] - - stf := typs[fieldID] - spf := poss[fieldID] - - var err2 error - curr, data, err2 = persistStoredFieldValues(fieldID, - storedFieldValues, stf, spf, curr, metaEncode, data) - if err2 != nil { - return 0, nil, err2 - } - } - - metaBytes := metaBuf.Bytes() - - compressed = snappy.Encode(compressed[:cap(compressed)], data) - - // record where we're about to start writing - docNumOffsets[newDocNum] = uint64(w.Count()) - - // write out the meta len and compressed data len - _, err = writeUvarints(w, - uint64(len(metaBytes)), - uint64(len(idFieldVal)+len(compressed))) - if err != nil { - return 0, nil, err - } - // now write the meta - _, err = w.Write(metaBytes) - if err != nil { - return 0, nil, err - } - // now write the _id field val (counted as part of the 'compressed' data) - _, err = w.Write(idFieldVal) - if err != nil { - return 0, nil, err - } - // now write the compressed data - _, err = w.Write(compressed) - if err != nil { - return 0, nil, err - } - - newDocNum++ - } - - rv = append(rv, segNewDocNums) - } - - // return value is the start of the stored index - storedIndexOffset := uint64(w.Count()) - - // now write out the stored doc index - for _, docNumOffset := range docNumOffsets { - err := binary.Write(w, binary.BigEndian, docNumOffset) - if err != nil { - return 0, nil, err - } - } - - return storedIndexOffset, rv, nil -} - -// copyStoredDocs writes out a segment's stored doc info, optimized by -// using a single Write() call for the entire set of bytes. The -// newDocNumOffsets is filled with the new offsets for each doc. -func (s *SegmentBase) copyStoredDocs(newDocNum uint64, newDocNumOffsets []uint64, - w *CountHashWriter) error { - if s.numDocs <= 0 { - return nil - } - - indexOffset0, storedOffset0, _, _, _ := - s.getDocStoredOffsets(0) // the segment's first doc - - indexOffsetN, storedOffsetN, readN, metaLenN, dataLenN := - s.getDocStoredOffsets(s.numDocs - 1) // the segment's last doc - - storedOffset0New := uint64(w.Count()) - - storedBytes := s.mem[storedOffset0 : storedOffsetN+readN+metaLenN+dataLenN] - _, err := w.Write(storedBytes) - if err != nil { - return err - } - - // remap the storedOffset's for the docs into new offsets relative - // to storedOffset0New, filling the given docNumOffsetsOut array - for indexOffset := indexOffset0; indexOffset <= indexOffsetN; indexOffset += 8 { - storedOffset := binary.BigEndian.Uint64(s.mem[indexOffset : indexOffset+8]) - storedOffsetNew := storedOffset - storedOffset0 + storedOffset0New - newDocNumOffsets[newDocNum] = storedOffsetNew - newDocNum += 1 - } - - return nil -} - -// mergeFields builds a unified list of fields used across all the -// input segments, and computes whether the fields are the same across -// segments (which depends on fields to be sorted in the same way -// across segments) -func mergeFields(segments []*SegmentBase) (bool, []string) { - fieldsSame := true - - var segment0Fields []string - if len(segments) > 0 { - segment0Fields = segments[0].Fields() - } - - fieldsExist := map[string]struct{}{} - for _, segment := range segments { - fields := segment.Fields() - for fieldi, field := range fields { - fieldsExist[field] = struct{}{} - if len(segment0Fields) != len(fields) || segment0Fields[fieldi] != field { - fieldsSame = false - } - } - } - - rv := make([]string, 0, len(fieldsExist)) - // ensure _id stays first - rv = append(rv, "_id") - for k := range fieldsExist { - if k != "_id" { - rv = append(rv, k) - } - } - - sort.Strings(rv[1:]) // leave _id as first - - return fieldsSame, rv -} - -func isClosed(closeCh chan struct{}) bool { - select { - case <-closeCh: - return true - default: - return false - } -} diff --git a/vendor/github.com/blevesearch/zap/v12/new.go b/vendor/github.com/blevesearch/zap/v12/new.go deleted file mode 100644 index 98158186..00000000 --- a/vendor/github.com/blevesearch/zap/v12/new.go +++ /dev/null @@ -1,860 +0,0 @@ -// Copyright (c) 2018 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "bytes" - "encoding/binary" - "math" - "sort" - "sync" - - "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/couchbase/vellum" - "github.com/golang/snappy" -) - -var NewSegmentBufferNumResultsBump int = 100 -var NewSegmentBufferNumResultsFactor float64 = 1.0 -var NewSegmentBufferAvgBytesPerDocFactor float64 = 1.0 - -// ValidateDocFields can be set by applications to perform additional checks -// on fields in a document being added to a new segment, by default it does -// nothing. -// This API is experimental and may be removed at any time. -var ValidateDocFields = func(field document.Field) error { - return nil -} - -// AnalysisResultsToSegmentBase produces an in-memory zap-encoded -// SegmentBase from analysis results -func (z *ZapPlugin) New(results []*index.AnalysisResult) ( - segment.Segment, uint64, error) { - return z.newWithChunkMode(results, DefaultChunkMode) -} - -func (*ZapPlugin) newWithChunkMode(results []*index.AnalysisResult, - chunkMode uint32) (segment.Segment, uint64, error) { - s := interimPool.Get().(*interim) - - var br bytes.Buffer - if s.lastNumDocs > 0 { - // use previous results to initialize the buf with an estimate - // size, but note that the interim instance comes from a - // global interimPool, so multiple scorch instances indexing - // different docs can lead to low quality estimates - estimateAvgBytesPerDoc := int(float64(s.lastOutSize/s.lastNumDocs) * - NewSegmentBufferNumResultsFactor) - estimateNumResults := int(float64(len(results)+NewSegmentBufferNumResultsBump) * - NewSegmentBufferAvgBytesPerDocFactor) - br.Grow(estimateAvgBytesPerDoc * estimateNumResults) - } - - s.results = results - s.chunkMode = chunkMode - s.w = NewCountHashWriter(&br) - - storedIndexOffset, fieldsIndexOffset, fdvIndexOffset, dictOffsets, - err := s.convert() - if err != nil { - return nil, uint64(0), err - } - - sb, err := InitSegmentBase(br.Bytes(), s.w.Sum32(), chunkMode, - s.FieldsMap, s.FieldsInv, uint64(len(results)), - storedIndexOffset, fieldsIndexOffset, fdvIndexOffset, dictOffsets) - - if err == nil && s.reset() == nil { - s.lastNumDocs = len(results) - s.lastOutSize = len(br.Bytes()) - interimPool.Put(s) - } - - return sb, uint64(len(br.Bytes())), err -} - -var interimPool = sync.Pool{New: func() interface{} { return &interim{} }} - -// interim holds temporary working data used while converting from -// analysis results to a zap-encoded segment -type interim struct { - results []*index.AnalysisResult - - chunkMode uint32 - - w *CountHashWriter - - // FieldsMap adds 1 to field id to avoid zero value issues - // name -> field id + 1 - FieldsMap map[string]uint16 - - // FieldsInv is the inverse of FieldsMap - // field id -> name - FieldsInv []string - - // Term dictionaries for each field - // field id -> term -> postings list id + 1 - Dicts []map[string]uint64 - - // Terms for each field, where terms are sorted ascending - // field id -> []term - DictKeys [][]string - - // Fields whose IncludeDocValues is true - // field id -> bool - IncludeDocValues []bool - - // postings id -> bitmap of docNums - Postings []*roaring.Bitmap - - // postings id -> freq/norm's, one for each docNum in postings - FreqNorms [][]interimFreqNorm - freqNormsBacking []interimFreqNorm - - // postings id -> locs, one for each freq - Locs [][]interimLoc - locsBacking []interimLoc - - numTermsPerPostingsList []int // key is postings list id - numLocsPerPostingsList []int // key is postings list id - - builder *vellum.Builder - builderBuf bytes.Buffer - - metaBuf bytes.Buffer - - tmp0 []byte - tmp1 []byte - - lastNumDocs int - lastOutSize int -} - -func (s *interim) reset() (err error) { - s.results = nil - s.chunkMode = 0 - s.w = nil - s.FieldsMap = nil - s.FieldsInv = nil - for i := range s.Dicts { - s.Dicts[i] = nil - } - s.Dicts = s.Dicts[:0] - for i := range s.DictKeys { - s.DictKeys[i] = s.DictKeys[i][:0] - } - s.DictKeys = s.DictKeys[:0] - for i := range s.IncludeDocValues { - s.IncludeDocValues[i] = false - } - s.IncludeDocValues = s.IncludeDocValues[:0] - for _, idn := range s.Postings { - idn.Clear() - } - s.Postings = s.Postings[:0] - s.FreqNorms = s.FreqNorms[:0] - for i := range s.freqNormsBacking { - s.freqNormsBacking[i] = interimFreqNorm{} - } - s.freqNormsBacking = s.freqNormsBacking[:0] - s.Locs = s.Locs[:0] - for i := range s.locsBacking { - s.locsBacking[i] = interimLoc{} - } - s.locsBacking = s.locsBacking[:0] - s.numTermsPerPostingsList = s.numTermsPerPostingsList[:0] - s.numLocsPerPostingsList = s.numLocsPerPostingsList[:0] - s.builderBuf.Reset() - if s.builder != nil { - err = s.builder.Reset(&s.builderBuf) - } - s.metaBuf.Reset() - s.tmp0 = s.tmp0[:0] - s.tmp1 = s.tmp1[:0] - s.lastNumDocs = 0 - s.lastOutSize = 0 - - return err -} - -func (s *interim) grabBuf(size int) []byte { - buf := s.tmp0 - if cap(buf) < size { - buf = make([]byte, size) - s.tmp0 = buf - } - return buf[0:size] -} - -type interimStoredField struct { - vals [][]byte - typs []byte - arrayposs [][]uint64 // array positions -} - -type interimFreqNorm struct { - freq uint64 - norm float32 - numLocs int -} - -type interimLoc struct { - fieldID uint16 - pos uint64 - start uint64 - end uint64 - arrayposs []uint64 -} - -func (s *interim) convert() (uint64, uint64, uint64, []uint64, error) { - s.FieldsMap = map[string]uint16{} - - s.getOrDefineField("_id") // _id field is fieldID 0 - - for _, result := range s.results { - for _, field := range result.Document.CompositeFields { - s.getOrDefineField(field.Name()) - } - for _, field := range result.Document.Fields { - s.getOrDefineField(field.Name()) - } - } - - sort.Strings(s.FieldsInv[1:]) // keep _id as first field - - for fieldID, fieldName := range s.FieldsInv { - s.FieldsMap[fieldName] = uint16(fieldID + 1) - } - - if cap(s.IncludeDocValues) >= len(s.FieldsInv) { - s.IncludeDocValues = s.IncludeDocValues[:len(s.FieldsInv)] - } else { - s.IncludeDocValues = make([]bool, len(s.FieldsInv)) - } - - s.prepareDicts() - - for _, dict := range s.DictKeys { - sort.Strings(dict) - } - - s.processDocuments() - - storedIndexOffset, err := s.writeStoredFields() - if err != nil { - return 0, 0, 0, nil, err - } - - var fdvIndexOffset uint64 - var dictOffsets []uint64 - - if len(s.results) > 0 { - fdvIndexOffset, dictOffsets, err = s.writeDicts() - if err != nil { - return 0, 0, 0, nil, err - } - } else { - dictOffsets = make([]uint64, len(s.FieldsInv)) - } - - fieldsIndexOffset, err := persistFields(s.FieldsInv, s.w, dictOffsets) - if err != nil { - return 0, 0, 0, nil, err - } - - return storedIndexOffset, fieldsIndexOffset, fdvIndexOffset, dictOffsets, nil -} - -func (s *interim) getOrDefineField(fieldName string) int { - fieldIDPlus1, exists := s.FieldsMap[fieldName] - if !exists { - fieldIDPlus1 = uint16(len(s.FieldsInv) + 1) - s.FieldsMap[fieldName] = fieldIDPlus1 - s.FieldsInv = append(s.FieldsInv, fieldName) - - s.Dicts = append(s.Dicts, make(map[string]uint64)) - - n := len(s.DictKeys) - if n < cap(s.DictKeys) { - s.DictKeys = s.DictKeys[:n+1] - s.DictKeys[n] = s.DictKeys[n][:0] - } else { - s.DictKeys = append(s.DictKeys, []string(nil)) - } - } - - return int(fieldIDPlus1 - 1) -} - -// fill Dicts and DictKeys from analysis results -func (s *interim) prepareDicts() { - var pidNext int - - var totTFs int - var totLocs int - - visitField := func(fieldID uint16, tfs analysis.TokenFrequencies) { - dict := s.Dicts[fieldID] - dictKeys := s.DictKeys[fieldID] - - for term, tf := range tfs { - pidPlus1, exists := dict[term] - if !exists { - pidNext++ - pidPlus1 = uint64(pidNext) - - dict[term] = pidPlus1 - dictKeys = append(dictKeys, term) - - s.numTermsPerPostingsList = append(s.numTermsPerPostingsList, 0) - s.numLocsPerPostingsList = append(s.numLocsPerPostingsList, 0) - } - - pid := pidPlus1 - 1 - - s.numTermsPerPostingsList[pid] += 1 - s.numLocsPerPostingsList[pid] += len(tf.Locations) - - totLocs += len(tf.Locations) - } - - totTFs += len(tfs) - - s.DictKeys[fieldID] = dictKeys - } - - for _, result := range s.results { - // walk each composite field - for _, field := range result.Document.CompositeFields { - fieldID := uint16(s.getOrDefineField(field.Name())) - _, tf := field.Analyze() - visitField(fieldID, tf) - } - - // walk each field - for i, field := range result.Document.Fields { - fieldID := uint16(s.getOrDefineField(field.Name())) - tf := result.Analyzed[i] - visitField(fieldID, tf) - } - } - - numPostingsLists := pidNext - - if cap(s.Postings) >= numPostingsLists { - s.Postings = s.Postings[:numPostingsLists] - } else { - postings := make([]*roaring.Bitmap, numPostingsLists) - copy(postings, s.Postings[:cap(s.Postings)]) - for i := 0; i < numPostingsLists; i++ { - if postings[i] == nil { - postings[i] = roaring.New() - } - } - s.Postings = postings - } - - if cap(s.FreqNorms) >= numPostingsLists { - s.FreqNorms = s.FreqNorms[:numPostingsLists] - } else { - s.FreqNorms = make([][]interimFreqNorm, numPostingsLists) - } - - if cap(s.freqNormsBacking) >= totTFs { - s.freqNormsBacking = s.freqNormsBacking[:totTFs] - } else { - s.freqNormsBacking = make([]interimFreqNorm, totTFs) - } - - freqNormsBacking := s.freqNormsBacking - for pid, numTerms := range s.numTermsPerPostingsList { - s.FreqNorms[pid] = freqNormsBacking[0:0] - freqNormsBacking = freqNormsBacking[numTerms:] - } - - if cap(s.Locs) >= numPostingsLists { - s.Locs = s.Locs[:numPostingsLists] - } else { - s.Locs = make([][]interimLoc, numPostingsLists) - } - - if cap(s.locsBacking) >= totLocs { - s.locsBacking = s.locsBacking[:totLocs] - } else { - s.locsBacking = make([]interimLoc, totLocs) - } - - locsBacking := s.locsBacking - for pid, numLocs := range s.numLocsPerPostingsList { - s.Locs[pid] = locsBacking[0:0] - locsBacking = locsBacking[numLocs:] - } -} - -func (s *interim) processDocuments() { - numFields := len(s.FieldsInv) - reuseFieldLens := make([]int, numFields) - reuseFieldTFs := make([]analysis.TokenFrequencies, numFields) - - for docNum, result := range s.results { - for i := 0; i < numFields; i++ { // clear these for reuse - reuseFieldLens[i] = 0 - reuseFieldTFs[i] = nil - } - - s.processDocument(uint64(docNum), result, - reuseFieldLens, reuseFieldTFs) - } -} - -func (s *interim) processDocument(docNum uint64, - result *index.AnalysisResult, - fieldLens []int, fieldTFs []analysis.TokenFrequencies) { - visitField := func(fieldID uint16, fieldName string, - ln int, tf analysis.TokenFrequencies) { - fieldLens[fieldID] += ln - - existingFreqs := fieldTFs[fieldID] - if existingFreqs != nil { - existingFreqs.MergeAll(fieldName, tf) - } else { - fieldTFs[fieldID] = tf - } - } - - // walk each composite field - for _, field := range result.Document.CompositeFields { - fieldID := uint16(s.getOrDefineField(field.Name())) - ln, tf := field.Analyze() - visitField(fieldID, field.Name(), ln, tf) - } - - // walk each field - for i, field := range result.Document.Fields { - fieldID := uint16(s.getOrDefineField(field.Name())) - ln := result.Length[i] - tf := result.Analyzed[i] - visitField(fieldID, field.Name(), ln, tf) - } - - // now that it's been rolled up into fieldTFs, walk that - for fieldID, tfs := range fieldTFs { - dict := s.Dicts[fieldID] - norm := float32(1.0 / math.Sqrt(float64(fieldLens[fieldID]))) - - for term, tf := range tfs { - pid := dict[term] - 1 - bs := s.Postings[pid] - bs.Add(uint32(docNum)) - - s.FreqNorms[pid] = append(s.FreqNorms[pid], - interimFreqNorm{ - freq: uint64(tf.Frequency()), - norm: norm, - numLocs: len(tf.Locations), - }) - - if len(tf.Locations) > 0 { - locs := s.Locs[pid] - - for _, loc := range tf.Locations { - var locf = uint16(fieldID) - if loc.Field != "" { - locf = uint16(s.getOrDefineField(loc.Field)) - } - var arrayposs []uint64 - if len(loc.ArrayPositions) > 0 { - arrayposs = loc.ArrayPositions - } - locs = append(locs, interimLoc{ - fieldID: locf, - pos: uint64(loc.Position), - start: uint64(loc.Start), - end: uint64(loc.End), - arrayposs: arrayposs, - }) - } - - s.Locs[pid] = locs - } - } - } -} - -func (s *interim) writeStoredFields() ( - storedIndexOffset uint64, err error) { - varBuf := make([]byte, binary.MaxVarintLen64) - metaEncode := func(val uint64) (int, error) { - wb := binary.PutUvarint(varBuf, val) - return s.metaBuf.Write(varBuf[:wb]) - } - - data, compressed := s.tmp0[:0], s.tmp1[:0] - defer func() { s.tmp0, s.tmp1 = data, compressed }() - - // keyed by docNum - docStoredOffsets := make([]uint64, len(s.results)) - - // keyed by fieldID, for the current doc in the loop - docStoredFields := map[uint16]interimStoredField{} - - for docNum, result := range s.results { - for fieldID := range docStoredFields { // reset for next doc - delete(docStoredFields, fieldID) - } - - for _, field := range result.Document.Fields { - fieldID := uint16(s.getOrDefineField(field.Name())) - - opts := field.Options() - - if opts.IsStored() { - isf := docStoredFields[fieldID] - isf.vals = append(isf.vals, field.Value()) - isf.typs = append(isf.typs, encodeFieldType(field)) - isf.arrayposs = append(isf.arrayposs, field.ArrayPositions()) - docStoredFields[fieldID] = isf - } - - if opts.IncludeDocValues() { - s.IncludeDocValues[fieldID] = true - } - - err := ValidateDocFields(field) - if err != nil { - return 0, err - } - } - - var curr int - - s.metaBuf.Reset() - data = data[:0] - - // _id field special case optimizes ExternalID() lookups - idFieldVal := docStoredFields[uint16(0)].vals[0] - _, err = metaEncode(uint64(len(idFieldVal))) - if err != nil { - return 0, err - } - - // handle non-"_id" fields - for fieldID := 1; fieldID < len(s.FieldsInv); fieldID++ { - isf, exists := docStoredFields[uint16(fieldID)] - if exists { - curr, data, err = persistStoredFieldValues( - fieldID, isf.vals, isf.typs, isf.arrayposs, - curr, metaEncode, data) - if err != nil { - return 0, err - } - } - } - - metaBytes := s.metaBuf.Bytes() - - compressed = snappy.Encode(compressed[:cap(compressed)], data) - - docStoredOffsets[docNum] = uint64(s.w.Count()) - - _, err := writeUvarints(s.w, - uint64(len(metaBytes)), - uint64(len(idFieldVal)+len(compressed))) - if err != nil { - return 0, err - } - - _, err = s.w.Write(metaBytes) - if err != nil { - return 0, err - } - - _, err = s.w.Write(idFieldVal) - if err != nil { - return 0, err - } - - _, err = s.w.Write(compressed) - if err != nil { - return 0, err - } - } - - storedIndexOffset = uint64(s.w.Count()) - - for _, docStoredOffset := range docStoredOffsets { - err = binary.Write(s.w, binary.BigEndian, docStoredOffset) - if err != nil { - return 0, err - } - } - - return storedIndexOffset, nil -} - -func (s *interim) writeDicts() (fdvIndexOffset uint64, dictOffsets []uint64, err error) { - dictOffsets = make([]uint64, len(s.FieldsInv)) - - fdvOffsetsStart := make([]uint64, len(s.FieldsInv)) - fdvOffsetsEnd := make([]uint64, len(s.FieldsInv)) - - buf := s.grabBuf(binary.MaxVarintLen64) - - // these int coders are initialized with chunk size 1024 - // however this will be reset to the correct chunk size - // while processing each individual field-term section - tfEncoder := newChunkedIntCoder(1024, uint64(len(s.results)-1)) - locEncoder := newChunkedIntCoder(1024, uint64(len(s.results)-1)) - - var docTermMap [][]byte - - if s.builder == nil { - s.builder, err = vellum.New(&s.builderBuf, nil) - if err != nil { - return 0, nil, err - } - } - - for fieldID, terms := range s.DictKeys { - if cap(docTermMap) < len(s.results) { - docTermMap = make([][]byte, len(s.results)) - } else { - docTermMap = docTermMap[0:len(s.results)] - for docNum := range docTermMap { // reset the docTermMap - docTermMap[docNum] = docTermMap[docNum][:0] - } - } - - dict := s.Dicts[fieldID] - - for _, term := range terms { // terms are already sorted - pid := dict[term] - 1 - - postingsBS := s.Postings[pid] - - freqNorms := s.FreqNorms[pid] - freqNormOffset := 0 - - locs := s.Locs[pid] - locOffset := 0 - - chunkSize, err := getChunkSize(s.chunkMode, postingsBS.GetCardinality(), uint64(len(s.results))) - if err != nil { - return 0, nil, err - } - tfEncoder.SetChunkSize(chunkSize, uint64(len(s.results)-1)) - locEncoder.SetChunkSize(chunkSize, uint64(len(s.results)-1)) - - postingsItr := postingsBS.Iterator() - for postingsItr.HasNext() { - docNum := uint64(postingsItr.Next()) - - freqNorm := freqNorms[freqNormOffset] - - err = tfEncoder.Add(docNum, - encodeFreqHasLocs(freqNorm.freq, freqNorm.numLocs > 0), - uint64(math.Float32bits(freqNorm.norm))) - if err != nil { - return 0, nil, err - } - - if freqNorm.numLocs > 0 { - numBytesLocs := 0 - for _, loc := range locs[locOffset : locOffset+freqNorm.numLocs] { - numBytesLocs += totalUvarintBytes( - uint64(loc.fieldID), loc.pos, loc.start, loc.end, - uint64(len(loc.arrayposs)), loc.arrayposs) - } - - err = locEncoder.Add(docNum, uint64(numBytesLocs)) - if err != nil { - return 0, nil, err - } - - for _, loc := range locs[locOffset : locOffset+freqNorm.numLocs] { - err = locEncoder.Add(docNum, - uint64(loc.fieldID), loc.pos, loc.start, loc.end, - uint64(len(loc.arrayposs))) - if err != nil { - return 0, nil, err - } - - err = locEncoder.Add(docNum, loc.arrayposs...) - if err != nil { - return 0, nil, err - } - } - - locOffset += freqNorm.numLocs - } - - freqNormOffset++ - - docTermMap[docNum] = append( - append(docTermMap[docNum], term...), - termSeparator) - } - - tfEncoder.Close() - locEncoder.Close() - - postingsOffset, err := - writePostings(postingsBS, tfEncoder, locEncoder, nil, s.w, buf) - if err != nil { - return 0, nil, err - } - - if postingsOffset > uint64(0) { - err = s.builder.Insert([]byte(term), postingsOffset) - if err != nil { - return 0, nil, err - } - } - - tfEncoder.Reset() - locEncoder.Reset() - } - - err = s.builder.Close() - if err != nil { - return 0, nil, err - } - - // record where this dictionary starts - dictOffsets[fieldID] = uint64(s.w.Count()) - - vellumData := s.builderBuf.Bytes() - - // write out the length of the vellum data - n := binary.PutUvarint(buf, uint64(len(vellumData))) - _, err = s.w.Write(buf[:n]) - if err != nil { - return 0, nil, err - } - - // write this vellum to disk - _, err = s.w.Write(vellumData) - if err != nil { - return 0, nil, err - } - - // reset vellum for reuse - s.builderBuf.Reset() - - err = s.builder.Reset(&s.builderBuf) - if err != nil { - return 0, nil, err - } - - // write the field doc values - // NOTE: doc values continue to use legacy chunk mode - chunkSize, err := getChunkSize(LegacyChunkMode, 0, 0) - if err != nil { - return 0, nil, err - } - fdvEncoder := newChunkedContentCoder(chunkSize, uint64(len(s.results)-1), s.w, false) - if s.IncludeDocValues[fieldID] { - for docNum, docTerms := range docTermMap { - if len(docTerms) > 0 { - err = fdvEncoder.Add(uint64(docNum), docTerms) - if err != nil { - return 0, nil, err - } - } - } - err = fdvEncoder.Close() - if err != nil { - return 0, nil, err - } - - fdvOffsetsStart[fieldID] = uint64(s.w.Count()) - - _, err = fdvEncoder.Write() - if err != nil { - return 0, nil, err - } - - fdvOffsetsEnd[fieldID] = uint64(s.w.Count()) - - fdvEncoder.Reset() - } else { - fdvOffsetsStart[fieldID] = fieldNotUninverted - fdvOffsetsEnd[fieldID] = fieldNotUninverted - } - } - - fdvIndexOffset = uint64(s.w.Count()) - - for i := 0; i < len(fdvOffsetsStart); i++ { - n := binary.PutUvarint(buf, fdvOffsetsStart[i]) - _, err := s.w.Write(buf[:n]) - if err != nil { - return 0, nil, err - } - n = binary.PutUvarint(buf, fdvOffsetsEnd[i]) - _, err = s.w.Write(buf[:n]) - if err != nil { - return 0, nil, err - } - } - - return fdvIndexOffset, dictOffsets, nil -} - -func encodeFieldType(f document.Field) byte { - fieldType := byte('x') - switch f.(type) { - case *document.TextField: - fieldType = 't' - case *document.NumericField: - fieldType = 'n' - case *document.DateTimeField: - fieldType = 'd' - case *document.BooleanField: - fieldType = 'b' - case *document.GeoPointField: - fieldType = 'g' - case *document.CompositeField: - fieldType = 'c' - } - return fieldType -} - -// returns the total # of bytes needed to encode the given uint64's -// into binary.PutUVarint() encoding -func totalUvarintBytes(a, b, c, d, e uint64, more []uint64) (n int) { - n = numUvarintBytes(a) - n += numUvarintBytes(b) - n += numUvarintBytes(c) - n += numUvarintBytes(d) - n += numUvarintBytes(e) - for _, v := range more { - n += numUvarintBytes(v) - } - return n -} - -// returns # of bytes needed to encode x in binary.PutUvarint() encoding -func numUvarintBytes(x uint64) (n int) { - for x >= 0x80 { - x >>= 7 - n++ - } - return n + 1 -} diff --git a/vendor/github.com/blevesearch/zap/v12/plugin.go b/vendor/github.com/blevesearch/zap/v12/plugin.go deleted file mode 100644 index 38a0638d..00000000 --- a/vendor/github.com/blevesearch/zap/v12/plugin.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (c) 2020 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "github.com/blevesearch/bleve/index/scorch/segment" -) - -// ZapPlugin implements the Plugin interface of -// the blevesearch/bleve/index/scorch/segment pkg -type ZapPlugin struct{} - -func (*ZapPlugin) Type() string { - return Type -} - -func (*ZapPlugin) Version() uint32 { - return Version -} - -// Plugin returns an instance segment.Plugin for use -// by the Scorch indexing scheme -func Plugin() segment.Plugin { - return &ZapPlugin{} -} diff --git a/vendor/github.com/blevesearch/zap/v12/posting.go b/vendor/github.com/blevesearch/zap/v12/posting.go deleted file mode 100644 index 3a6ee548..00000000 --- a/vendor/github.com/blevesearch/zap/v12/posting.go +++ /dev/null @@ -1,798 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "encoding/binary" - "fmt" - "math" - "reflect" - - "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/blevesearch/bleve/size" -) - -var reflectStaticSizePostingsList int -var reflectStaticSizePostingsIterator int -var reflectStaticSizePosting int -var reflectStaticSizeLocation int - -func init() { - var pl PostingsList - reflectStaticSizePostingsList = int(reflect.TypeOf(pl).Size()) - var pi PostingsIterator - reflectStaticSizePostingsIterator = int(reflect.TypeOf(pi).Size()) - var p Posting - reflectStaticSizePosting = int(reflect.TypeOf(p).Size()) - var l Location - reflectStaticSizeLocation = int(reflect.TypeOf(l).Size()) -} - -// FST or vellum value (uint64) encoding is determined by the top two -// highest-order or most significant bits... -// -// encoding : MSB -// name : 63 62 61...to...bit #0 (LSB) -// ----------+---+---+--------------------------------------------------- -// general : 0 | 0 | 62-bits of postingsOffset. -// ~ : 0 | 1 | reserved for future. -// 1-hit : 1 | 0 | 31-bits of positive float31 norm | 31-bits docNum. -// ~ : 1 | 1 | reserved for future. -// -// Encoding "general" is able to handle all cases, where the -// postingsOffset points to more information about the postings for -// the term. -// -// Encoding "1-hit" is used to optimize a commonly seen case when a -// term has only a single hit. For example, a term in the _id field -// will have only 1 hit. The "1-hit" encoding is used for a term -// in a field when... -// -// - term vector info is disabled for that field; -// - and, the term appears in only a single doc for that field; -// - and, the term's freq is exactly 1 in that single doc for that field; -// - and, the docNum must fit into 31-bits; -// -// Otherwise, the "general" encoding is used instead. -// -// In the "1-hit" encoding, the field in that single doc may have -// other terms, which is supported in the "1-hit" encoding by the -// positive float31 norm. - -const FSTValEncodingMask = uint64(0xc000000000000000) -const FSTValEncodingGeneral = uint64(0x0000000000000000) -const FSTValEncoding1Hit = uint64(0x8000000000000000) - -func FSTValEncode1Hit(docNum uint64, normBits uint64) uint64 { - return FSTValEncoding1Hit | ((mask31Bits & normBits) << 31) | (mask31Bits & docNum) -} - -func FSTValDecode1Hit(v uint64) (docNum uint64, normBits uint64) { - return (mask31Bits & v), (mask31Bits & (v >> 31)) -} - -const mask31Bits = uint64(0x000000007fffffff) - -func under32Bits(x uint64) bool { - return x <= mask31Bits -} - -const DocNum1HitFinished = math.MaxUint64 - -var NormBits1Hit = uint64(math.Float32bits(float32(1))) - -// PostingsList is an in-memory representation of a postings list -type PostingsList struct { - sb *SegmentBase - postingsOffset uint64 - freqOffset uint64 - locOffset uint64 - postings *roaring.Bitmap - except *roaring.Bitmap - - // when normBits1Hit != 0, then this postings list came from a - // 1-hit encoding, and only the docNum1Hit & normBits1Hit apply - docNum1Hit uint64 - normBits1Hit uint64 -} - -// represents an immutable, empty postings list -var emptyPostingsList = &PostingsList{} - -func (p *PostingsList) Size() int { - sizeInBytes := reflectStaticSizePostingsList + size.SizeOfPtr - - if p.except != nil { - sizeInBytes += int(p.except.GetSizeInBytes()) - } - - return sizeInBytes -} - -func (p *PostingsList) OrInto(receiver *roaring.Bitmap) { - if p.normBits1Hit != 0 { - receiver.Add(uint32(p.docNum1Hit)) - return - } - - if p.postings != nil { - receiver.Or(p.postings) - } -} - -// Iterator returns an iterator for this postings list -func (p *PostingsList) Iterator(includeFreq, includeNorm, includeLocs bool, - prealloc segment.PostingsIterator) segment.PostingsIterator { - if p.normBits1Hit == 0 && p.postings == nil { - return emptyPostingsIterator - } - - var preallocPI *PostingsIterator - pi, ok := prealloc.(*PostingsIterator) - if ok && pi != nil { - preallocPI = pi - } - if preallocPI == emptyPostingsIterator { - preallocPI = nil - } - - return p.iterator(includeFreq, includeNorm, includeLocs, preallocPI) -} - -func (p *PostingsList) iterator(includeFreq, includeNorm, includeLocs bool, - rv *PostingsIterator) *PostingsIterator { - if rv == nil { - rv = &PostingsIterator{} - } else { - freqNormReader := rv.freqNormReader - if freqNormReader != nil { - freqNormReader.reset() - } - - locReader := rv.locReader - if locReader != nil { - locReader.reset() - } - - nextLocs := rv.nextLocs[:0] - nextSegmentLocs := rv.nextSegmentLocs[:0] - - buf := rv.buf - - *rv = PostingsIterator{} // clear the struct - - rv.freqNormReader = freqNormReader - rv.locReader = locReader - - rv.nextLocs = nextLocs - rv.nextSegmentLocs = nextSegmentLocs - - rv.buf = buf - } - - rv.postings = p - rv.includeFreqNorm = includeFreq || includeNorm || includeLocs - rv.includeLocs = includeLocs - - if p.normBits1Hit != 0 { - // "1-hit" encoding - rv.docNum1Hit = p.docNum1Hit - rv.normBits1Hit = p.normBits1Hit - - if p.except != nil && p.except.Contains(uint32(rv.docNum1Hit)) { - rv.docNum1Hit = DocNum1HitFinished - } - - return rv - } - - // "general" encoding, check if empty - if p.postings == nil { - return rv - } - - // initialize freq chunk reader - if rv.includeFreqNorm { - rv.freqNormReader = newChunkedIntDecoder(p.sb.mem, p.freqOffset) - } - - // initialize the loc chunk reader - if rv.includeLocs { - rv.locReader = newChunkedIntDecoder(p.sb.mem, p.locOffset) - } - - rv.all = p.postings.Iterator() - if p.except != nil { - rv.ActualBM = roaring.AndNot(p.postings, p.except) - rv.Actual = rv.ActualBM.Iterator() - } else { - rv.ActualBM = p.postings - rv.Actual = rv.all // Optimize to use same iterator for all & Actual. - } - - return rv -} - -// Count returns the number of items on this postings list -func (p *PostingsList) Count() uint64 { - var n, e uint64 - if p.normBits1Hit != 0 { - n = 1 - if p.except != nil && p.except.Contains(uint32(p.docNum1Hit)) { - e = 1 - } - } else if p.postings != nil { - n = p.postings.GetCardinality() - if p.except != nil { - e = p.postings.AndCardinality(p.except) - } - } - return n - e -} - -func (rv *PostingsList) read(postingsOffset uint64, d *Dictionary) error { - rv.postingsOffset = postingsOffset - - // handle "1-hit" encoding special case - if rv.postingsOffset&FSTValEncodingMask == FSTValEncoding1Hit { - return rv.init1Hit(postingsOffset) - } - - // read the location of the freq/norm details - var n uint64 - var read int - - rv.freqOffset, read = binary.Uvarint(d.sb.mem[postingsOffset+n : postingsOffset+binary.MaxVarintLen64]) - n += uint64(read) - - rv.locOffset, read = binary.Uvarint(d.sb.mem[postingsOffset+n : postingsOffset+n+binary.MaxVarintLen64]) - n += uint64(read) - - var postingsLen uint64 - postingsLen, read = binary.Uvarint(d.sb.mem[postingsOffset+n : postingsOffset+n+binary.MaxVarintLen64]) - n += uint64(read) - - roaringBytes := d.sb.mem[postingsOffset+n : postingsOffset+n+postingsLen] - - if rv.postings == nil { - rv.postings = roaring.NewBitmap() - } - _, err := rv.postings.FromBuffer(roaringBytes) - if err != nil { - return fmt.Errorf("error loading roaring bitmap: %v", err) - } - - return nil -} - -func (rv *PostingsList) init1Hit(fstVal uint64) error { - docNum, normBits := FSTValDecode1Hit(fstVal) - - rv.docNum1Hit = docNum - rv.normBits1Hit = normBits - - return nil -} - -// PostingsIterator provides a way to iterate through the postings list -type PostingsIterator struct { - postings *PostingsList - all roaring.IntPeekable - Actual roaring.IntPeekable - ActualBM *roaring.Bitmap - - currChunk uint32 - freqNormReader *chunkedIntDecoder - locReader *chunkedIntDecoder - - next Posting // reused across Next() calls - nextLocs []Location // reused across Next() calls - nextSegmentLocs []segment.Location // reused across Next() calls - - docNum1Hit uint64 - normBits1Hit uint64 - - buf []byte - - includeFreqNorm bool - includeLocs bool -} - -var emptyPostingsIterator = &PostingsIterator{} - -func (i *PostingsIterator) Size() int { - sizeInBytes := reflectStaticSizePostingsIterator + size.SizeOfPtr + - i.next.Size() - // account for freqNormReader, locReader if we start using this. - for _, entry := range i.nextLocs { - sizeInBytes += entry.Size() - } - - return sizeInBytes -} - -func (i *PostingsIterator) loadChunk(chunk int) error { - if i.includeFreqNorm { - err := i.freqNormReader.loadChunk(chunk) - if err != nil { - return err - } - } - - if i.includeLocs { - err := i.locReader.loadChunk(chunk) - if err != nil { - return err - } - } - - i.currChunk = uint32(chunk) - return nil -} - -func (i *PostingsIterator) readFreqNormHasLocs() (uint64, uint64, bool, error) { - if i.normBits1Hit != 0 { - return 1, i.normBits1Hit, false, nil - } - - freqHasLocs, err := i.freqNormReader.readUvarint() - if err != nil { - return 0, 0, false, fmt.Errorf("error reading frequency: %v", err) - } - - freq, hasLocs := decodeFreqHasLocs(freqHasLocs) - - normBits, err := i.freqNormReader.readUvarint() - if err != nil { - return 0, 0, false, fmt.Errorf("error reading norm: %v", err) - } - - return freq, normBits, hasLocs, nil -} - -func (i *PostingsIterator) skipFreqNormReadHasLocs() (bool, error) { - if i.normBits1Hit != 0 { - return false, nil - } - - freqHasLocs, err := i.freqNormReader.readUvarint() - if err != nil { - return false, fmt.Errorf("error reading freqHasLocs: %v", err) - } - - i.freqNormReader.SkipUvarint() // Skip normBits. - - return freqHasLocs&0x01 != 0, nil // See decodeFreqHasLocs() / hasLocs. -} - -func encodeFreqHasLocs(freq uint64, hasLocs bool) uint64 { - rv := freq << 1 - if hasLocs { - rv = rv | 0x01 // 0'th LSB encodes whether there are locations - } - return rv -} - -func decodeFreqHasLocs(freqHasLocs uint64) (uint64, bool) { - freq := freqHasLocs >> 1 - hasLocs := freqHasLocs&0x01 != 0 - return freq, hasLocs -} - -// readLocation processes all the integers on the stream representing a single -// location. -func (i *PostingsIterator) readLocation(l *Location) error { - // read off field - fieldID, err := i.locReader.readUvarint() - if err != nil { - return fmt.Errorf("error reading location field: %v", err) - } - // read off pos - pos, err := i.locReader.readUvarint() - if err != nil { - return fmt.Errorf("error reading location pos: %v", err) - } - // read off start - start, err := i.locReader.readUvarint() - if err != nil { - return fmt.Errorf("error reading location start: %v", err) - } - // read off end - end, err := i.locReader.readUvarint() - if err != nil { - return fmt.Errorf("error reading location end: %v", err) - } - // read off num array pos - numArrayPos, err := i.locReader.readUvarint() - if err != nil { - return fmt.Errorf("error reading location num array pos: %v", err) - } - - l.field = i.postings.sb.fieldsInv[fieldID] - l.pos = pos - l.start = start - l.end = end - - if cap(l.ap) < int(numArrayPos) { - l.ap = make([]uint64, int(numArrayPos)) - } else { - l.ap = l.ap[:int(numArrayPos)] - } - - // read off array positions - for k := 0; k < int(numArrayPos); k++ { - ap, err := i.locReader.readUvarint() - if err != nil { - return fmt.Errorf("error reading array position: %v", err) - } - - l.ap[k] = ap - } - - return nil -} - -// Next returns the next posting on the postings list, or nil at the end -func (i *PostingsIterator) Next() (segment.Posting, error) { - return i.nextAtOrAfter(0) -} - -// Advance returns the posting at the specified docNum or it is not present -// the next posting, or if the end is reached, nil -func (i *PostingsIterator) Advance(docNum uint64) (segment.Posting, error) { - return i.nextAtOrAfter(docNum) -} - -// Next returns the next posting on the postings list, or nil at the end -func (i *PostingsIterator) nextAtOrAfter(atOrAfter uint64) (segment.Posting, error) { - docNum, exists, err := i.nextDocNumAtOrAfter(atOrAfter) - if err != nil || !exists { - return nil, err - } - - i.next = Posting{} // clear the struct - rv := &i.next - rv.docNum = docNum - - if !i.includeFreqNorm { - return rv, nil - } - - var normBits uint64 - var hasLocs bool - - rv.freq, normBits, hasLocs, err = i.readFreqNormHasLocs() - if err != nil { - return nil, err - } - - rv.norm = math.Float32frombits(uint32(normBits)) - - if i.includeLocs && hasLocs { - // prepare locations into reused slices, where we assume - // rv.freq >= "number of locs", since in a composite field, - // some component fields might have their IncludeTermVector - // flags disabled while other component fields are enabled - if cap(i.nextLocs) >= int(rv.freq) { - i.nextLocs = i.nextLocs[0:rv.freq] - } else { - i.nextLocs = make([]Location, rv.freq, rv.freq*2) - } - if cap(i.nextSegmentLocs) < int(rv.freq) { - i.nextSegmentLocs = make([]segment.Location, rv.freq, rv.freq*2) - } - rv.locs = i.nextSegmentLocs[:0] - - numLocsBytes, err := i.locReader.readUvarint() - if err != nil { - return nil, fmt.Errorf("error reading location numLocsBytes: %v", err) - } - - j := 0 - startBytesRemaining := i.locReader.Len() // # bytes remaining in the locReader - for startBytesRemaining-i.locReader.Len() < int(numLocsBytes) { - err := i.readLocation(&i.nextLocs[j]) - if err != nil { - return nil, err - } - rv.locs = append(rv.locs, &i.nextLocs[j]) - j++ - } - } - - return rv, nil -} - -// nextDocNum returns the next docNum on the postings list, and also -// sets up the currChunk / loc related fields of the iterator. -func (i *PostingsIterator) nextDocNumAtOrAfter(atOrAfter uint64) (uint64, bool, error) { - if i.normBits1Hit != 0 { - if i.docNum1Hit == DocNum1HitFinished { - return 0, false, nil - } - if i.docNum1Hit < atOrAfter { - // advanced past our 1-hit - i.docNum1Hit = DocNum1HitFinished // consume our 1-hit docNum - return 0, false, nil - } - docNum := i.docNum1Hit - i.docNum1Hit = DocNum1HitFinished // consume our 1-hit docNum - return docNum, true, nil - } - - if i.Actual == nil || !i.Actual.HasNext() { - return 0, false, nil - } - - if i.postings == nil || i.postings.postings == i.ActualBM { - return i.nextDocNumAtOrAfterClean(atOrAfter) - } - - i.Actual.AdvanceIfNeeded(uint32(atOrAfter)) - - if !i.Actual.HasNext() { - // couldn't find anything - return 0, false, nil - } - - n := i.Actual.Next() - allN := i.all.Next() - - chunkSize, err := getChunkSize(i.postings.sb.chunkMode, i.postings.postings.GetCardinality(), i.postings.sb.numDocs) - if err != nil { - return 0, false, err - } - nChunk := n / uint32(chunkSize) - - // when allN becomes >= to here, then allN is in the same chunk as nChunk. - allNReachesNChunk := nChunk * uint32(chunkSize) - - // n is the next actual hit (excluding some postings), and - // allN is the next hit in the full postings, and - // if they don't match, move 'all' forwards until they do - for allN != n { - // we've reached same chunk, so move the freq/norm/loc decoders forward - if i.includeFreqNorm && allN >= allNReachesNChunk { - err := i.currChunkNext(nChunk) - if err != nil { - return 0, false, err - } - } - - allN = i.all.Next() - } - - if i.includeFreqNorm && (i.currChunk != nChunk || i.freqNormReader.isNil()) { - err := i.loadChunk(int(nChunk)) - if err != nil { - return 0, false, fmt.Errorf("error loading chunk: %v", err) - } - } - - return uint64(n), true, nil -} - -// optimization when the postings list is "clean" (e.g., no updates & -// no deletions) where the all bitmap is the same as the actual bitmap -func (i *PostingsIterator) nextDocNumAtOrAfterClean( - atOrAfter uint64) (uint64, bool, error) { - - if !i.includeFreqNorm { - i.Actual.AdvanceIfNeeded(uint32(atOrAfter)) - - if !i.Actual.HasNext() { - return 0, false, nil // couldn't find anything - } - - return uint64(i.Actual.Next()), true, nil - } - - chunkSize, err := getChunkSize(i.postings.sb.chunkMode, i.postings.postings.GetCardinality(), i.postings.sb.numDocs) - if err != nil { - return 0, false, err - } - - // freq-norm's needed, so maintain freq-norm chunk reader - sameChunkNexts := 0 // # of times we called Next() in the same chunk - n := i.Actual.Next() - nChunk := n / uint32(chunkSize) - - for uint64(n) < atOrAfter && i.Actual.HasNext() { - n = i.Actual.Next() - - nChunkPrev := nChunk - nChunk = n / uint32(chunkSize) - - if nChunk != nChunkPrev { - sameChunkNexts = 0 - } else { - sameChunkNexts += 1 - } - } - - if uint64(n) < atOrAfter { - // couldn't find anything - return 0, false, nil - } - - for j := 0; j < sameChunkNexts; j++ { - err := i.currChunkNext(nChunk) - if err != nil { - return 0, false, fmt.Errorf("error optimized currChunkNext: %v", err) - } - } - - if i.currChunk != nChunk || i.freqNormReader.isNil() { - err := i.loadChunk(int(nChunk)) - if err != nil { - return 0, false, fmt.Errorf("error loading chunk: %v", err) - } - } - - return uint64(n), true, nil -} - -func (i *PostingsIterator) currChunkNext(nChunk uint32) error { - if i.currChunk != nChunk || i.freqNormReader.isNil() { - err := i.loadChunk(int(nChunk)) - if err != nil { - return fmt.Errorf("error loading chunk: %v", err) - } - } - - // read off freq/offsets even though we don't care about them - hasLocs, err := i.skipFreqNormReadHasLocs() - if err != nil { - return err - } - - if i.includeLocs && hasLocs { - numLocsBytes, err := i.locReader.readUvarint() - if err != nil { - return fmt.Errorf("error reading location numLocsBytes: %v", err) - } - - // skip over all the location bytes - i.locReader.SkipBytes(int(numLocsBytes)) - } - - return nil -} - -// DocNum1Hit returns the docNum and true if this is "1-hit" optimized -// and the docNum is available. -func (p *PostingsIterator) DocNum1Hit() (uint64, bool) { - if p.normBits1Hit != 0 && p.docNum1Hit != DocNum1HitFinished { - return p.docNum1Hit, true - } - return 0, false -} - -// ActualBitmap returns the underlying actual bitmap -// which can be used up the stack for optimizations -func (p *PostingsIterator) ActualBitmap() *roaring.Bitmap { - return p.ActualBM -} - -// ReplaceActual replaces the ActualBM with the provided -// bitmap -func (p *PostingsIterator) ReplaceActual(abm *roaring.Bitmap) { - p.ActualBM = abm - p.Actual = abm.Iterator() -} - -// PostingsIteratorFromBitmap constructs a PostingsIterator given an -// "actual" bitmap. -func PostingsIteratorFromBitmap(bm *roaring.Bitmap, - includeFreqNorm, includeLocs bool) (segment.PostingsIterator, error) { - return &PostingsIterator{ - ActualBM: bm, - Actual: bm.Iterator(), - includeFreqNorm: includeFreqNorm, - includeLocs: includeLocs, - }, nil -} - -// PostingsIteratorFrom1Hit constructs a PostingsIterator given a -// 1-hit docNum. -func PostingsIteratorFrom1Hit(docNum1Hit uint64, - includeFreqNorm, includeLocs bool) (segment.PostingsIterator, error) { - return &PostingsIterator{ - docNum1Hit: docNum1Hit, - normBits1Hit: NormBits1Hit, - includeFreqNorm: includeFreqNorm, - includeLocs: includeLocs, - }, nil -} - -// Posting is a single entry in a postings list -type Posting struct { - docNum uint64 - freq uint64 - norm float32 - locs []segment.Location -} - -func (p *Posting) Size() int { - sizeInBytes := reflectStaticSizePosting - - for _, entry := range p.locs { - sizeInBytes += entry.Size() - } - - return sizeInBytes -} - -// Number returns the document number of this posting in this segment -func (p *Posting) Number() uint64 { - return p.docNum -} - -// Frequency returns the frequencies of occurrence of this term in this doc/field -func (p *Posting) Frequency() uint64 { - return p.freq -} - -// Norm returns the normalization factor for this posting -func (p *Posting) Norm() float64 { - return float64(p.norm) -} - -// Locations returns the location information for each occurrence -func (p *Posting) Locations() []segment.Location { - return p.locs -} - -// Location represents the location of a single occurrence -type Location struct { - field string - pos uint64 - start uint64 - end uint64 - ap []uint64 -} - -func (l *Location) Size() int { - return reflectStaticSizeLocation + - len(l.field) + - len(l.ap)*size.SizeOfUint64 -} - -// Field returns the name of the field (useful in composite fields to know -// which original field the value came from) -func (l *Location) Field() string { - return l.field -} - -// Start returns the start byte offset of this occurrence -func (l *Location) Start() uint64 { - return l.start -} - -// End returns the end byte offset of this occurrence -func (l *Location) End() uint64 { - return l.end -} - -// Pos returns the 1-based phrase position of this occurrence -func (l *Location) Pos() uint64 { - return l.pos -} - -// ArrayPositions returns the array position vector associated with this occurrence -func (l *Location) ArrayPositions() []uint64 { - return l.ap -} diff --git a/vendor/github.com/blevesearch/zap/v12/segment.go b/vendor/github.com/blevesearch/zap/v12/segment.go deleted file mode 100644 index e8b1f067..00000000 --- a/vendor/github.com/blevesearch/zap/v12/segment.go +++ /dev/null @@ -1,572 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "bytes" - "encoding/binary" - "fmt" - "io" - "os" - "sync" - "unsafe" - - "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/blevesearch/bleve/size" - "github.com/couchbase/vellum" - mmap "github.com/blevesearch/mmap-go" - "github.com/golang/snappy" -) - -var reflectStaticSizeSegmentBase int - -func init() { - var sb SegmentBase - reflectStaticSizeSegmentBase = int(unsafe.Sizeof(sb)) -} - -// Open returns a zap impl of a segment -func (*ZapPlugin) Open(path string) (segment.Segment, error) { - f, err := os.Open(path) - if err != nil { - return nil, err - } - mm, err := mmap.Map(f, mmap.RDONLY, 0) - if err != nil { - // mmap failed, try to close the file - _ = f.Close() - return nil, err - } - - rv := &Segment{ - SegmentBase: SegmentBase{ - mem: mm[0 : len(mm)-FooterSize], - fieldsMap: make(map[string]uint16), - fieldDvReaders: make(map[uint16]*docValueReader), - fieldFSTs: make(map[uint16]*vellum.FST), - }, - f: f, - mm: mm, - path: path, - refs: 1, - } - rv.SegmentBase.updateSize() - - err = rv.loadConfig() - if err != nil { - _ = rv.Close() - return nil, err - } - - err = rv.loadFields() - if err != nil { - _ = rv.Close() - return nil, err - } - - err = rv.loadDvReaders() - if err != nil { - _ = rv.Close() - return nil, err - } - - return rv, nil -} - -// SegmentBase is a memory only, read-only implementation of the -// segment.Segment interface, using zap's data representation. -type SegmentBase struct { - mem []byte - memCRC uint32 - chunkMode uint32 - fieldsMap map[string]uint16 // fieldName -> fieldID+1 - fieldsInv []string // fieldID -> fieldName - numDocs uint64 - storedIndexOffset uint64 - fieldsIndexOffset uint64 - docValueOffset uint64 - dictLocs []uint64 - fieldDvReaders map[uint16]*docValueReader // naive chunk cache per field - fieldDvNames []string // field names cached in fieldDvReaders - size uint64 - - m sync.Mutex - fieldFSTs map[uint16]*vellum.FST -} - -func (sb *SegmentBase) Size() int { - return int(sb.size) -} - -func (sb *SegmentBase) updateSize() { - sizeInBytes := reflectStaticSizeSegmentBase + - cap(sb.mem) - - // fieldsMap - for k := range sb.fieldsMap { - sizeInBytes += (len(k) + size.SizeOfString) + size.SizeOfUint16 - } - - // fieldsInv, dictLocs - for _, entry := range sb.fieldsInv { - sizeInBytes += len(entry) + size.SizeOfString - } - sizeInBytes += len(sb.dictLocs) * size.SizeOfUint64 - - // fieldDvReaders - for _, v := range sb.fieldDvReaders { - sizeInBytes += size.SizeOfUint16 + size.SizeOfPtr - if v != nil { - sizeInBytes += v.size() - } - } - - sb.size = uint64(sizeInBytes) -} - -func (sb *SegmentBase) AddRef() {} -func (sb *SegmentBase) DecRef() (err error) { return nil } -func (sb *SegmentBase) Close() (err error) { return nil } - -// Segment implements a persisted segment.Segment interface, by -// embedding an mmap()'ed SegmentBase. -type Segment struct { - SegmentBase - - f *os.File - mm mmap.MMap - path string - version uint32 - crc uint32 - - m sync.Mutex // Protects the fields that follow. - refs int64 -} - -func (s *Segment) Size() int { - // 8 /* size of file pointer */ - // 4 /* size of version -> uint32 */ - // 4 /* size of crc -> uint32 */ - sizeOfUints := 16 - - sizeInBytes := (len(s.path) + size.SizeOfString) + sizeOfUints - - // mutex, refs -> int64 - sizeInBytes += 16 - - // do not include the mmap'ed part - return sizeInBytes + s.SegmentBase.Size() - cap(s.mem) -} - -func (s *Segment) AddRef() { - s.m.Lock() - s.refs++ - s.m.Unlock() -} - -func (s *Segment) DecRef() (err error) { - s.m.Lock() - s.refs-- - if s.refs == 0 { - err = s.closeActual() - } - s.m.Unlock() - return err -} - -func (s *Segment) loadConfig() error { - crcOffset := len(s.mm) - 4 - s.crc = binary.BigEndian.Uint32(s.mm[crcOffset : crcOffset+4]) - - verOffset := crcOffset - 4 - s.version = binary.BigEndian.Uint32(s.mm[verOffset : verOffset+4]) - if s.version != Version { - return fmt.Errorf("unsupported version %d", s.version) - } - - chunkOffset := verOffset - 4 - s.chunkMode = binary.BigEndian.Uint32(s.mm[chunkOffset : chunkOffset+4]) - - docValueOffset := chunkOffset - 8 - s.docValueOffset = binary.BigEndian.Uint64(s.mm[docValueOffset : docValueOffset+8]) - - fieldsIndexOffset := docValueOffset - 8 - s.fieldsIndexOffset = binary.BigEndian.Uint64(s.mm[fieldsIndexOffset : fieldsIndexOffset+8]) - - storedIndexOffset := fieldsIndexOffset - 8 - s.storedIndexOffset = binary.BigEndian.Uint64(s.mm[storedIndexOffset : storedIndexOffset+8]) - - numDocsOffset := storedIndexOffset - 8 - s.numDocs = binary.BigEndian.Uint64(s.mm[numDocsOffset : numDocsOffset+8]) - return nil -} - -func (s *SegmentBase) loadFields() error { - // NOTE for now we assume the fields index immediately precedes - // the footer, and if this changes, need to adjust accordingly (or - // store explicit length), where s.mem was sliced from s.mm in Open(). - fieldsIndexEnd := uint64(len(s.mem)) - - // iterate through fields index - var fieldID uint64 - for s.fieldsIndexOffset+(8*fieldID) < fieldsIndexEnd { - addr := binary.BigEndian.Uint64(s.mem[s.fieldsIndexOffset+(8*fieldID) : s.fieldsIndexOffset+(8*fieldID)+8]) - - dictLoc, read := binary.Uvarint(s.mem[addr:fieldsIndexEnd]) - n := uint64(read) - s.dictLocs = append(s.dictLocs, dictLoc) - - var nameLen uint64 - nameLen, read = binary.Uvarint(s.mem[addr+n : fieldsIndexEnd]) - n += uint64(read) - - name := string(s.mem[addr+n : addr+n+nameLen]) - s.fieldsInv = append(s.fieldsInv, name) - s.fieldsMap[name] = uint16(fieldID + 1) - - fieldID++ - } - return nil -} - -// Dictionary returns the term dictionary for the specified field -func (s *SegmentBase) Dictionary(field string) (segment.TermDictionary, error) { - dict, err := s.dictionary(field) - if err == nil && dict == nil { - return &segment.EmptyDictionary{}, nil - } - return dict, err -} - -func (sb *SegmentBase) dictionary(field string) (rv *Dictionary, err error) { - fieldIDPlus1 := sb.fieldsMap[field] - if fieldIDPlus1 > 0 { - rv = &Dictionary{ - sb: sb, - field: field, - fieldID: fieldIDPlus1 - 1, - } - - dictStart := sb.dictLocs[rv.fieldID] - if dictStart > 0 { - var ok bool - sb.m.Lock() - if rv.fst, ok = sb.fieldFSTs[rv.fieldID]; !ok { - // read the length of the vellum data - vellumLen, read := binary.Uvarint(sb.mem[dictStart : dictStart+binary.MaxVarintLen64]) - fstBytes := sb.mem[dictStart+uint64(read) : dictStart+uint64(read)+vellumLen] - rv.fst, err = vellum.Load(fstBytes) - if err != nil { - sb.m.Unlock() - return nil, fmt.Errorf("dictionary field %s vellum err: %v", field, err) - } - - sb.fieldFSTs[rv.fieldID] = rv.fst - } - - sb.m.Unlock() - rv.fstReader, err = rv.fst.Reader() - if err != nil { - return nil, fmt.Errorf("dictionary field %s vellum reader err: %v", field, err) - } - - } - } - - return rv, nil -} - -// visitDocumentCtx holds data structures that are reusable across -// multiple VisitDocument() calls to avoid memory allocations -type visitDocumentCtx struct { - buf []byte - reader bytes.Reader - arrayPos []uint64 -} - -var visitDocumentCtxPool = sync.Pool{ - New: func() interface{} { - reuse := &visitDocumentCtx{} - return reuse - }, -} - -// VisitDocument invokes the DocFieldValueVistor for each stored field -// for the specified doc number -func (s *SegmentBase) VisitDocument(num uint64, visitor segment.DocumentFieldValueVisitor) error { - vdc := visitDocumentCtxPool.Get().(*visitDocumentCtx) - defer visitDocumentCtxPool.Put(vdc) - return s.visitDocument(vdc, num, visitor) -} - -func (s *SegmentBase) visitDocument(vdc *visitDocumentCtx, num uint64, - visitor segment.DocumentFieldValueVisitor) error { - // first make sure this is a valid number in this segment - if num < s.numDocs { - meta, compressed := s.getDocStoredMetaAndCompressed(num) - - vdc.reader.Reset(meta) - - // handle _id field special case - idFieldValLen, err := binary.ReadUvarint(&vdc.reader) - if err != nil { - return err - } - idFieldVal := compressed[:idFieldValLen] - - keepGoing := visitor("_id", byte('t'), idFieldVal, nil) - if !keepGoing { - visitDocumentCtxPool.Put(vdc) - return nil - } - - // handle non-"_id" fields - compressed = compressed[idFieldValLen:] - - uncompressed, err := snappy.Decode(vdc.buf[:cap(vdc.buf)], compressed) - if err != nil { - return err - } - - for keepGoing { - field, err := binary.ReadUvarint(&vdc.reader) - if err == io.EOF { - break - } - if err != nil { - return err - } - typ, err := binary.ReadUvarint(&vdc.reader) - if err != nil { - return err - } - offset, err := binary.ReadUvarint(&vdc.reader) - if err != nil { - return err - } - l, err := binary.ReadUvarint(&vdc.reader) - if err != nil { - return err - } - numap, err := binary.ReadUvarint(&vdc.reader) - if err != nil { - return err - } - var arrayPos []uint64 - if numap > 0 { - if cap(vdc.arrayPos) < int(numap) { - vdc.arrayPos = make([]uint64, numap) - } - arrayPos = vdc.arrayPos[:numap] - for i := 0; i < int(numap); i++ { - ap, err := binary.ReadUvarint(&vdc.reader) - if err != nil { - return err - } - arrayPos[i] = ap - } - } - - value := uncompressed[offset : offset+l] - keepGoing = visitor(s.fieldsInv[field], byte(typ), value, arrayPos) - } - - vdc.buf = uncompressed - } - return nil -} - -// DocID returns the value of the _id field for the given docNum -func (s *SegmentBase) DocID(num uint64) ([]byte, error) { - if num >= s.numDocs { - return nil, nil - } - - vdc := visitDocumentCtxPool.Get().(*visitDocumentCtx) - - meta, compressed := s.getDocStoredMetaAndCompressed(num) - - vdc.reader.Reset(meta) - - // handle _id field special case - idFieldValLen, err := binary.ReadUvarint(&vdc.reader) - if err != nil { - return nil, err - } - idFieldVal := compressed[:idFieldValLen] - - visitDocumentCtxPool.Put(vdc) - - return idFieldVal, nil -} - -// Count returns the number of documents in this segment. -func (s *SegmentBase) Count() uint64 { - return s.numDocs -} - -// DocNumbers returns a bitset corresponding to the doc numbers of all the -// provided _id strings -func (s *SegmentBase) DocNumbers(ids []string) (*roaring.Bitmap, error) { - rv := roaring.New() - - if len(s.fieldsMap) > 0 { - idDict, err := s.dictionary("_id") - if err != nil { - return nil, err - } - - postingsList := emptyPostingsList - - sMax, err := idDict.fst.GetMaxKey() - if err != nil { - return nil, err - } - sMaxStr := string(sMax) - filteredIds := make([]string, 0, len(ids)) - for _, id := range ids { - if id <= sMaxStr { - filteredIds = append(filteredIds, id) - } - } - - for _, id := range filteredIds { - postingsList, err = idDict.postingsList([]byte(id), nil, postingsList) - if err != nil { - return nil, err - } - postingsList.OrInto(rv) - } - } - - return rv, nil -} - -// Fields returns the field names used in this segment -func (s *SegmentBase) Fields() []string { - return s.fieldsInv -} - -// Path returns the path of this segment on disk -func (s *Segment) Path() string { - return s.path -} - -// Close releases all resources associated with this segment -func (s *Segment) Close() (err error) { - return s.DecRef() -} - -func (s *Segment) closeActual() (err error) { - if s.mm != nil { - err = s.mm.Unmap() - } - // try to close file even if unmap failed - if s.f != nil { - err2 := s.f.Close() - if err == nil { - // try to return first error - err = err2 - } - } - return -} - -// some helpers i started adding for the command-line utility - -// Data returns the underlying mmaped data slice -func (s *Segment) Data() []byte { - return s.mm -} - -// CRC returns the CRC value stored in the file footer -func (s *Segment) CRC() uint32 { - return s.crc -} - -// Version returns the file version in the file footer -func (s *Segment) Version() uint32 { - return s.version -} - -// ChunkFactor returns the chunk factor in the file footer -func (s *Segment) ChunkMode() uint32 { - return s.chunkMode -} - -// FieldsIndexOffset returns the fields index offset in the file footer -func (s *Segment) FieldsIndexOffset() uint64 { - return s.fieldsIndexOffset -} - -// StoredIndexOffset returns the stored value index offset in the file footer -func (s *Segment) StoredIndexOffset() uint64 { - return s.storedIndexOffset -} - -// DocValueOffset returns the docValue offset in the file footer -func (s *Segment) DocValueOffset() uint64 { - return s.docValueOffset -} - -// NumDocs returns the number of documents in the file footer -func (s *Segment) NumDocs() uint64 { - return s.numDocs -} - -// DictAddr is a helper function to compute the file offset where the -// dictionary is stored for the specified field. -func (s *Segment) DictAddr(field string) (uint64, error) { - fieldIDPlus1, ok := s.fieldsMap[field] - if !ok { - return 0, fmt.Errorf("no such field '%s'", field) - } - - return s.dictLocs[fieldIDPlus1-1], nil -} - -func (s *SegmentBase) loadDvReaders() error { - if s.docValueOffset == fieldNotUninverted || s.numDocs == 0 { - return nil - } - - var read uint64 - for fieldID, field := range s.fieldsInv { - var fieldLocStart, fieldLocEnd uint64 - var n int - fieldLocStart, n = binary.Uvarint(s.mem[s.docValueOffset+read : s.docValueOffset+read+binary.MaxVarintLen64]) - if n <= 0 { - return fmt.Errorf("loadDvReaders: failed to read the docvalue offset start for field %d", fieldID) - } - read += uint64(n) - fieldLocEnd, n = binary.Uvarint(s.mem[s.docValueOffset+read : s.docValueOffset+read+binary.MaxVarintLen64]) - if n <= 0 { - return fmt.Errorf("loadDvReaders: failed to read the docvalue offset end for field %d", fieldID) - } - read += uint64(n) - - fieldDvReader, err := s.loadFieldDocValueReader(field, fieldLocStart, fieldLocEnd) - if err != nil { - return err - } - if fieldDvReader != nil { - s.fieldDvReaders[uint16(fieldID)] = fieldDvReader - s.fieldDvNames = append(s.fieldDvNames, field) - } - } - - return nil -} diff --git a/vendor/github.com/blevesearch/zap/v13/README.md b/vendor/github.com/blevesearch/zap/v13/README.md deleted file mode 100644 index 0facb669..00000000 --- a/vendor/github.com/blevesearch/zap/v13/README.md +++ /dev/null @@ -1,158 +0,0 @@ -# zap file format - -Advanced ZAP File Format Documentation is [here](zap.md). - -The file is written in the reverse order that we typically access data. This helps us write in one pass since later sections of the file require file offsets of things we've already written. - -Current usage: - -- mmap the entire file -- crc-32 bytes and version are in fixed position at end of the file -- reading remainder of footer could be version specific -- remainder of footer gives us: - - 3 important offsets (docValue , fields index and stored data index) - - 2 important values (number of docs and chunk factor) -- field data is processed once and memoized onto the heap so that we never have to go back to disk for it -- access to stored data by doc number means first navigating to the stored data index, then accessing a fixed position offset into that slice, which gives us the actual address of the data. the first bytes of that section tell us the size of data so that we know where it ends. -- access to all other indexed data follows the following pattern: - - first know the field name -> convert to id - - next navigate to term dictionary for that field - - some operations stop here and do dictionary ops - - next use dictionary to navigate to posting list for a specific term - - walk posting list - - if necessary, walk posting details as we go - - if location info is desired, consult location bitmap to see if it is there - -## stored fields section - -- for each document - - preparation phase: - - produce a slice of metadata bytes and data bytes - - produce these slices in field id order - - field value is appended to the data slice - - metadata slice is varint encoded with the following values for each field value - - field id (uint16) - - field type (byte) - - field value start offset in uncompressed data slice (uint64) - - field value length (uint64) - - field number of array positions (uint64) - - one additional value for each array position (uint64) - - compress the data slice using snappy - - file writing phase: - - remember the start offset for this document - - write out meta data length (varint uint64) - - write out compressed data length (varint uint64) - - write out the metadata bytes - - write out the compressed data bytes - -## stored fields idx - -- for each document - - write start offset (remembered from previous section) of stored data (big endian uint64) - -With this index and a known document number, we have direct access to all the stored field data. - -## posting details (freq/norm) section - -- for each posting list - - produce a slice containing multiple consecutive chunks (each chunk is varint stream) - - produce a slice remembering offsets of where each chunk starts - - preparation phase: - - for each hit in the posting list - - if this hit is in next chunk close out encoding of last chunk and record offset start of next - - encode term frequency (uint64) - - encode norm factor (float32) - - file writing phase: - - remember start position for this posting list details - - write out number of chunks that follow (varint uint64) - - write out length of each chunk (each a varint uint64) - - write out the byte slice containing all the chunk data - -If you know the doc number you're interested in, this format lets you jump to the correct chunk (docNum/chunkFactor) directly and then seek within that chunk until you find it. - -## posting details (location) section - -- for each posting list - - produce a slice containing multiple consecutive chunks (each chunk is varint stream) - - produce a slice remembering offsets of where each chunk starts - - preparation phase: - - for each hit in the posting list - - if this hit is in next chunk close out encoding of last chunk and record offset start of next - - encode field (uint16) - - encode field pos (uint64) - - encode field start (uint64) - - encode field end (uint64) - - encode number of array positions to follow (uint64) - - encode each array position (each uint64) - - file writing phase: - - remember start position for this posting list details - - write out number of chunks that follow (varint uint64) - - write out length of each chunk (each a varint uint64) - - write out the byte slice containing all the chunk data - -If you know the doc number you're interested in, this format lets you jump to the correct chunk (docNum/chunkFactor) directly and then seek within that chunk until you find it. - -## postings list section - -- for each posting list - - preparation phase: - - encode roaring bitmap posting list to bytes (so we know the length) - - file writing phase: - - remember the start position for this posting list - - write freq/norm details offset (remembered from previous, as varint uint64) - - write location details offset (remembered from previous, as varint uint64) - - write length of encoded roaring bitmap - - write the serialized roaring bitmap data - -## dictionary - -- for each field - - preparation phase: - - encode vellum FST with dictionary data pointing to file offset of posting list (remembered from previous) - - file writing phase: - - remember the start position of this persistDictionary - - write length of vellum data (varint uint64) - - write out vellum data - -## fields section - -- for each field - - file writing phase: - - remember start offset for each field - - write dictionary address (remembered from previous) (varint uint64) - - write length of field name (varint uint64) - - write field name bytes - -## fields idx - -- for each field - - file writing phase: - - write big endian uint64 of start offset for each field - -NOTE: currently we don't know or record the length of this fields index. Instead we rely on the fact that we know it immediately precedes a footer of known size. - -## fields DocValue - -- for each field - - preparation phase: - - produce a slice containing multiple consecutive chunks, where each chunk is composed of a meta section followed by compressed columnar field data - - produce a slice remembering the length of each chunk - - file writing phase: - - remember the start position of this first field DocValue offset in the footer - - write out number of chunks that follow (varint uint64) - - write out length of each chunk (each a varint uint64) - - write out the byte slice containing all the chunk data - -NOTE: currently the meta header inside each chunk gives clue to the location offsets and size of the data pertaining to a given docID and any -read operation leverage that meta information to extract the document specific data from the file. - -## footer - -- file writing phase - - write number of docs (big endian uint64) - - write stored field index location (big endian uint64) - - write field index location (big endian uint64) - - write field docValue location (big endian uint64) - - write out chunk factor (big endian uint32) - - write out version (big endian uint32) - - write out file CRC of everything preceding this (big endian uint32) diff --git a/vendor/github.com/blevesearch/zap/v13/build.go b/vendor/github.com/blevesearch/zap/v13/build.go deleted file mode 100644 index 58d829f0..00000000 --- a/vendor/github.com/blevesearch/zap/v13/build.go +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "bufio" - "math" - "os" - - "github.com/couchbase/vellum" -) - -const Version uint32 = 13 - -const Type string = "zap" - -const fieldNotUninverted = math.MaxUint64 - -func (sb *SegmentBase) Persist(path string) error { - return PersistSegmentBase(sb, path) -} - -// PersistSegmentBase persists SegmentBase in the zap file format. -func PersistSegmentBase(sb *SegmentBase, path string) error { - flag := os.O_RDWR | os.O_CREATE - - f, err := os.OpenFile(path, flag, 0600) - if err != nil { - return err - } - - cleanup := func() { - _ = f.Close() - _ = os.Remove(path) - } - - br := bufio.NewWriter(f) - - _, err = br.Write(sb.mem) - if err != nil { - cleanup() - return err - } - - err = persistFooter(sb.numDocs, sb.storedIndexOffset, sb.fieldsIndexOffset, sb.docValueOffset, - sb.chunkMode, sb.memCRC, br) - if err != nil { - cleanup() - return err - } - - err = br.Flush() - if err != nil { - cleanup() - return err - } - - err = f.Sync() - if err != nil { - cleanup() - return err - } - - err = f.Close() - if err != nil { - cleanup() - return err - } - - return nil -} - -func persistStoredFieldValues(fieldID int, - storedFieldValues [][]byte, stf []byte, spf [][]uint64, - curr int, metaEncode varintEncoder, data []byte) ( - int, []byte, error) { - for i := 0; i < len(storedFieldValues); i++ { - // encode field - _, err := metaEncode(uint64(fieldID)) - if err != nil { - return 0, nil, err - } - // encode type - _, err = metaEncode(uint64(stf[i])) - if err != nil { - return 0, nil, err - } - // encode start offset - _, err = metaEncode(uint64(curr)) - if err != nil { - return 0, nil, err - } - // end len - _, err = metaEncode(uint64(len(storedFieldValues[i]))) - if err != nil { - return 0, nil, err - } - // encode number of array pos - _, err = metaEncode(uint64(len(spf[i]))) - if err != nil { - return 0, nil, err - } - // encode all array positions - for _, pos := range spf[i] { - _, err = metaEncode(pos) - if err != nil { - return 0, nil, err - } - } - - data = append(data, storedFieldValues[i]...) - curr += len(storedFieldValues[i]) - } - - return curr, data, nil -} - -func InitSegmentBase(mem []byte, memCRC uint32, chunkMode uint32, - fieldsMap map[string]uint16, fieldsInv []string, numDocs uint64, - storedIndexOffset uint64, fieldsIndexOffset uint64, docValueOffset uint64, - dictLocs []uint64) (*SegmentBase, error) { - sb := &SegmentBase{ - mem: mem, - memCRC: memCRC, - chunkMode: chunkMode, - fieldsMap: fieldsMap, - fieldsInv: fieldsInv, - numDocs: numDocs, - storedIndexOffset: storedIndexOffset, - fieldsIndexOffset: fieldsIndexOffset, - docValueOffset: docValueOffset, - dictLocs: dictLocs, - fieldDvReaders: make(map[uint16]*docValueReader), - fieldFSTs: make(map[uint16]*vellum.FST), - } - sb.updateSize() - - err := sb.loadDvReaders() - if err != nil { - return nil, err - } - - return sb, nil -} diff --git a/vendor/github.com/blevesearch/zap/v13/count.go b/vendor/github.com/blevesearch/zap/v13/count.go deleted file mode 100644 index 50290f88..00000000 --- a/vendor/github.com/blevesearch/zap/v13/count.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "hash/crc32" - "io" - - "github.com/blevesearch/bleve/index/scorch/segment" -) - -// CountHashWriter is a wrapper around a Writer which counts the number of -// bytes which have been written and computes a crc32 hash -type CountHashWriter struct { - w io.Writer - crc uint32 - n int - s segment.StatsReporter -} - -// NewCountHashWriter returns a CountHashWriter which wraps the provided Writer -func NewCountHashWriter(w io.Writer) *CountHashWriter { - return &CountHashWriter{w: w} -} - -func NewCountHashWriterWithStatsReporter(w io.Writer, s segment.StatsReporter) *CountHashWriter { - return &CountHashWriter{w: w, s: s} -} - -// Write writes the provided bytes to the wrapped writer and counts the bytes -func (c *CountHashWriter) Write(b []byte) (int, error) { - n, err := c.w.Write(b) - c.crc = crc32.Update(c.crc, crc32.IEEETable, b[:n]) - c.n += n - if c.s != nil { - c.s.ReportBytesWritten(uint64(n)) - } - return n, err -} - -// Count returns the number of bytes written -func (c *CountHashWriter) Count() int { - return c.n -} - -// Sum32 returns the CRC-32 hash of the content written to this writer -func (c *CountHashWriter) Sum32() uint32 { - return c.crc -} diff --git a/vendor/github.com/blevesearch/zap/v13/dict.go b/vendor/github.com/blevesearch/zap/v13/dict.go deleted file mode 100644 index ad4a8f8d..00000000 --- a/vendor/github.com/blevesearch/zap/v13/dict.go +++ /dev/null @@ -1,263 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "bytes" - "fmt" - - "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/couchbase/vellum" -) - -// Dictionary is the zap representation of the term dictionary -type Dictionary struct { - sb *SegmentBase - field string - fieldID uint16 - fst *vellum.FST - fstReader *vellum.Reader -} - -// PostingsList returns the postings list for the specified term -func (d *Dictionary) PostingsList(term []byte, except *roaring.Bitmap, - prealloc segment.PostingsList) (segment.PostingsList, error) { - var preallocPL *PostingsList - pl, ok := prealloc.(*PostingsList) - if ok && pl != nil { - preallocPL = pl - } - return d.postingsList(term, except, preallocPL) -} - -func (d *Dictionary) postingsList(term []byte, except *roaring.Bitmap, rv *PostingsList) (*PostingsList, error) { - if d.fstReader == nil { - if rv == nil || rv == emptyPostingsList { - return emptyPostingsList, nil - } - return d.postingsListInit(rv, except), nil - } - - postingsOffset, exists, err := d.fstReader.Get(term) - if err != nil { - return nil, fmt.Errorf("vellum err: %v", err) - } - if !exists { - if rv == nil || rv == emptyPostingsList { - return emptyPostingsList, nil - } - return d.postingsListInit(rv, except), nil - } - - return d.postingsListFromOffset(postingsOffset, except, rv) -} - -func (d *Dictionary) postingsListFromOffset(postingsOffset uint64, except *roaring.Bitmap, rv *PostingsList) (*PostingsList, error) { - rv = d.postingsListInit(rv, except) - - err := rv.read(postingsOffset, d) - if err != nil { - return nil, err - } - - return rv, nil -} - -func (d *Dictionary) postingsListInit(rv *PostingsList, except *roaring.Bitmap) *PostingsList { - if rv == nil || rv == emptyPostingsList { - rv = &PostingsList{} - } else { - postings := rv.postings - if postings != nil { - postings.Clear() - } - - *rv = PostingsList{} // clear the struct - - rv.postings = postings - } - rv.sb = d.sb - rv.except = except - return rv -} - -func (d *Dictionary) Contains(key []byte) (bool, error) { - return d.fst.Contains(key) -} - -// Iterator returns an iterator for this dictionary -func (d *Dictionary) Iterator() segment.DictionaryIterator { - rv := &DictionaryIterator{ - d: d, - } - - if d.fst != nil { - itr, err := d.fst.Iterator(nil, nil) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - } - - return rv -} - -// PrefixIterator returns an iterator which only visits terms having the -// the specified prefix -func (d *Dictionary) PrefixIterator(prefix string) segment.DictionaryIterator { - rv := &DictionaryIterator{ - d: d, - } - - kBeg := []byte(prefix) - kEnd := segment.IncrementBytes(kBeg) - - if d.fst != nil { - itr, err := d.fst.Iterator(kBeg, kEnd) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - } - - return rv -} - -// RangeIterator returns an iterator which only visits terms between the -// start and end terms. NOTE: bleve.index API specifies the end is inclusive. -func (d *Dictionary) RangeIterator(start, end string) segment.DictionaryIterator { - rv := &DictionaryIterator{ - d: d, - } - - // need to increment the end position to be inclusive - var endBytes []byte - if len(end) > 0 { - endBytes = []byte(end) - if endBytes[len(endBytes)-1] < 0xff { - endBytes[len(endBytes)-1]++ - } else { - endBytes = append(endBytes, 0xff) - } - } - - if d.fst != nil { - itr, err := d.fst.Iterator([]byte(start), endBytes) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - } - - return rv -} - -// AutomatonIterator returns an iterator which only visits terms -// having the the vellum automaton and start/end key range -func (d *Dictionary) AutomatonIterator(a vellum.Automaton, - startKeyInclusive, endKeyExclusive []byte) segment.DictionaryIterator { - rv := &DictionaryIterator{ - d: d, - } - - if d.fst != nil { - itr, err := d.fst.Search(a, startKeyInclusive, endKeyExclusive) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - } - - return rv -} - -func (d *Dictionary) OnlyIterator(onlyTerms [][]byte, - includeCount bool) segment.DictionaryIterator { - - rv := &DictionaryIterator{ - d: d, - omitCount: !includeCount, - } - - var buf bytes.Buffer - builder, err := vellum.New(&buf, nil) - if err != nil { - rv.err = err - return rv - } - for _, term := range onlyTerms { - err = builder.Insert(term, 0) - if err != nil { - rv.err = err - return rv - } - } - err = builder.Close() - if err != nil { - rv.err = err - return rv - } - - onlyFST, err := vellum.Load(buf.Bytes()) - if err != nil { - rv.err = err - return rv - } - - itr, err := d.fst.Search(onlyFST, nil, nil) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - - return rv -} - -// DictionaryIterator is an iterator for term dictionary -type DictionaryIterator struct { - d *Dictionary - itr vellum.Iterator - err error - tmp PostingsList - entry index.DictEntry - omitCount bool -} - -// Next returns the next entry in the dictionary -func (i *DictionaryIterator) Next() (*index.DictEntry, error) { - if i.err != nil && i.err != vellum.ErrIteratorDone { - return nil, i.err - } else if i.itr == nil || i.err == vellum.ErrIteratorDone { - return nil, nil - } - term, postingsOffset := i.itr.Current() - i.entry.Term = string(term) - if !i.omitCount { - i.err = i.tmp.read(postingsOffset, i.d) - if i.err != nil { - return nil, i.err - } - i.entry.Count = i.tmp.Count() - } - i.err = i.itr.Next() - return &i.entry, nil -} diff --git a/vendor/github.com/blevesearch/zap/v13/docvalues.go b/vendor/github.com/blevesearch/zap/v13/docvalues.go deleted file mode 100644 index 793797bd..00000000 --- a/vendor/github.com/blevesearch/zap/v13/docvalues.go +++ /dev/null @@ -1,312 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "bytes" - "encoding/binary" - "fmt" - "math" - "reflect" - "sort" - - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/blevesearch/bleve/size" - "github.com/golang/snappy" -) - -var reflectStaticSizedocValueReader int - -func init() { - var dvi docValueReader - reflectStaticSizedocValueReader = int(reflect.TypeOf(dvi).Size()) -} - -type docNumTermsVisitor func(docNum uint64, terms []byte) error - -type docVisitState struct { - dvrs map[uint16]*docValueReader - segment *SegmentBase -} - -type docValueReader struct { - field string - curChunkNum uint64 - chunkOffsets []uint64 - dvDataLoc uint64 - curChunkHeader []MetaData - curChunkData []byte // compressed data cache - uncompressed []byte // temp buf for snappy decompression -} - -func (di *docValueReader) size() int { - return reflectStaticSizedocValueReader + size.SizeOfPtr + - len(di.field) + - len(di.chunkOffsets)*size.SizeOfUint64 + - len(di.curChunkHeader)*reflectStaticSizeMetaData + - len(di.curChunkData) -} - -func (di *docValueReader) cloneInto(rv *docValueReader) *docValueReader { - if rv == nil { - rv = &docValueReader{} - } - - rv.field = di.field - rv.curChunkNum = math.MaxUint64 - rv.chunkOffsets = di.chunkOffsets // immutable, so it's sharable - rv.dvDataLoc = di.dvDataLoc - rv.curChunkHeader = rv.curChunkHeader[:0] - rv.curChunkData = nil - rv.uncompressed = rv.uncompressed[:0] - - return rv -} - -func (di *docValueReader) curChunkNumber() uint64 { - return di.curChunkNum -} - -func (s *SegmentBase) loadFieldDocValueReader(field string, - fieldDvLocStart, fieldDvLocEnd uint64) (*docValueReader, error) { - // get the docValue offset for the given fields - if fieldDvLocStart == fieldNotUninverted { - // no docValues found, nothing to do - return nil, nil - } - - // read the number of chunks, and chunk offsets position - var numChunks, chunkOffsetsPosition uint64 - - if fieldDvLocEnd-fieldDvLocStart > 16 { - numChunks = binary.BigEndian.Uint64(s.mem[fieldDvLocEnd-8 : fieldDvLocEnd]) - // read the length of chunk offsets - chunkOffsetsLen := binary.BigEndian.Uint64(s.mem[fieldDvLocEnd-16 : fieldDvLocEnd-8]) - // acquire position of chunk offsets - chunkOffsetsPosition = (fieldDvLocEnd - 16) - chunkOffsetsLen - } else { - return nil, fmt.Errorf("loadFieldDocValueReader: fieldDvLoc too small: %d-%d", fieldDvLocEnd, fieldDvLocStart) - } - - fdvIter := &docValueReader{ - curChunkNum: math.MaxUint64, - field: field, - chunkOffsets: make([]uint64, int(numChunks)), - } - - // read the chunk offsets - var offset uint64 - for i := 0; i < int(numChunks); i++ { - loc, read := binary.Uvarint(s.mem[chunkOffsetsPosition+offset : chunkOffsetsPosition+offset+binary.MaxVarintLen64]) - if read <= 0 { - return nil, fmt.Errorf("corrupted chunk offset during segment load") - } - fdvIter.chunkOffsets[i] = loc - offset += uint64(read) - } - - // set the data offset - fdvIter.dvDataLoc = fieldDvLocStart - - return fdvIter, nil -} - -func (di *docValueReader) loadDvChunk(chunkNumber uint64, s *SegmentBase) error { - // advance to the chunk where the docValues - // reside for the given docNum - destChunkDataLoc, curChunkEnd := di.dvDataLoc, di.dvDataLoc - start, end := readChunkBoundary(int(chunkNumber), di.chunkOffsets) - if start >= end { - di.curChunkHeader = di.curChunkHeader[:0] - di.curChunkData = nil - di.curChunkNum = chunkNumber - di.uncompressed = di.uncompressed[:0] - return nil - } - - destChunkDataLoc += start - curChunkEnd += end - - // read the number of docs reside in the chunk - numDocs, read := binary.Uvarint(s.mem[destChunkDataLoc : destChunkDataLoc+binary.MaxVarintLen64]) - if read <= 0 { - return fmt.Errorf("failed to read the chunk") - } - chunkMetaLoc := destChunkDataLoc + uint64(read) - - offset := uint64(0) - if cap(di.curChunkHeader) < int(numDocs) { - di.curChunkHeader = make([]MetaData, int(numDocs)) - } else { - di.curChunkHeader = di.curChunkHeader[:int(numDocs)] - } - for i := 0; i < int(numDocs); i++ { - di.curChunkHeader[i].DocNum, read = binary.Uvarint(s.mem[chunkMetaLoc+offset : chunkMetaLoc+offset+binary.MaxVarintLen64]) - offset += uint64(read) - di.curChunkHeader[i].DocDvOffset, read = binary.Uvarint(s.mem[chunkMetaLoc+offset : chunkMetaLoc+offset+binary.MaxVarintLen64]) - offset += uint64(read) - } - - compressedDataLoc := chunkMetaLoc + offset - dataLength := curChunkEnd - compressedDataLoc - di.curChunkData = s.mem[compressedDataLoc : compressedDataLoc+dataLength] - di.curChunkNum = chunkNumber - di.uncompressed = di.uncompressed[:0] - return nil -} - -func (di *docValueReader) iterateAllDocValues(s *SegmentBase, visitor docNumTermsVisitor) error { - for i := 0; i < len(di.chunkOffsets); i++ { - err := di.loadDvChunk(uint64(i), s) - if err != nil { - return err - } - if di.curChunkData == nil || len(di.curChunkHeader) == 0 { - continue - } - - // uncompress the already loaded data - uncompressed, err := snappy.Decode(di.uncompressed[:cap(di.uncompressed)], di.curChunkData) - if err != nil { - return err - } - di.uncompressed = uncompressed - - start := uint64(0) - for _, entry := range di.curChunkHeader { - err = visitor(entry.DocNum, uncompressed[start:entry.DocDvOffset]) - if err != nil { - return err - } - - start = entry.DocDvOffset - } - } - - return nil -} - -func (di *docValueReader) visitDocValues(docNum uint64, - visitor index.DocumentFieldTermVisitor) error { - // binary search the term locations for the docNum - start, end := di.getDocValueLocs(docNum) - if start == math.MaxUint64 || end == math.MaxUint64 || start == end { - return nil - } - - var uncompressed []byte - var err error - // use the uncompressed copy if available - if len(di.uncompressed) > 0 { - uncompressed = di.uncompressed - } else { - // uncompress the already loaded data - uncompressed, err = snappy.Decode(di.uncompressed[:cap(di.uncompressed)], di.curChunkData) - if err != nil { - return err - } - di.uncompressed = uncompressed - } - - // pick the terms for the given docNum - uncompressed = uncompressed[start:end] - for { - i := bytes.Index(uncompressed, termSeparatorSplitSlice) - if i < 0 { - break - } - - visitor(di.field, uncompressed[0:i]) - uncompressed = uncompressed[i+1:] - } - - return nil -} - -func (di *docValueReader) getDocValueLocs(docNum uint64) (uint64, uint64) { - i := sort.Search(len(di.curChunkHeader), func(i int) bool { - return di.curChunkHeader[i].DocNum >= docNum - }) - if i < len(di.curChunkHeader) && di.curChunkHeader[i].DocNum == docNum { - return ReadDocValueBoundary(i, di.curChunkHeader) - } - return math.MaxUint64, math.MaxUint64 -} - -// VisitDocumentFieldTerms is an implementation of the -// DocumentFieldTermVisitable interface -func (s *SegmentBase) VisitDocumentFieldTerms(localDocNum uint64, fields []string, - visitor index.DocumentFieldTermVisitor, dvsIn segment.DocVisitState) ( - segment.DocVisitState, error) { - dvs, ok := dvsIn.(*docVisitState) - if !ok || dvs == nil { - dvs = &docVisitState{} - } else { - if dvs.segment != s { - dvs.segment = s - dvs.dvrs = nil - } - } - - var fieldIDPlus1 uint16 - if dvs.dvrs == nil { - dvs.dvrs = make(map[uint16]*docValueReader, len(fields)) - for _, field := range fields { - if fieldIDPlus1, ok = s.fieldsMap[field]; !ok { - continue - } - fieldID := fieldIDPlus1 - 1 - if dvIter, exists := s.fieldDvReaders[fieldID]; exists && - dvIter != nil { - dvs.dvrs[fieldID] = dvIter.cloneInto(dvs.dvrs[fieldID]) - } - } - } - - // find the chunkNumber where the docValues are stored - // NOTE: doc values continue to use legacy chunk mode - chunkFactor, err := getChunkSize(LegacyChunkMode, 0, 0) - if err != nil { - return nil, err - } - docInChunk := localDocNum / chunkFactor - var dvr *docValueReader - for _, field := range fields { - if fieldIDPlus1, ok = s.fieldsMap[field]; !ok { - continue - } - fieldID := fieldIDPlus1 - 1 - if dvr, ok = dvs.dvrs[fieldID]; ok && dvr != nil { - // check if the chunk is already loaded - if docInChunk != dvr.curChunkNumber() { - err := dvr.loadDvChunk(docInChunk, s) - if err != nil { - return dvs, err - } - } - - _ = dvr.visitDocValues(localDocNum, visitor) - } - } - return dvs, nil -} - -// VisitableDocValueFields returns the list of fields with -// persisted doc value terms ready to be visitable using the -// VisitDocumentFieldTerms method. -func (s *SegmentBase) VisitableDocValueFields() ([]string, error) { - return s.fieldDvNames, nil -} diff --git a/vendor/github.com/blevesearch/zap/v13/enumerator.go b/vendor/github.com/blevesearch/zap/v13/enumerator.go deleted file mode 100644 index bc5b7e62..00000000 --- a/vendor/github.com/blevesearch/zap/v13/enumerator.go +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright (c) 2018 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "bytes" - - "github.com/couchbase/vellum" -) - -// enumerator provides an ordered traversal of multiple vellum -// iterators. Like JOIN of iterators, the enumerator produces a -// sequence of (key, iteratorIndex, value) tuples, sorted by key ASC, -// then iteratorIndex ASC, where the same key might be seen or -// repeated across multiple child iterators. -type enumerator struct { - itrs []vellum.Iterator - currKs [][]byte - currVs []uint64 - - lowK []byte - lowIdxs []int - lowCurr int -} - -// newEnumerator returns a new enumerator over the vellum Iterators -func newEnumerator(itrs []vellum.Iterator) (*enumerator, error) { - rv := &enumerator{ - itrs: itrs, - currKs: make([][]byte, len(itrs)), - currVs: make([]uint64, len(itrs)), - lowIdxs: make([]int, 0, len(itrs)), - } - for i, itr := range rv.itrs { - rv.currKs[i], rv.currVs[i] = itr.Current() - } - rv.updateMatches(false) - if rv.lowK == nil && len(rv.lowIdxs) == 0 { - return rv, vellum.ErrIteratorDone - } - return rv, nil -} - -// updateMatches maintains the low key matches based on the currKs -func (m *enumerator) updateMatches(skipEmptyKey bool) { - m.lowK = nil - m.lowIdxs = m.lowIdxs[:0] - m.lowCurr = 0 - - for i, key := range m.currKs { - if (key == nil && m.currVs[i] == 0) || // in case of empty iterator - (len(key) == 0 && skipEmptyKey) { // skip empty keys - continue - } - - cmp := bytes.Compare(key, m.lowK) - if cmp < 0 || len(m.lowIdxs) == 0 { - // reached a new low - m.lowK = key - m.lowIdxs = m.lowIdxs[:0] - m.lowIdxs = append(m.lowIdxs, i) - } else if cmp == 0 { - m.lowIdxs = append(m.lowIdxs, i) - } - } -} - -// Current returns the enumerator's current key, iterator-index, and -// value. If the enumerator is not pointing at a valid value (because -// Next returned an error previously), Current will return nil,0,0. -func (m *enumerator) Current() ([]byte, int, uint64) { - var i int - var v uint64 - if m.lowCurr < len(m.lowIdxs) { - i = m.lowIdxs[m.lowCurr] - v = m.currVs[i] - } - return m.lowK, i, v -} - -// GetLowIdxsAndValues will return all of the iterator indices -// which point to the current key, and their corresponding -// values. This can be used by advanced caller which may need -// to peek into these other sets of data before processing. -func (m *enumerator) GetLowIdxsAndValues() ([]int, []uint64) { - values := make([]uint64, 0, len(m.lowIdxs)) - for _, idx := range m.lowIdxs { - values = append(values, m.currVs[idx]) - } - return m.lowIdxs, values -} - -// Next advances the enumerator to the next key/iterator/value result, -// else vellum.ErrIteratorDone is returned. -func (m *enumerator) Next() error { - m.lowCurr += 1 - if m.lowCurr >= len(m.lowIdxs) { - // move all the current low iterators forwards - for _, vi := range m.lowIdxs { - err := m.itrs[vi].Next() - if err != nil && err != vellum.ErrIteratorDone { - return err - } - m.currKs[vi], m.currVs[vi] = m.itrs[vi].Current() - } - // can skip any empty keys encountered at this point - m.updateMatches(true) - } - if m.lowK == nil && len(m.lowIdxs) == 0 { - return vellum.ErrIteratorDone - } - return nil -} - -// Close all the underlying Iterators. The first error, if any, will -// be returned. -func (m *enumerator) Close() error { - var rv error - for _, itr := range m.itrs { - err := itr.Close() - if rv == nil { - rv = err - } - } - return rv -} diff --git a/vendor/github.com/blevesearch/zap/v13/intDecoder.go b/vendor/github.com/blevesearch/zap/v13/intDecoder.go deleted file mode 100644 index 4cd008ff..00000000 --- a/vendor/github.com/blevesearch/zap/v13/intDecoder.go +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright (c) 2019 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "encoding/binary" - "fmt" - - "github.com/blevesearch/bleve/index/scorch/segment" -) - -type chunkedIntDecoder struct { - startOffset uint64 - dataStartOffset uint64 - chunkOffsets []uint64 - curChunkBytes []byte - data []byte - r *segment.MemUvarintReader -} - -func newChunkedIntDecoder(buf []byte, offset uint64) *chunkedIntDecoder { - rv := &chunkedIntDecoder{startOffset: offset, data: buf} - var n, numChunks uint64 - var read int - if offset == termNotEncoded { - numChunks = 0 - } else { - numChunks, read = binary.Uvarint(buf[offset+n : offset+n+binary.MaxVarintLen64]) - } - - n += uint64(read) - if cap(rv.chunkOffsets) >= int(numChunks) { - rv.chunkOffsets = rv.chunkOffsets[:int(numChunks)] - } else { - rv.chunkOffsets = make([]uint64, int(numChunks)) - } - for i := 0; i < int(numChunks); i++ { - rv.chunkOffsets[i], read = binary.Uvarint(buf[offset+n : offset+n+binary.MaxVarintLen64]) - n += uint64(read) - } - rv.dataStartOffset = offset + n - return rv -} - -func (d *chunkedIntDecoder) loadChunk(chunk int) error { - if d.startOffset == termNotEncoded { - d.r = segment.NewMemUvarintReader([]byte(nil)) - return nil - } - - if chunk >= len(d.chunkOffsets) { - return fmt.Errorf("tried to load freq chunk that doesn't exist %d/(%d)", - chunk, len(d.chunkOffsets)) - } - - end, start := d.dataStartOffset, d.dataStartOffset - s, e := readChunkBoundary(chunk, d.chunkOffsets) - start += s - end += e - d.curChunkBytes = d.data[start:end] - if d.r == nil { - d.r = segment.NewMemUvarintReader(d.curChunkBytes) - } else { - d.r.Reset(d.curChunkBytes) - } - - return nil -} - -func (d *chunkedIntDecoder) reset() { - d.startOffset = 0 - d.dataStartOffset = 0 - d.chunkOffsets = d.chunkOffsets[:0] - d.curChunkBytes = d.curChunkBytes[:0] - d.data = d.data[:0] - if d.r != nil { - d.r.Reset([]byte(nil)) - } -} - -func (d *chunkedIntDecoder) isNil() bool { - return d.curChunkBytes == nil -} - -func (d *chunkedIntDecoder) readUvarint() (uint64, error) { - return d.r.ReadUvarint() -} - -func (d *chunkedIntDecoder) SkipUvarint() { - d.r.SkipUvarint() -} - -func (d *chunkedIntDecoder) SkipBytes(count int) { - d.r.SkipBytes(count) -} - -func (d *chunkedIntDecoder) Len() int { - return d.r.Len() -} diff --git a/vendor/github.com/blevesearch/zap/v13/merge.go b/vendor/github.com/blevesearch/zap/v13/merge.go deleted file mode 100644 index 805100fb..00000000 --- a/vendor/github.com/blevesearch/zap/v13/merge.go +++ /dev/null @@ -1,847 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "bufio" - "bytes" - "encoding/binary" - "fmt" - "math" - "os" - "sort" - - "github.com/RoaringBitmap/roaring" - seg "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/couchbase/vellum" - "github.com/golang/snappy" -) - -var DefaultFileMergerBufferSize = 1024 * 1024 - -const docDropped = math.MaxUint64 // sentinel docNum to represent a deleted doc - -// Merge takes a slice of segments and bit masks describing which -// documents may be dropped, and creates a new segment containing the -// remaining data. This new segment is built at the specified path. -func (*ZapPlugin) Merge(segments []seg.Segment, drops []*roaring.Bitmap, path string, - closeCh chan struct{}, s seg.StatsReporter) ( - [][]uint64, uint64, error) { - - segmentBases := make([]*SegmentBase, len(segments)) - for segmenti, segment := range segments { - switch segmentx := segment.(type) { - case *Segment: - segmentBases[segmenti] = &segmentx.SegmentBase - case *SegmentBase: - segmentBases[segmenti] = segmentx - default: - panic(fmt.Sprintf("oops, unexpected segment type: %T", segment)) - } - } - return mergeSegmentBases(segmentBases, drops, path, DefaultChunkMode, closeCh, s) -} - -func mergeSegmentBases(segmentBases []*SegmentBase, drops []*roaring.Bitmap, path string, - chunkMode uint32, closeCh chan struct{}, s seg.StatsReporter) ( - [][]uint64, uint64, error) { - flag := os.O_RDWR | os.O_CREATE - - f, err := os.OpenFile(path, flag, 0600) - if err != nil { - return nil, 0, err - } - - cleanup := func() { - _ = f.Close() - _ = os.Remove(path) - } - - // buffer the output - br := bufio.NewWriterSize(f, DefaultFileMergerBufferSize) - - // wrap it for counting (tracking offsets) - cr := NewCountHashWriterWithStatsReporter(br, s) - - newDocNums, numDocs, storedIndexOffset, fieldsIndexOffset, docValueOffset, _, _, _, err := - MergeToWriter(segmentBases, drops, chunkMode, cr, closeCh) - if err != nil { - cleanup() - return nil, 0, err - } - - err = persistFooter(numDocs, storedIndexOffset, fieldsIndexOffset, - docValueOffset, chunkMode, cr.Sum32(), cr) - if err != nil { - cleanup() - return nil, 0, err - } - - err = br.Flush() - if err != nil { - cleanup() - return nil, 0, err - } - - err = f.Sync() - if err != nil { - cleanup() - return nil, 0, err - } - - err = f.Close() - if err != nil { - cleanup() - return nil, 0, err - } - - return newDocNums, uint64(cr.Count()), nil -} - -func MergeToWriter(segments []*SegmentBase, drops []*roaring.Bitmap, - chunkMode uint32, cr *CountHashWriter, closeCh chan struct{}) ( - newDocNums [][]uint64, - numDocs, storedIndexOffset, fieldsIndexOffset, docValueOffset uint64, - dictLocs []uint64, fieldsInv []string, fieldsMap map[string]uint16, - err error) { - docValueOffset = uint64(fieldNotUninverted) - - var fieldsSame bool - fieldsSame, fieldsInv = mergeFields(segments) - fieldsMap = mapFields(fieldsInv) - - numDocs = computeNewDocCount(segments, drops) - - if isClosed(closeCh) { - return nil, 0, 0, 0, 0, nil, nil, nil, seg.ErrClosed - } - - if numDocs > 0 { - storedIndexOffset, newDocNums, err = mergeStoredAndRemap(segments, drops, - fieldsMap, fieldsInv, fieldsSame, numDocs, cr, closeCh) - if err != nil { - return nil, 0, 0, 0, 0, nil, nil, nil, err - } - - dictLocs, docValueOffset, err = persistMergedRest(segments, drops, - fieldsInv, fieldsMap, fieldsSame, - newDocNums, numDocs, chunkMode, cr, closeCh) - if err != nil { - return nil, 0, 0, 0, 0, nil, nil, nil, err - } - } else { - dictLocs = make([]uint64, len(fieldsInv)) - } - - fieldsIndexOffset, err = persistFields(fieldsInv, cr, dictLocs) - if err != nil { - return nil, 0, 0, 0, 0, nil, nil, nil, err - } - - return newDocNums, numDocs, storedIndexOffset, fieldsIndexOffset, docValueOffset, dictLocs, fieldsInv, fieldsMap, nil -} - -// mapFields takes the fieldsInv list and returns a map of fieldName -// to fieldID+1 -func mapFields(fields []string) map[string]uint16 { - rv := make(map[string]uint16, len(fields)) - for i, fieldName := range fields { - rv[fieldName] = uint16(i) + 1 - } - return rv -} - -// computeNewDocCount determines how many documents will be in the newly -// merged segment when obsoleted docs are dropped -func computeNewDocCount(segments []*SegmentBase, drops []*roaring.Bitmap) uint64 { - var newDocCount uint64 - for segI, segment := range segments { - newDocCount += segment.numDocs - if drops[segI] != nil { - newDocCount -= drops[segI].GetCardinality() - } - } - return newDocCount -} - -func persistMergedRest(segments []*SegmentBase, dropsIn []*roaring.Bitmap, - fieldsInv []string, fieldsMap map[string]uint16, fieldsSame bool, - newDocNumsIn [][]uint64, newSegDocCount uint64, chunkMode uint32, - w *CountHashWriter, closeCh chan struct{}) ([]uint64, uint64, error) { - - var bufMaxVarintLen64 []byte = make([]byte, binary.MaxVarintLen64) - var bufLoc []uint64 - - var postings *PostingsList - var postItr *PostingsIterator - - rv := make([]uint64, len(fieldsInv)) - fieldDvLocsStart := make([]uint64, len(fieldsInv)) - fieldDvLocsEnd := make([]uint64, len(fieldsInv)) - - // these int coders are initialized with chunk size 1024 - // however this will be reset to the correct chunk size - // while processing each individual field-term section - tfEncoder := newChunkedIntCoder(1024, newSegDocCount-1) - locEncoder := newChunkedIntCoder(1024, newSegDocCount-1) - - var vellumBuf bytes.Buffer - newVellum, err := vellum.New(&vellumBuf, nil) - if err != nil { - return nil, 0, err - } - - newRoaring := roaring.NewBitmap() - - // for each field - for fieldID, fieldName := range fieldsInv { - - // collect FST iterators from all active segments for this field - var newDocNums [][]uint64 - var drops []*roaring.Bitmap - var dicts []*Dictionary - var itrs []vellum.Iterator - - var segmentsInFocus []*SegmentBase - - for segmentI, segment := range segments { - - // check for the closure in meantime - if isClosed(closeCh) { - return nil, 0, seg.ErrClosed - } - - dict, err2 := segment.dictionary(fieldName) - if err2 != nil { - return nil, 0, err2 - } - if dict != nil && dict.fst != nil { - itr, err2 := dict.fst.Iterator(nil, nil) - if err2 != nil && err2 != vellum.ErrIteratorDone { - return nil, 0, err2 - } - if itr != nil { - newDocNums = append(newDocNums, newDocNumsIn[segmentI]) - if dropsIn[segmentI] != nil && !dropsIn[segmentI].IsEmpty() { - drops = append(drops, dropsIn[segmentI]) - } else { - drops = append(drops, nil) - } - dicts = append(dicts, dict) - itrs = append(itrs, itr) - segmentsInFocus = append(segmentsInFocus, segment) - } - } - } - - var prevTerm []byte - - newRoaring.Clear() - - var lastDocNum, lastFreq, lastNorm uint64 - - // determines whether to use "1-hit" encoding optimization - // when a term appears in only 1 doc, with no loc info, - // has freq of 1, and the docNum fits into 31-bits - use1HitEncoding := func(termCardinality uint64) (bool, uint64, uint64) { - if termCardinality == uint64(1) && locEncoder.FinalSize() <= 0 { - docNum := uint64(newRoaring.Minimum()) - if under32Bits(docNum) && docNum == lastDocNum && lastFreq == 1 { - return true, docNum, lastNorm - } - } - return false, 0, 0 - } - - finishTerm := func(term []byte) error { - tfEncoder.Close() - locEncoder.Close() - - postingsOffset, err := writePostings(newRoaring, - tfEncoder, locEncoder, use1HitEncoding, w, bufMaxVarintLen64) - if err != nil { - return err - } - - if postingsOffset > 0 { - err = newVellum.Insert(term, postingsOffset) - if err != nil { - return err - } - } - - newRoaring.Clear() - - tfEncoder.Reset() - locEncoder.Reset() - - lastDocNum = 0 - lastFreq = 0 - lastNorm = 0 - - return nil - } - - enumerator, err := newEnumerator(itrs) - - for err == nil { - term, itrI, postingsOffset := enumerator.Current() - - if !bytes.Equal(prevTerm, term) { - // check for the closure in meantime - if isClosed(closeCh) { - return nil, 0, seg.ErrClosed - } - - // if the term changed, write out the info collected - // for the previous term - err = finishTerm(prevTerm) - if err != nil { - return nil, 0, err - } - } - if !bytes.Equal(prevTerm, term) || prevTerm == nil { - // compute cardinality of field-term in new seg - var newCard uint64 - lowItrIdxs, lowItrVals := enumerator.GetLowIdxsAndValues() - for i, idx := range lowItrIdxs { - pl, err := dicts[idx].postingsListFromOffset(lowItrVals[i], drops[idx], nil) - if err != nil { - return nil, 0, err - } - newCard += pl.Count() - } - // compute correct chunk size with this - chunkSize, err := getChunkSize(chunkMode, newCard, newSegDocCount) - if err != nil { - return nil, 0, err - } - // update encoders chunk - tfEncoder.SetChunkSize(chunkSize, newSegDocCount-1) - locEncoder.SetChunkSize(chunkSize, newSegDocCount-1) - } - - postings, err = dicts[itrI].postingsListFromOffset( - postingsOffset, drops[itrI], postings) - if err != nil { - return nil, 0, err - } - - postItr = postings.iterator(true, true, true, postItr) - - // can no longer optimize by copying, since chunk factor could have changed - lastDocNum, lastFreq, lastNorm, bufLoc, err = mergeTermFreqNormLocs( - fieldsMap, term, postItr, newDocNums[itrI], newRoaring, - tfEncoder, locEncoder, bufLoc) - - if err != nil { - return nil, 0, err - } - - prevTerm = prevTerm[:0] // copy to prevTerm in case Next() reuses term mem - prevTerm = append(prevTerm, term...) - - err = enumerator.Next() - } - if err != vellum.ErrIteratorDone { - return nil, 0, err - } - - err = finishTerm(prevTerm) - if err != nil { - return nil, 0, err - } - - dictOffset := uint64(w.Count()) - - err = newVellum.Close() - if err != nil { - return nil, 0, err - } - vellumData := vellumBuf.Bytes() - - // write out the length of the vellum data - n := binary.PutUvarint(bufMaxVarintLen64, uint64(len(vellumData))) - _, err = w.Write(bufMaxVarintLen64[:n]) - if err != nil { - return nil, 0, err - } - - // write this vellum to disk - _, err = w.Write(vellumData) - if err != nil { - return nil, 0, err - } - - rv[fieldID] = dictOffset - - // get the field doc value offset (start) - fieldDvLocsStart[fieldID] = uint64(w.Count()) - - // update the field doc values - // NOTE: doc values continue to use legacy chunk mode - chunkSize, err := getChunkSize(LegacyChunkMode, 0, 0) - if err != nil { - return nil, 0, err - } - fdvEncoder := newChunkedContentCoder(chunkSize, newSegDocCount-1, w, true) - - fdvReadersAvailable := false - var dvIterClone *docValueReader - for segmentI, segment := range segmentsInFocus { - // check for the closure in meantime - if isClosed(closeCh) { - return nil, 0, seg.ErrClosed - } - - fieldIDPlus1 := uint16(segment.fieldsMap[fieldName]) - if dvIter, exists := segment.fieldDvReaders[fieldIDPlus1-1]; exists && - dvIter != nil { - fdvReadersAvailable = true - dvIterClone = dvIter.cloneInto(dvIterClone) - err = dvIterClone.iterateAllDocValues(segment, func(docNum uint64, terms []byte) error { - if newDocNums[segmentI][docNum] == docDropped { - return nil - } - err := fdvEncoder.Add(newDocNums[segmentI][docNum], terms) - if err != nil { - return err - } - return nil - }) - if err != nil { - return nil, 0, err - } - } - } - - if fdvReadersAvailable { - err = fdvEncoder.Close() - if err != nil { - return nil, 0, err - } - - // persist the doc value details for this field - _, err = fdvEncoder.Write() - if err != nil { - return nil, 0, err - } - - // get the field doc value offset (end) - fieldDvLocsEnd[fieldID] = uint64(w.Count()) - } else { - fieldDvLocsStart[fieldID] = fieldNotUninverted - fieldDvLocsEnd[fieldID] = fieldNotUninverted - } - - // reset vellum buffer and vellum builder - vellumBuf.Reset() - err = newVellum.Reset(&vellumBuf) - if err != nil { - return nil, 0, err - } - } - - fieldDvLocsOffset := uint64(w.Count()) - - buf := bufMaxVarintLen64 - for i := 0; i < len(fieldDvLocsStart); i++ { - n := binary.PutUvarint(buf, fieldDvLocsStart[i]) - _, err := w.Write(buf[:n]) - if err != nil { - return nil, 0, err - } - n = binary.PutUvarint(buf, fieldDvLocsEnd[i]) - _, err = w.Write(buf[:n]) - if err != nil { - return nil, 0, err - } - } - - return rv, fieldDvLocsOffset, nil -} - -func mergeTermFreqNormLocs(fieldsMap map[string]uint16, term []byte, postItr *PostingsIterator, - newDocNums []uint64, newRoaring *roaring.Bitmap, - tfEncoder *chunkedIntCoder, locEncoder *chunkedIntCoder, bufLoc []uint64) ( - lastDocNum uint64, lastFreq uint64, lastNorm uint64, bufLocOut []uint64, err error) { - next, err := postItr.Next() - for next != nil && err == nil { - hitNewDocNum := newDocNums[next.Number()] - if hitNewDocNum == docDropped { - return 0, 0, 0, nil, fmt.Errorf("see hit with dropped docNum") - } - - newRoaring.Add(uint32(hitNewDocNum)) - - nextFreq := next.Frequency() - nextNorm := uint64(math.Float32bits(float32(next.Norm()))) - - locs := next.Locations() - - err = tfEncoder.Add(hitNewDocNum, - encodeFreqHasLocs(nextFreq, len(locs) > 0), nextNorm) - if err != nil { - return 0, 0, 0, nil, err - } - - if len(locs) > 0 { - numBytesLocs := 0 - for _, loc := range locs { - ap := loc.ArrayPositions() - numBytesLocs += totalUvarintBytes(uint64(fieldsMap[loc.Field()]-1), - loc.Pos(), loc.Start(), loc.End(), uint64(len(ap)), ap) - } - - err = locEncoder.Add(hitNewDocNum, uint64(numBytesLocs)) - if err != nil { - return 0, 0, 0, nil, err - } - - for _, loc := range locs { - ap := loc.ArrayPositions() - if cap(bufLoc) < 5+len(ap) { - bufLoc = make([]uint64, 0, 5+len(ap)) - } - args := bufLoc[0:5] - args[0] = uint64(fieldsMap[loc.Field()] - 1) - args[1] = loc.Pos() - args[2] = loc.Start() - args[3] = loc.End() - args[4] = uint64(len(ap)) - args = append(args, ap...) - err = locEncoder.Add(hitNewDocNum, args...) - if err != nil { - return 0, 0, 0, nil, err - } - } - } - - lastDocNum = hitNewDocNum - lastFreq = nextFreq - lastNorm = nextNorm - - next, err = postItr.Next() - } - - return lastDocNum, lastFreq, lastNorm, bufLoc, err -} - -func writePostings(postings *roaring.Bitmap, tfEncoder, locEncoder *chunkedIntCoder, - use1HitEncoding func(uint64) (bool, uint64, uint64), - w *CountHashWriter, bufMaxVarintLen64 []byte) ( - offset uint64, err error) { - termCardinality := postings.GetCardinality() - if termCardinality <= 0 { - return 0, nil - } - - if use1HitEncoding != nil { - encodeAs1Hit, docNum1Hit, normBits1Hit := use1HitEncoding(termCardinality) - if encodeAs1Hit { - return FSTValEncode1Hit(docNum1Hit, normBits1Hit), nil - } - } - - var tfOffset uint64 - tfOffset, _, err = tfEncoder.writeAt(w) - if err != nil { - return 0, err - } - - var locOffset uint64 - locOffset, _, err = locEncoder.writeAt(w) - if err != nil { - return 0, err - } - - postingsOffset := uint64(w.Count()) - - n := binary.PutUvarint(bufMaxVarintLen64, tfOffset) - _, err = w.Write(bufMaxVarintLen64[:n]) - if err != nil { - return 0, err - } - - n = binary.PutUvarint(bufMaxVarintLen64, locOffset) - _, err = w.Write(bufMaxVarintLen64[:n]) - if err != nil { - return 0, err - } - - _, err = writeRoaringWithLen(postings, w, bufMaxVarintLen64) - if err != nil { - return 0, err - } - - return postingsOffset, nil -} - -type varintEncoder func(uint64) (int, error) - -func mergeStoredAndRemap(segments []*SegmentBase, drops []*roaring.Bitmap, - fieldsMap map[string]uint16, fieldsInv []string, fieldsSame bool, newSegDocCount uint64, - w *CountHashWriter, closeCh chan struct{}) (uint64, [][]uint64, error) { - var rv [][]uint64 // The remapped or newDocNums for each segment. - - var newDocNum uint64 - - var curr int - var data, compressed []byte - var metaBuf bytes.Buffer - varBuf := make([]byte, binary.MaxVarintLen64) - metaEncode := func(val uint64) (int, error) { - wb := binary.PutUvarint(varBuf, val) - return metaBuf.Write(varBuf[:wb]) - } - - vals := make([][][]byte, len(fieldsInv)) - typs := make([][]byte, len(fieldsInv)) - poss := make([][][]uint64, len(fieldsInv)) - - var posBuf []uint64 - - docNumOffsets := make([]uint64, newSegDocCount) - - vdc := visitDocumentCtxPool.Get().(*visitDocumentCtx) - defer visitDocumentCtxPool.Put(vdc) - - // for each segment - for segI, segment := range segments { - // check for the closure in meantime - if isClosed(closeCh) { - return 0, nil, seg.ErrClosed - } - - segNewDocNums := make([]uint64, segment.numDocs) - - dropsI := drops[segI] - - // optimize when the field mapping is the same across all - // segments and there are no deletions, via byte-copying - // of stored docs bytes directly to the writer - if fieldsSame && (dropsI == nil || dropsI.GetCardinality() == 0) { - err := segment.copyStoredDocs(newDocNum, docNumOffsets, w) - if err != nil { - return 0, nil, err - } - - for i := uint64(0); i < segment.numDocs; i++ { - segNewDocNums[i] = newDocNum - newDocNum++ - } - rv = append(rv, segNewDocNums) - - continue - } - - // for each doc num - for docNum := uint64(0); docNum < segment.numDocs; docNum++ { - // TODO: roaring's API limits docNums to 32-bits? - if dropsI != nil && dropsI.Contains(uint32(docNum)) { - segNewDocNums[docNum] = docDropped - continue - } - - segNewDocNums[docNum] = newDocNum - - curr = 0 - metaBuf.Reset() - data = data[:0] - - posTemp := posBuf - - // collect all the data - for i := 0; i < len(fieldsInv); i++ { - vals[i] = vals[i][:0] - typs[i] = typs[i][:0] - poss[i] = poss[i][:0] - } - err := segment.visitDocument(vdc, docNum, func(field string, typ byte, value []byte, pos []uint64) bool { - fieldID := int(fieldsMap[field]) - 1 - vals[fieldID] = append(vals[fieldID], value) - typs[fieldID] = append(typs[fieldID], typ) - - // copy array positions to preserve them beyond the scope of this callback - var curPos []uint64 - if len(pos) > 0 { - if cap(posTemp) < len(pos) { - posBuf = make([]uint64, len(pos)*len(fieldsInv)) - posTemp = posBuf - } - curPos = posTemp[0:len(pos)] - copy(curPos, pos) - posTemp = posTemp[len(pos):] - } - poss[fieldID] = append(poss[fieldID], curPos) - - return true - }) - if err != nil { - return 0, nil, err - } - - // _id field special case optimizes ExternalID() lookups - idFieldVal := vals[uint16(0)][0] - _, err = metaEncode(uint64(len(idFieldVal))) - if err != nil { - return 0, nil, err - } - - // now walk the non-"_id" fields in order - for fieldID := 1; fieldID < len(fieldsInv); fieldID++ { - storedFieldValues := vals[fieldID] - - stf := typs[fieldID] - spf := poss[fieldID] - - var err2 error - curr, data, err2 = persistStoredFieldValues(fieldID, - storedFieldValues, stf, spf, curr, metaEncode, data) - if err2 != nil { - return 0, nil, err2 - } - } - - metaBytes := metaBuf.Bytes() - - compressed = snappy.Encode(compressed[:cap(compressed)], data) - - // record where we're about to start writing - docNumOffsets[newDocNum] = uint64(w.Count()) - - // write out the meta len and compressed data len - _, err = writeUvarints(w, - uint64(len(metaBytes)), - uint64(len(idFieldVal)+len(compressed))) - if err != nil { - return 0, nil, err - } - // now write the meta - _, err = w.Write(metaBytes) - if err != nil { - return 0, nil, err - } - // now write the _id field val (counted as part of the 'compressed' data) - _, err = w.Write(idFieldVal) - if err != nil { - return 0, nil, err - } - // now write the compressed data - _, err = w.Write(compressed) - if err != nil { - return 0, nil, err - } - - newDocNum++ - } - - rv = append(rv, segNewDocNums) - } - - // return value is the start of the stored index - storedIndexOffset := uint64(w.Count()) - - // now write out the stored doc index - for _, docNumOffset := range docNumOffsets { - err := binary.Write(w, binary.BigEndian, docNumOffset) - if err != nil { - return 0, nil, err - } - } - - return storedIndexOffset, rv, nil -} - -// copyStoredDocs writes out a segment's stored doc info, optimized by -// using a single Write() call for the entire set of bytes. The -// newDocNumOffsets is filled with the new offsets for each doc. -func (s *SegmentBase) copyStoredDocs(newDocNum uint64, newDocNumOffsets []uint64, - w *CountHashWriter) error { - if s.numDocs <= 0 { - return nil - } - - indexOffset0, storedOffset0, _, _, _ := - s.getDocStoredOffsets(0) // the segment's first doc - - indexOffsetN, storedOffsetN, readN, metaLenN, dataLenN := - s.getDocStoredOffsets(s.numDocs - 1) // the segment's last doc - - storedOffset0New := uint64(w.Count()) - - storedBytes := s.mem[storedOffset0 : storedOffsetN+readN+metaLenN+dataLenN] - _, err := w.Write(storedBytes) - if err != nil { - return err - } - - // remap the storedOffset's for the docs into new offsets relative - // to storedOffset0New, filling the given docNumOffsetsOut array - for indexOffset := indexOffset0; indexOffset <= indexOffsetN; indexOffset += 8 { - storedOffset := binary.BigEndian.Uint64(s.mem[indexOffset : indexOffset+8]) - storedOffsetNew := storedOffset - storedOffset0 + storedOffset0New - newDocNumOffsets[newDocNum] = storedOffsetNew - newDocNum += 1 - } - - return nil -} - -// mergeFields builds a unified list of fields used across all the -// input segments, and computes whether the fields are the same across -// segments (which depends on fields to be sorted in the same way -// across segments) -func mergeFields(segments []*SegmentBase) (bool, []string) { - fieldsSame := true - - var segment0Fields []string - if len(segments) > 0 { - segment0Fields = segments[0].Fields() - } - - fieldsExist := map[string]struct{}{} - for _, segment := range segments { - fields := segment.Fields() - for fieldi, field := range fields { - fieldsExist[field] = struct{}{} - if len(segment0Fields) != len(fields) || segment0Fields[fieldi] != field { - fieldsSame = false - } - } - } - - rv := make([]string, 0, len(fieldsExist)) - // ensure _id stays first - rv = append(rv, "_id") - for k := range fieldsExist { - if k != "_id" { - rv = append(rv, k) - } - } - - sort.Strings(rv[1:]) // leave _id as first - - return fieldsSame, rv -} - -func isClosed(closeCh chan struct{}) bool { - select { - case <-closeCh: - return true - default: - return false - } -} diff --git a/vendor/github.com/blevesearch/zap/v13/new.go b/vendor/github.com/blevesearch/zap/v13/new.go deleted file mode 100644 index 98158186..00000000 --- a/vendor/github.com/blevesearch/zap/v13/new.go +++ /dev/null @@ -1,860 +0,0 @@ -// Copyright (c) 2018 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "bytes" - "encoding/binary" - "math" - "sort" - "sync" - - "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/couchbase/vellum" - "github.com/golang/snappy" -) - -var NewSegmentBufferNumResultsBump int = 100 -var NewSegmentBufferNumResultsFactor float64 = 1.0 -var NewSegmentBufferAvgBytesPerDocFactor float64 = 1.0 - -// ValidateDocFields can be set by applications to perform additional checks -// on fields in a document being added to a new segment, by default it does -// nothing. -// This API is experimental and may be removed at any time. -var ValidateDocFields = func(field document.Field) error { - return nil -} - -// AnalysisResultsToSegmentBase produces an in-memory zap-encoded -// SegmentBase from analysis results -func (z *ZapPlugin) New(results []*index.AnalysisResult) ( - segment.Segment, uint64, error) { - return z.newWithChunkMode(results, DefaultChunkMode) -} - -func (*ZapPlugin) newWithChunkMode(results []*index.AnalysisResult, - chunkMode uint32) (segment.Segment, uint64, error) { - s := interimPool.Get().(*interim) - - var br bytes.Buffer - if s.lastNumDocs > 0 { - // use previous results to initialize the buf with an estimate - // size, but note that the interim instance comes from a - // global interimPool, so multiple scorch instances indexing - // different docs can lead to low quality estimates - estimateAvgBytesPerDoc := int(float64(s.lastOutSize/s.lastNumDocs) * - NewSegmentBufferNumResultsFactor) - estimateNumResults := int(float64(len(results)+NewSegmentBufferNumResultsBump) * - NewSegmentBufferAvgBytesPerDocFactor) - br.Grow(estimateAvgBytesPerDoc * estimateNumResults) - } - - s.results = results - s.chunkMode = chunkMode - s.w = NewCountHashWriter(&br) - - storedIndexOffset, fieldsIndexOffset, fdvIndexOffset, dictOffsets, - err := s.convert() - if err != nil { - return nil, uint64(0), err - } - - sb, err := InitSegmentBase(br.Bytes(), s.w.Sum32(), chunkMode, - s.FieldsMap, s.FieldsInv, uint64(len(results)), - storedIndexOffset, fieldsIndexOffset, fdvIndexOffset, dictOffsets) - - if err == nil && s.reset() == nil { - s.lastNumDocs = len(results) - s.lastOutSize = len(br.Bytes()) - interimPool.Put(s) - } - - return sb, uint64(len(br.Bytes())), err -} - -var interimPool = sync.Pool{New: func() interface{} { return &interim{} }} - -// interim holds temporary working data used while converting from -// analysis results to a zap-encoded segment -type interim struct { - results []*index.AnalysisResult - - chunkMode uint32 - - w *CountHashWriter - - // FieldsMap adds 1 to field id to avoid zero value issues - // name -> field id + 1 - FieldsMap map[string]uint16 - - // FieldsInv is the inverse of FieldsMap - // field id -> name - FieldsInv []string - - // Term dictionaries for each field - // field id -> term -> postings list id + 1 - Dicts []map[string]uint64 - - // Terms for each field, where terms are sorted ascending - // field id -> []term - DictKeys [][]string - - // Fields whose IncludeDocValues is true - // field id -> bool - IncludeDocValues []bool - - // postings id -> bitmap of docNums - Postings []*roaring.Bitmap - - // postings id -> freq/norm's, one for each docNum in postings - FreqNorms [][]interimFreqNorm - freqNormsBacking []interimFreqNorm - - // postings id -> locs, one for each freq - Locs [][]interimLoc - locsBacking []interimLoc - - numTermsPerPostingsList []int // key is postings list id - numLocsPerPostingsList []int // key is postings list id - - builder *vellum.Builder - builderBuf bytes.Buffer - - metaBuf bytes.Buffer - - tmp0 []byte - tmp1 []byte - - lastNumDocs int - lastOutSize int -} - -func (s *interim) reset() (err error) { - s.results = nil - s.chunkMode = 0 - s.w = nil - s.FieldsMap = nil - s.FieldsInv = nil - for i := range s.Dicts { - s.Dicts[i] = nil - } - s.Dicts = s.Dicts[:0] - for i := range s.DictKeys { - s.DictKeys[i] = s.DictKeys[i][:0] - } - s.DictKeys = s.DictKeys[:0] - for i := range s.IncludeDocValues { - s.IncludeDocValues[i] = false - } - s.IncludeDocValues = s.IncludeDocValues[:0] - for _, idn := range s.Postings { - idn.Clear() - } - s.Postings = s.Postings[:0] - s.FreqNorms = s.FreqNorms[:0] - for i := range s.freqNormsBacking { - s.freqNormsBacking[i] = interimFreqNorm{} - } - s.freqNormsBacking = s.freqNormsBacking[:0] - s.Locs = s.Locs[:0] - for i := range s.locsBacking { - s.locsBacking[i] = interimLoc{} - } - s.locsBacking = s.locsBacking[:0] - s.numTermsPerPostingsList = s.numTermsPerPostingsList[:0] - s.numLocsPerPostingsList = s.numLocsPerPostingsList[:0] - s.builderBuf.Reset() - if s.builder != nil { - err = s.builder.Reset(&s.builderBuf) - } - s.metaBuf.Reset() - s.tmp0 = s.tmp0[:0] - s.tmp1 = s.tmp1[:0] - s.lastNumDocs = 0 - s.lastOutSize = 0 - - return err -} - -func (s *interim) grabBuf(size int) []byte { - buf := s.tmp0 - if cap(buf) < size { - buf = make([]byte, size) - s.tmp0 = buf - } - return buf[0:size] -} - -type interimStoredField struct { - vals [][]byte - typs []byte - arrayposs [][]uint64 // array positions -} - -type interimFreqNorm struct { - freq uint64 - norm float32 - numLocs int -} - -type interimLoc struct { - fieldID uint16 - pos uint64 - start uint64 - end uint64 - arrayposs []uint64 -} - -func (s *interim) convert() (uint64, uint64, uint64, []uint64, error) { - s.FieldsMap = map[string]uint16{} - - s.getOrDefineField("_id") // _id field is fieldID 0 - - for _, result := range s.results { - for _, field := range result.Document.CompositeFields { - s.getOrDefineField(field.Name()) - } - for _, field := range result.Document.Fields { - s.getOrDefineField(field.Name()) - } - } - - sort.Strings(s.FieldsInv[1:]) // keep _id as first field - - for fieldID, fieldName := range s.FieldsInv { - s.FieldsMap[fieldName] = uint16(fieldID + 1) - } - - if cap(s.IncludeDocValues) >= len(s.FieldsInv) { - s.IncludeDocValues = s.IncludeDocValues[:len(s.FieldsInv)] - } else { - s.IncludeDocValues = make([]bool, len(s.FieldsInv)) - } - - s.prepareDicts() - - for _, dict := range s.DictKeys { - sort.Strings(dict) - } - - s.processDocuments() - - storedIndexOffset, err := s.writeStoredFields() - if err != nil { - return 0, 0, 0, nil, err - } - - var fdvIndexOffset uint64 - var dictOffsets []uint64 - - if len(s.results) > 0 { - fdvIndexOffset, dictOffsets, err = s.writeDicts() - if err != nil { - return 0, 0, 0, nil, err - } - } else { - dictOffsets = make([]uint64, len(s.FieldsInv)) - } - - fieldsIndexOffset, err := persistFields(s.FieldsInv, s.w, dictOffsets) - if err != nil { - return 0, 0, 0, nil, err - } - - return storedIndexOffset, fieldsIndexOffset, fdvIndexOffset, dictOffsets, nil -} - -func (s *interim) getOrDefineField(fieldName string) int { - fieldIDPlus1, exists := s.FieldsMap[fieldName] - if !exists { - fieldIDPlus1 = uint16(len(s.FieldsInv) + 1) - s.FieldsMap[fieldName] = fieldIDPlus1 - s.FieldsInv = append(s.FieldsInv, fieldName) - - s.Dicts = append(s.Dicts, make(map[string]uint64)) - - n := len(s.DictKeys) - if n < cap(s.DictKeys) { - s.DictKeys = s.DictKeys[:n+1] - s.DictKeys[n] = s.DictKeys[n][:0] - } else { - s.DictKeys = append(s.DictKeys, []string(nil)) - } - } - - return int(fieldIDPlus1 - 1) -} - -// fill Dicts and DictKeys from analysis results -func (s *interim) prepareDicts() { - var pidNext int - - var totTFs int - var totLocs int - - visitField := func(fieldID uint16, tfs analysis.TokenFrequencies) { - dict := s.Dicts[fieldID] - dictKeys := s.DictKeys[fieldID] - - for term, tf := range tfs { - pidPlus1, exists := dict[term] - if !exists { - pidNext++ - pidPlus1 = uint64(pidNext) - - dict[term] = pidPlus1 - dictKeys = append(dictKeys, term) - - s.numTermsPerPostingsList = append(s.numTermsPerPostingsList, 0) - s.numLocsPerPostingsList = append(s.numLocsPerPostingsList, 0) - } - - pid := pidPlus1 - 1 - - s.numTermsPerPostingsList[pid] += 1 - s.numLocsPerPostingsList[pid] += len(tf.Locations) - - totLocs += len(tf.Locations) - } - - totTFs += len(tfs) - - s.DictKeys[fieldID] = dictKeys - } - - for _, result := range s.results { - // walk each composite field - for _, field := range result.Document.CompositeFields { - fieldID := uint16(s.getOrDefineField(field.Name())) - _, tf := field.Analyze() - visitField(fieldID, tf) - } - - // walk each field - for i, field := range result.Document.Fields { - fieldID := uint16(s.getOrDefineField(field.Name())) - tf := result.Analyzed[i] - visitField(fieldID, tf) - } - } - - numPostingsLists := pidNext - - if cap(s.Postings) >= numPostingsLists { - s.Postings = s.Postings[:numPostingsLists] - } else { - postings := make([]*roaring.Bitmap, numPostingsLists) - copy(postings, s.Postings[:cap(s.Postings)]) - for i := 0; i < numPostingsLists; i++ { - if postings[i] == nil { - postings[i] = roaring.New() - } - } - s.Postings = postings - } - - if cap(s.FreqNorms) >= numPostingsLists { - s.FreqNorms = s.FreqNorms[:numPostingsLists] - } else { - s.FreqNorms = make([][]interimFreqNorm, numPostingsLists) - } - - if cap(s.freqNormsBacking) >= totTFs { - s.freqNormsBacking = s.freqNormsBacking[:totTFs] - } else { - s.freqNormsBacking = make([]interimFreqNorm, totTFs) - } - - freqNormsBacking := s.freqNormsBacking - for pid, numTerms := range s.numTermsPerPostingsList { - s.FreqNorms[pid] = freqNormsBacking[0:0] - freqNormsBacking = freqNormsBacking[numTerms:] - } - - if cap(s.Locs) >= numPostingsLists { - s.Locs = s.Locs[:numPostingsLists] - } else { - s.Locs = make([][]interimLoc, numPostingsLists) - } - - if cap(s.locsBacking) >= totLocs { - s.locsBacking = s.locsBacking[:totLocs] - } else { - s.locsBacking = make([]interimLoc, totLocs) - } - - locsBacking := s.locsBacking - for pid, numLocs := range s.numLocsPerPostingsList { - s.Locs[pid] = locsBacking[0:0] - locsBacking = locsBacking[numLocs:] - } -} - -func (s *interim) processDocuments() { - numFields := len(s.FieldsInv) - reuseFieldLens := make([]int, numFields) - reuseFieldTFs := make([]analysis.TokenFrequencies, numFields) - - for docNum, result := range s.results { - for i := 0; i < numFields; i++ { // clear these for reuse - reuseFieldLens[i] = 0 - reuseFieldTFs[i] = nil - } - - s.processDocument(uint64(docNum), result, - reuseFieldLens, reuseFieldTFs) - } -} - -func (s *interim) processDocument(docNum uint64, - result *index.AnalysisResult, - fieldLens []int, fieldTFs []analysis.TokenFrequencies) { - visitField := func(fieldID uint16, fieldName string, - ln int, tf analysis.TokenFrequencies) { - fieldLens[fieldID] += ln - - existingFreqs := fieldTFs[fieldID] - if existingFreqs != nil { - existingFreqs.MergeAll(fieldName, tf) - } else { - fieldTFs[fieldID] = tf - } - } - - // walk each composite field - for _, field := range result.Document.CompositeFields { - fieldID := uint16(s.getOrDefineField(field.Name())) - ln, tf := field.Analyze() - visitField(fieldID, field.Name(), ln, tf) - } - - // walk each field - for i, field := range result.Document.Fields { - fieldID := uint16(s.getOrDefineField(field.Name())) - ln := result.Length[i] - tf := result.Analyzed[i] - visitField(fieldID, field.Name(), ln, tf) - } - - // now that it's been rolled up into fieldTFs, walk that - for fieldID, tfs := range fieldTFs { - dict := s.Dicts[fieldID] - norm := float32(1.0 / math.Sqrt(float64(fieldLens[fieldID]))) - - for term, tf := range tfs { - pid := dict[term] - 1 - bs := s.Postings[pid] - bs.Add(uint32(docNum)) - - s.FreqNorms[pid] = append(s.FreqNorms[pid], - interimFreqNorm{ - freq: uint64(tf.Frequency()), - norm: norm, - numLocs: len(tf.Locations), - }) - - if len(tf.Locations) > 0 { - locs := s.Locs[pid] - - for _, loc := range tf.Locations { - var locf = uint16(fieldID) - if loc.Field != "" { - locf = uint16(s.getOrDefineField(loc.Field)) - } - var arrayposs []uint64 - if len(loc.ArrayPositions) > 0 { - arrayposs = loc.ArrayPositions - } - locs = append(locs, interimLoc{ - fieldID: locf, - pos: uint64(loc.Position), - start: uint64(loc.Start), - end: uint64(loc.End), - arrayposs: arrayposs, - }) - } - - s.Locs[pid] = locs - } - } - } -} - -func (s *interim) writeStoredFields() ( - storedIndexOffset uint64, err error) { - varBuf := make([]byte, binary.MaxVarintLen64) - metaEncode := func(val uint64) (int, error) { - wb := binary.PutUvarint(varBuf, val) - return s.metaBuf.Write(varBuf[:wb]) - } - - data, compressed := s.tmp0[:0], s.tmp1[:0] - defer func() { s.tmp0, s.tmp1 = data, compressed }() - - // keyed by docNum - docStoredOffsets := make([]uint64, len(s.results)) - - // keyed by fieldID, for the current doc in the loop - docStoredFields := map[uint16]interimStoredField{} - - for docNum, result := range s.results { - for fieldID := range docStoredFields { // reset for next doc - delete(docStoredFields, fieldID) - } - - for _, field := range result.Document.Fields { - fieldID := uint16(s.getOrDefineField(field.Name())) - - opts := field.Options() - - if opts.IsStored() { - isf := docStoredFields[fieldID] - isf.vals = append(isf.vals, field.Value()) - isf.typs = append(isf.typs, encodeFieldType(field)) - isf.arrayposs = append(isf.arrayposs, field.ArrayPositions()) - docStoredFields[fieldID] = isf - } - - if opts.IncludeDocValues() { - s.IncludeDocValues[fieldID] = true - } - - err := ValidateDocFields(field) - if err != nil { - return 0, err - } - } - - var curr int - - s.metaBuf.Reset() - data = data[:0] - - // _id field special case optimizes ExternalID() lookups - idFieldVal := docStoredFields[uint16(0)].vals[0] - _, err = metaEncode(uint64(len(idFieldVal))) - if err != nil { - return 0, err - } - - // handle non-"_id" fields - for fieldID := 1; fieldID < len(s.FieldsInv); fieldID++ { - isf, exists := docStoredFields[uint16(fieldID)] - if exists { - curr, data, err = persistStoredFieldValues( - fieldID, isf.vals, isf.typs, isf.arrayposs, - curr, metaEncode, data) - if err != nil { - return 0, err - } - } - } - - metaBytes := s.metaBuf.Bytes() - - compressed = snappy.Encode(compressed[:cap(compressed)], data) - - docStoredOffsets[docNum] = uint64(s.w.Count()) - - _, err := writeUvarints(s.w, - uint64(len(metaBytes)), - uint64(len(idFieldVal)+len(compressed))) - if err != nil { - return 0, err - } - - _, err = s.w.Write(metaBytes) - if err != nil { - return 0, err - } - - _, err = s.w.Write(idFieldVal) - if err != nil { - return 0, err - } - - _, err = s.w.Write(compressed) - if err != nil { - return 0, err - } - } - - storedIndexOffset = uint64(s.w.Count()) - - for _, docStoredOffset := range docStoredOffsets { - err = binary.Write(s.w, binary.BigEndian, docStoredOffset) - if err != nil { - return 0, err - } - } - - return storedIndexOffset, nil -} - -func (s *interim) writeDicts() (fdvIndexOffset uint64, dictOffsets []uint64, err error) { - dictOffsets = make([]uint64, len(s.FieldsInv)) - - fdvOffsetsStart := make([]uint64, len(s.FieldsInv)) - fdvOffsetsEnd := make([]uint64, len(s.FieldsInv)) - - buf := s.grabBuf(binary.MaxVarintLen64) - - // these int coders are initialized with chunk size 1024 - // however this will be reset to the correct chunk size - // while processing each individual field-term section - tfEncoder := newChunkedIntCoder(1024, uint64(len(s.results)-1)) - locEncoder := newChunkedIntCoder(1024, uint64(len(s.results)-1)) - - var docTermMap [][]byte - - if s.builder == nil { - s.builder, err = vellum.New(&s.builderBuf, nil) - if err != nil { - return 0, nil, err - } - } - - for fieldID, terms := range s.DictKeys { - if cap(docTermMap) < len(s.results) { - docTermMap = make([][]byte, len(s.results)) - } else { - docTermMap = docTermMap[0:len(s.results)] - for docNum := range docTermMap { // reset the docTermMap - docTermMap[docNum] = docTermMap[docNum][:0] - } - } - - dict := s.Dicts[fieldID] - - for _, term := range terms { // terms are already sorted - pid := dict[term] - 1 - - postingsBS := s.Postings[pid] - - freqNorms := s.FreqNorms[pid] - freqNormOffset := 0 - - locs := s.Locs[pid] - locOffset := 0 - - chunkSize, err := getChunkSize(s.chunkMode, postingsBS.GetCardinality(), uint64(len(s.results))) - if err != nil { - return 0, nil, err - } - tfEncoder.SetChunkSize(chunkSize, uint64(len(s.results)-1)) - locEncoder.SetChunkSize(chunkSize, uint64(len(s.results)-1)) - - postingsItr := postingsBS.Iterator() - for postingsItr.HasNext() { - docNum := uint64(postingsItr.Next()) - - freqNorm := freqNorms[freqNormOffset] - - err = tfEncoder.Add(docNum, - encodeFreqHasLocs(freqNorm.freq, freqNorm.numLocs > 0), - uint64(math.Float32bits(freqNorm.norm))) - if err != nil { - return 0, nil, err - } - - if freqNorm.numLocs > 0 { - numBytesLocs := 0 - for _, loc := range locs[locOffset : locOffset+freqNorm.numLocs] { - numBytesLocs += totalUvarintBytes( - uint64(loc.fieldID), loc.pos, loc.start, loc.end, - uint64(len(loc.arrayposs)), loc.arrayposs) - } - - err = locEncoder.Add(docNum, uint64(numBytesLocs)) - if err != nil { - return 0, nil, err - } - - for _, loc := range locs[locOffset : locOffset+freqNorm.numLocs] { - err = locEncoder.Add(docNum, - uint64(loc.fieldID), loc.pos, loc.start, loc.end, - uint64(len(loc.arrayposs))) - if err != nil { - return 0, nil, err - } - - err = locEncoder.Add(docNum, loc.arrayposs...) - if err != nil { - return 0, nil, err - } - } - - locOffset += freqNorm.numLocs - } - - freqNormOffset++ - - docTermMap[docNum] = append( - append(docTermMap[docNum], term...), - termSeparator) - } - - tfEncoder.Close() - locEncoder.Close() - - postingsOffset, err := - writePostings(postingsBS, tfEncoder, locEncoder, nil, s.w, buf) - if err != nil { - return 0, nil, err - } - - if postingsOffset > uint64(0) { - err = s.builder.Insert([]byte(term), postingsOffset) - if err != nil { - return 0, nil, err - } - } - - tfEncoder.Reset() - locEncoder.Reset() - } - - err = s.builder.Close() - if err != nil { - return 0, nil, err - } - - // record where this dictionary starts - dictOffsets[fieldID] = uint64(s.w.Count()) - - vellumData := s.builderBuf.Bytes() - - // write out the length of the vellum data - n := binary.PutUvarint(buf, uint64(len(vellumData))) - _, err = s.w.Write(buf[:n]) - if err != nil { - return 0, nil, err - } - - // write this vellum to disk - _, err = s.w.Write(vellumData) - if err != nil { - return 0, nil, err - } - - // reset vellum for reuse - s.builderBuf.Reset() - - err = s.builder.Reset(&s.builderBuf) - if err != nil { - return 0, nil, err - } - - // write the field doc values - // NOTE: doc values continue to use legacy chunk mode - chunkSize, err := getChunkSize(LegacyChunkMode, 0, 0) - if err != nil { - return 0, nil, err - } - fdvEncoder := newChunkedContentCoder(chunkSize, uint64(len(s.results)-1), s.w, false) - if s.IncludeDocValues[fieldID] { - for docNum, docTerms := range docTermMap { - if len(docTerms) > 0 { - err = fdvEncoder.Add(uint64(docNum), docTerms) - if err != nil { - return 0, nil, err - } - } - } - err = fdvEncoder.Close() - if err != nil { - return 0, nil, err - } - - fdvOffsetsStart[fieldID] = uint64(s.w.Count()) - - _, err = fdvEncoder.Write() - if err != nil { - return 0, nil, err - } - - fdvOffsetsEnd[fieldID] = uint64(s.w.Count()) - - fdvEncoder.Reset() - } else { - fdvOffsetsStart[fieldID] = fieldNotUninverted - fdvOffsetsEnd[fieldID] = fieldNotUninverted - } - } - - fdvIndexOffset = uint64(s.w.Count()) - - for i := 0; i < len(fdvOffsetsStart); i++ { - n := binary.PutUvarint(buf, fdvOffsetsStart[i]) - _, err := s.w.Write(buf[:n]) - if err != nil { - return 0, nil, err - } - n = binary.PutUvarint(buf, fdvOffsetsEnd[i]) - _, err = s.w.Write(buf[:n]) - if err != nil { - return 0, nil, err - } - } - - return fdvIndexOffset, dictOffsets, nil -} - -func encodeFieldType(f document.Field) byte { - fieldType := byte('x') - switch f.(type) { - case *document.TextField: - fieldType = 't' - case *document.NumericField: - fieldType = 'n' - case *document.DateTimeField: - fieldType = 'd' - case *document.BooleanField: - fieldType = 'b' - case *document.GeoPointField: - fieldType = 'g' - case *document.CompositeField: - fieldType = 'c' - } - return fieldType -} - -// returns the total # of bytes needed to encode the given uint64's -// into binary.PutUVarint() encoding -func totalUvarintBytes(a, b, c, d, e uint64, more []uint64) (n int) { - n = numUvarintBytes(a) - n += numUvarintBytes(b) - n += numUvarintBytes(c) - n += numUvarintBytes(d) - n += numUvarintBytes(e) - for _, v := range more { - n += numUvarintBytes(v) - } - return n -} - -// returns # of bytes needed to encode x in binary.PutUvarint() encoding -func numUvarintBytes(x uint64) (n int) { - for x >= 0x80 { - x >>= 7 - n++ - } - return n + 1 -} diff --git a/vendor/github.com/blevesearch/zap/v13/plugin.go b/vendor/github.com/blevesearch/zap/v13/plugin.go deleted file mode 100644 index 38a0638d..00000000 --- a/vendor/github.com/blevesearch/zap/v13/plugin.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (c) 2020 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "github.com/blevesearch/bleve/index/scorch/segment" -) - -// ZapPlugin implements the Plugin interface of -// the blevesearch/bleve/index/scorch/segment pkg -type ZapPlugin struct{} - -func (*ZapPlugin) Type() string { - return Type -} - -func (*ZapPlugin) Version() uint32 { - return Version -} - -// Plugin returns an instance segment.Plugin for use -// by the Scorch indexing scheme -func Plugin() segment.Plugin { - return &ZapPlugin{} -} diff --git a/vendor/github.com/blevesearch/zap/v13/posting.go b/vendor/github.com/blevesearch/zap/v13/posting.go deleted file mode 100644 index 3a6ee548..00000000 --- a/vendor/github.com/blevesearch/zap/v13/posting.go +++ /dev/null @@ -1,798 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "encoding/binary" - "fmt" - "math" - "reflect" - - "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/blevesearch/bleve/size" -) - -var reflectStaticSizePostingsList int -var reflectStaticSizePostingsIterator int -var reflectStaticSizePosting int -var reflectStaticSizeLocation int - -func init() { - var pl PostingsList - reflectStaticSizePostingsList = int(reflect.TypeOf(pl).Size()) - var pi PostingsIterator - reflectStaticSizePostingsIterator = int(reflect.TypeOf(pi).Size()) - var p Posting - reflectStaticSizePosting = int(reflect.TypeOf(p).Size()) - var l Location - reflectStaticSizeLocation = int(reflect.TypeOf(l).Size()) -} - -// FST or vellum value (uint64) encoding is determined by the top two -// highest-order or most significant bits... -// -// encoding : MSB -// name : 63 62 61...to...bit #0 (LSB) -// ----------+---+---+--------------------------------------------------- -// general : 0 | 0 | 62-bits of postingsOffset. -// ~ : 0 | 1 | reserved for future. -// 1-hit : 1 | 0 | 31-bits of positive float31 norm | 31-bits docNum. -// ~ : 1 | 1 | reserved for future. -// -// Encoding "general" is able to handle all cases, where the -// postingsOffset points to more information about the postings for -// the term. -// -// Encoding "1-hit" is used to optimize a commonly seen case when a -// term has only a single hit. For example, a term in the _id field -// will have only 1 hit. The "1-hit" encoding is used for a term -// in a field when... -// -// - term vector info is disabled for that field; -// - and, the term appears in only a single doc for that field; -// - and, the term's freq is exactly 1 in that single doc for that field; -// - and, the docNum must fit into 31-bits; -// -// Otherwise, the "general" encoding is used instead. -// -// In the "1-hit" encoding, the field in that single doc may have -// other terms, which is supported in the "1-hit" encoding by the -// positive float31 norm. - -const FSTValEncodingMask = uint64(0xc000000000000000) -const FSTValEncodingGeneral = uint64(0x0000000000000000) -const FSTValEncoding1Hit = uint64(0x8000000000000000) - -func FSTValEncode1Hit(docNum uint64, normBits uint64) uint64 { - return FSTValEncoding1Hit | ((mask31Bits & normBits) << 31) | (mask31Bits & docNum) -} - -func FSTValDecode1Hit(v uint64) (docNum uint64, normBits uint64) { - return (mask31Bits & v), (mask31Bits & (v >> 31)) -} - -const mask31Bits = uint64(0x000000007fffffff) - -func under32Bits(x uint64) bool { - return x <= mask31Bits -} - -const DocNum1HitFinished = math.MaxUint64 - -var NormBits1Hit = uint64(math.Float32bits(float32(1))) - -// PostingsList is an in-memory representation of a postings list -type PostingsList struct { - sb *SegmentBase - postingsOffset uint64 - freqOffset uint64 - locOffset uint64 - postings *roaring.Bitmap - except *roaring.Bitmap - - // when normBits1Hit != 0, then this postings list came from a - // 1-hit encoding, and only the docNum1Hit & normBits1Hit apply - docNum1Hit uint64 - normBits1Hit uint64 -} - -// represents an immutable, empty postings list -var emptyPostingsList = &PostingsList{} - -func (p *PostingsList) Size() int { - sizeInBytes := reflectStaticSizePostingsList + size.SizeOfPtr - - if p.except != nil { - sizeInBytes += int(p.except.GetSizeInBytes()) - } - - return sizeInBytes -} - -func (p *PostingsList) OrInto(receiver *roaring.Bitmap) { - if p.normBits1Hit != 0 { - receiver.Add(uint32(p.docNum1Hit)) - return - } - - if p.postings != nil { - receiver.Or(p.postings) - } -} - -// Iterator returns an iterator for this postings list -func (p *PostingsList) Iterator(includeFreq, includeNorm, includeLocs bool, - prealloc segment.PostingsIterator) segment.PostingsIterator { - if p.normBits1Hit == 0 && p.postings == nil { - return emptyPostingsIterator - } - - var preallocPI *PostingsIterator - pi, ok := prealloc.(*PostingsIterator) - if ok && pi != nil { - preallocPI = pi - } - if preallocPI == emptyPostingsIterator { - preallocPI = nil - } - - return p.iterator(includeFreq, includeNorm, includeLocs, preallocPI) -} - -func (p *PostingsList) iterator(includeFreq, includeNorm, includeLocs bool, - rv *PostingsIterator) *PostingsIterator { - if rv == nil { - rv = &PostingsIterator{} - } else { - freqNormReader := rv.freqNormReader - if freqNormReader != nil { - freqNormReader.reset() - } - - locReader := rv.locReader - if locReader != nil { - locReader.reset() - } - - nextLocs := rv.nextLocs[:0] - nextSegmentLocs := rv.nextSegmentLocs[:0] - - buf := rv.buf - - *rv = PostingsIterator{} // clear the struct - - rv.freqNormReader = freqNormReader - rv.locReader = locReader - - rv.nextLocs = nextLocs - rv.nextSegmentLocs = nextSegmentLocs - - rv.buf = buf - } - - rv.postings = p - rv.includeFreqNorm = includeFreq || includeNorm || includeLocs - rv.includeLocs = includeLocs - - if p.normBits1Hit != 0 { - // "1-hit" encoding - rv.docNum1Hit = p.docNum1Hit - rv.normBits1Hit = p.normBits1Hit - - if p.except != nil && p.except.Contains(uint32(rv.docNum1Hit)) { - rv.docNum1Hit = DocNum1HitFinished - } - - return rv - } - - // "general" encoding, check if empty - if p.postings == nil { - return rv - } - - // initialize freq chunk reader - if rv.includeFreqNorm { - rv.freqNormReader = newChunkedIntDecoder(p.sb.mem, p.freqOffset) - } - - // initialize the loc chunk reader - if rv.includeLocs { - rv.locReader = newChunkedIntDecoder(p.sb.mem, p.locOffset) - } - - rv.all = p.postings.Iterator() - if p.except != nil { - rv.ActualBM = roaring.AndNot(p.postings, p.except) - rv.Actual = rv.ActualBM.Iterator() - } else { - rv.ActualBM = p.postings - rv.Actual = rv.all // Optimize to use same iterator for all & Actual. - } - - return rv -} - -// Count returns the number of items on this postings list -func (p *PostingsList) Count() uint64 { - var n, e uint64 - if p.normBits1Hit != 0 { - n = 1 - if p.except != nil && p.except.Contains(uint32(p.docNum1Hit)) { - e = 1 - } - } else if p.postings != nil { - n = p.postings.GetCardinality() - if p.except != nil { - e = p.postings.AndCardinality(p.except) - } - } - return n - e -} - -func (rv *PostingsList) read(postingsOffset uint64, d *Dictionary) error { - rv.postingsOffset = postingsOffset - - // handle "1-hit" encoding special case - if rv.postingsOffset&FSTValEncodingMask == FSTValEncoding1Hit { - return rv.init1Hit(postingsOffset) - } - - // read the location of the freq/norm details - var n uint64 - var read int - - rv.freqOffset, read = binary.Uvarint(d.sb.mem[postingsOffset+n : postingsOffset+binary.MaxVarintLen64]) - n += uint64(read) - - rv.locOffset, read = binary.Uvarint(d.sb.mem[postingsOffset+n : postingsOffset+n+binary.MaxVarintLen64]) - n += uint64(read) - - var postingsLen uint64 - postingsLen, read = binary.Uvarint(d.sb.mem[postingsOffset+n : postingsOffset+n+binary.MaxVarintLen64]) - n += uint64(read) - - roaringBytes := d.sb.mem[postingsOffset+n : postingsOffset+n+postingsLen] - - if rv.postings == nil { - rv.postings = roaring.NewBitmap() - } - _, err := rv.postings.FromBuffer(roaringBytes) - if err != nil { - return fmt.Errorf("error loading roaring bitmap: %v", err) - } - - return nil -} - -func (rv *PostingsList) init1Hit(fstVal uint64) error { - docNum, normBits := FSTValDecode1Hit(fstVal) - - rv.docNum1Hit = docNum - rv.normBits1Hit = normBits - - return nil -} - -// PostingsIterator provides a way to iterate through the postings list -type PostingsIterator struct { - postings *PostingsList - all roaring.IntPeekable - Actual roaring.IntPeekable - ActualBM *roaring.Bitmap - - currChunk uint32 - freqNormReader *chunkedIntDecoder - locReader *chunkedIntDecoder - - next Posting // reused across Next() calls - nextLocs []Location // reused across Next() calls - nextSegmentLocs []segment.Location // reused across Next() calls - - docNum1Hit uint64 - normBits1Hit uint64 - - buf []byte - - includeFreqNorm bool - includeLocs bool -} - -var emptyPostingsIterator = &PostingsIterator{} - -func (i *PostingsIterator) Size() int { - sizeInBytes := reflectStaticSizePostingsIterator + size.SizeOfPtr + - i.next.Size() - // account for freqNormReader, locReader if we start using this. - for _, entry := range i.nextLocs { - sizeInBytes += entry.Size() - } - - return sizeInBytes -} - -func (i *PostingsIterator) loadChunk(chunk int) error { - if i.includeFreqNorm { - err := i.freqNormReader.loadChunk(chunk) - if err != nil { - return err - } - } - - if i.includeLocs { - err := i.locReader.loadChunk(chunk) - if err != nil { - return err - } - } - - i.currChunk = uint32(chunk) - return nil -} - -func (i *PostingsIterator) readFreqNormHasLocs() (uint64, uint64, bool, error) { - if i.normBits1Hit != 0 { - return 1, i.normBits1Hit, false, nil - } - - freqHasLocs, err := i.freqNormReader.readUvarint() - if err != nil { - return 0, 0, false, fmt.Errorf("error reading frequency: %v", err) - } - - freq, hasLocs := decodeFreqHasLocs(freqHasLocs) - - normBits, err := i.freqNormReader.readUvarint() - if err != nil { - return 0, 0, false, fmt.Errorf("error reading norm: %v", err) - } - - return freq, normBits, hasLocs, nil -} - -func (i *PostingsIterator) skipFreqNormReadHasLocs() (bool, error) { - if i.normBits1Hit != 0 { - return false, nil - } - - freqHasLocs, err := i.freqNormReader.readUvarint() - if err != nil { - return false, fmt.Errorf("error reading freqHasLocs: %v", err) - } - - i.freqNormReader.SkipUvarint() // Skip normBits. - - return freqHasLocs&0x01 != 0, nil // See decodeFreqHasLocs() / hasLocs. -} - -func encodeFreqHasLocs(freq uint64, hasLocs bool) uint64 { - rv := freq << 1 - if hasLocs { - rv = rv | 0x01 // 0'th LSB encodes whether there are locations - } - return rv -} - -func decodeFreqHasLocs(freqHasLocs uint64) (uint64, bool) { - freq := freqHasLocs >> 1 - hasLocs := freqHasLocs&0x01 != 0 - return freq, hasLocs -} - -// readLocation processes all the integers on the stream representing a single -// location. -func (i *PostingsIterator) readLocation(l *Location) error { - // read off field - fieldID, err := i.locReader.readUvarint() - if err != nil { - return fmt.Errorf("error reading location field: %v", err) - } - // read off pos - pos, err := i.locReader.readUvarint() - if err != nil { - return fmt.Errorf("error reading location pos: %v", err) - } - // read off start - start, err := i.locReader.readUvarint() - if err != nil { - return fmt.Errorf("error reading location start: %v", err) - } - // read off end - end, err := i.locReader.readUvarint() - if err != nil { - return fmt.Errorf("error reading location end: %v", err) - } - // read off num array pos - numArrayPos, err := i.locReader.readUvarint() - if err != nil { - return fmt.Errorf("error reading location num array pos: %v", err) - } - - l.field = i.postings.sb.fieldsInv[fieldID] - l.pos = pos - l.start = start - l.end = end - - if cap(l.ap) < int(numArrayPos) { - l.ap = make([]uint64, int(numArrayPos)) - } else { - l.ap = l.ap[:int(numArrayPos)] - } - - // read off array positions - for k := 0; k < int(numArrayPos); k++ { - ap, err := i.locReader.readUvarint() - if err != nil { - return fmt.Errorf("error reading array position: %v", err) - } - - l.ap[k] = ap - } - - return nil -} - -// Next returns the next posting on the postings list, or nil at the end -func (i *PostingsIterator) Next() (segment.Posting, error) { - return i.nextAtOrAfter(0) -} - -// Advance returns the posting at the specified docNum or it is not present -// the next posting, or if the end is reached, nil -func (i *PostingsIterator) Advance(docNum uint64) (segment.Posting, error) { - return i.nextAtOrAfter(docNum) -} - -// Next returns the next posting on the postings list, or nil at the end -func (i *PostingsIterator) nextAtOrAfter(atOrAfter uint64) (segment.Posting, error) { - docNum, exists, err := i.nextDocNumAtOrAfter(atOrAfter) - if err != nil || !exists { - return nil, err - } - - i.next = Posting{} // clear the struct - rv := &i.next - rv.docNum = docNum - - if !i.includeFreqNorm { - return rv, nil - } - - var normBits uint64 - var hasLocs bool - - rv.freq, normBits, hasLocs, err = i.readFreqNormHasLocs() - if err != nil { - return nil, err - } - - rv.norm = math.Float32frombits(uint32(normBits)) - - if i.includeLocs && hasLocs { - // prepare locations into reused slices, where we assume - // rv.freq >= "number of locs", since in a composite field, - // some component fields might have their IncludeTermVector - // flags disabled while other component fields are enabled - if cap(i.nextLocs) >= int(rv.freq) { - i.nextLocs = i.nextLocs[0:rv.freq] - } else { - i.nextLocs = make([]Location, rv.freq, rv.freq*2) - } - if cap(i.nextSegmentLocs) < int(rv.freq) { - i.nextSegmentLocs = make([]segment.Location, rv.freq, rv.freq*2) - } - rv.locs = i.nextSegmentLocs[:0] - - numLocsBytes, err := i.locReader.readUvarint() - if err != nil { - return nil, fmt.Errorf("error reading location numLocsBytes: %v", err) - } - - j := 0 - startBytesRemaining := i.locReader.Len() // # bytes remaining in the locReader - for startBytesRemaining-i.locReader.Len() < int(numLocsBytes) { - err := i.readLocation(&i.nextLocs[j]) - if err != nil { - return nil, err - } - rv.locs = append(rv.locs, &i.nextLocs[j]) - j++ - } - } - - return rv, nil -} - -// nextDocNum returns the next docNum on the postings list, and also -// sets up the currChunk / loc related fields of the iterator. -func (i *PostingsIterator) nextDocNumAtOrAfter(atOrAfter uint64) (uint64, bool, error) { - if i.normBits1Hit != 0 { - if i.docNum1Hit == DocNum1HitFinished { - return 0, false, nil - } - if i.docNum1Hit < atOrAfter { - // advanced past our 1-hit - i.docNum1Hit = DocNum1HitFinished // consume our 1-hit docNum - return 0, false, nil - } - docNum := i.docNum1Hit - i.docNum1Hit = DocNum1HitFinished // consume our 1-hit docNum - return docNum, true, nil - } - - if i.Actual == nil || !i.Actual.HasNext() { - return 0, false, nil - } - - if i.postings == nil || i.postings.postings == i.ActualBM { - return i.nextDocNumAtOrAfterClean(atOrAfter) - } - - i.Actual.AdvanceIfNeeded(uint32(atOrAfter)) - - if !i.Actual.HasNext() { - // couldn't find anything - return 0, false, nil - } - - n := i.Actual.Next() - allN := i.all.Next() - - chunkSize, err := getChunkSize(i.postings.sb.chunkMode, i.postings.postings.GetCardinality(), i.postings.sb.numDocs) - if err != nil { - return 0, false, err - } - nChunk := n / uint32(chunkSize) - - // when allN becomes >= to here, then allN is in the same chunk as nChunk. - allNReachesNChunk := nChunk * uint32(chunkSize) - - // n is the next actual hit (excluding some postings), and - // allN is the next hit in the full postings, and - // if they don't match, move 'all' forwards until they do - for allN != n { - // we've reached same chunk, so move the freq/norm/loc decoders forward - if i.includeFreqNorm && allN >= allNReachesNChunk { - err := i.currChunkNext(nChunk) - if err != nil { - return 0, false, err - } - } - - allN = i.all.Next() - } - - if i.includeFreqNorm && (i.currChunk != nChunk || i.freqNormReader.isNil()) { - err := i.loadChunk(int(nChunk)) - if err != nil { - return 0, false, fmt.Errorf("error loading chunk: %v", err) - } - } - - return uint64(n), true, nil -} - -// optimization when the postings list is "clean" (e.g., no updates & -// no deletions) where the all bitmap is the same as the actual bitmap -func (i *PostingsIterator) nextDocNumAtOrAfterClean( - atOrAfter uint64) (uint64, bool, error) { - - if !i.includeFreqNorm { - i.Actual.AdvanceIfNeeded(uint32(atOrAfter)) - - if !i.Actual.HasNext() { - return 0, false, nil // couldn't find anything - } - - return uint64(i.Actual.Next()), true, nil - } - - chunkSize, err := getChunkSize(i.postings.sb.chunkMode, i.postings.postings.GetCardinality(), i.postings.sb.numDocs) - if err != nil { - return 0, false, err - } - - // freq-norm's needed, so maintain freq-norm chunk reader - sameChunkNexts := 0 // # of times we called Next() in the same chunk - n := i.Actual.Next() - nChunk := n / uint32(chunkSize) - - for uint64(n) < atOrAfter && i.Actual.HasNext() { - n = i.Actual.Next() - - nChunkPrev := nChunk - nChunk = n / uint32(chunkSize) - - if nChunk != nChunkPrev { - sameChunkNexts = 0 - } else { - sameChunkNexts += 1 - } - } - - if uint64(n) < atOrAfter { - // couldn't find anything - return 0, false, nil - } - - for j := 0; j < sameChunkNexts; j++ { - err := i.currChunkNext(nChunk) - if err != nil { - return 0, false, fmt.Errorf("error optimized currChunkNext: %v", err) - } - } - - if i.currChunk != nChunk || i.freqNormReader.isNil() { - err := i.loadChunk(int(nChunk)) - if err != nil { - return 0, false, fmt.Errorf("error loading chunk: %v", err) - } - } - - return uint64(n), true, nil -} - -func (i *PostingsIterator) currChunkNext(nChunk uint32) error { - if i.currChunk != nChunk || i.freqNormReader.isNil() { - err := i.loadChunk(int(nChunk)) - if err != nil { - return fmt.Errorf("error loading chunk: %v", err) - } - } - - // read off freq/offsets even though we don't care about them - hasLocs, err := i.skipFreqNormReadHasLocs() - if err != nil { - return err - } - - if i.includeLocs && hasLocs { - numLocsBytes, err := i.locReader.readUvarint() - if err != nil { - return fmt.Errorf("error reading location numLocsBytes: %v", err) - } - - // skip over all the location bytes - i.locReader.SkipBytes(int(numLocsBytes)) - } - - return nil -} - -// DocNum1Hit returns the docNum and true if this is "1-hit" optimized -// and the docNum is available. -func (p *PostingsIterator) DocNum1Hit() (uint64, bool) { - if p.normBits1Hit != 0 && p.docNum1Hit != DocNum1HitFinished { - return p.docNum1Hit, true - } - return 0, false -} - -// ActualBitmap returns the underlying actual bitmap -// which can be used up the stack for optimizations -func (p *PostingsIterator) ActualBitmap() *roaring.Bitmap { - return p.ActualBM -} - -// ReplaceActual replaces the ActualBM with the provided -// bitmap -func (p *PostingsIterator) ReplaceActual(abm *roaring.Bitmap) { - p.ActualBM = abm - p.Actual = abm.Iterator() -} - -// PostingsIteratorFromBitmap constructs a PostingsIterator given an -// "actual" bitmap. -func PostingsIteratorFromBitmap(bm *roaring.Bitmap, - includeFreqNorm, includeLocs bool) (segment.PostingsIterator, error) { - return &PostingsIterator{ - ActualBM: bm, - Actual: bm.Iterator(), - includeFreqNorm: includeFreqNorm, - includeLocs: includeLocs, - }, nil -} - -// PostingsIteratorFrom1Hit constructs a PostingsIterator given a -// 1-hit docNum. -func PostingsIteratorFrom1Hit(docNum1Hit uint64, - includeFreqNorm, includeLocs bool) (segment.PostingsIterator, error) { - return &PostingsIterator{ - docNum1Hit: docNum1Hit, - normBits1Hit: NormBits1Hit, - includeFreqNorm: includeFreqNorm, - includeLocs: includeLocs, - }, nil -} - -// Posting is a single entry in a postings list -type Posting struct { - docNum uint64 - freq uint64 - norm float32 - locs []segment.Location -} - -func (p *Posting) Size() int { - sizeInBytes := reflectStaticSizePosting - - for _, entry := range p.locs { - sizeInBytes += entry.Size() - } - - return sizeInBytes -} - -// Number returns the document number of this posting in this segment -func (p *Posting) Number() uint64 { - return p.docNum -} - -// Frequency returns the frequencies of occurrence of this term in this doc/field -func (p *Posting) Frequency() uint64 { - return p.freq -} - -// Norm returns the normalization factor for this posting -func (p *Posting) Norm() float64 { - return float64(p.norm) -} - -// Locations returns the location information for each occurrence -func (p *Posting) Locations() []segment.Location { - return p.locs -} - -// Location represents the location of a single occurrence -type Location struct { - field string - pos uint64 - start uint64 - end uint64 - ap []uint64 -} - -func (l *Location) Size() int { - return reflectStaticSizeLocation + - len(l.field) + - len(l.ap)*size.SizeOfUint64 -} - -// Field returns the name of the field (useful in composite fields to know -// which original field the value came from) -func (l *Location) Field() string { - return l.field -} - -// Start returns the start byte offset of this occurrence -func (l *Location) Start() uint64 { - return l.start -} - -// End returns the end byte offset of this occurrence -func (l *Location) End() uint64 { - return l.end -} - -// Pos returns the 1-based phrase position of this occurrence -func (l *Location) Pos() uint64 { - return l.pos -} - -// ArrayPositions returns the array position vector associated with this occurrence -func (l *Location) ArrayPositions() []uint64 { - return l.ap -} diff --git a/vendor/github.com/blevesearch/zap/v13/segment.go b/vendor/github.com/blevesearch/zap/v13/segment.go deleted file mode 100644 index e8b1f067..00000000 --- a/vendor/github.com/blevesearch/zap/v13/segment.go +++ /dev/null @@ -1,572 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "bytes" - "encoding/binary" - "fmt" - "io" - "os" - "sync" - "unsafe" - - "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/blevesearch/bleve/size" - "github.com/couchbase/vellum" - mmap "github.com/blevesearch/mmap-go" - "github.com/golang/snappy" -) - -var reflectStaticSizeSegmentBase int - -func init() { - var sb SegmentBase - reflectStaticSizeSegmentBase = int(unsafe.Sizeof(sb)) -} - -// Open returns a zap impl of a segment -func (*ZapPlugin) Open(path string) (segment.Segment, error) { - f, err := os.Open(path) - if err != nil { - return nil, err - } - mm, err := mmap.Map(f, mmap.RDONLY, 0) - if err != nil { - // mmap failed, try to close the file - _ = f.Close() - return nil, err - } - - rv := &Segment{ - SegmentBase: SegmentBase{ - mem: mm[0 : len(mm)-FooterSize], - fieldsMap: make(map[string]uint16), - fieldDvReaders: make(map[uint16]*docValueReader), - fieldFSTs: make(map[uint16]*vellum.FST), - }, - f: f, - mm: mm, - path: path, - refs: 1, - } - rv.SegmentBase.updateSize() - - err = rv.loadConfig() - if err != nil { - _ = rv.Close() - return nil, err - } - - err = rv.loadFields() - if err != nil { - _ = rv.Close() - return nil, err - } - - err = rv.loadDvReaders() - if err != nil { - _ = rv.Close() - return nil, err - } - - return rv, nil -} - -// SegmentBase is a memory only, read-only implementation of the -// segment.Segment interface, using zap's data representation. -type SegmentBase struct { - mem []byte - memCRC uint32 - chunkMode uint32 - fieldsMap map[string]uint16 // fieldName -> fieldID+1 - fieldsInv []string // fieldID -> fieldName - numDocs uint64 - storedIndexOffset uint64 - fieldsIndexOffset uint64 - docValueOffset uint64 - dictLocs []uint64 - fieldDvReaders map[uint16]*docValueReader // naive chunk cache per field - fieldDvNames []string // field names cached in fieldDvReaders - size uint64 - - m sync.Mutex - fieldFSTs map[uint16]*vellum.FST -} - -func (sb *SegmentBase) Size() int { - return int(sb.size) -} - -func (sb *SegmentBase) updateSize() { - sizeInBytes := reflectStaticSizeSegmentBase + - cap(sb.mem) - - // fieldsMap - for k := range sb.fieldsMap { - sizeInBytes += (len(k) + size.SizeOfString) + size.SizeOfUint16 - } - - // fieldsInv, dictLocs - for _, entry := range sb.fieldsInv { - sizeInBytes += len(entry) + size.SizeOfString - } - sizeInBytes += len(sb.dictLocs) * size.SizeOfUint64 - - // fieldDvReaders - for _, v := range sb.fieldDvReaders { - sizeInBytes += size.SizeOfUint16 + size.SizeOfPtr - if v != nil { - sizeInBytes += v.size() - } - } - - sb.size = uint64(sizeInBytes) -} - -func (sb *SegmentBase) AddRef() {} -func (sb *SegmentBase) DecRef() (err error) { return nil } -func (sb *SegmentBase) Close() (err error) { return nil } - -// Segment implements a persisted segment.Segment interface, by -// embedding an mmap()'ed SegmentBase. -type Segment struct { - SegmentBase - - f *os.File - mm mmap.MMap - path string - version uint32 - crc uint32 - - m sync.Mutex // Protects the fields that follow. - refs int64 -} - -func (s *Segment) Size() int { - // 8 /* size of file pointer */ - // 4 /* size of version -> uint32 */ - // 4 /* size of crc -> uint32 */ - sizeOfUints := 16 - - sizeInBytes := (len(s.path) + size.SizeOfString) + sizeOfUints - - // mutex, refs -> int64 - sizeInBytes += 16 - - // do not include the mmap'ed part - return sizeInBytes + s.SegmentBase.Size() - cap(s.mem) -} - -func (s *Segment) AddRef() { - s.m.Lock() - s.refs++ - s.m.Unlock() -} - -func (s *Segment) DecRef() (err error) { - s.m.Lock() - s.refs-- - if s.refs == 0 { - err = s.closeActual() - } - s.m.Unlock() - return err -} - -func (s *Segment) loadConfig() error { - crcOffset := len(s.mm) - 4 - s.crc = binary.BigEndian.Uint32(s.mm[crcOffset : crcOffset+4]) - - verOffset := crcOffset - 4 - s.version = binary.BigEndian.Uint32(s.mm[verOffset : verOffset+4]) - if s.version != Version { - return fmt.Errorf("unsupported version %d", s.version) - } - - chunkOffset := verOffset - 4 - s.chunkMode = binary.BigEndian.Uint32(s.mm[chunkOffset : chunkOffset+4]) - - docValueOffset := chunkOffset - 8 - s.docValueOffset = binary.BigEndian.Uint64(s.mm[docValueOffset : docValueOffset+8]) - - fieldsIndexOffset := docValueOffset - 8 - s.fieldsIndexOffset = binary.BigEndian.Uint64(s.mm[fieldsIndexOffset : fieldsIndexOffset+8]) - - storedIndexOffset := fieldsIndexOffset - 8 - s.storedIndexOffset = binary.BigEndian.Uint64(s.mm[storedIndexOffset : storedIndexOffset+8]) - - numDocsOffset := storedIndexOffset - 8 - s.numDocs = binary.BigEndian.Uint64(s.mm[numDocsOffset : numDocsOffset+8]) - return nil -} - -func (s *SegmentBase) loadFields() error { - // NOTE for now we assume the fields index immediately precedes - // the footer, and if this changes, need to adjust accordingly (or - // store explicit length), where s.mem was sliced from s.mm in Open(). - fieldsIndexEnd := uint64(len(s.mem)) - - // iterate through fields index - var fieldID uint64 - for s.fieldsIndexOffset+(8*fieldID) < fieldsIndexEnd { - addr := binary.BigEndian.Uint64(s.mem[s.fieldsIndexOffset+(8*fieldID) : s.fieldsIndexOffset+(8*fieldID)+8]) - - dictLoc, read := binary.Uvarint(s.mem[addr:fieldsIndexEnd]) - n := uint64(read) - s.dictLocs = append(s.dictLocs, dictLoc) - - var nameLen uint64 - nameLen, read = binary.Uvarint(s.mem[addr+n : fieldsIndexEnd]) - n += uint64(read) - - name := string(s.mem[addr+n : addr+n+nameLen]) - s.fieldsInv = append(s.fieldsInv, name) - s.fieldsMap[name] = uint16(fieldID + 1) - - fieldID++ - } - return nil -} - -// Dictionary returns the term dictionary for the specified field -func (s *SegmentBase) Dictionary(field string) (segment.TermDictionary, error) { - dict, err := s.dictionary(field) - if err == nil && dict == nil { - return &segment.EmptyDictionary{}, nil - } - return dict, err -} - -func (sb *SegmentBase) dictionary(field string) (rv *Dictionary, err error) { - fieldIDPlus1 := sb.fieldsMap[field] - if fieldIDPlus1 > 0 { - rv = &Dictionary{ - sb: sb, - field: field, - fieldID: fieldIDPlus1 - 1, - } - - dictStart := sb.dictLocs[rv.fieldID] - if dictStart > 0 { - var ok bool - sb.m.Lock() - if rv.fst, ok = sb.fieldFSTs[rv.fieldID]; !ok { - // read the length of the vellum data - vellumLen, read := binary.Uvarint(sb.mem[dictStart : dictStart+binary.MaxVarintLen64]) - fstBytes := sb.mem[dictStart+uint64(read) : dictStart+uint64(read)+vellumLen] - rv.fst, err = vellum.Load(fstBytes) - if err != nil { - sb.m.Unlock() - return nil, fmt.Errorf("dictionary field %s vellum err: %v", field, err) - } - - sb.fieldFSTs[rv.fieldID] = rv.fst - } - - sb.m.Unlock() - rv.fstReader, err = rv.fst.Reader() - if err != nil { - return nil, fmt.Errorf("dictionary field %s vellum reader err: %v", field, err) - } - - } - } - - return rv, nil -} - -// visitDocumentCtx holds data structures that are reusable across -// multiple VisitDocument() calls to avoid memory allocations -type visitDocumentCtx struct { - buf []byte - reader bytes.Reader - arrayPos []uint64 -} - -var visitDocumentCtxPool = sync.Pool{ - New: func() interface{} { - reuse := &visitDocumentCtx{} - return reuse - }, -} - -// VisitDocument invokes the DocFieldValueVistor for each stored field -// for the specified doc number -func (s *SegmentBase) VisitDocument(num uint64, visitor segment.DocumentFieldValueVisitor) error { - vdc := visitDocumentCtxPool.Get().(*visitDocumentCtx) - defer visitDocumentCtxPool.Put(vdc) - return s.visitDocument(vdc, num, visitor) -} - -func (s *SegmentBase) visitDocument(vdc *visitDocumentCtx, num uint64, - visitor segment.DocumentFieldValueVisitor) error { - // first make sure this is a valid number in this segment - if num < s.numDocs { - meta, compressed := s.getDocStoredMetaAndCompressed(num) - - vdc.reader.Reset(meta) - - // handle _id field special case - idFieldValLen, err := binary.ReadUvarint(&vdc.reader) - if err != nil { - return err - } - idFieldVal := compressed[:idFieldValLen] - - keepGoing := visitor("_id", byte('t'), idFieldVal, nil) - if !keepGoing { - visitDocumentCtxPool.Put(vdc) - return nil - } - - // handle non-"_id" fields - compressed = compressed[idFieldValLen:] - - uncompressed, err := snappy.Decode(vdc.buf[:cap(vdc.buf)], compressed) - if err != nil { - return err - } - - for keepGoing { - field, err := binary.ReadUvarint(&vdc.reader) - if err == io.EOF { - break - } - if err != nil { - return err - } - typ, err := binary.ReadUvarint(&vdc.reader) - if err != nil { - return err - } - offset, err := binary.ReadUvarint(&vdc.reader) - if err != nil { - return err - } - l, err := binary.ReadUvarint(&vdc.reader) - if err != nil { - return err - } - numap, err := binary.ReadUvarint(&vdc.reader) - if err != nil { - return err - } - var arrayPos []uint64 - if numap > 0 { - if cap(vdc.arrayPos) < int(numap) { - vdc.arrayPos = make([]uint64, numap) - } - arrayPos = vdc.arrayPos[:numap] - for i := 0; i < int(numap); i++ { - ap, err := binary.ReadUvarint(&vdc.reader) - if err != nil { - return err - } - arrayPos[i] = ap - } - } - - value := uncompressed[offset : offset+l] - keepGoing = visitor(s.fieldsInv[field], byte(typ), value, arrayPos) - } - - vdc.buf = uncompressed - } - return nil -} - -// DocID returns the value of the _id field for the given docNum -func (s *SegmentBase) DocID(num uint64) ([]byte, error) { - if num >= s.numDocs { - return nil, nil - } - - vdc := visitDocumentCtxPool.Get().(*visitDocumentCtx) - - meta, compressed := s.getDocStoredMetaAndCompressed(num) - - vdc.reader.Reset(meta) - - // handle _id field special case - idFieldValLen, err := binary.ReadUvarint(&vdc.reader) - if err != nil { - return nil, err - } - idFieldVal := compressed[:idFieldValLen] - - visitDocumentCtxPool.Put(vdc) - - return idFieldVal, nil -} - -// Count returns the number of documents in this segment. -func (s *SegmentBase) Count() uint64 { - return s.numDocs -} - -// DocNumbers returns a bitset corresponding to the doc numbers of all the -// provided _id strings -func (s *SegmentBase) DocNumbers(ids []string) (*roaring.Bitmap, error) { - rv := roaring.New() - - if len(s.fieldsMap) > 0 { - idDict, err := s.dictionary("_id") - if err != nil { - return nil, err - } - - postingsList := emptyPostingsList - - sMax, err := idDict.fst.GetMaxKey() - if err != nil { - return nil, err - } - sMaxStr := string(sMax) - filteredIds := make([]string, 0, len(ids)) - for _, id := range ids { - if id <= sMaxStr { - filteredIds = append(filteredIds, id) - } - } - - for _, id := range filteredIds { - postingsList, err = idDict.postingsList([]byte(id), nil, postingsList) - if err != nil { - return nil, err - } - postingsList.OrInto(rv) - } - } - - return rv, nil -} - -// Fields returns the field names used in this segment -func (s *SegmentBase) Fields() []string { - return s.fieldsInv -} - -// Path returns the path of this segment on disk -func (s *Segment) Path() string { - return s.path -} - -// Close releases all resources associated with this segment -func (s *Segment) Close() (err error) { - return s.DecRef() -} - -func (s *Segment) closeActual() (err error) { - if s.mm != nil { - err = s.mm.Unmap() - } - // try to close file even if unmap failed - if s.f != nil { - err2 := s.f.Close() - if err == nil { - // try to return first error - err = err2 - } - } - return -} - -// some helpers i started adding for the command-line utility - -// Data returns the underlying mmaped data slice -func (s *Segment) Data() []byte { - return s.mm -} - -// CRC returns the CRC value stored in the file footer -func (s *Segment) CRC() uint32 { - return s.crc -} - -// Version returns the file version in the file footer -func (s *Segment) Version() uint32 { - return s.version -} - -// ChunkFactor returns the chunk factor in the file footer -func (s *Segment) ChunkMode() uint32 { - return s.chunkMode -} - -// FieldsIndexOffset returns the fields index offset in the file footer -func (s *Segment) FieldsIndexOffset() uint64 { - return s.fieldsIndexOffset -} - -// StoredIndexOffset returns the stored value index offset in the file footer -func (s *Segment) StoredIndexOffset() uint64 { - return s.storedIndexOffset -} - -// DocValueOffset returns the docValue offset in the file footer -func (s *Segment) DocValueOffset() uint64 { - return s.docValueOffset -} - -// NumDocs returns the number of documents in the file footer -func (s *Segment) NumDocs() uint64 { - return s.numDocs -} - -// DictAddr is a helper function to compute the file offset where the -// dictionary is stored for the specified field. -func (s *Segment) DictAddr(field string) (uint64, error) { - fieldIDPlus1, ok := s.fieldsMap[field] - if !ok { - return 0, fmt.Errorf("no such field '%s'", field) - } - - return s.dictLocs[fieldIDPlus1-1], nil -} - -func (s *SegmentBase) loadDvReaders() error { - if s.docValueOffset == fieldNotUninverted || s.numDocs == 0 { - return nil - } - - var read uint64 - for fieldID, field := range s.fieldsInv { - var fieldLocStart, fieldLocEnd uint64 - var n int - fieldLocStart, n = binary.Uvarint(s.mem[s.docValueOffset+read : s.docValueOffset+read+binary.MaxVarintLen64]) - if n <= 0 { - return fmt.Errorf("loadDvReaders: failed to read the docvalue offset start for field %d", fieldID) - } - read += uint64(n) - fieldLocEnd, n = binary.Uvarint(s.mem[s.docValueOffset+read : s.docValueOffset+read+binary.MaxVarintLen64]) - if n <= 0 { - return fmt.Errorf("loadDvReaders: failed to read the docvalue offset end for field %d", fieldID) - } - read += uint64(n) - - fieldDvReader, err := s.loadFieldDocValueReader(field, fieldLocStart, fieldLocEnd) - if err != nil { - return err - } - if fieldDvReader != nil { - s.fieldDvReaders[uint16(fieldID)] = fieldDvReader - s.fieldDvNames = append(s.fieldDvNames, field) - } - } - - return nil -} diff --git a/vendor/github.com/blevesearch/zap/v14/README.md b/vendor/github.com/blevesearch/zap/v14/README.md deleted file mode 100644 index 0facb669..00000000 --- a/vendor/github.com/blevesearch/zap/v14/README.md +++ /dev/null @@ -1,158 +0,0 @@ -# zap file format - -Advanced ZAP File Format Documentation is [here](zap.md). - -The file is written in the reverse order that we typically access data. This helps us write in one pass since later sections of the file require file offsets of things we've already written. - -Current usage: - -- mmap the entire file -- crc-32 bytes and version are in fixed position at end of the file -- reading remainder of footer could be version specific -- remainder of footer gives us: - - 3 important offsets (docValue , fields index and stored data index) - - 2 important values (number of docs and chunk factor) -- field data is processed once and memoized onto the heap so that we never have to go back to disk for it -- access to stored data by doc number means first navigating to the stored data index, then accessing a fixed position offset into that slice, which gives us the actual address of the data. the first bytes of that section tell us the size of data so that we know where it ends. -- access to all other indexed data follows the following pattern: - - first know the field name -> convert to id - - next navigate to term dictionary for that field - - some operations stop here and do dictionary ops - - next use dictionary to navigate to posting list for a specific term - - walk posting list - - if necessary, walk posting details as we go - - if location info is desired, consult location bitmap to see if it is there - -## stored fields section - -- for each document - - preparation phase: - - produce a slice of metadata bytes and data bytes - - produce these slices in field id order - - field value is appended to the data slice - - metadata slice is varint encoded with the following values for each field value - - field id (uint16) - - field type (byte) - - field value start offset in uncompressed data slice (uint64) - - field value length (uint64) - - field number of array positions (uint64) - - one additional value for each array position (uint64) - - compress the data slice using snappy - - file writing phase: - - remember the start offset for this document - - write out meta data length (varint uint64) - - write out compressed data length (varint uint64) - - write out the metadata bytes - - write out the compressed data bytes - -## stored fields idx - -- for each document - - write start offset (remembered from previous section) of stored data (big endian uint64) - -With this index and a known document number, we have direct access to all the stored field data. - -## posting details (freq/norm) section - -- for each posting list - - produce a slice containing multiple consecutive chunks (each chunk is varint stream) - - produce a slice remembering offsets of where each chunk starts - - preparation phase: - - for each hit in the posting list - - if this hit is in next chunk close out encoding of last chunk and record offset start of next - - encode term frequency (uint64) - - encode norm factor (float32) - - file writing phase: - - remember start position for this posting list details - - write out number of chunks that follow (varint uint64) - - write out length of each chunk (each a varint uint64) - - write out the byte slice containing all the chunk data - -If you know the doc number you're interested in, this format lets you jump to the correct chunk (docNum/chunkFactor) directly and then seek within that chunk until you find it. - -## posting details (location) section - -- for each posting list - - produce a slice containing multiple consecutive chunks (each chunk is varint stream) - - produce a slice remembering offsets of where each chunk starts - - preparation phase: - - for each hit in the posting list - - if this hit is in next chunk close out encoding of last chunk and record offset start of next - - encode field (uint16) - - encode field pos (uint64) - - encode field start (uint64) - - encode field end (uint64) - - encode number of array positions to follow (uint64) - - encode each array position (each uint64) - - file writing phase: - - remember start position for this posting list details - - write out number of chunks that follow (varint uint64) - - write out length of each chunk (each a varint uint64) - - write out the byte slice containing all the chunk data - -If you know the doc number you're interested in, this format lets you jump to the correct chunk (docNum/chunkFactor) directly and then seek within that chunk until you find it. - -## postings list section - -- for each posting list - - preparation phase: - - encode roaring bitmap posting list to bytes (so we know the length) - - file writing phase: - - remember the start position for this posting list - - write freq/norm details offset (remembered from previous, as varint uint64) - - write location details offset (remembered from previous, as varint uint64) - - write length of encoded roaring bitmap - - write the serialized roaring bitmap data - -## dictionary - -- for each field - - preparation phase: - - encode vellum FST with dictionary data pointing to file offset of posting list (remembered from previous) - - file writing phase: - - remember the start position of this persistDictionary - - write length of vellum data (varint uint64) - - write out vellum data - -## fields section - -- for each field - - file writing phase: - - remember start offset for each field - - write dictionary address (remembered from previous) (varint uint64) - - write length of field name (varint uint64) - - write field name bytes - -## fields idx - -- for each field - - file writing phase: - - write big endian uint64 of start offset for each field - -NOTE: currently we don't know or record the length of this fields index. Instead we rely on the fact that we know it immediately precedes a footer of known size. - -## fields DocValue - -- for each field - - preparation phase: - - produce a slice containing multiple consecutive chunks, where each chunk is composed of a meta section followed by compressed columnar field data - - produce a slice remembering the length of each chunk - - file writing phase: - - remember the start position of this first field DocValue offset in the footer - - write out number of chunks that follow (varint uint64) - - write out length of each chunk (each a varint uint64) - - write out the byte slice containing all the chunk data - -NOTE: currently the meta header inside each chunk gives clue to the location offsets and size of the data pertaining to a given docID and any -read operation leverage that meta information to extract the document specific data from the file. - -## footer - -- file writing phase - - write number of docs (big endian uint64) - - write stored field index location (big endian uint64) - - write field index location (big endian uint64) - - write field docValue location (big endian uint64) - - write out chunk factor (big endian uint32) - - write out version (big endian uint32) - - write out file CRC of everything preceding this (big endian uint32) diff --git a/vendor/github.com/blevesearch/zap/v14/build.go b/vendor/github.com/blevesearch/zap/v14/build.go deleted file mode 100644 index 7a8dce0d..00000000 --- a/vendor/github.com/blevesearch/zap/v14/build.go +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "bufio" - "math" - "os" - - "github.com/couchbase/vellum" -) - -const Version uint32 = 14 - -const Type string = "zap" - -const fieldNotUninverted = math.MaxUint64 - -func (sb *SegmentBase) Persist(path string) error { - return PersistSegmentBase(sb, path) -} - -// PersistSegmentBase persists SegmentBase in the zap file format. -func PersistSegmentBase(sb *SegmentBase, path string) error { - flag := os.O_RDWR | os.O_CREATE - - f, err := os.OpenFile(path, flag, 0600) - if err != nil { - return err - } - - cleanup := func() { - _ = f.Close() - _ = os.Remove(path) - } - - br := bufio.NewWriter(f) - - _, err = br.Write(sb.mem) - if err != nil { - cleanup() - return err - } - - err = persistFooter(sb.numDocs, sb.storedIndexOffset, sb.fieldsIndexOffset, sb.docValueOffset, - sb.chunkMode, sb.memCRC, br) - if err != nil { - cleanup() - return err - } - - err = br.Flush() - if err != nil { - cleanup() - return err - } - - err = f.Sync() - if err != nil { - cleanup() - return err - } - - err = f.Close() - if err != nil { - cleanup() - return err - } - - return nil -} - -func persistStoredFieldValues(fieldID int, - storedFieldValues [][]byte, stf []byte, spf [][]uint64, - curr int, metaEncode varintEncoder, data []byte) ( - int, []byte, error) { - for i := 0; i < len(storedFieldValues); i++ { - // encode field - _, err := metaEncode(uint64(fieldID)) - if err != nil { - return 0, nil, err - } - // encode type - _, err = metaEncode(uint64(stf[i])) - if err != nil { - return 0, nil, err - } - // encode start offset - _, err = metaEncode(uint64(curr)) - if err != nil { - return 0, nil, err - } - // end len - _, err = metaEncode(uint64(len(storedFieldValues[i]))) - if err != nil { - return 0, nil, err - } - // encode number of array pos - _, err = metaEncode(uint64(len(spf[i]))) - if err != nil { - return 0, nil, err - } - // encode all array positions - for _, pos := range spf[i] { - _, err = metaEncode(pos) - if err != nil { - return 0, nil, err - } - } - - data = append(data, storedFieldValues[i]...) - curr += len(storedFieldValues[i]) - } - - return curr, data, nil -} - -func InitSegmentBase(mem []byte, memCRC uint32, chunkMode uint32, - fieldsMap map[string]uint16, fieldsInv []string, numDocs uint64, - storedIndexOffset uint64, fieldsIndexOffset uint64, docValueOffset uint64, - dictLocs []uint64) (*SegmentBase, error) { - sb := &SegmentBase{ - mem: mem, - memCRC: memCRC, - chunkMode: chunkMode, - fieldsMap: fieldsMap, - fieldsInv: fieldsInv, - numDocs: numDocs, - storedIndexOffset: storedIndexOffset, - fieldsIndexOffset: fieldsIndexOffset, - docValueOffset: docValueOffset, - dictLocs: dictLocs, - fieldDvReaders: make(map[uint16]*docValueReader), - fieldFSTs: make(map[uint16]*vellum.FST), - } - sb.updateSize() - - err := sb.loadDvReaders() - if err != nil { - return nil, err - } - - return sb, nil -} diff --git a/vendor/github.com/blevesearch/zap/v14/count.go b/vendor/github.com/blevesearch/zap/v14/count.go deleted file mode 100644 index 50290f88..00000000 --- a/vendor/github.com/blevesearch/zap/v14/count.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "hash/crc32" - "io" - - "github.com/blevesearch/bleve/index/scorch/segment" -) - -// CountHashWriter is a wrapper around a Writer which counts the number of -// bytes which have been written and computes a crc32 hash -type CountHashWriter struct { - w io.Writer - crc uint32 - n int - s segment.StatsReporter -} - -// NewCountHashWriter returns a CountHashWriter which wraps the provided Writer -func NewCountHashWriter(w io.Writer) *CountHashWriter { - return &CountHashWriter{w: w} -} - -func NewCountHashWriterWithStatsReporter(w io.Writer, s segment.StatsReporter) *CountHashWriter { - return &CountHashWriter{w: w, s: s} -} - -// Write writes the provided bytes to the wrapped writer and counts the bytes -func (c *CountHashWriter) Write(b []byte) (int, error) { - n, err := c.w.Write(b) - c.crc = crc32.Update(c.crc, crc32.IEEETable, b[:n]) - c.n += n - if c.s != nil { - c.s.ReportBytesWritten(uint64(n)) - } - return n, err -} - -// Count returns the number of bytes written -func (c *CountHashWriter) Count() int { - return c.n -} - -// Sum32 returns the CRC-32 hash of the content written to this writer -func (c *CountHashWriter) Sum32() uint32 { - return c.crc -} diff --git a/vendor/github.com/blevesearch/zap/v14/dict.go b/vendor/github.com/blevesearch/zap/v14/dict.go deleted file mode 100644 index ad4a8f8d..00000000 --- a/vendor/github.com/blevesearch/zap/v14/dict.go +++ /dev/null @@ -1,263 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "bytes" - "fmt" - - "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/couchbase/vellum" -) - -// Dictionary is the zap representation of the term dictionary -type Dictionary struct { - sb *SegmentBase - field string - fieldID uint16 - fst *vellum.FST - fstReader *vellum.Reader -} - -// PostingsList returns the postings list for the specified term -func (d *Dictionary) PostingsList(term []byte, except *roaring.Bitmap, - prealloc segment.PostingsList) (segment.PostingsList, error) { - var preallocPL *PostingsList - pl, ok := prealloc.(*PostingsList) - if ok && pl != nil { - preallocPL = pl - } - return d.postingsList(term, except, preallocPL) -} - -func (d *Dictionary) postingsList(term []byte, except *roaring.Bitmap, rv *PostingsList) (*PostingsList, error) { - if d.fstReader == nil { - if rv == nil || rv == emptyPostingsList { - return emptyPostingsList, nil - } - return d.postingsListInit(rv, except), nil - } - - postingsOffset, exists, err := d.fstReader.Get(term) - if err != nil { - return nil, fmt.Errorf("vellum err: %v", err) - } - if !exists { - if rv == nil || rv == emptyPostingsList { - return emptyPostingsList, nil - } - return d.postingsListInit(rv, except), nil - } - - return d.postingsListFromOffset(postingsOffset, except, rv) -} - -func (d *Dictionary) postingsListFromOffset(postingsOffset uint64, except *roaring.Bitmap, rv *PostingsList) (*PostingsList, error) { - rv = d.postingsListInit(rv, except) - - err := rv.read(postingsOffset, d) - if err != nil { - return nil, err - } - - return rv, nil -} - -func (d *Dictionary) postingsListInit(rv *PostingsList, except *roaring.Bitmap) *PostingsList { - if rv == nil || rv == emptyPostingsList { - rv = &PostingsList{} - } else { - postings := rv.postings - if postings != nil { - postings.Clear() - } - - *rv = PostingsList{} // clear the struct - - rv.postings = postings - } - rv.sb = d.sb - rv.except = except - return rv -} - -func (d *Dictionary) Contains(key []byte) (bool, error) { - return d.fst.Contains(key) -} - -// Iterator returns an iterator for this dictionary -func (d *Dictionary) Iterator() segment.DictionaryIterator { - rv := &DictionaryIterator{ - d: d, - } - - if d.fst != nil { - itr, err := d.fst.Iterator(nil, nil) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - } - - return rv -} - -// PrefixIterator returns an iterator which only visits terms having the -// the specified prefix -func (d *Dictionary) PrefixIterator(prefix string) segment.DictionaryIterator { - rv := &DictionaryIterator{ - d: d, - } - - kBeg := []byte(prefix) - kEnd := segment.IncrementBytes(kBeg) - - if d.fst != nil { - itr, err := d.fst.Iterator(kBeg, kEnd) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - } - - return rv -} - -// RangeIterator returns an iterator which only visits terms between the -// start and end terms. NOTE: bleve.index API specifies the end is inclusive. -func (d *Dictionary) RangeIterator(start, end string) segment.DictionaryIterator { - rv := &DictionaryIterator{ - d: d, - } - - // need to increment the end position to be inclusive - var endBytes []byte - if len(end) > 0 { - endBytes = []byte(end) - if endBytes[len(endBytes)-1] < 0xff { - endBytes[len(endBytes)-1]++ - } else { - endBytes = append(endBytes, 0xff) - } - } - - if d.fst != nil { - itr, err := d.fst.Iterator([]byte(start), endBytes) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - } - - return rv -} - -// AutomatonIterator returns an iterator which only visits terms -// having the the vellum automaton and start/end key range -func (d *Dictionary) AutomatonIterator(a vellum.Automaton, - startKeyInclusive, endKeyExclusive []byte) segment.DictionaryIterator { - rv := &DictionaryIterator{ - d: d, - } - - if d.fst != nil { - itr, err := d.fst.Search(a, startKeyInclusive, endKeyExclusive) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - } - - return rv -} - -func (d *Dictionary) OnlyIterator(onlyTerms [][]byte, - includeCount bool) segment.DictionaryIterator { - - rv := &DictionaryIterator{ - d: d, - omitCount: !includeCount, - } - - var buf bytes.Buffer - builder, err := vellum.New(&buf, nil) - if err != nil { - rv.err = err - return rv - } - for _, term := range onlyTerms { - err = builder.Insert(term, 0) - if err != nil { - rv.err = err - return rv - } - } - err = builder.Close() - if err != nil { - rv.err = err - return rv - } - - onlyFST, err := vellum.Load(buf.Bytes()) - if err != nil { - rv.err = err - return rv - } - - itr, err := d.fst.Search(onlyFST, nil, nil) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - - return rv -} - -// DictionaryIterator is an iterator for term dictionary -type DictionaryIterator struct { - d *Dictionary - itr vellum.Iterator - err error - tmp PostingsList - entry index.DictEntry - omitCount bool -} - -// Next returns the next entry in the dictionary -func (i *DictionaryIterator) Next() (*index.DictEntry, error) { - if i.err != nil && i.err != vellum.ErrIteratorDone { - return nil, i.err - } else if i.itr == nil || i.err == vellum.ErrIteratorDone { - return nil, nil - } - term, postingsOffset := i.itr.Current() - i.entry.Term = string(term) - if !i.omitCount { - i.err = i.tmp.read(postingsOffset, i.d) - if i.err != nil { - return nil, i.err - } - i.entry.Count = i.tmp.Count() - } - i.err = i.itr.Next() - return &i.entry, nil -} diff --git a/vendor/github.com/blevesearch/zap/v14/docvalues.go b/vendor/github.com/blevesearch/zap/v14/docvalues.go deleted file mode 100644 index 793797bd..00000000 --- a/vendor/github.com/blevesearch/zap/v14/docvalues.go +++ /dev/null @@ -1,312 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "bytes" - "encoding/binary" - "fmt" - "math" - "reflect" - "sort" - - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/blevesearch/bleve/size" - "github.com/golang/snappy" -) - -var reflectStaticSizedocValueReader int - -func init() { - var dvi docValueReader - reflectStaticSizedocValueReader = int(reflect.TypeOf(dvi).Size()) -} - -type docNumTermsVisitor func(docNum uint64, terms []byte) error - -type docVisitState struct { - dvrs map[uint16]*docValueReader - segment *SegmentBase -} - -type docValueReader struct { - field string - curChunkNum uint64 - chunkOffsets []uint64 - dvDataLoc uint64 - curChunkHeader []MetaData - curChunkData []byte // compressed data cache - uncompressed []byte // temp buf for snappy decompression -} - -func (di *docValueReader) size() int { - return reflectStaticSizedocValueReader + size.SizeOfPtr + - len(di.field) + - len(di.chunkOffsets)*size.SizeOfUint64 + - len(di.curChunkHeader)*reflectStaticSizeMetaData + - len(di.curChunkData) -} - -func (di *docValueReader) cloneInto(rv *docValueReader) *docValueReader { - if rv == nil { - rv = &docValueReader{} - } - - rv.field = di.field - rv.curChunkNum = math.MaxUint64 - rv.chunkOffsets = di.chunkOffsets // immutable, so it's sharable - rv.dvDataLoc = di.dvDataLoc - rv.curChunkHeader = rv.curChunkHeader[:0] - rv.curChunkData = nil - rv.uncompressed = rv.uncompressed[:0] - - return rv -} - -func (di *docValueReader) curChunkNumber() uint64 { - return di.curChunkNum -} - -func (s *SegmentBase) loadFieldDocValueReader(field string, - fieldDvLocStart, fieldDvLocEnd uint64) (*docValueReader, error) { - // get the docValue offset for the given fields - if fieldDvLocStart == fieldNotUninverted { - // no docValues found, nothing to do - return nil, nil - } - - // read the number of chunks, and chunk offsets position - var numChunks, chunkOffsetsPosition uint64 - - if fieldDvLocEnd-fieldDvLocStart > 16 { - numChunks = binary.BigEndian.Uint64(s.mem[fieldDvLocEnd-8 : fieldDvLocEnd]) - // read the length of chunk offsets - chunkOffsetsLen := binary.BigEndian.Uint64(s.mem[fieldDvLocEnd-16 : fieldDvLocEnd-8]) - // acquire position of chunk offsets - chunkOffsetsPosition = (fieldDvLocEnd - 16) - chunkOffsetsLen - } else { - return nil, fmt.Errorf("loadFieldDocValueReader: fieldDvLoc too small: %d-%d", fieldDvLocEnd, fieldDvLocStart) - } - - fdvIter := &docValueReader{ - curChunkNum: math.MaxUint64, - field: field, - chunkOffsets: make([]uint64, int(numChunks)), - } - - // read the chunk offsets - var offset uint64 - for i := 0; i < int(numChunks); i++ { - loc, read := binary.Uvarint(s.mem[chunkOffsetsPosition+offset : chunkOffsetsPosition+offset+binary.MaxVarintLen64]) - if read <= 0 { - return nil, fmt.Errorf("corrupted chunk offset during segment load") - } - fdvIter.chunkOffsets[i] = loc - offset += uint64(read) - } - - // set the data offset - fdvIter.dvDataLoc = fieldDvLocStart - - return fdvIter, nil -} - -func (di *docValueReader) loadDvChunk(chunkNumber uint64, s *SegmentBase) error { - // advance to the chunk where the docValues - // reside for the given docNum - destChunkDataLoc, curChunkEnd := di.dvDataLoc, di.dvDataLoc - start, end := readChunkBoundary(int(chunkNumber), di.chunkOffsets) - if start >= end { - di.curChunkHeader = di.curChunkHeader[:0] - di.curChunkData = nil - di.curChunkNum = chunkNumber - di.uncompressed = di.uncompressed[:0] - return nil - } - - destChunkDataLoc += start - curChunkEnd += end - - // read the number of docs reside in the chunk - numDocs, read := binary.Uvarint(s.mem[destChunkDataLoc : destChunkDataLoc+binary.MaxVarintLen64]) - if read <= 0 { - return fmt.Errorf("failed to read the chunk") - } - chunkMetaLoc := destChunkDataLoc + uint64(read) - - offset := uint64(0) - if cap(di.curChunkHeader) < int(numDocs) { - di.curChunkHeader = make([]MetaData, int(numDocs)) - } else { - di.curChunkHeader = di.curChunkHeader[:int(numDocs)] - } - for i := 0; i < int(numDocs); i++ { - di.curChunkHeader[i].DocNum, read = binary.Uvarint(s.mem[chunkMetaLoc+offset : chunkMetaLoc+offset+binary.MaxVarintLen64]) - offset += uint64(read) - di.curChunkHeader[i].DocDvOffset, read = binary.Uvarint(s.mem[chunkMetaLoc+offset : chunkMetaLoc+offset+binary.MaxVarintLen64]) - offset += uint64(read) - } - - compressedDataLoc := chunkMetaLoc + offset - dataLength := curChunkEnd - compressedDataLoc - di.curChunkData = s.mem[compressedDataLoc : compressedDataLoc+dataLength] - di.curChunkNum = chunkNumber - di.uncompressed = di.uncompressed[:0] - return nil -} - -func (di *docValueReader) iterateAllDocValues(s *SegmentBase, visitor docNumTermsVisitor) error { - for i := 0; i < len(di.chunkOffsets); i++ { - err := di.loadDvChunk(uint64(i), s) - if err != nil { - return err - } - if di.curChunkData == nil || len(di.curChunkHeader) == 0 { - continue - } - - // uncompress the already loaded data - uncompressed, err := snappy.Decode(di.uncompressed[:cap(di.uncompressed)], di.curChunkData) - if err != nil { - return err - } - di.uncompressed = uncompressed - - start := uint64(0) - for _, entry := range di.curChunkHeader { - err = visitor(entry.DocNum, uncompressed[start:entry.DocDvOffset]) - if err != nil { - return err - } - - start = entry.DocDvOffset - } - } - - return nil -} - -func (di *docValueReader) visitDocValues(docNum uint64, - visitor index.DocumentFieldTermVisitor) error { - // binary search the term locations for the docNum - start, end := di.getDocValueLocs(docNum) - if start == math.MaxUint64 || end == math.MaxUint64 || start == end { - return nil - } - - var uncompressed []byte - var err error - // use the uncompressed copy if available - if len(di.uncompressed) > 0 { - uncompressed = di.uncompressed - } else { - // uncompress the already loaded data - uncompressed, err = snappy.Decode(di.uncompressed[:cap(di.uncompressed)], di.curChunkData) - if err != nil { - return err - } - di.uncompressed = uncompressed - } - - // pick the terms for the given docNum - uncompressed = uncompressed[start:end] - for { - i := bytes.Index(uncompressed, termSeparatorSplitSlice) - if i < 0 { - break - } - - visitor(di.field, uncompressed[0:i]) - uncompressed = uncompressed[i+1:] - } - - return nil -} - -func (di *docValueReader) getDocValueLocs(docNum uint64) (uint64, uint64) { - i := sort.Search(len(di.curChunkHeader), func(i int) bool { - return di.curChunkHeader[i].DocNum >= docNum - }) - if i < len(di.curChunkHeader) && di.curChunkHeader[i].DocNum == docNum { - return ReadDocValueBoundary(i, di.curChunkHeader) - } - return math.MaxUint64, math.MaxUint64 -} - -// VisitDocumentFieldTerms is an implementation of the -// DocumentFieldTermVisitable interface -func (s *SegmentBase) VisitDocumentFieldTerms(localDocNum uint64, fields []string, - visitor index.DocumentFieldTermVisitor, dvsIn segment.DocVisitState) ( - segment.DocVisitState, error) { - dvs, ok := dvsIn.(*docVisitState) - if !ok || dvs == nil { - dvs = &docVisitState{} - } else { - if dvs.segment != s { - dvs.segment = s - dvs.dvrs = nil - } - } - - var fieldIDPlus1 uint16 - if dvs.dvrs == nil { - dvs.dvrs = make(map[uint16]*docValueReader, len(fields)) - for _, field := range fields { - if fieldIDPlus1, ok = s.fieldsMap[field]; !ok { - continue - } - fieldID := fieldIDPlus1 - 1 - if dvIter, exists := s.fieldDvReaders[fieldID]; exists && - dvIter != nil { - dvs.dvrs[fieldID] = dvIter.cloneInto(dvs.dvrs[fieldID]) - } - } - } - - // find the chunkNumber where the docValues are stored - // NOTE: doc values continue to use legacy chunk mode - chunkFactor, err := getChunkSize(LegacyChunkMode, 0, 0) - if err != nil { - return nil, err - } - docInChunk := localDocNum / chunkFactor - var dvr *docValueReader - for _, field := range fields { - if fieldIDPlus1, ok = s.fieldsMap[field]; !ok { - continue - } - fieldID := fieldIDPlus1 - 1 - if dvr, ok = dvs.dvrs[fieldID]; ok && dvr != nil { - // check if the chunk is already loaded - if docInChunk != dvr.curChunkNumber() { - err := dvr.loadDvChunk(docInChunk, s) - if err != nil { - return dvs, err - } - } - - _ = dvr.visitDocValues(localDocNum, visitor) - } - } - return dvs, nil -} - -// VisitableDocValueFields returns the list of fields with -// persisted doc value terms ready to be visitable using the -// VisitDocumentFieldTerms method. -func (s *SegmentBase) VisitableDocValueFields() ([]string, error) { - return s.fieldDvNames, nil -} diff --git a/vendor/github.com/blevesearch/zap/v14/enumerator.go b/vendor/github.com/blevesearch/zap/v14/enumerator.go deleted file mode 100644 index bc5b7e62..00000000 --- a/vendor/github.com/blevesearch/zap/v14/enumerator.go +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright (c) 2018 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "bytes" - - "github.com/couchbase/vellum" -) - -// enumerator provides an ordered traversal of multiple vellum -// iterators. Like JOIN of iterators, the enumerator produces a -// sequence of (key, iteratorIndex, value) tuples, sorted by key ASC, -// then iteratorIndex ASC, where the same key might be seen or -// repeated across multiple child iterators. -type enumerator struct { - itrs []vellum.Iterator - currKs [][]byte - currVs []uint64 - - lowK []byte - lowIdxs []int - lowCurr int -} - -// newEnumerator returns a new enumerator over the vellum Iterators -func newEnumerator(itrs []vellum.Iterator) (*enumerator, error) { - rv := &enumerator{ - itrs: itrs, - currKs: make([][]byte, len(itrs)), - currVs: make([]uint64, len(itrs)), - lowIdxs: make([]int, 0, len(itrs)), - } - for i, itr := range rv.itrs { - rv.currKs[i], rv.currVs[i] = itr.Current() - } - rv.updateMatches(false) - if rv.lowK == nil && len(rv.lowIdxs) == 0 { - return rv, vellum.ErrIteratorDone - } - return rv, nil -} - -// updateMatches maintains the low key matches based on the currKs -func (m *enumerator) updateMatches(skipEmptyKey bool) { - m.lowK = nil - m.lowIdxs = m.lowIdxs[:0] - m.lowCurr = 0 - - for i, key := range m.currKs { - if (key == nil && m.currVs[i] == 0) || // in case of empty iterator - (len(key) == 0 && skipEmptyKey) { // skip empty keys - continue - } - - cmp := bytes.Compare(key, m.lowK) - if cmp < 0 || len(m.lowIdxs) == 0 { - // reached a new low - m.lowK = key - m.lowIdxs = m.lowIdxs[:0] - m.lowIdxs = append(m.lowIdxs, i) - } else if cmp == 0 { - m.lowIdxs = append(m.lowIdxs, i) - } - } -} - -// Current returns the enumerator's current key, iterator-index, and -// value. If the enumerator is not pointing at a valid value (because -// Next returned an error previously), Current will return nil,0,0. -func (m *enumerator) Current() ([]byte, int, uint64) { - var i int - var v uint64 - if m.lowCurr < len(m.lowIdxs) { - i = m.lowIdxs[m.lowCurr] - v = m.currVs[i] - } - return m.lowK, i, v -} - -// GetLowIdxsAndValues will return all of the iterator indices -// which point to the current key, and their corresponding -// values. This can be used by advanced caller which may need -// to peek into these other sets of data before processing. -func (m *enumerator) GetLowIdxsAndValues() ([]int, []uint64) { - values := make([]uint64, 0, len(m.lowIdxs)) - for _, idx := range m.lowIdxs { - values = append(values, m.currVs[idx]) - } - return m.lowIdxs, values -} - -// Next advances the enumerator to the next key/iterator/value result, -// else vellum.ErrIteratorDone is returned. -func (m *enumerator) Next() error { - m.lowCurr += 1 - if m.lowCurr >= len(m.lowIdxs) { - // move all the current low iterators forwards - for _, vi := range m.lowIdxs { - err := m.itrs[vi].Next() - if err != nil && err != vellum.ErrIteratorDone { - return err - } - m.currKs[vi], m.currVs[vi] = m.itrs[vi].Current() - } - // can skip any empty keys encountered at this point - m.updateMatches(true) - } - if m.lowK == nil && len(m.lowIdxs) == 0 { - return vellum.ErrIteratorDone - } - return nil -} - -// Close all the underlying Iterators. The first error, if any, will -// be returned. -func (m *enumerator) Close() error { - var rv error - for _, itr := range m.itrs { - err := itr.Close() - if rv == nil { - rv = err - } - } - return rv -} diff --git a/vendor/github.com/blevesearch/zap/v14/intDecoder.go b/vendor/github.com/blevesearch/zap/v14/intDecoder.go deleted file mode 100644 index 3baa0c2a..00000000 --- a/vendor/github.com/blevesearch/zap/v14/intDecoder.go +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright (c) 2019 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "encoding/binary" - "fmt" - - "github.com/blevesearch/bleve/index/scorch/segment" -) - -type chunkedIntDecoder struct { - startOffset uint64 - dataStartOffset uint64 - chunkOffsets []uint64 - curChunkBytes []byte - data []byte - r *segment.MemUvarintReader -} - -// newChunkedIntDecoder expects an optional or reset chunkedIntDecoder for better reuse. -func newChunkedIntDecoder(buf []byte, offset uint64, rv *chunkedIntDecoder) *chunkedIntDecoder { - if rv == nil { - rv = &chunkedIntDecoder{startOffset: offset, data: buf} - } else { - rv.startOffset = offset - rv.data = buf - } - - var n, numChunks uint64 - var read int - if offset == termNotEncoded { - numChunks = 0 - } else { - numChunks, read = binary.Uvarint(buf[offset+n : offset+n+binary.MaxVarintLen64]) - } - - n += uint64(read) - if cap(rv.chunkOffsets) >= int(numChunks) { - rv.chunkOffsets = rv.chunkOffsets[:int(numChunks)] - } else { - rv.chunkOffsets = make([]uint64, int(numChunks)) - } - for i := 0; i < int(numChunks); i++ { - rv.chunkOffsets[i], read = binary.Uvarint(buf[offset+n : offset+n+binary.MaxVarintLen64]) - n += uint64(read) - } - rv.dataStartOffset = offset + n - return rv -} - -func (d *chunkedIntDecoder) loadChunk(chunk int) error { - if d.startOffset == termNotEncoded { - d.r = segment.NewMemUvarintReader([]byte(nil)) - return nil - } - - if chunk >= len(d.chunkOffsets) { - return fmt.Errorf("tried to load freq chunk that doesn't exist %d/(%d)", - chunk, len(d.chunkOffsets)) - } - - end, start := d.dataStartOffset, d.dataStartOffset - s, e := readChunkBoundary(chunk, d.chunkOffsets) - start += s - end += e - d.curChunkBytes = d.data[start:end] - if d.r == nil { - d.r = segment.NewMemUvarintReader(d.curChunkBytes) - } else { - d.r.Reset(d.curChunkBytes) - } - - return nil -} - -func (d *chunkedIntDecoder) reset() { - d.startOffset = 0 - d.dataStartOffset = 0 - d.chunkOffsets = d.chunkOffsets[:0] - d.curChunkBytes = d.curChunkBytes[:0] - d.data = d.data[:0] - if d.r != nil { - d.r.Reset([]byte(nil)) - } -} - -func (d *chunkedIntDecoder) isNil() bool { - return d.curChunkBytes == nil || len(d.curChunkBytes) == 0 -} - -func (d *chunkedIntDecoder) readUvarint() (uint64, error) { - return d.r.ReadUvarint() -} - -func (d *chunkedIntDecoder) SkipUvarint() { - d.r.SkipUvarint() -} - -func (d *chunkedIntDecoder) SkipBytes(count int) { - d.r.SkipBytes(count) -} - -func (d *chunkedIntDecoder) Len() int { - return d.r.Len() -} diff --git a/vendor/github.com/blevesearch/zap/v14/merge.go b/vendor/github.com/blevesearch/zap/v14/merge.go deleted file mode 100644 index 805100fb..00000000 --- a/vendor/github.com/blevesearch/zap/v14/merge.go +++ /dev/null @@ -1,847 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "bufio" - "bytes" - "encoding/binary" - "fmt" - "math" - "os" - "sort" - - "github.com/RoaringBitmap/roaring" - seg "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/couchbase/vellum" - "github.com/golang/snappy" -) - -var DefaultFileMergerBufferSize = 1024 * 1024 - -const docDropped = math.MaxUint64 // sentinel docNum to represent a deleted doc - -// Merge takes a slice of segments and bit masks describing which -// documents may be dropped, and creates a new segment containing the -// remaining data. This new segment is built at the specified path. -func (*ZapPlugin) Merge(segments []seg.Segment, drops []*roaring.Bitmap, path string, - closeCh chan struct{}, s seg.StatsReporter) ( - [][]uint64, uint64, error) { - - segmentBases := make([]*SegmentBase, len(segments)) - for segmenti, segment := range segments { - switch segmentx := segment.(type) { - case *Segment: - segmentBases[segmenti] = &segmentx.SegmentBase - case *SegmentBase: - segmentBases[segmenti] = segmentx - default: - panic(fmt.Sprintf("oops, unexpected segment type: %T", segment)) - } - } - return mergeSegmentBases(segmentBases, drops, path, DefaultChunkMode, closeCh, s) -} - -func mergeSegmentBases(segmentBases []*SegmentBase, drops []*roaring.Bitmap, path string, - chunkMode uint32, closeCh chan struct{}, s seg.StatsReporter) ( - [][]uint64, uint64, error) { - flag := os.O_RDWR | os.O_CREATE - - f, err := os.OpenFile(path, flag, 0600) - if err != nil { - return nil, 0, err - } - - cleanup := func() { - _ = f.Close() - _ = os.Remove(path) - } - - // buffer the output - br := bufio.NewWriterSize(f, DefaultFileMergerBufferSize) - - // wrap it for counting (tracking offsets) - cr := NewCountHashWriterWithStatsReporter(br, s) - - newDocNums, numDocs, storedIndexOffset, fieldsIndexOffset, docValueOffset, _, _, _, err := - MergeToWriter(segmentBases, drops, chunkMode, cr, closeCh) - if err != nil { - cleanup() - return nil, 0, err - } - - err = persistFooter(numDocs, storedIndexOffset, fieldsIndexOffset, - docValueOffset, chunkMode, cr.Sum32(), cr) - if err != nil { - cleanup() - return nil, 0, err - } - - err = br.Flush() - if err != nil { - cleanup() - return nil, 0, err - } - - err = f.Sync() - if err != nil { - cleanup() - return nil, 0, err - } - - err = f.Close() - if err != nil { - cleanup() - return nil, 0, err - } - - return newDocNums, uint64(cr.Count()), nil -} - -func MergeToWriter(segments []*SegmentBase, drops []*roaring.Bitmap, - chunkMode uint32, cr *CountHashWriter, closeCh chan struct{}) ( - newDocNums [][]uint64, - numDocs, storedIndexOffset, fieldsIndexOffset, docValueOffset uint64, - dictLocs []uint64, fieldsInv []string, fieldsMap map[string]uint16, - err error) { - docValueOffset = uint64(fieldNotUninverted) - - var fieldsSame bool - fieldsSame, fieldsInv = mergeFields(segments) - fieldsMap = mapFields(fieldsInv) - - numDocs = computeNewDocCount(segments, drops) - - if isClosed(closeCh) { - return nil, 0, 0, 0, 0, nil, nil, nil, seg.ErrClosed - } - - if numDocs > 0 { - storedIndexOffset, newDocNums, err = mergeStoredAndRemap(segments, drops, - fieldsMap, fieldsInv, fieldsSame, numDocs, cr, closeCh) - if err != nil { - return nil, 0, 0, 0, 0, nil, nil, nil, err - } - - dictLocs, docValueOffset, err = persistMergedRest(segments, drops, - fieldsInv, fieldsMap, fieldsSame, - newDocNums, numDocs, chunkMode, cr, closeCh) - if err != nil { - return nil, 0, 0, 0, 0, nil, nil, nil, err - } - } else { - dictLocs = make([]uint64, len(fieldsInv)) - } - - fieldsIndexOffset, err = persistFields(fieldsInv, cr, dictLocs) - if err != nil { - return nil, 0, 0, 0, 0, nil, nil, nil, err - } - - return newDocNums, numDocs, storedIndexOffset, fieldsIndexOffset, docValueOffset, dictLocs, fieldsInv, fieldsMap, nil -} - -// mapFields takes the fieldsInv list and returns a map of fieldName -// to fieldID+1 -func mapFields(fields []string) map[string]uint16 { - rv := make(map[string]uint16, len(fields)) - for i, fieldName := range fields { - rv[fieldName] = uint16(i) + 1 - } - return rv -} - -// computeNewDocCount determines how many documents will be in the newly -// merged segment when obsoleted docs are dropped -func computeNewDocCount(segments []*SegmentBase, drops []*roaring.Bitmap) uint64 { - var newDocCount uint64 - for segI, segment := range segments { - newDocCount += segment.numDocs - if drops[segI] != nil { - newDocCount -= drops[segI].GetCardinality() - } - } - return newDocCount -} - -func persistMergedRest(segments []*SegmentBase, dropsIn []*roaring.Bitmap, - fieldsInv []string, fieldsMap map[string]uint16, fieldsSame bool, - newDocNumsIn [][]uint64, newSegDocCount uint64, chunkMode uint32, - w *CountHashWriter, closeCh chan struct{}) ([]uint64, uint64, error) { - - var bufMaxVarintLen64 []byte = make([]byte, binary.MaxVarintLen64) - var bufLoc []uint64 - - var postings *PostingsList - var postItr *PostingsIterator - - rv := make([]uint64, len(fieldsInv)) - fieldDvLocsStart := make([]uint64, len(fieldsInv)) - fieldDvLocsEnd := make([]uint64, len(fieldsInv)) - - // these int coders are initialized with chunk size 1024 - // however this will be reset to the correct chunk size - // while processing each individual field-term section - tfEncoder := newChunkedIntCoder(1024, newSegDocCount-1) - locEncoder := newChunkedIntCoder(1024, newSegDocCount-1) - - var vellumBuf bytes.Buffer - newVellum, err := vellum.New(&vellumBuf, nil) - if err != nil { - return nil, 0, err - } - - newRoaring := roaring.NewBitmap() - - // for each field - for fieldID, fieldName := range fieldsInv { - - // collect FST iterators from all active segments for this field - var newDocNums [][]uint64 - var drops []*roaring.Bitmap - var dicts []*Dictionary - var itrs []vellum.Iterator - - var segmentsInFocus []*SegmentBase - - for segmentI, segment := range segments { - - // check for the closure in meantime - if isClosed(closeCh) { - return nil, 0, seg.ErrClosed - } - - dict, err2 := segment.dictionary(fieldName) - if err2 != nil { - return nil, 0, err2 - } - if dict != nil && dict.fst != nil { - itr, err2 := dict.fst.Iterator(nil, nil) - if err2 != nil && err2 != vellum.ErrIteratorDone { - return nil, 0, err2 - } - if itr != nil { - newDocNums = append(newDocNums, newDocNumsIn[segmentI]) - if dropsIn[segmentI] != nil && !dropsIn[segmentI].IsEmpty() { - drops = append(drops, dropsIn[segmentI]) - } else { - drops = append(drops, nil) - } - dicts = append(dicts, dict) - itrs = append(itrs, itr) - segmentsInFocus = append(segmentsInFocus, segment) - } - } - } - - var prevTerm []byte - - newRoaring.Clear() - - var lastDocNum, lastFreq, lastNorm uint64 - - // determines whether to use "1-hit" encoding optimization - // when a term appears in only 1 doc, with no loc info, - // has freq of 1, and the docNum fits into 31-bits - use1HitEncoding := func(termCardinality uint64) (bool, uint64, uint64) { - if termCardinality == uint64(1) && locEncoder.FinalSize() <= 0 { - docNum := uint64(newRoaring.Minimum()) - if under32Bits(docNum) && docNum == lastDocNum && lastFreq == 1 { - return true, docNum, lastNorm - } - } - return false, 0, 0 - } - - finishTerm := func(term []byte) error { - tfEncoder.Close() - locEncoder.Close() - - postingsOffset, err := writePostings(newRoaring, - tfEncoder, locEncoder, use1HitEncoding, w, bufMaxVarintLen64) - if err != nil { - return err - } - - if postingsOffset > 0 { - err = newVellum.Insert(term, postingsOffset) - if err != nil { - return err - } - } - - newRoaring.Clear() - - tfEncoder.Reset() - locEncoder.Reset() - - lastDocNum = 0 - lastFreq = 0 - lastNorm = 0 - - return nil - } - - enumerator, err := newEnumerator(itrs) - - for err == nil { - term, itrI, postingsOffset := enumerator.Current() - - if !bytes.Equal(prevTerm, term) { - // check for the closure in meantime - if isClosed(closeCh) { - return nil, 0, seg.ErrClosed - } - - // if the term changed, write out the info collected - // for the previous term - err = finishTerm(prevTerm) - if err != nil { - return nil, 0, err - } - } - if !bytes.Equal(prevTerm, term) || prevTerm == nil { - // compute cardinality of field-term in new seg - var newCard uint64 - lowItrIdxs, lowItrVals := enumerator.GetLowIdxsAndValues() - for i, idx := range lowItrIdxs { - pl, err := dicts[idx].postingsListFromOffset(lowItrVals[i], drops[idx], nil) - if err != nil { - return nil, 0, err - } - newCard += pl.Count() - } - // compute correct chunk size with this - chunkSize, err := getChunkSize(chunkMode, newCard, newSegDocCount) - if err != nil { - return nil, 0, err - } - // update encoders chunk - tfEncoder.SetChunkSize(chunkSize, newSegDocCount-1) - locEncoder.SetChunkSize(chunkSize, newSegDocCount-1) - } - - postings, err = dicts[itrI].postingsListFromOffset( - postingsOffset, drops[itrI], postings) - if err != nil { - return nil, 0, err - } - - postItr = postings.iterator(true, true, true, postItr) - - // can no longer optimize by copying, since chunk factor could have changed - lastDocNum, lastFreq, lastNorm, bufLoc, err = mergeTermFreqNormLocs( - fieldsMap, term, postItr, newDocNums[itrI], newRoaring, - tfEncoder, locEncoder, bufLoc) - - if err != nil { - return nil, 0, err - } - - prevTerm = prevTerm[:0] // copy to prevTerm in case Next() reuses term mem - prevTerm = append(prevTerm, term...) - - err = enumerator.Next() - } - if err != vellum.ErrIteratorDone { - return nil, 0, err - } - - err = finishTerm(prevTerm) - if err != nil { - return nil, 0, err - } - - dictOffset := uint64(w.Count()) - - err = newVellum.Close() - if err != nil { - return nil, 0, err - } - vellumData := vellumBuf.Bytes() - - // write out the length of the vellum data - n := binary.PutUvarint(bufMaxVarintLen64, uint64(len(vellumData))) - _, err = w.Write(bufMaxVarintLen64[:n]) - if err != nil { - return nil, 0, err - } - - // write this vellum to disk - _, err = w.Write(vellumData) - if err != nil { - return nil, 0, err - } - - rv[fieldID] = dictOffset - - // get the field doc value offset (start) - fieldDvLocsStart[fieldID] = uint64(w.Count()) - - // update the field doc values - // NOTE: doc values continue to use legacy chunk mode - chunkSize, err := getChunkSize(LegacyChunkMode, 0, 0) - if err != nil { - return nil, 0, err - } - fdvEncoder := newChunkedContentCoder(chunkSize, newSegDocCount-1, w, true) - - fdvReadersAvailable := false - var dvIterClone *docValueReader - for segmentI, segment := range segmentsInFocus { - // check for the closure in meantime - if isClosed(closeCh) { - return nil, 0, seg.ErrClosed - } - - fieldIDPlus1 := uint16(segment.fieldsMap[fieldName]) - if dvIter, exists := segment.fieldDvReaders[fieldIDPlus1-1]; exists && - dvIter != nil { - fdvReadersAvailable = true - dvIterClone = dvIter.cloneInto(dvIterClone) - err = dvIterClone.iterateAllDocValues(segment, func(docNum uint64, terms []byte) error { - if newDocNums[segmentI][docNum] == docDropped { - return nil - } - err := fdvEncoder.Add(newDocNums[segmentI][docNum], terms) - if err != nil { - return err - } - return nil - }) - if err != nil { - return nil, 0, err - } - } - } - - if fdvReadersAvailable { - err = fdvEncoder.Close() - if err != nil { - return nil, 0, err - } - - // persist the doc value details for this field - _, err = fdvEncoder.Write() - if err != nil { - return nil, 0, err - } - - // get the field doc value offset (end) - fieldDvLocsEnd[fieldID] = uint64(w.Count()) - } else { - fieldDvLocsStart[fieldID] = fieldNotUninverted - fieldDvLocsEnd[fieldID] = fieldNotUninverted - } - - // reset vellum buffer and vellum builder - vellumBuf.Reset() - err = newVellum.Reset(&vellumBuf) - if err != nil { - return nil, 0, err - } - } - - fieldDvLocsOffset := uint64(w.Count()) - - buf := bufMaxVarintLen64 - for i := 0; i < len(fieldDvLocsStart); i++ { - n := binary.PutUvarint(buf, fieldDvLocsStart[i]) - _, err := w.Write(buf[:n]) - if err != nil { - return nil, 0, err - } - n = binary.PutUvarint(buf, fieldDvLocsEnd[i]) - _, err = w.Write(buf[:n]) - if err != nil { - return nil, 0, err - } - } - - return rv, fieldDvLocsOffset, nil -} - -func mergeTermFreqNormLocs(fieldsMap map[string]uint16, term []byte, postItr *PostingsIterator, - newDocNums []uint64, newRoaring *roaring.Bitmap, - tfEncoder *chunkedIntCoder, locEncoder *chunkedIntCoder, bufLoc []uint64) ( - lastDocNum uint64, lastFreq uint64, lastNorm uint64, bufLocOut []uint64, err error) { - next, err := postItr.Next() - for next != nil && err == nil { - hitNewDocNum := newDocNums[next.Number()] - if hitNewDocNum == docDropped { - return 0, 0, 0, nil, fmt.Errorf("see hit with dropped docNum") - } - - newRoaring.Add(uint32(hitNewDocNum)) - - nextFreq := next.Frequency() - nextNorm := uint64(math.Float32bits(float32(next.Norm()))) - - locs := next.Locations() - - err = tfEncoder.Add(hitNewDocNum, - encodeFreqHasLocs(nextFreq, len(locs) > 0), nextNorm) - if err != nil { - return 0, 0, 0, nil, err - } - - if len(locs) > 0 { - numBytesLocs := 0 - for _, loc := range locs { - ap := loc.ArrayPositions() - numBytesLocs += totalUvarintBytes(uint64(fieldsMap[loc.Field()]-1), - loc.Pos(), loc.Start(), loc.End(), uint64(len(ap)), ap) - } - - err = locEncoder.Add(hitNewDocNum, uint64(numBytesLocs)) - if err != nil { - return 0, 0, 0, nil, err - } - - for _, loc := range locs { - ap := loc.ArrayPositions() - if cap(bufLoc) < 5+len(ap) { - bufLoc = make([]uint64, 0, 5+len(ap)) - } - args := bufLoc[0:5] - args[0] = uint64(fieldsMap[loc.Field()] - 1) - args[1] = loc.Pos() - args[2] = loc.Start() - args[3] = loc.End() - args[4] = uint64(len(ap)) - args = append(args, ap...) - err = locEncoder.Add(hitNewDocNum, args...) - if err != nil { - return 0, 0, 0, nil, err - } - } - } - - lastDocNum = hitNewDocNum - lastFreq = nextFreq - lastNorm = nextNorm - - next, err = postItr.Next() - } - - return lastDocNum, lastFreq, lastNorm, bufLoc, err -} - -func writePostings(postings *roaring.Bitmap, tfEncoder, locEncoder *chunkedIntCoder, - use1HitEncoding func(uint64) (bool, uint64, uint64), - w *CountHashWriter, bufMaxVarintLen64 []byte) ( - offset uint64, err error) { - termCardinality := postings.GetCardinality() - if termCardinality <= 0 { - return 0, nil - } - - if use1HitEncoding != nil { - encodeAs1Hit, docNum1Hit, normBits1Hit := use1HitEncoding(termCardinality) - if encodeAs1Hit { - return FSTValEncode1Hit(docNum1Hit, normBits1Hit), nil - } - } - - var tfOffset uint64 - tfOffset, _, err = tfEncoder.writeAt(w) - if err != nil { - return 0, err - } - - var locOffset uint64 - locOffset, _, err = locEncoder.writeAt(w) - if err != nil { - return 0, err - } - - postingsOffset := uint64(w.Count()) - - n := binary.PutUvarint(bufMaxVarintLen64, tfOffset) - _, err = w.Write(bufMaxVarintLen64[:n]) - if err != nil { - return 0, err - } - - n = binary.PutUvarint(bufMaxVarintLen64, locOffset) - _, err = w.Write(bufMaxVarintLen64[:n]) - if err != nil { - return 0, err - } - - _, err = writeRoaringWithLen(postings, w, bufMaxVarintLen64) - if err != nil { - return 0, err - } - - return postingsOffset, nil -} - -type varintEncoder func(uint64) (int, error) - -func mergeStoredAndRemap(segments []*SegmentBase, drops []*roaring.Bitmap, - fieldsMap map[string]uint16, fieldsInv []string, fieldsSame bool, newSegDocCount uint64, - w *CountHashWriter, closeCh chan struct{}) (uint64, [][]uint64, error) { - var rv [][]uint64 // The remapped or newDocNums for each segment. - - var newDocNum uint64 - - var curr int - var data, compressed []byte - var metaBuf bytes.Buffer - varBuf := make([]byte, binary.MaxVarintLen64) - metaEncode := func(val uint64) (int, error) { - wb := binary.PutUvarint(varBuf, val) - return metaBuf.Write(varBuf[:wb]) - } - - vals := make([][][]byte, len(fieldsInv)) - typs := make([][]byte, len(fieldsInv)) - poss := make([][][]uint64, len(fieldsInv)) - - var posBuf []uint64 - - docNumOffsets := make([]uint64, newSegDocCount) - - vdc := visitDocumentCtxPool.Get().(*visitDocumentCtx) - defer visitDocumentCtxPool.Put(vdc) - - // for each segment - for segI, segment := range segments { - // check for the closure in meantime - if isClosed(closeCh) { - return 0, nil, seg.ErrClosed - } - - segNewDocNums := make([]uint64, segment.numDocs) - - dropsI := drops[segI] - - // optimize when the field mapping is the same across all - // segments and there are no deletions, via byte-copying - // of stored docs bytes directly to the writer - if fieldsSame && (dropsI == nil || dropsI.GetCardinality() == 0) { - err := segment.copyStoredDocs(newDocNum, docNumOffsets, w) - if err != nil { - return 0, nil, err - } - - for i := uint64(0); i < segment.numDocs; i++ { - segNewDocNums[i] = newDocNum - newDocNum++ - } - rv = append(rv, segNewDocNums) - - continue - } - - // for each doc num - for docNum := uint64(0); docNum < segment.numDocs; docNum++ { - // TODO: roaring's API limits docNums to 32-bits? - if dropsI != nil && dropsI.Contains(uint32(docNum)) { - segNewDocNums[docNum] = docDropped - continue - } - - segNewDocNums[docNum] = newDocNum - - curr = 0 - metaBuf.Reset() - data = data[:0] - - posTemp := posBuf - - // collect all the data - for i := 0; i < len(fieldsInv); i++ { - vals[i] = vals[i][:0] - typs[i] = typs[i][:0] - poss[i] = poss[i][:0] - } - err := segment.visitDocument(vdc, docNum, func(field string, typ byte, value []byte, pos []uint64) bool { - fieldID := int(fieldsMap[field]) - 1 - vals[fieldID] = append(vals[fieldID], value) - typs[fieldID] = append(typs[fieldID], typ) - - // copy array positions to preserve them beyond the scope of this callback - var curPos []uint64 - if len(pos) > 0 { - if cap(posTemp) < len(pos) { - posBuf = make([]uint64, len(pos)*len(fieldsInv)) - posTemp = posBuf - } - curPos = posTemp[0:len(pos)] - copy(curPos, pos) - posTemp = posTemp[len(pos):] - } - poss[fieldID] = append(poss[fieldID], curPos) - - return true - }) - if err != nil { - return 0, nil, err - } - - // _id field special case optimizes ExternalID() lookups - idFieldVal := vals[uint16(0)][0] - _, err = metaEncode(uint64(len(idFieldVal))) - if err != nil { - return 0, nil, err - } - - // now walk the non-"_id" fields in order - for fieldID := 1; fieldID < len(fieldsInv); fieldID++ { - storedFieldValues := vals[fieldID] - - stf := typs[fieldID] - spf := poss[fieldID] - - var err2 error - curr, data, err2 = persistStoredFieldValues(fieldID, - storedFieldValues, stf, spf, curr, metaEncode, data) - if err2 != nil { - return 0, nil, err2 - } - } - - metaBytes := metaBuf.Bytes() - - compressed = snappy.Encode(compressed[:cap(compressed)], data) - - // record where we're about to start writing - docNumOffsets[newDocNum] = uint64(w.Count()) - - // write out the meta len and compressed data len - _, err = writeUvarints(w, - uint64(len(metaBytes)), - uint64(len(idFieldVal)+len(compressed))) - if err != nil { - return 0, nil, err - } - // now write the meta - _, err = w.Write(metaBytes) - if err != nil { - return 0, nil, err - } - // now write the _id field val (counted as part of the 'compressed' data) - _, err = w.Write(idFieldVal) - if err != nil { - return 0, nil, err - } - // now write the compressed data - _, err = w.Write(compressed) - if err != nil { - return 0, nil, err - } - - newDocNum++ - } - - rv = append(rv, segNewDocNums) - } - - // return value is the start of the stored index - storedIndexOffset := uint64(w.Count()) - - // now write out the stored doc index - for _, docNumOffset := range docNumOffsets { - err := binary.Write(w, binary.BigEndian, docNumOffset) - if err != nil { - return 0, nil, err - } - } - - return storedIndexOffset, rv, nil -} - -// copyStoredDocs writes out a segment's stored doc info, optimized by -// using a single Write() call for the entire set of bytes. The -// newDocNumOffsets is filled with the new offsets for each doc. -func (s *SegmentBase) copyStoredDocs(newDocNum uint64, newDocNumOffsets []uint64, - w *CountHashWriter) error { - if s.numDocs <= 0 { - return nil - } - - indexOffset0, storedOffset0, _, _, _ := - s.getDocStoredOffsets(0) // the segment's first doc - - indexOffsetN, storedOffsetN, readN, metaLenN, dataLenN := - s.getDocStoredOffsets(s.numDocs - 1) // the segment's last doc - - storedOffset0New := uint64(w.Count()) - - storedBytes := s.mem[storedOffset0 : storedOffsetN+readN+metaLenN+dataLenN] - _, err := w.Write(storedBytes) - if err != nil { - return err - } - - // remap the storedOffset's for the docs into new offsets relative - // to storedOffset0New, filling the given docNumOffsetsOut array - for indexOffset := indexOffset0; indexOffset <= indexOffsetN; indexOffset += 8 { - storedOffset := binary.BigEndian.Uint64(s.mem[indexOffset : indexOffset+8]) - storedOffsetNew := storedOffset - storedOffset0 + storedOffset0New - newDocNumOffsets[newDocNum] = storedOffsetNew - newDocNum += 1 - } - - return nil -} - -// mergeFields builds a unified list of fields used across all the -// input segments, and computes whether the fields are the same across -// segments (which depends on fields to be sorted in the same way -// across segments) -func mergeFields(segments []*SegmentBase) (bool, []string) { - fieldsSame := true - - var segment0Fields []string - if len(segments) > 0 { - segment0Fields = segments[0].Fields() - } - - fieldsExist := map[string]struct{}{} - for _, segment := range segments { - fields := segment.Fields() - for fieldi, field := range fields { - fieldsExist[field] = struct{}{} - if len(segment0Fields) != len(fields) || segment0Fields[fieldi] != field { - fieldsSame = false - } - } - } - - rv := make([]string, 0, len(fieldsExist)) - // ensure _id stays first - rv = append(rv, "_id") - for k := range fieldsExist { - if k != "_id" { - rv = append(rv, k) - } - } - - sort.Strings(rv[1:]) // leave _id as first - - return fieldsSame, rv -} - -func isClosed(closeCh chan struct{}) bool { - select { - case <-closeCh: - return true - default: - return false - } -} diff --git a/vendor/github.com/blevesearch/zap/v14/new.go b/vendor/github.com/blevesearch/zap/v14/new.go deleted file mode 100644 index 98158186..00000000 --- a/vendor/github.com/blevesearch/zap/v14/new.go +++ /dev/null @@ -1,860 +0,0 @@ -// Copyright (c) 2018 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "bytes" - "encoding/binary" - "math" - "sort" - "sync" - - "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/couchbase/vellum" - "github.com/golang/snappy" -) - -var NewSegmentBufferNumResultsBump int = 100 -var NewSegmentBufferNumResultsFactor float64 = 1.0 -var NewSegmentBufferAvgBytesPerDocFactor float64 = 1.0 - -// ValidateDocFields can be set by applications to perform additional checks -// on fields in a document being added to a new segment, by default it does -// nothing. -// This API is experimental and may be removed at any time. -var ValidateDocFields = func(field document.Field) error { - return nil -} - -// AnalysisResultsToSegmentBase produces an in-memory zap-encoded -// SegmentBase from analysis results -func (z *ZapPlugin) New(results []*index.AnalysisResult) ( - segment.Segment, uint64, error) { - return z.newWithChunkMode(results, DefaultChunkMode) -} - -func (*ZapPlugin) newWithChunkMode(results []*index.AnalysisResult, - chunkMode uint32) (segment.Segment, uint64, error) { - s := interimPool.Get().(*interim) - - var br bytes.Buffer - if s.lastNumDocs > 0 { - // use previous results to initialize the buf with an estimate - // size, but note that the interim instance comes from a - // global interimPool, so multiple scorch instances indexing - // different docs can lead to low quality estimates - estimateAvgBytesPerDoc := int(float64(s.lastOutSize/s.lastNumDocs) * - NewSegmentBufferNumResultsFactor) - estimateNumResults := int(float64(len(results)+NewSegmentBufferNumResultsBump) * - NewSegmentBufferAvgBytesPerDocFactor) - br.Grow(estimateAvgBytesPerDoc * estimateNumResults) - } - - s.results = results - s.chunkMode = chunkMode - s.w = NewCountHashWriter(&br) - - storedIndexOffset, fieldsIndexOffset, fdvIndexOffset, dictOffsets, - err := s.convert() - if err != nil { - return nil, uint64(0), err - } - - sb, err := InitSegmentBase(br.Bytes(), s.w.Sum32(), chunkMode, - s.FieldsMap, s.FieldsInv, uint64(len(results)), - storedIndexOffset, fieldsIndexOffset, fdvIndexOffset, dictOffsets) - - if err == nil && s.reset() == nil { - s.lastNumDocs = len(results) - s.lastOutSize = len(br.Bytes()) - interimPool.Put(s) - } - - return sb, uint64(len(br.Bytes())), err -} - -var interimPool = sync.Pool{New: func() interface{} { return &interim{} }} - -// interim holds temporary working data used while converting from -// analysis results to a zap-encoded segment -type interim struct { - results []*index.AnalysisResult - - chunkMode uint32 - - w *CountHashWriter - - // FieldsMap adds 1 to field id to avoid zero value issues - // name -> field id + 1 - FieldsMap map[string]uint16 - - // FieldsInv is the inverse of FieldsMap - // field id -> name - FieldsInv []string - - // Term dictionaries for each field - // field id -> term -> postings list id + 1 - Dicts []map[string]uint64 - - // Terms for each field, where terms are sorted ascending - // field id -> []term - DictKeys [][]string - - // Fields whose IncludeDocValues is true - // field id -> bool - IncludeDocValues []bool - - // postings id -> bitmap of docNums - Postings []*roaring.Bitmap - - // postings id -> freq/norm's, one for each docNum in postings - FreqNorms [][]interimFreqNorm - freqNormsBacking []interimFreqNorm - - // postings id -> locs, one for each freq - Locs [][]interimLoc - locsBacking []interimLoc - - numTermsPerPostingsList []int // key is postings list id - numLocsPerPostingsList []int // key is postings list id - - builder *vellum.Builder - builderBuf bytes.Buffer - - metaBuf bytes.Buffer - - tmp0 []byte - tmp1 []byte - - lastNumDocs int - lastOutSize int -} - -func (s *interim) reset() (err error) { - s.results = nil - s.chunkMode = 0 - s.w = nil - s.FieldsMap = nil - s.FieldsInv = nil - for i := range s.Dicts { - s.Dicts[i] = nil - } - s.Dicts = s.Dicts[:0] - for i := range s.DictKeys { - s.DictKeys[i] = s.DictKeys[i][:0] - } - s.DictKeys = s.DictKeys[:0] - for i := range s.IncludeDocValues { - s.IncludeDocValues[i] = false - } - s.IncludeDocValues = s.IncludeDocValues[:0] - for _, idn := range s.Postings { - idn.Clear() - } - s.Postings = s.Postings[:0] - s.FreqNorms = s.FreqNorms[:0] - for i := range s.freqNormsBacking { - s.freqNormsBacking[i] = interimFreqNorm{} - } - s.freqNormsBacking = s.freqNormsBacking[:0] - s.Locs = s.Locs[:0] - for i := range s.locsBacking { - s.locsBacking[i] = interimLoc{} - } - s.locsBacking = s.locsBacking[:0] - s.numTermsPerPostingsList = s.numTermsPerPostingsList[:0] - s.numLocsPerPostingsList = s.numLocsPerPostingsList[:0] - s.builderBuf.Reset() - if s.builder != nil { - err = s.builder.Reset(&s.builderBuf) - } - s.metaBuf.Reset() - s.tmp0 = s.tmp0[:0] - s.tmp1 = s.tmp1[:0] - s.lastNumDocs = 0 - s.lastOutSize = 0 - - return err -} - -func (s *interim) grabBuf(size int) []byte { - buf := s.tmp0 - if cap(buf) < size { - buf = make([]byte, size) - s.tmp0 = buf - } - return buf[0:size] -} - -type interimStoredField struct { - vals [][]byte - typs []byte - arrayposs [][]uint64 // array positions -} - -type interimFreqNorm struct { - freq uint64 - norm float32 - numLocs int -} - -type interimLoc struct { - fieldID uint16 - pos uint64 - start uint64 - end uint64 - arrayposs []uint64 -} - -func (s *interim) convert() (uint64, uint64, uint64, []uint64, error) { - s.FieldsMap = map[string]uint16{} - - s.getOrDefineField("_id") // _id field is fieldID 0 - - for _, result := range s.results { - for _, field := range result.Document.CompositeFields { - s.getOrDefineField(field.Name()) - } - for _, field := range result.Document.Fields { - s.getOrDefineField(field.Name()) - } - } - - sort.Strings(s.FieldsInv[1:]) // keep _id as first field - - for fieldID, fieldName := range s.FieldsInv { - s.FieldsMap[fieldName] = uint16(fieldID + 1) - } - - if cap(s.IncludeDocValues) >= len(s.FieldsInv) { - s.IncludeDocValues = s.IncludeDocValues[:len(s.FieldsInv)] - } else { - s.IncludeDocValues = make([]bool, len(s.FieldsInv)) - } - - s.prepareDicts() - - for _, dict := range s.DictKeys { - sort.Strings(dict) - } - - s.processDocuments() - - storedIndexOffset, err := s.writeStoredFields() - if err != nil { - return 0, 0, 0, nil, err - } - - var fdvIndexOffset uint64 - var dictOffsets []uint64 - - if len(s.results) > 0 { - fdvIndexOffset, dictOffsets, err = s.writeDicts() - if err != nil { - return 0, 0, 0, nil, err - } - } else { - dictOffsets = make([]uint64, len(s.FieldsInv)) - } - - fieldsIndexOffset, err := persistFields(s.FieldsInv, s.w, dictOffsets) - if err != nil { - return 0, 0, 0, nil, err - } - - return storedIndexOffset, fieldsIndexOffset, fdvIndexOffset, dictOffsets, nil -} - -func (s *interim) getOrDefineField(fieldName string) int { - fieldIDPlus1, exists := s.FieldsMap[fieldName] - if !exists { - fieldIDPlus1 = uint16(len(s.FieldsInv) + 1) - s.FieldsMap[fieldName] = fieldIDPlus1 - s.FieldsInv = append(s.FieldsInv, fieldName) - - s.Dicts = append(s.Dicts, make(map[string]uint64)) - - n := len(s.DictKeys) - if n < cap(s.DictKeys) { - s.DictKeys = s.DictKeys[:n+1] - s.DictKeys[n] = s.DictKeys[n][:0] - } else { - s.DictKeys = append(s.DictKeys, []string(nil)) - } - } - - return int(fieldIDPlus1 - 1) -} - -// fill Dicts and DictKeys from analysis results -func (s *interim) prepareDicts() { - var pidNext int - - var totTFs int - var totLocs int - - visitField := func(fieldID uint16, tfs analysis.TokenFrequencies) { - dict := s.Dicts[fieldID] - dictKeys := s.DictKeys[fieldID] - - for term, tf := range tfs { - pidPlus1, exists := dict[term] - if !exists { - pidNext++ - pidPlus1 = uint64(pidNext) - - dict[term] = pidPlus1 - dictKeys = append(dictKeys, term) - - s.numTermsPerPostingsList = append(s.numTermsPerPostingsList, 0) - s.numLocsPerPostingsList = append(s.numLocsPerPostingsList, 0) - } - - pid := pidPlus1 - 1 - - s.numTermsPerPostingsList[pid] += 1 - s.numLocsPerPostingsList[pid] += len(tf.Locations) - - totLocs += len(tf.Locations) - } - - totTFs += len(tfs) - - s.DictKeys[fieldID] = dictKeys - } - - for _, result := range s.results { - // walk each composite field - for _, field := range result.Document.CompositeFields { - fieldID := uint16(s.getOrDefineField(field.Name())) - _, tf := field.Analyze() - visitField(fieldID, tf) - } - - // walk each field - for i, field := range result.Document.Fields { - fieldID := uint16(s.getOrDefineField(field.Name())) - tf := result.Analyzed[i] - visitField(fieldID, tf) - } - } - - numPostingsLists := pidNext - - if cap(s.Postings) >= numPostingsLists { - s.Postings = s.Postings[:numPostingsLists] - } else { - postings := make([]*roaring.Bitmap, numPostingsLists) - copy(postings, s.Postings[:cap(s.Postings)]) - for i := 0; i < numPostingsLists; i++ { - if postings[i] == nil { - postings[i] = roaring.New() - } - } - s.Postings = postings - } - - if cap(s.FreqNorms) >= numPostingsLists { - s.FreqNorms = s.FreqNorms[:numPostingsLists] - } else { - s.FreqNorms = make([][]interimFreqNorm, numPostingsLists) - } - - if cap(s.freqNormsBacking) >= totTFs { - s.freqNormsBacking = s.freqNormsBacking[:totTFs] - } else { - s.freqNormsBacking = make([]interimFreqNorm, totTFs) - } - - freqNormsBacking := s.freqNormsBacking - for pid, numTerms := range s.numTermsPerPostingsList { - s.FreqNorms[pid] = freqNormsBacking[0:0] - freqNormsBacking = freqNormsBacking[numTerms:] - } - - if cap(s.Locs) >= numPostingsLists { - s.Locs = s.Locs[:numPostingsLists] - } else { - s.Locs = make([][]interimLoc, numPostingsLists) - } - - if cap(s.locsBacking) >= totLocs { - s.locsBacking = s.locsBacking[:totLocs] - } else { - s.locsBacking = make([]interimLoc, totLocs) - } - - locsBacking := s.locsBacking - for pid, numLocs := range s.numLocsPerPostingsList { - s.Locs[pid] = locsBacking[0:0] - locsBacking = locsBacking[numLocs:] - } -} - -func (s *interim) processDocuments() { - numFields := len(s.FieldsInv) - reuseFieldLens := make([]int, numFields) - reuseFieldTFs := make([]analysis.TokenFrequencies, numFields) - - for docNum, result := range s.results { - for i := 0; i < numFields; i++ { // clear these for reuse - reuseFieldLens[i] = 0 - reuseFieldTFs[i] = nil - } - - s.processDocument(uint64(docNum), result, - reuseFieldLens, reuseFieldTFs) - } -} - -func (s *interim) processDocument(docNum uint64, - result *index.AnalysisResult, - fieldLens []int, fieldTFs []analysis.TokenFrequencies) { - visitField := func(fieldID uint16, fieldName string, - ln int, tf analysis.TokenFrequencies) { - fieldLens[fieldID] += ln - - existingFreqs := fieldTFs[fieldID] - if existingFreqs != nil { - existingFreqs.MergeAll(fieldName, tf) - } else { - fieldTFs[fieldID] = tf - } - } - - // walk each composite field - for _, field := range result.Document.CompositeFields { - fieldID := uint16(s.getOrDefineField(field.Name())) - ln, tf := field.Analyze() - visitField(fieldID, field.Name(), ln, tf) - } - - // walk each field - for i, field := range result.Document.Fields { - fieldID := uint16(s.getOrDefineField(field.Name())) - ln := result.Length[i] - tf := result.Analyzed[i] - visitField(fieldID, field.Name(), ln, tf) - } - - // now that it's been rolled up into fieldTFs, walk that - for fieldID, tfs := range fieldTFs { - dict := s.Dicts[fieldID] - norm := float32(1.0 / math.Sqrt(float64(fieldLens[fieldID]))) - - for term, tf := range tfs { - pid := dict[term] - 1 - bs := s.Postings[pid] - bs.Add(uint32(docNum)) - - s.FreqNorms[pid] = append(s.FreqNorms[pid], - interimFreqNorm{ - freq: uint64(tf.Frequency()), - norm: norm, - numLocs: len(tf.Locations), - }) - - if len(tf.Locations) > 0 { - locs := s.Locs[pid] - - for _, loc := range tf.Locations { - var locf = uint16(fieldID) - if loc.Field != "" { - locf = uint16(s.getOrDefineField(loc.Field)) - } - var arrayposs []uint64 - if len(loc.ArrayPositions) > 0 { - arrayposs = loc.ArrayPositions - } - locs = append(locs, interimLoc{ - fieldID: locf, - pos: uint64(loc.Position), - start: uint64(loc.Start), - end: uint64(loc.End), - arrayposs: arrayposs, - }) - } - - s.Locs[pid] = locs - } - } - } -} - -func (s *interim) writeStoredFields() ( - storedIndexOffset uint64, err error) { - varBuf := make([]byte, binary.MaxVarintLen64) - metaEncode := func(val uint64) (int, error) { - wb := binary.PutUvarint(varBuf, val) - return s.metaBuf.Write(varBuf[:wb]) - } - - data, compressed := s.tmp0[:0], s.tmp1[:0] - defer func() { s.tmp0, s.tmp1 = data, compressed }() - - // keyed by docNum - docStoredOffsets := make([]uint64, len(s.results)) - - // keyed by fieldID, for the current doc in the loop - docStoredFields := map[uint16]interimStoredField{} - - for docNum, result := range s.results { - for fieldID := range docStoredFields { // reset for next doc - delete(docStoredFields, fieldID) - } - - for _, field := range result.Document.Fields { - fieldID := uint16(s.getOrDefineField(field.Name())) - - opts := field.Options() - - if opts.IsStored() { - isf := docStoredFields[fieldID] - isf.vals = append(isf.vals, field.Value()) - isf.typs = append(isf.typs, encodeFieldType(field)) - isf.arrayposs = append(isf.arrayposs, field.ArrayPositions()) - docStoredFields[fieldID] = isf - } - - if opts.IncludeDocValues() { - s.IncludeDocValues[fieldID] = true - } - - err := ValidateDocFields(field) - if err != nil { - return 0, err - } - } - - var curr int - - s.metaBuf.Reset() - data = data[:0] - - // _id field special case optimizes ExternalID() lookups - idFieldVal := docStoredFields[uint16(0)].vals[0] - _, err = metaEncode(uint64(len(idFieldVal))) - if err != nil { - return 0, err - } - - // handle non-"_id" fields - for fieldID := 1; fieldID < len(s.FieldsInv); fieldID++ { - isf, exists := docStoredFields[uint16(fieldID)] - if exists { - curr, data, err = persistStoredFieldValues( - fieldID, isf.vals, isf.typs, isf.arrayposs, - curr, metaEncode, data) - if err != nil { - return 0, err - } - } - } - - metaBytes := s.metaBuf.Bytes() - - compressed = snappy.Encode(compressed[:cap(compressed)], data) - - docStoredOffsets[docNum] = uint64(s.w.Count()) - - _, err := writeUvarints(s.w, - uint64(len(metaBytes)), - uint64(len(idFieldVal)+len(compressed))) - if err != nil { - return 0, err - } - - _, err = s.w.Write(metaBytes) - if err != nil { - return 0, err - } - - _, err = s.w.Write(idFieldVal) - if err != nil { - return 0, err - } - - _, err = s.w.Write(compressed) - if err != nil { - return 0, err - } - } - - storedIndexOffset = uint64(s.w.Count()) - - for _, docStoredOffset := range docStoredOffsets { - err = binary.Write(s.w, binary.BigEndian, docStoredOffset) - if err != nil { - return 0, err - } - } - - return storedIndexOffset, nil -} - -func (s *interim) writeDicts() (fdvIndexOffset uint64, dictOffsets []uint64, err error) { - dictOffsets = make([]uint64, len(s.FieldsInv)) - - fdvOffsetsStart := make([]uint64, len(s.FieldsInv)) - fdvOffsetsEnd := make([]uint64, len(s.FieldsInv)) - - buf := s.grabBuf(binary.MaxVarintLen64) - - // these int coders are initialized with chunk size 1024 - // however this will be reset to the correct chunk size - // while processing each individual field-term section - tfEncoder := newChunkedIntCoder(1024, uint64(len(s.results)-1)) - locEncoder := newChunkedIntCoder(1024, uint64(len(s.results)-1)) - - var docTermMap [][]byte - - if s.builder == nil { - s.builder, err = vellum.New(&s.builderBuf, nil) - if err != nil { - return 0, nil, err - } - } - - for fieldID, terms := range s.DictKeys { - if cap(docTermMap) < len(s.results) { - docTermMap = make([][]byte, len(s.results)) - } else { - docTermMap = docTermMap[0:len(s.results)] - for docNum := range docTermMap { // reset the docTermMap - docTermMap[docNum] = docTermMap[docNum][:0] - } - } - - dict := s.Dicts[fieldID] - - for _, term := range terms { // terms are already sorted - pid := dict[term] - 1 - - postingsBS := s.Postings[pid] - - freqNorms := s.FreqNorms[pid] - freqNormOffset := 0 - - locs := s.Locs[pid] - locOffset := 0 - - chunkSize, err := getChunkSize(s.chunkMode, postingsBS.GetCardinality(), uint64(len(s.results))) - if err != nil { - return 0, nil, err - } - tfEncoder.SetChunkSize(chunkSize, uint64(len(s.results)-1)) - locEncoder.SetChunkSize(chunkSize, uint64(len(s.results)-1)) - - postingsItr := postingsBS.Iterator() - for postingsItr.HasNext() { - docNum := uint64(postingsItr.Next()) - - freqNorm := freqNorms[freqNormOffset] - - err = tfEncoder.Add(docNum, - encodeFreqHasLocs(freqNorm.freq, freqNorm.numLocs > 0), - uint64(math.Float32bits(freqNorm.norm))) - if err != nil { - return 0, nil, err - } - - if freqNorm.numLocs > 0 { - numBytesLocs := 0 - for _, loc := range locs[locOffset : locOffset+freqNorm.numLocs] { - numBytesLocs += totalUvarintBytes( - uint64(loc.fieldID), loc.pos, loc.start, loc.end, - uint64(len(loc.arrayposs)), loc.arrayposs) - } - - err = locEncoder.Add(docNum, uint64(numBytesLocs)) - if err != nil { - return 0, nil, err - } - - for _, loc := range locs[locOffset : locOffset+freqNorm.numLocs] { - err = locEncoder.Add(docNum, - uint64(loc.fieldID), loc.pos, loc.start, loc.end, - uint64(len(loc.arrayposs))) - if err != nil { - return 0, nil, err - } - - err = locEncoder.Add(docNum, loc.arrayposs...) - if err != nil { - return 0, nil, err - } - } - - locOffset += freqNorm.numLocs - } - - freqNormOffset++ - - docTermMap[docNum] = append( - append(docTermMap[docNum], term...), - termSeparator) - } - - tfEncoder.Close() - locEncoder.Close() - - postingsOffset, err := - writePostings(postingsBS, tfEncoder, locEncoder, nil, s.w, buf) - if err != nil { - return 0, nil, err - } - - if postingsOffset > uint64(0) { - err = s.builder.Insert([]byte(term), postingsOffset) - if err != nil { - return 0, nil, err - } - } - - tfEncoder.Reset() - locEncoder.Reset() - } - - err = s.builder.Close() - if err != nil { - return 0, nil, err - } - - // record where this dictionary starts - dictOffsets[fieldID] = uint64(s.w.Count()) - - vellumData := s.builderBuf.Bytes() - - // write out the length of the vellum data - n := binary.PutUvarint(buf, uint64(len(vellumData))) - _, err = s.w.Write(buf[:n]) - if err != nil { - return 0, nil, err - } - - // write this vellum to disk - _, err = s.w.Write(vellumData) - if err != nil { - return 0, nil, err - } - - // reset vellum for reuse - s.builderBuf.Reset() - - err = s.builder.Reset(&s.builderBuf) - if err != nil { - return 0, nil, err - } - - // write the field doc values - // NOTE: doc values continue to use legacy chunk mode - chunkSize, err := getChunkSize(LegacyChunkMode, 0, 0) - if err != nil { - return 0, nil, err - } - fdvEncoder := newChunkedContentCoder(chunkSize, uint64(len(s.results)-1), s.w, false) - if s.IncludeDocValues[fieldID] { - for docNum, docTerms := range docTermMap { - if len(docTerms) > 0 { - err = fdvEncoder.Add(uint64(docNum), docTerms) - if err != nil { - return 0, nil, err - } - } - } - err = fdvEncoder.Close() - if err != nil { - return 0, nil, err - } - - fdvOffsetsStart[fieldID] = uint64(s.w.Count()) - - _, err = fdvEncoder.Write() - if err != nil { - return 0, nil, err - } - - fdvOffsetsEnd[fieldID] = uint64(s.w.Count()) - - fdvEncoder.Reset() - } else { - fdvOffsetsStart[fieldID] = fieldNotUninverted - fdvOffsetsEnd[fieldID] = fieldNotUninverted - } - } - - fdvIndexOffset = uint64(s.w.Count()) - - for i := 0; i < len(fdvOffsetsStart); i++ { - n := binary.PutUvarint(buf, fdvOffsetsStart[i]) - _, err := s.w.Write(buf[:n]) - if err != nil { - return 0, nil, err - } - n = binary.PutUvarint(buf, fdvOffsetsEnd[i]) - _, err = s.w.Write(buf[:n]) - if err != nil { - return 0, nil, err - } - } - - return fdvIndexOffset, dictOffsets, nil -} - -func encodeFieldType(f document.Field) byte { - fieldType := byte('x') - switch f.(type) { - case *document.TextField: - fieldType = 't' - case *document.NumericField: - fieldType = 'n' - case *document.DateTimeField: - fieldType = 'd' - case *document.BooleanField: - fieldType = 'b' - case *document.GeoPointField: - fieldType = 'g' - case *document.CompositeField: - fieldType = 'c' - } - return fieldType -} - -// returns the total # of bytes needed to encode the given uint64's -// into binary.PutUVarint() encoding -func totalUvarintBytes(a, b, c, d, e uint64, more []uint64) (n int) { - n = numUvarintBytes(a) - n += numUvarintBytes(b) - n += numUvarintBytes(c) - n += numUvarintBytes(d) - n += numUvarintBytes(e) - for _, v := range more { - n += numUvarintBytes(v) - } - return n -} - -// returns # of bytes needed to encode x in binary.PutUvarint() encoding -func numUvarintBytes(x uint64) (n int) { - for x >= 0x80 { - x >>= 7 - n++ - } - return n + 1 -} diff --git a/vendor/github.com/blevesearch/zap/v14/plugin.go b/vendor/github.com/blevesearch/zap/v14/plugin.go deleted file mode 100644 index 38a0638d..00000000 --- a/vendor/github.com/blevesearch/zap/v14/plugin.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (c) 2020 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "github.com/blevesearch/bleve/index/scorch/segment" -) - -// ZapPlugin implements the Plugin interface of -// the blevesearch/bleve/index/scorch/segment pkg -type ZapPlugin struct{} - -func (*ZapPlugin) Type() string { - return Type -} - -func (*ZapPlugin) Version() uint32 { - return Version -} - -// Plugin returns an instance segment.Plugin for use -// by the Scorch indexing scheme -func Plugin() segment.Plugin { - return &ZapPlugin{} -} diff --git a/vendor/github.com/blevesearch/zap/v14/posting.go b/vendor/github.com/blevesearch/zap/v14/posting.go deleted file mode 100644 index 88b2b953..00000000 --- a/vendor/github.com/blevesearch/zap/v14/posting.go +++ /dev/null @@ -1,796 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "encoding/binary" - "fmt" - "math" - "reflect" - - "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/blevesearch/bleve/size" -) - -var reflectStaticSizePostingsList int -var reflectStaticSizePostingsIterator int -var reflectStaticSizePosting int -var reflectStaticSizeLocation int - -func init() { - var pl PostingsList - reflectStaticSizePostingsList = int(reflect.TypeOf(pl).Size()) - var pi PostingsIterator - reflectStaticSizePostingsIterator = int(reflect.TypeOf(pi).Size()) - var p Posting - reflectStaticSizePosting = int(reflect.TypeOf(p).Size()) - var l Location - reflectStaticSizeLocation = int(reflect.TypeOf(l).Size()) -} - -// FST or vellum value (uint64) encoding is determined by the top two -// highest-order or most significant bits... -// -// encoding : MSB -// name : 63 62 61...to...bit #0 (LSB) -// ----------+---+---+--------------------------------------------------- -// general : 0 | 0 | 62-bits of postingsOffset. -// ~ : 0 | 1 | reserved for future. -// 1-hit : 1 | 0 | 31-bits of positive float31 norm | 31-bits docNum. -// ~ : 1 | 1 | reserved for future. -// -// Encoding "general" is able to handle all cases, where the -// postingsOffset points to more information about the postings for -// the term. -// -// Encoding "1-hit" is used to optimize a commonly seen case when a -// term has only a single hit. For example, a term in the _id field -// will have only 1 hit. The "1-hit" encoding is used for a term -// in a field when... -// -// - term vector info is disabled for that field; -// - and, the term appears in only a single doc for that field; -// - and, the term's freq is exactly 1 in that single doc for that field; -// - and, the docNum must fit into 31-bits; -// -// Otherwise, the "general" encoding is used instead. -// -// In the "1-hit" encoding, the field in that single doc may have -// other terms, which is supported in the "1-hit" encoding by the -// positive float31 norm. - -const FSTValEncodingMask = uint64(0xc000000000000000) -const FSTValEncodingGeneral = uint64(0x0000000000000000) -const FSTValEncoding1Hit = uint64(0x8000000000000000) - -func FSTValEncode1Hit(docNum uint64, normBits uint64) uint64 { - return FSTValEncoding1Hit | ((mask31Bits & normBits) << 31) | (mask31Bits & docNum) -} - -func FSTValDecode1Hit(v uint64) (docNum uint64, normBits uint64) { - return (mask31Bits & v), (mask31Bits & (v >> 31)) -} - -const mask31Bits = uint64(0x000000007fffffff) - -func under32Bits(x uint64) bool { - return x <= mask31Bits -} - -const DocNum1HitFinished = math.MaxUint64 - -var NormBits1Hit = uint64(math.Float32bits(float32(1))) - -// PostingsList is an in-memory representation of a postings list -type PostingsList struct { - sb *SegmentBase - postingsOffset uint64 - freqOffset uint64 - locOffset uint64 - postings *roaring.Bitmap - except *roaring.Bitmap - - // when normBits1Hit != 0, then this postings list came from a - // 1-hit encoding, and only the docNum1Hit & normBits1Hit apply - docNum1Hit uint64 - normBits1Hit uint64 - - chunkSize uint64 -} - -// represents an immutable, empty postings list -var emptyPostingsList = &PostingsList{} - -func (p *PostingsList) Size() int { - sizeInBytes := reflectStaticSizePostingsList + size.SizeOfPtr - - if p.except != nil { - sizeInBytes += int(p.except.GetSizeInBytes()) - } - - return sizeInBytes -} - -func (p *PostingsList) OrInto(receiver *roaring.Bitmap) { - if p.normBits1Hit != 0 { - receiver.Add(uint32(p.docNum1Hit)) - return - } - - if p.postings != nil { - receiver.Or(p.postings) - } -} - -// Iterator returns an iterator for this postings list -func (p *PostingsList) Iterator(includeFreq, includeNorm, includeLocs bool, - prealloc segment.PostingsIterator) segment.PostingsIterator { - if p.normBits1Hit == 0 && p.postings == nil { - return emptyPostingsIterator - } - - var preallocPI *PostingsIterator - pi, ok := prealloc.(*PostingsIterator) - if ok && pi != nil { - preallocPI = pi - } - if preallocPI == emptyPostingsIterator { - preallocPI = nil - } - - return p.iterator(includeFreq, includeNorm, includeLocs, preallocPI) -} - -func (p *PostingsList) iterator(includeFreq, includeNorm, includeLocs bool, - rv *PostingsIterator) *PostingsIterator { - if rv == nil { - rv = &PostingsIterator{} - } else { - freqNormReader := rv.freqNormReader - if freqNormReader != nil { - freqNormReader.reset() - } - - locReader := rv.locReader - if locReader != nil { - locReader.reset() - } - - nextLocs := rv.nextLocs[:0] - nextSegmentLocs := rv.nextSegmentLocs[:0] - - buf := rv.buf - - *rv = PostingsIterator{} // clear the struct - - rv.freqNormReader = freqNormReader - rv.locReader = locReader - - rv.nextLocs = nextLocs - rv.nextSegmentLocs = nextSegmentLocs - - rv.buf = buf - } - - rv.postings = p - rv.includeFreqNorm = includeFreq || includeNorm || includeLocs - rv.includeLocs = includeLocs - - if p.normBits1Hit != 0 { - // "1-hit" encoding - rv.docNum1Hit = p.docNum1Hit - rv.normBits1Hit = p.normBits1Hit - - if p.except != nil && p.except.Contains(uint32(rv.docNum1Hit)) { - rv.docNum1Hit = DocNum1HitFinished - } - - return rv - } - - // "general" encoding, check if empty - if p.postings == nil { - return rv - } - - // initialize freq chunk reader - if rv.includeFreqNorm { - rv.freqNormReader = newChunkedIntDecoder(p.sb.mem, p.freqOffset, rv.freqNormReader) - } - - // initialize the loc chunk reader - if rv.includeLocs { - rv.locReader = newChunkedIntDecoder(p.sb.mem, p.locOffset, rv.locReader) - } - - rv.all = p.postings.Iterator() - if p.except != nil { - rv.ActualBM = roaring.AndNot(p.postings, p.except) - rv.Actual = rv.ActualBM.Iterator() - } else { - rv.ActualBM = p.postings - rv.Actual = rv.all // Optimize to use same iterator for all & Actual. - } - - return rv -} - -// Count returns the number of items on this postings list -func (p *PostingsList) Count() uint64 { - var n, e uint64 - if p.normBits1Hit != 0 { - n = 1 - if p.except != nil && p.except.Contains(uint32(p.docNum1Hit)) { - e = 1 - } - } else if p.postings != nil { - n = p.postings.GetCardinality() - if p.except != nil { - e = p.postings.AndCardinality(p.except) - } - } - return n - e -} - -func (rv *PostingsList) read(postingsOffset uint64, d *Dictionary) error { - rv.postingsOffset = postingsOffset - - // handle "1-hit" encoding special case - if rv.postingsOffset&FSTValEncodingMask == FSTValEncoding1Hit { - return rv.init1Hit(postingsOffset) - } - - // read the location of the freq/norm details - var n uint64 - var read int - - rv.freqOffset, read = binary.Uvarint(d.sb.mem[postingsOffset+n : postingsOffset+binary.MaxVarintLen64]) - n += uint64(read) - - rv.locOffset, read = binary.Uvarint(d.sb.mem[postingsOffset+n : postingsOffset+n+binary.MaxVarintLen64]) - n += uint64(read) - - var postingsLen uint64 - postingsLen, read = binary.Uvarint(d.sb.mem[postingsOffset+n : postingsOffset+n+binary.MaxVarintLen64]) - n += uint64(read) - - roaringBytes := d.sb.mem[postingsOffset+n : postingsOffset+n+postingsLen] - - if rv.postings == nil { - rv.postings = roaring.NewBitmap() - } - _, err := rv.postings.FromBuffer(roaringBytes) - if err != nil { - return fmt.Errorf("error loading roaring bitmap: %v", err) - } - - rv.chunkSize, err = getChunkSize(d.sb.chunkMode, - rv.postings.GetCardinality(), d.sb.numDocs) - if err != nil { - return err - } - - return nil -} - -func (rv *PostingsList) init1Hit(fstVal uint64) error { - docNum, normBits := FSTValDecode1Hit(fstVal) - - rv.docNum1Hit = docNum - rv.normBits1Hit = normBits - - return nil -} - -// PostingsIterator provides a way to iterate through the postings list -type PostingsIterator struct { - postings *PostingsList - all roaring.IntPeekable - Actual roaring.IntPeekable - ActualBM *roaring.Bitmap - - currChunk uint32 - freqNormReader *chunkedIntDecoder - locReader *chunkedIntDecoder - - next Posting // reused across Next() calls - nextLocs []Location // reused across Next() calls - nextSegmentLocs []segment.Location // reused across Next() calls - - docNum1Hit uint64 - normBits1Hit uint64 - - buf []byte - - includeFreqNorm bool - includeLocs bool -} - -var emptyPostingsIterator = &PostingsIterator{} - -func (i *PostingsIterator) Size() int { - sizeInBytes := reflectStaticSizePostingsIterator + size.SizeOfPtr + - i.next.Size() - // account for freqNormReader, locReader if we start using this. - for _, entry := range i.nextLocs { - sizeInBytes += entry.Size() - } - - return sizeInBytes -} - -func (i *PostingsIterator) loadChunk(chunk int) error { - if i.includeFreqNorm { - err := i.freqNormReader.loadChunk(chunk) - if err != nil { - return err - } - } - - if i.includeLocs { - err := i.locReader.loadChunk(chunk) - if err != nil { - return err - } - } - - i.currChunk = uint32(chunk) - return nil -} - -func (i *PostingsIterator) readFreqNormHasLocs() (uint64, uint64, bool, error) { - if i.normBits1Hit != 0 { - return 1, i.normBits1Hit, false, nil - } - - freqHasLocs, err := i.freqNormReader.readUvarint() - if err != nil { - return 0, 0, false, fmt.Errorf("error reading frequency: %v", err) - } - - freq, hasLocs := decodeFreqHasLocs(freqHasLocs) - - normBits, err := i.freqNormReader.readUvarint() - if err != nil { - return 0, 0, false, fmt.Errorf("error reading norm: %v", err) - } - - return freq, normBits, hasLocs, nil -} - -func (i *PostingsIterator) skipFreqNormReadHasLocs() (bool, error) { - if i.normBits1Hit != 0 { - return false, nil - } - - freqHasLocs, err := i.freqNormReader.readUvarint() - if err != nil { - return false, fmt.Errorf("error reading freqHasLocs: %v", err) - } - - i.freqNormReader.SkipUvarint() // Skip normBits. - - return freqHasLocs&0x01 != 0, nil // See decodeFreqHasLocs() / hasLocs. -} - -func encodeFreqHasLocs(freq uint64, hasLocs bool) uint64 { - rv := freq << 1 - if hasLocs { - rv = rv | 0x01 // 0'th LSB encodes whether there are locations - } - return rv -} - -func decodeFreqHasLocs(freqHasLocs uint64) (uint64, bool) { - freq := freqHasLocs >> 1 - hasLocs := freqHasLocs&0x01 != 0 - return freq, hasLocs -} - -// readLocation processes all the integers on the stream representing a single -// location. -func (i *PostingsIterator) readLocation(l *Location) error { - // read off field - fieldID, err := i.locReader.readUvarint() - if err != nil { - return fmt.Errorf("error reading location field: %v", err) - } - // read off pos - pos, err := i.locReader.readUvarint() - if err != nil { - return fmt.Errorf("error reading location pos: %v", err) - } - // read off start - start, err := i.locReader.readUvarint() - if err != nil { - return fmt.Errorf("error reading location start: %v", err) - } - // read off end - end, err := i.locReader.readUvarint() - if err != nil { - return fmt.Errorf("error reading location end: %v", err) - } - // read off num array pos - numArrayPos, err := i.locReader.readUvarint() - if err != nil { - return fmt.Errorf("error reading location num array pos: %v", err) - } - - l.field = i.postings.sb.fieldsInv[fieldID] - l.pos = pos - l.start = start - l.end = end - - if cap(l.ap) < int(numArrayPos) { - l.ap = make([]uint64, int(numArrayPos)) - } else { - l.ap = l.ap[:int(numArrayPos)] - } - - // read off array positions - for k := 0; k < int(numArrayPos); k++ { - ap, err := i.locReader.readUvarint() - if err != nil { - return fmt.Errorf("error reading array position: %v", err) - } - - l.ap[k] = ap - } - - return nil -} - -// Next returns the next posting on the postings list, or nil at the end -func (i *PostingsIterator) Next() (segment.Posting, error) { - return i.nextAtOrAfter(0) -} - -// Advance returns the posting at the specified docNum or it is not present -// the next posting, or if the end is reached, nil -func (i *PostingsIterator) Advance(docNum uint64) (segment.Posting, error) { - return i.nextAtOrAfter(docNum) -} - -// Next returns the next posting on the postings list, or nil at the end -func (i *PostingsIterator) nextAtOrAfter(atOrAfter uint64) (segment.Posting, error) { - docNum, exists, err := i.nextDocNumAtOrAfter(atOrAfter) - if err != nil || !exists { - return nil, err - } - - i.next = Posting{} // clear the struct - rv := &i.next - rv.docNum = docNum - - if !i.includeFreqNorm { - return rv, nil - } - - var normBits uint64 - var hasLocs bool - - rv.freq, normBits, hasLocs, err = i.readFreqNormHasLocs() - if err != nil { - return nil, err - } - - rv.norm = math.Float32frombits(uint32(normBits)) - - if i.includeLocs && hasLocs { - // prepare locations into reused slices, where we assume - // rv.freq >= "number of locs", since in a composite field, - // some component fields might have their IncludeTermVector - // flags disabled while other component fields are enabled - if cap(i.nextLocs) >= int(rv.freq) { - i.nextLocs = i.nextLocs[0:rv.freq] - } else { - i.nextLocs = make([]Location, rv.freq, rv.freq*2) - } - if cap(i.nextSegmentLocs) < int(rv.freq) { - i.nextSegmentLocs = make([]segment.Location, rv.freq, rv.freq*2) - } - rv.locs = i.nextSegmentLocs[:0] - - numLocsBytes, err := i.locReader.readUvarint() - if err != nil { - return nil, fmt.Errorf("error reading location numLocsBytes: %v", err) - } - - j := 0 - startBytesRemaining := i.locReader.Len() // # bytes remaining in the locReader - for startBytesRemaining-i.locReader.Len() < int(numLocsBytes) { - err := i.readLocation(&i.nextLocs[j]) - if err != nil { - return nil, err - } - rv.locs = append(rv.locs, &i.nextLocs[j]) - j++ - } - } - - return rv, nil -} - -// nextDocNum returns the next docNum on the postings list, and also -// sets up the currChunk / loc related fields of the iterator. -func (i *PostingsIterator) nextDocNumAtOrAfter(atOrAfter uint64) (uint64, bool, error) { - if i.normBits1Hit != 0 { - if i.docNum1Hit == DocNum1HitFinished { - return 0, false, nil - } - if i.docNum1Hit < atOrAfter { - // advanced past our 1-hit - i.docNum1Hit = DocNum1HitFinished // consume our 1-hit docNum - return 0, false, nil - } - docNum := i.docNum1Hit - i.docNum1Hit = DocNum1HitFinished // consume our 1-hit docNum - return docNum, true, nil - } - - if i.Actual == nil || !i.Actual.HasNext() { - return 0, false, nil - } - - if i.postings == nil || i.postings.postings == i.ActualBM { - return i.nextDocNumAtOrAfterClean(atOrAfter) - } - - i.Actual.AdvanceIfNeeded(uint32(atOrAfter)) - - if !i.Actual.HasNext() { - // couldn't find anything - return 0, false, nil - } - - n := i.Actual.Next() - allN := i.all.Next() - nChunk := n / uint32(i.postings.chunkSize) - - // when allN becomes >= to here, then allN is in the same chunk as nChunk. - allNReachesNChunk := nChunk * uint32(i.postings.chunkSize) - - // n is the next actual hit (excluding some postings), and - // allN is the next hit in the full postings, and - // if they don't match, move 'all' forwards until they do - for allN != n { - // we've reached same chunk, so move the freq/norm/loc decoders forward - if i.includeFreqNorm && allN >= allNReachesNChunk { - err := i.currChunkNext(nChunk) - if err != nil { - return 0, false, err - } - } - - allN = i.all.Next() - } - - if i.includeFreqNorm && (i.currChunk != nChunk || i.freqNormReader.isNil()) { - err := i.loadChunk(int(nChunk)) - if err != nil { - return 0, false, fmt.Errorf("error loading chunk: %v", err) - } - } - - return uint64(n), true, nil -} - -// optimization when the postings list is "clean" (e.g., no updates & -// no deletions) where the all bitmap is the same as the actual bitmap -func (i *PostingsIterator) nextDocNumAtOrAfterClean( - atOrAfter uint64) (uint64, bool, error) { - - if !i.includeFreqNorm { - i.Actual.AdvanceIfNeeded(uint32(atOrAfter)) - - if !i.Actual.HasNext() { - return 0, false, nil // couldn't find anything - } - - return uint64(i.Actual.Next()), true, nil - } - - // freq-norm's needed, so maintain freq-norm chunk reader - sameChunkNexts := 0 // # of times we called Next() in the same chunk - n := i.Actual.Next() - nChunk := n / uint32(i.postings.chunkSize) - - for uint64(n) < atOrAfter && i.Actual.HasNext() { - n = i.Actual.Next() - - nChunkPrev := nChunk - nChunk = n / uint32(i.postings.chunkSize) - - if nChunk != nChunkPrev { - sameChunkNexts = 0 - } else { - sameChunkNexts += 1 - } - } - - if uint64(n) < atOrAfter { - // couldn't find anything - return 0, false, nil - } - - for j := 0; j < sameChunkNexts; j++ { - err := i.currChunkNext(nChunk) - if err != nil { - return 0, false, fmt.Errorf("error optimized currChunkNext: %v", err) - } - } - - if i.currChunk != nChunk || i.freqNormReader.isNil() { - err := i.loadChunk(int(nChunk)) - if err != nil { - return 0, false, fmt.Errorf("error loading chunk: %v", err) - } - } - - return uint64(n), true, nil -} - -func (i *PostingsIterator) currChunkNext(nChunk uint32) error { - if i.currChunk != nChunk || i.freqNormReader.isNil() { - err := i.loadChunk(int(nChunk)) - if err != nil { - return fmt.Errorf("error loading chunk: %v", err) - } - } - - // read off freq/offsets even though we don't care about them - hasLocs, err := i.skipFreqNormReadHasLocs() - if err != nil { - return err - } - - if i.includeLocs && hasLocs { - numLocsBytes, err := i.locReader.readUvarint() - if err != nil { - return fmt.Errorf("error reading location numLocsBytes: %v", err) - } - - // skip over all the location bytes - i.locReader.SkipBytes(int(numLocsBytes)) - } - - return nil -} - -// DocNum1Hit returns the docNum and true if this is "1-hit" optimized -// and the docNum is available. -func (p *PostingsIterator) DocNum1Hit() (uint64, bool) { - if p.normBits1Hit != 0 && p.docNum1Hit != DocNum1HitFinished { - return p.docNum1Hit, true - } - return 0, false -} - -// ActualBitmap returns the underlying actual bitmap -// which can be used up the stack for optimizations -func (p *PostingsIterator) ActualBitmap() *roaring.Bitmap { - return p.ActualBM -} - -// ReplaceActual replaces the ActualBM with the provided -// bitmap -func (p *PostingsIterator) ReplaceActual(abm *roaring.Bitmap) { - p.ActualBM = abm - p.Actual = abm.Iterator() -} - -// PostingsIteratorFromBitmap constructs a PostingsIterator given an -// "actual" bitmap. -func PostingsIteratorFromBitmap(bm *roaring.Bitmap, - includeFreqNorm, includeLocs bool) (segment.PostingsIterator, error) { - return &PostingsIterator{ - ActualBM: bm, - Actual: bm.Iterator(), - includeFreqNorm: includeFreqNorm, - includeLocs: includeLocs, - }, nil -} - -// PostingsIteratorFrom1Hit constructs a PostingsIterator given a -// 1-hit docNum. -func PostingsIteratorFrom1Hit(docNum1Hit uint64, - includeFreqNorm, includeLocs bool) (segment.PostingsIterator, error) { - return &PostingsIterator{ - docNum1Hit: docNum1Hit, - normBits1Hit: NormBits1Hit, - includeFreqNorm: includeFreqNorm, - includeLocs: includeLocs, - }, nil -} - -// Posting is a single entry in a postings list -type Posting struct { - docNum uint64 - freq uint64 - norm float32 - locs []segment.Location -} - -func (p *Posting) Size() int { - sizeInBytes := reflectStaticSizePosting - - for _, entry := range p.locs { - sizeInBytes += entry.Size() - } - - return sizeInBytes -} - -// Number returns the document number of this posting in this segment -func (p *Posting) Number() uint64 { - return p.docNum -} - -// Frequency returns the frequencies of occurrence of this term in this doc/field -func (p *Posting) Frequency() uint64 { - return p.freq -} - -// Norm returns the normalization factor for this posting -func (p *Posting) Norm() float64 { - return float64(p.norm) -} - -// Locations returns the location information for each occurrence -func (p *Posting) Locations() []segment.Location { - return p.locs -} - -// Location represents the location of a single occurrence -type Location struct { - field string - pos uint64 - start uint64 - end uint64 - ap []uint64 -} - -func (l *Location) Size() int { - return reflectStaticSizeLocation + - len(l.field) + - len(l.ap)*size.SizeOfUint64 -} - -// Field returns the name of the field (useful in composite fields to know -// which original field the value came from) -func (l *Location) Field() string { - return l.field -} - -// Start returns the start byte offset of this occurrence -func (l *Location) Start() uint64 { - return l.start -} - -// End returns the end byte offset of this occurrence -func (l *Location) End() uint64 { - return l.end -} - -// Pos returns the 1-based phrase position of this occurrence -func (l *Location) Pos() uint64 { - return l.pos -} - -// ArrayPositions returns the array position vector associated with this occurrence -func (l *Location) ArrayPositions() []uint64 { - return l.ap -} diff --git a/vendor/github.com/blevesearch/zap/v14/segment.go b/vendor/github.com/blevesearch/zap/v14/segment.go deleted file mode 100644 index e8b1f067..00000000 --- a/vendor/github.com/blevesearch/zap/v14/segment.go +++ /dev/null @@ -1,572 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "bytes" - "encoding/binary" - "fmt" - "io" - "os" - "sync" - "unsafe" - - "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/blevesearch/bleve/size" - "github.com/couchbase/vellum" - mmap "github.com/blevesearch/mmap-go" - "github.com/golang/snappy" -) - -var reflectStaticSizeSegmentBase int - -func init() { - var sb SegmentBase - reflectStaticSizeSegmentBase = int(unsafe.Sizeof(sb)) -} - -// Open returns a zap impl of a segment -func (*ZapPlugin) Open(path string) (segment.Segment, error) { - f, err := os.Open(path) - if err != nil { - return nil, err - } - mm, err := mmap.Map(f, mmap.RDONLY, 0) - if err != nil { - // mmap failed, try to close the file - _ = f.Close() - return nil, err - } - - rv := &Segment{ - SegmentBase: SegmentBase{ - mem: mm[0 : len(mm)-FooterSize], - fieldsMap: make(map[string]uint16), - fieldDvReaders: make(map[uint16]*docValueReader), - fieldFSTs: make(map[uint16]*vellum.FST), - }, - f: f, - mm: mm, - path: path, - refs: 1, - } - rv.SegmentBase.updateSize() - - err = rv.loadConfig() - if err != nil { - _ = rv.Close() - return nil, err - } - - err = rv.loadFields() - if err != nil { - _ = rv.Close() - return nil, err - } - - err = rv.loadDvReaders() - if err != nil { - _ = rv.Close() - return nil, err - } - - return rv, nil -} - -// SegmentBase is a memory only, read-only implementation of the -// segment.Segment interface, using zap's data representation. -type SegmentBase struct { - mem []byte - memCRC uint32 - chunkMode uint32 - fieldsMap map[string]uint16 // fieldName -> fieldID+1 - fieldsInv []string // fieldID -> fieldName - numDocs uint64 - storedIndexOffset uint64 - fieldsIndexOffset uint64 - docValueOffset uint64 - dictLocs []uint64 - fieldDvReaders map[uint16]*docValueReader // naive chunk cache per field - fieldDvNames []string // field names cached in fieldDvReaders - size uint64 - - m sync.Mutex - fieldFSTs map[uint16]*vellum.FST -} - -func (sb *SegmentBase) Size() int { - return int(sb.size) -} - -func (sb *SegmentBase) updateSize() { - sizeInBytes := reflectStaticSizeSegmentBase + - cap(sb.mem) - - // fieldsMap - for k := range sb.fieldsMap { - sizeInBytes += (len(k) + size.SizeOfString) + size.SizeOfUint16 - } - - // fieldsInv, dictLocs - for _, entry := range sb.fieldsInv { - sizeInBytes += len(entry) + size.SizeOfString - } - sizeInBytes += len(sb.dictLocs) * size.SizeOfUint64 - - // fieldDvReaders - for _, v := range sb.fieldDvReaders { - sizeInBytes += size.SizeOfUint16 + size.SizeOfPtr - if v != nil { - sizeInBytes += v.size() - } - } - - sb.size = uint64(sizeInBytes) -} - -func (sb *SegmentBase) AddRef() {} -func (sb *SegmentBase) DecRef() (err error) { return nil } -func (sb *SegmentBase) Close() (err error) { return nil } - -// Segment implements a persisted segment.Segment interface, by -// embedding an mmap()'ed SegmentBase. -type Segment struct { - SegmentBase - - f *os.File - mm mmap.MMap - path string - version uint32 - crc uint32 - - m sync.Mutex // Protects the fields that follow. - refs int64 -} - -func (s *Segment) Size() int { - // 8 /* size of file pointer */ - // 4 /* size of version -> uint32 */ - // 4 /* size of crc -> uint32 */ - sizeOfUints := 16 - - sizeInBytes := (len(s.path) + size.SizeOfString) + sizeOfUints - - // mutex, refs -> int64 - sizeInBytes += 16 - - // do not include the mmap'ed part - return sizeInBytes + s.SegmentBase.Size() - cap(s.mem) -} - -func (s *Segment) AddRef() { - s.m.Lock() - s.refs++ - s.m.Unlock() -} - -func (s *Segment) DecRef() (err error) { - s.m.Lock() - s.refs-- - if s.refs == 0 { - err = s.closeActual() - } - s.m.Unlock() - return err -} - -func (s *Segment) loadConfig() error { - crcOffset := len(s.mm) - 4 - s.crc = binary.BigEndian.Uint32(s.mm[crcOffset : crcOffset+4]) - - verOffset := crcOffset - 4 - s.version = binary.BigEndian.Uint32(s.mm[verOffset : verOffset+4]) - if s.version != Version { - return fmt.Errorf("unsupported version %d", s.version) - } - - chunkOffset := verOffset - 4 - s.chunkMode = binary.BigEndian.Uint32(s.mm[chunkOffset : chunkOffset+4]) - - docValueOffset := chunkOffset - 8 - s.docValueOffset = binary.BigEndian.Uint64(s.mm[docValueOffset : docValueOffset+8]) - - fieldsIndexOffset := docValueOffset - 8 - s.fieldsIndexOffset = binary.BigEndian.Uint64(s.mm[fieldsIndexOffset : fieldsIndexOffset+8]) - - storedIndexOffset := fieldsIndexOffset - 8 - s.storedIndexOffset = binary.BigEndian.Uint64(s.mm[storedIndexOffset : storedIndexOffset+8]) - - numDocsOffset := storedIndexOffset - 8 - s.numDocs = binary.BigEndian.Uint64(s.mm[numDocsOffset : numDocsOffset+8]) - return nil -} - -func (s *SegmentBase) loadFields() error { - // NOTE for now we assume the fields index immediately precedes - // the footer, and if this changes, need to adjust accordingly (or - // store explicit length), where s.mem was sliced from s.mm in Open(). - fieldsIndexEnd := uint64(len(s.mem)) - - // iterate through fields index - var fieldID uint64 - for s.fieldsIndexOffset+(8*fieldID) < fieldsIndexEnd { - addr := binary.BigEndian.Uint64(s.mem[s.fieldsIndexOffset+(8*fieldID) : s.fieldsIndexOffset+(8*fieldID)+8]) - - dictLoc, read := binary.Uvarint(s.mem[addr:fieldsIndexEnd]) - n := uint64(read) - s.dictLocs = append(s.dictLocs, dictLoc) - - var nameLen uint64 - nameLen, read = binary.Uvarint(s.mem[addr+n : fieldsIndexEnd]) - n += uint64(read) - - name := string(s.mem[addr+n : addr+n+nameLen]) - s.fieldsInv = append(s.fieldsInv, name) - s.fieldsMap[name] = uint16(fieldID + 1) - - fieldID++ - } - return nil -} - -// Dictionary returns the term dictionary for the specified field -func (s *SegmentBase) Dictionary(field string) (segment.TermDictionary, error) { - dict, err := s.dictionary(field) - if err == nil && dict == nil { - return &segment.EmptyDictionary{}, nil - } - return dict, err -} - -func (sb *SegmentBase) dictionary(field string) (rv *Dictionary, err error) { - fieldIDPlus1 := sb.fieldsMap[field] - if fieldIDPlus1 > 0 { - rv = &Dictionary{ - sb: sb, - field: field, - fieldID: fieldIDPlus1 - 1, - } - - dictStart := sb.dictLocs[rv.fieldID] - if dictStart > 0 { - var ok bool - sb.m.Lock() - if rv.fst, ok = sb.fieldFSTs[rv.fieldID]; !ok { - // read the length of the vellum data - vellumLen, read := binary.Uvarint(sb.mem[dictStart : dictStart+binary.MaxVarintLen64]) - fstBytes := sb.mem[dictStart+uint64(read) : dictStart+uint64(read)+vellumLen] - rv.fst, err = vellum.Load(fstBytes) - if err != nil { - sb.m.Unlock() - return nil, fmt.Errorf("dictionary field %s vellum err: %v", field, err) - } - - sb.fieldFSTs[rv.fieldID] = rv.fst - } - - sb.m.Unlock() - rv.fstReader, err = rv.fst.Reader() - if err != nil { - return nil, fmt.Errorf("dictionary field %s vellum reader err: %v", field, err) - } - - } - } - - return rv, nil -} - -// visitDocumentCtx holds data structures that are reusable across -// multiple VisitDocument() calls to avoid memory allocations -type visitDocumentCtx struct { - buf []byte - reader bytes.Reader - arrayPos []uint64 -} - -var visitDocumentCtxPool = sync.Pool{ - New: func() interface{} { - reuse := &visitDocumentCtx{} - return reuse - }, -} - -// VisitDocument invokes the DocFieldValueVistor for each stored field -// for the specified doc number -func (s *SegmentBase) VisitDocument(num uint64, visitor segment.DocumentFieldValueVisitor) error { - vdc := visitDocumentCtxPool.Get().(*visitDocumentCtx) - defer visitDocumentCtxPool.Put(vdc) - return s.visitDocument(vdc, num, visitor) -} - -func (s *SegmentBase) visitDocument(vdc *visitDocumentCtx, num uint64, - visitor segment.DocumentFieldValueVisitor) error { - // first make sure this is a valid number in this segment - if num < s.numDocs { - meta, compressed := s.getDocStoredMetaAndCompressed(num) - - vdc.reader.Reset(meta) - - // handle _id field special case - idFieldValLen, err := binary.ReadUvarint(&vdc.reader) - if err != nil { - return err - } - idFieldVal := compressed[:idFieldValLen] - - keepGoing := visitor("_id", byte('t'), idFieldVal, nil) - if !keepGoing { - visitDocumentCtxPool.Put(vdc) - return nil - } - - // handle non-"_id" fields - compressed = compressed[idFieldValLen:] - - uncompressed, err := snappy.Decode(vdc.buf[:cap(vdc.buf)], compressed) - if err != nil { - return err - } - - for keepGoing { - field, err := binary.ReadUvarint(&vdc.reader) - if err == io.EOF { - break - } - if err != nil { - return err - } - typ, err := binary.ReadUvarint(&vdc.reader) - if err != nil { - return err - } - offset, err := binary.ReadUvarint(&vdc.reader) - if err != nil { - return err - } - l, err := binary.ReadUvarint(&vdc.reader) - if err != nil { - return err - } - numap, err := binary.ReadUvarint(&vdc.reader) - if err != nil { - return err - } - var arrayPos []uint64 - if numap > 0 { - if cap(vdc.arrayPos) < int(numap) { - vdc.arrayPos = make([]uint64, numap) - } - arrayPos = vdc.arrayPos[:numap] - for i := 0; i < int(numap); i++ { - ap, err := binary.ReadUvarint(&vdc.reader) - if err != nil { - return err - } - arrayPos[i] = ap - } - } - - value := uncompressed[offset : offset+l] - keepGoing = visitor(s.fieldsInv[field], byte(typ), value, arrayPos) - } - - vdc.buf = uncompressed - } - return nil -} - -// DocID returns the value of the _id field for the given docNum -func (s *SegmentBase) DocID(num uint64) ([]byte, error) { - if num >= s.numDocs { - return nil, nil - } - - vdc := visitDocumentCtxPool.Get().(*visitDocumentCtx) - - meta, compressed := s.getDocStoredMetaAndCompressed(num) - - vdc.reader.Reset(meta) - - // handle _id field special case - idFieldValLen, err := binary.ReadUvarint(&vdc.reader) - if err != nil { - return nil, err - } - idFieldVal := compressed[:idFieldValLen] - - visitDocumentCtxPool.Put(vdc) - - return idFieldVal, nil -} - -// Count returns the number of documents in this segment. -func (s *SegmentBase) Count() uint64 { - return s.numDocs -} - -// DocNumbers returns a bitset corresponding to the doc numbers of all the -// provided _id strings -func (s *SegmentBase) DocNumbers(ids []string) (*roaring.Bitmap, error) { - rv := roaring.New() - - if len(s.fieldsMap) > 0 { - idDict, err := s.dictionary("_id") - if err != nil { - return nil, err - } - - postingsList := emptyPostingsList - - sMax, err := idDict.fst.GetMaxKey() - if err != nil { - return nil, err - } - sMaxStr := string(sMax) - filteredIds := make([]string, 0, len(ids)) - for _, id := range ids { - if id <= sMaxStr { - filteredIds = append(filteredIds, id) - } - } - - for _, id := range filteredIds { - postingsList, err = idDict.postingsList([]byte(id), nil, postingsList) - if err != nil { - return nil, err - } - postingsList.OrInto(rv) - } - } - - return rv, nil -} - -// Fields returns the field names used in this segment -func (s *SegmentBase) Fields() []string { - return s.fieldsInv -} - -// Path returns the path of this segment on disk -func (s *Segment) Path() string { - return s.path -} - -// Close releases all resources associated with this segment -func (s *Segment) Close() (err error) { - return s.DecRef() -} - -func (s *Segment) closeActual() (err error) { - if s.mm != nil { - err = s.mm.Unmap() - } - // try to close file even if unmap failed - if s.f != nil { - err2 := s.f.Close() - if err == nil { - // try to return first error - err = err2 - } - } - return -} - -// some helpers i started adding for the command-line utility - -// Data returns the underlying mmaped data slice -func (s *Segment) Data() []byte { - return s.mm -} - -// CRC returns the CRC value stored in the file footer -func (s *Segment) CRC() uint32 { - return s.crc -} - -// Version returns the file version in the file footer -func (s *Segment) Version() uint32 { - return s.version -} - -// ChunkFactor returns the chunk factor in the file footer -func (s *Segment) ChunkMode() uint32 { - return s.chunkMode -} - -// FieldsIndexOffset returns the fields index offset in the file footer -func (s *Segment) FieldsIndexOffset() uint64 { - return s.fieldsIndexOffset -} - -// StoredIndexOffset returns the stored value index offset in the file footer -func (s *Segment) StoredIndexOffset() uint64 { - return s.storedIndexOffset -} - -// DocValueOffset returns the docValue offset in the file footer -func (s *Segment) DocValueOffset() uint64 { - return s.docValueOffset -} - -// NumDocs returns the number of documents in the file footer -func (s *Segment) NumDocs() uint64 { - return s.numDocs -} - -// DictAddr is a helper function to compute the file offset where the -// dictionary is stored for the specified field. -func (s *Segment) DictAddr(field string) (uint64, error) { - fieldIDPlus1, ok := s.fieldsMap[field] - if !ok { - return 0, fmt.Errorf("no such field '%s'", field) - } - - return s.dictLocs[fieldIDPlus1-1], nil -} - -func (s *SegmentBase) loadDvReaders() error { - if s.docValueOffset == fieldNotUninverted || s.numDocs == 0 { - return nil - } - - var read uint64 - for fieldID, field := range s.fieldsInv { - var fieldLocStart, fieldLocEnd uint64 - var n int - fieldLocStart, n = binary.Uvarint(s.mem[s.docValueOffset+read : s.docValueOffset+read+binary.MaxVarintLen64]) - if n <= 0 { - return fmt.Errorf("loadDvReaders: failed to read the docvalue offset start for field %d", fieldID) - } - read += uint64(n) - fieldLocEnd, n = binary.Uvarint(s.mem[s.docValueOffset+read : s.docValueOffset+read+binary.MaxVarintLen64]) - if n <= 0 { - return fmt.Errorf("loadDvReaders: failed to read the docvalue offset end for field %d", fieldID) - } - read += uint64(n) - - fieldDvReader, err := s.loadFieldDocValueReader(field, fieldLocStart, fieldLocEnd) - if err != nil { - return err - } - if fieldDvReader != nil { - s.fieldDvReaders[uint16(fieldID)] = fieldDvReader - s.fieldDvNames = append(s.fieldDvNames, field) - } - } - - return nil -} diff --git a/vendor/github.com/blevesearch/zap/v15/README.md b/vendor/github.com/blevesearch/zap/v15/README.md deleted file mode 100644 index 0facb669..00000000 --- a/vendor/github.com/blevesearch/zap/v15/README.md +++ /dev/null @@ -1,158 +0,0 @@ -# zap file format - -Advanced ZAP File Format Documentation is [here](zap.md). - -The file is written in the reverse order that we typically access data. This helps us write in one pass since later sections of the file require file offsets of things we've already written. - -Current usage: - -- mmap the entire file -- crc-32 bytes and version are in fixed position at end of the file -- reading remainder of footer could be version specific -- remainder of footer gives us: - - 3 important offsets (docValue , fields index and stored data index) - - 2 important values (number of docs and chunk factor) -- field data is processed once and memoized onto the heap so that we never have to go back to disk for it -- access to stored data by doc number means first navigating to the stored data index, then accessing a fixed position offset into that slice, which gives us the actual address of the data. the first bytes of that section tell us the size of data so that we know where it ends. -- access to all other indexed data follows the following pattern: - - first know the field name -> convert to id - - next navigate to term dictionary for that field - - some operations stop here and do dictionary ops - - next use dictionary to navigate to posting list for a specific term - - walk posting list - - if necessary, walk posting details as we go - - if location info is desired, consult location bitmap to see if it is there - -## stored fields section - -- for each document - - preparation phase: - - produce a slice of metadata bytes and data bytes - - produce these slices in field id order - - field value is appended to the data slice - - metadata slice is varint encoded with the following values for each field value - - field id (uint16) - - field type (byte) - - field value start offset in uncompressed data slice (uint64) - - field value length (uint64) - - field number of array positions (uint64) - - one additional value for each array position (uint64) - - compress the data slice using snappy - - file writing phase: - - remember the start offset for this document - - write out meta data length (varint uint64) - - write out compressed data length (varint uint64) - - write out the metadata bytes - - write out the compressed data bytes - -## stored fields idx - -- for each document - - write start offset (remembered from previous section) of stored data (big endian uint64) - -With this index and a known document number, we have direct access to all the stored field data. - -## posting details (freq/norm) section - -- for each posting list - - produce a slice containing multiple consecutive chunks (each chunk is varint stream) - - produce a slice remembering offsets of where each chunk starts - - preparation phase: - - for each hit in the posting list - - if this hit is in next chunk close out encoding of last chunk and record offset start of next - - encode term frequency (uint64) - - encode norm factor (float32) - - file writing phase: - - remember start position for this posting list details - - write out number of chunks that follow (varint uint64) - - write out length of each chunk (each a varint uint64) - - write out the byte slice containing all the chunk data - -If you know the doc number you're interested in, this format lets you jump to the correct chunk (docNum/chunkFactor) directly and then seek within that chunk until you find it. - -## posting details (location) section - -- for each posting list - - produce a slice containing multiple consecutive chunks (each chunk is varint stream) - - produce a slice remembering offsets of where each chunk starts - - preparation phase: - - for each hit in the posting list - - if this hit is in next chunk close out encoding of last chunk and record offset start of next - - encode field (uint16) - - encode field pos (uint64) - - encode field start (uint64) - - encode field end (uint64) - - encode number of array positions to follow (uint64) - - encode each array position (each uint64) - - file writing phase: - - remember start position for this posting list details - - write out number of chunks that follow (varint uint64) - - write out length of each chunk (each a varint uint64) - - write out the byte slice containing all the chunk data - -If you know the doc number you're interested in, this format lets you jump to the correct chunk (docNum/chunkFactor) directly and then seek within that chunk until you find it. - -## postings list section - -- for each posting list - - preparation phase: - - encode roaring bitmap posting list to bytes (so we know the length) - - file writing phase: - - remember the start position for this posting list - - write freq/norm details offset (remembered from previous, as varint uint64) - - write location details offset (remembered from previous, as varint uint64) - - write length of encoded roaring bitmap - - write the serialized roaring bitmap data - -## dictionary - -- for each field - - preparation phase: - - encode vellum FST with dictionary data pointing to file offset of posting list (remembered from previous) - - file writing phase: - - remember the start position of this persistDictionary - - write length of vellum data (varint uint64) - - write out vellum data - -## fields section - -- for each field - - file writing phase: - - remember start offset for each field - - write dictionary address (remembered from previous) (varint uint64) - - write length of field name (varint uint64) - - write field name bytes - -## fields idx - -- for each field - - file writing phase: - - write big endian uint64 of start offset for each field - -NOTE: currently we don't know or record the length of this fields index. Instead we rely on the fact that we know it immediately precedes a footer of known size. - -## fields DocValue - -- for each field - - preparation phase: - - produce a slice containing multiple consecutive chunks, where each chunk is composed of a meta section followed by compressed columnar field data - - produce a slice remembering the length of each chunk - - file writing phase: - - remember the start position of this first field DocValue offset in the footer - - write out number of chunks that follow (varint uint64) - - write out length of each chunk (each a varint uint64) - - write out the byte slice containing all the chunk data - -NOTE: currently the meta header inside each chunk gives clue to the location offsets and size of the data pertaining to a given docID and any -read operation leverage that meta information to extract the document specific data from the file. - -## footer - -- file writing phase - - write number of docs (big endian uint64) - - write stored field index location (big endian uint64) - - write field index location (big endian uint64) - - write field docValue location (big endian uint64) - - write out chunk factor (big endian uint32) - - write out version (big endian uint32) - - write out file CRC of everything preceding this (big endian uint32) diff --git a/vendor/github.com/blevesearch/zap/v15/build.go b/vendor/github.com/blevesearch/zap/v15/build.go deleted file mode 100644 index 1b731837..00000000 --- a/vendor/github.com/blevesearch/zap/v15/build.go +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "bufio" - "math" - "os" - - "github.com/couchbase/vellum" -) - -const Version uint32 = 15 - -const Type string = "zap" - -const fieldNotUninverted = math.MaxUint64 - -func (sb *SegmentBase) Persist(path string) error { - return PersistSegmentBase(sb, path) -} - -// PersistSegmentBase persists SegmentBase in the zap file format. -func PersistSegmentBase(sb *SegmentBase, path string) error { - flag := os.O_RDWR | os.O_CREATE - - f, err := os.OpenFile(path, flag, 0600) - if err != nil { - return err - } - - cleanup := func() { - _ = f.Close() - _ = os.Remove(path) - } - - br := bufio.NewWriter(f) - - _, err = br.Write(sb.mem) - if err != nil { - cleanup() - return err - } - - err = persistFooter(sb.numDocs, sb.storedIndexOffset, sb.fieldsIndexOffset, sb.docValueOffset, - sb.chunkMode, sb.memCRC, br) - if err != nil { - cleanup() - return err - } - - err = br.Flush() - if err != nil { - cleanup() - return err - } - - err = f.Sync() - if err != nil { - cleanup() - return err - } - - err = f.Close() - if err != nil { - cleanup() - return err - } - - return nil -} - -func persistStoredFieldValues(fieldID int, - storedFieldValues [][]byte, stf []byte, spf [][]uint64, - curr int, metaEncode varintEncoder, data []byte) ( - int, []byte, error) { - for i := 0; i < len(storedFieldValues); i++ { - // encode field - _, err := metaEncode(uint64(fieldID)) - if err != nil { - return 0, nil, err - } - // encode type - _, err = metaEncode(uint64(stf[i])) - if err != nil { - return 0, nil, err - } - // encode start offset - _, err = metaEncode(uint64(curr)) - if err != nil { - return 0, nil, err - } - // end len - _, err = metaEncode(uint64(len(storedFieldValues[i]))) - if err != nil { - return 0, nil, err - } - // encode number of array pos - _, err = metaEncode(uint64(len(spf[i]))) - if err != nil { - return 0, nil, err - } - // encode all array positions - for _, pos := range spf[i] { - _, err = metaEncode(pos) - if err != nil { - return 0, nil, err - } - } - - data = append(data, storedFieldValues[i]...) - curr += len(storedFieldValues[i]) - } - - return curr, data, nil -} - -func InitSegmentBase(mem []byte, memCRC uint32, chunkMode uint32, - fieldsMap map[string]uint16, fieldsInv []string, numDocs uint64, - storedIndexOffset uint64, fieldsIndexOffset uint64, docValueOffset uint64, - dictLocs []uint64) (*SegmentBase, error) { - sb := &SegmentBase{ - mem: mem, - memCRC: memCRC, - chunkMode: chunkMode, - fieldsMap: fieldsMap, - fieldsInv: fieldsInv, - numDocs: numDocs, - storedIndexOffset: storedIndexOffset, - fieldsIndexOffset: fieldsIndexOffset, - docValueOffset: docValueOffset, - dictLocs: dictLocs, - fieldDvReaders: make(map[uint16]*docValueReader), - fieldFSTs: make(map[uint16]*vellum.FST), - } - sb.updateSize() - - err := sb.loadDvReaders() - if err != nil { - return nil, err - } - - return sb, nil -} diff --git a/vendor/github.com/blevesearch/zap/v15/count.go b/vendor/github.com/blevesearch/zap/v15/count.go deleted file mode 100644 index 50290f88..00000000 --- a/vendor/github.com/blevesearch/zap/v15/count.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "hash/crc32" - "io" - - "github.com/blevesearch/bleve/index/scorch/segment" -) - -// CountHashWriter is a wrapper around a Writer which counts the number of -// bytes which have been written and computes a crc32 hash -type CountHashWriter struct { - w io.Writer - crc uint32 - n int - s segment.StatsReporter -} - -// NewCountHashWriter returns a CountHashWriter which wraps the provided Writer -func NewCountHashWriter(w io.Writer) *CountHashWriter { - return &CountHashWriter{w: w} -} - -func NewCountHashWriterWithStatsReporter(w io.Writer, s segment.StatsReporter) *CountHashWriter { - return &CountHashWriter{w: w, s: s} -} - -// Write writes the provided bytes to the wrapped writer and counts the bytes -func (c *CountHashWriter) Write(b []byte) (int, error) { - n, err := c.w.Write(b) - c.crc = crc32.Update(c.crc, crc32.IEEETable, b[:n]) - c.n += n - if c.s != nil { - c.s.ReportBytesWritten(uint64(n)) - } - return n, err -} - -// Count returns the number of bytes written -func (c *CountHashWriter) Count() int { - return c.n -} - -// Sum32 returns the CRC-32 hash of the content written to this writer -func (c *CountHashWriter) Sum32() uint32 { - return c.crc -} diff --git a/vendor/github.com/blevesearch/zap/v15/dict.go b/vendor/github.com/blevesearch/zap/v15/dict.go deleted file mode 100644 index ad4a8f8d..00000000 --- a/vendor/github.com/blevesearch/zap/v15/dict.go +++ /dev/null @@ -1,263 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "bytes" - "fmt" - - "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/couchbase/vellum" -) - -// Dictionary is the zap representation of the term dictionary -type Dictionary struct { - sb *SegmentBase - field string - fieldID uint16 - fst *vellum.FST - fstReader *vellum.Reader -} - -// PostingsList returns the postings list for the specified term -func (d *Dictionary) PostingsList(term []byte, except *roaring.Bitmap, - prealloc segment.PostingsList) (segment.PostingsList, error) { - var preallocPL *PostingsList - pl, ok := prealloc.(*PostingsList) - if ok && pl != nil { - preallocPL = pl - } - return d.postingsList(term, except, preallocPL) -} - -func (d *Dictionary) postingsList(term []byte, except *roaring.Bitmap, rv *PostingsList) (*PostingsList, error) { - if d.fstReader == nil { - if rv == nil || rv == emptyPostingsList { - return emptyPostingsList, nil - } - return d.postingsListInit(rv, except), nil - } - - postingsOffset, exists, err := d.fstReader.Get(term) - if err != nil { - return nil, fmt.Errorf("vellum err: %v", err) - } - if !exists { - if rv == nil || rv == emptyPostingsList { - return emptyPostingsList, nil - } - return d.postingsListInit(rv, except), nil - } - - return d.postingsListFromOffset(postingsOffset, except, rv) -} - -func (d *Dictionary) postingsListFromOffset(postingsOffset uint64, except *roaring.Bitmap, rv *PostingsList) (*PostingsList, error) { - rv = d.postingsListInit(rv, except) - - err := rv.read(postingsOffset, d) - if err != nil { - return nil, err - } - - return rv, nil -} - -func (d *Dictionary) postingsListInit(rv *PostingsList, except *roaring.Bitmap) *PostingsList { - if rv == nil || rv == emptyPostingsList { - rv = &PostingsList{} - } else { - postings := rv.postings - if postings != nil { - postings.Clear() - } - - *rv = PostingsList{} // clear the struct - - rv.postings = postings - } - rv.sb = d.sb - rv.except = except - return rv -} - -func (d *Dictionary) Contains(key []byte) (bool, error) { - return d.fst.Contains(key) -} - -// Iterator returns an iterator for this dictionary -func (d *Dictionary) Iterator() segment.DictionaryIterator { - rv := &DictionaryIterator{ - d: d, - } - - if d.fst != nil { - itr, err := d.fst.Iterator(nil, nil) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - } - - return rv -} - -// PrefixIterator returns an iterator which only visits terms having the -// the specified prefix -func (d *Dictionary) PrefixIterator(prefix string) segment.DictionaryIterator { - rv := &DictionaryIterator{ - d: d, - } - - kBeg := []byte(prefix) - kEnd := segment.IncrementBytes(kBeg) - - if d.fst != nil { - itr, err := d.fst.Iterator(kBeg, kEnd) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - } - - return rv -} - -// RangeIterator returns an iterator which only visits terms between the -// start and end terms. NOTE: bleve.index API specifies the end is inclusive. -func (d *Dictionary) RangeIterator(start, end string) segment.DictionaryIterator { - rv := &DictionaryIterator{ - d: d, - } - - // need to increment the end position to be inclusive - var endBytes []byte - if len(end) > 0 { - endBytes = []byte(end) - if endBytes[len(endBytes)-1] < 0xff { - endBytes[len(endBytes)-1]++ - } else { - endBytes = append(endBytes, 0xff) - } - } - - if d.fst != nil { - itr, err := d.fst.Iterator([]byte(start), endBytes) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - } - - return rv -} - -// AutomatonIterator returns an iterator which only visits terms -// having the the vellum automaton and start/end key range -func (d *Dictionary) AutomatonIterator(a vellum.Automaton, - startKeyInclusive, endKeyExclusive []byte) segment.DictionaryIterator { - rv := &DictionaryIterator{ - d: d, - } - - if d.fst != nil { - itr, err := d.fst.Search(a, startKeyInclusive, endKeyExclusive) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - } - - return rv -} - -func (d *Dictionary) OnlyIterator(onlyTerms [][]byte, - includeCount bool) segment.DictionaryIterator { - - rv := &DictionaryIterator{ - d: d, - omitCount: !includeCount, - } - - var buf bytes.Buffer - builder, err := vellum.New(&buf, nil) - if err != nil { - rv.err = err - return rv - } - for _, term := range onlyTerms { - err = builder.Insert(term, 0) - if err != nil { - rv.err = err - return rv - } - } - err = builder.Close() - if err != nil { - rv.err = err - return rv - } - - onlyFST, err := vellum.Load(buf.Bytes()) - if err != nil { - rv.err = err - return rv - } - - itr, err := d.fst.Search(onlyFST, nil, nil) - if err == nil { - rv.itr = itr - } else if err != vellum.ErrIteratorDone { - rv.err = err - } - - return rv -} - -// DictionaryIterator is an iterator for term dictionary -type DictionaryIterator struct { - d *Dictionary - itr vellum.Iterator - err error - tmp PostingsList - entry index.DictEntry - omitCount bool -} - -// Next returns the next entry in the dictionary -func (i *DictionaryIterator) Next() (*index.DictEntry, error) { - if i.err != nil && i.err != vellum.ErrIteratorDone { - return nil, i.err - } else if i.itr == nil || i.err == vellum.ErrIteratorDone { - return nil, nil - } - term, postingsOffset := i.itr.Current() - i.entry.Term = string(term) - if !i.omitCount { - i.err = i.tmp.read(postingsOffset, i.d) - if i.err != nil { - return nil, i.err - } - i.entry.Count = i.tmp.Count() - } - i.err = i.itr.Next() - return &i.entry, nil -} diff --git a/vendor/github.com/blevesearch/zap/v15/docvalues.go b/vendor/github.com/blevesearch/zap/v15/docvalues.go deleted file mode 100644 index 793797bd..00000000 --- a/vendor/github.com/blevesearch/zap/v15/docvalues.go +++ /dev/null @@ -1,312 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "bytes" - "encoding/binary" - "fmt" - "math" - "reflect" - "sort" - - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/blevesearch/bleve/size" - "github.com/golang/snappy" -) - -var reflectStaticSizedocValueReader int - -func init() { - var dvi docValueReader - reflectStaticSizedocValueReader = int(reflect.TypeOf(dvi).Size()) -} - -type docNumTermsVisitor func(docNum uint64, terms []byte) error - -type docVisitState struct { - dvrs map[uint16]*docValueReader - segment *SegmentBase -} - -type docValueReader struct { - field string - curChunkNum uint64 - chunkOffsets []uint64 - dvDataLoc uint64 - curChunkHeader []MetaData - curChunkData []byte // compressed data cache - uncompressed []byte // temp buf for snappy decompression -} - -func (di *docValueReader) size() int { - return reflectStaticSizedocValueReader + size.SizeOfPtr + - len(di.field) + - len(di.chunkOffsets)*size.SizeOfUint64 + - len(di.curChunkHeader)*reflectStaticSizeMetaData + - len(di.curChunkData) -} - -func (di *docValueReader) cloneInto(rv *docValueReader) *docValueReader { - if rv == nil { - rv = &docValueReader{} - } - - rv.field = di.field - rv.curChunkNum = math.MaxUint64 - rv.chunkOffsets = di.chunkOffsets // immutable, so it's sharable - rv.dvDataLoc = di.dvDataLoc - rv.curChunkHeader = rv.curChunkHeader[:0] - rv.curChunkData = nil - rv.uncompressed = rv.uncompressed[:0] - - return rv -} - -func (di *docValueReader) curChunkNumber() uint64 { - return di.curChunkNum -} - -func (s *SegmentBase) loadFieldDocValueReader(field string, - fieldDvLocStart, fieldDvLocEnd uint64) (*docValueReader, error) { - // get the docValue offset for the given fields - if fieldDvLocStart == fieldNotUninverted { - // no docValues found, nothing to do - return nil, nil - } - - // read the number of chunks, and chunk offsets position - var numChunks, chunkOffsetsPosition uint64 - - if fieldDvLocEnd-fieldDvLocStart > 16 { - numChunks = binary.BigEndian.Uint64(s.mem[fieldDvLocEnd-8 : fieldDvLocEnd]) - // read the length of chunk offsets - chunkOffsetsLen := binary.BigEndian.Uint64(s.mem[fieldDvLocEnd-16 : fieldDvLocEnd-8]) - // acquire position of chunk offsets - chunkOffsetsPosition = (fieldDvLocEnd - 16) - chunkOffsetsLen - } else { - return nil, fmt.Errorf("loadFieldDocValueReader: fieldDvLoc too small: %d-%d", fieldDvLocEnd, fieldDvLocStart) - } - - fdvIter := &docValueReader{ - curChunkNum: math.MaxUint64, - field: field, - chunkOffsets: make([]uint64, int(numChunks)), - } - - // read the chunk offsets - var offset uint64 - for i := 0; i < int(numChunks); i++ { - loc, read := binary.Uvarint(s.mem[chunkOffsetsPosition+offset : chunkOffsetsPosition+offset+binary.MaxVarintLen64]) - if read <= 0 { - return nil, fmt.Errorf("corrupted chunk offset during segment load") - } - fdvIter.chunkOffsets[i] = loc - offset += uint64(read) - } - - // set the data offset - fdvIter.dvDataLoc = fieldDvLocStart - - return fdvIter, nil -} - -func (di *docValueReader) loadDvChunk(chunkNumber uint64, s *SegmentBase) error { - // advance to the chunk where the docValues - // reside for the given docNum - destChunkDataLoc, curChunkEnd := di.dvDataLoc, di.dvDataLoc - start, end := readChunkBoundary(int(chunkNumber), di.chunkOffsets) - if start >= end { - di.curChunkHeader = di.curChunkHeader[:0] - di.curChunkData = nil - di.curChunkNum = chunkNumber - di.uncompressed = di.uncompressed[:0] - return nil - } - - destChunkDataLoc += start - curChunkEnd += end - - // read the number of docs reside in the chunk - numDocs, read := binary.Uvarint(s.mem[destChunkDataLoc : destChunkDataLoc+binary.MaxVarintLen64]) - if read <= 0 { - return fmt.Errorf("failed to read the chunk") - } - chunkMetaLoc := destChunkDataLoc + uint64(read) - - offset := uint64(0) - if cap(di.curChunkHeader) < int(numDocs) { - di.curChunkHeader = make([]MetaData, int(numDocs)) - } else { - di.curChunkHeader = di.curChunkHeader[:int(numDocs)] - } - for i := 0; i < int(numDocs); i++ { - di.curChunkHeader[i].DocNum, read = binary.Uvarint(s.mem[chunkMetaLoc+offset : chunkMetaLoc+offset+binary.MaxVarintLen64]) - offset += uint64(read) - di.curChunkHeader[i].DocDvOffset, read = binary.Uvarint(s.mem[chunkMetaLoc+offset : chunkMetaLoc+offset+binary.MaxVarintLen64]) - offset += uint64(read) - } - - compressedDataLoc := chunkMetaLoc + offset - dataLength := curChunkEnd - compressedDataLoc - di.curChunkData = s.mem[compressedDataLoc : compressedDataLoc+dataLength] - di.curChunkNum = chunkNumber - di.uncompressed = di.uncompressed[:0] - return nil -} - -func (di *docValueReader) iterateAllDocValues(s *SegmentBase, visitor docNumTermsVisitor) error { - for i := 0; i < len(di.chunkOffsets); i++ { - err := di.loadDvChunk(uint64(i), s) - if err != nil { - return err - } - if di.curChunkData == nil || len(di.curChunkHeader) == 0 { - continue - } - - // uncompress the already loaded data - uncompressed, err := snappy.Decode(di.uncompressed[:cap(di.uncompressed)], di.curChunkData) - if err != nil { - return err - } - di.uncompressed = uncompressed - - start := uint64(0) - for _, entry := range di.curChunkHeader { - err = visitor(entry.DocNum, uncompressed[start:entry.DocDvOffset]) - if err != nil { - return err - } - - start = entry.DocDvOffset - } - } - - return nil -} - -func (di *docValueReader) visitDocValues(docNum uint64, - visitor index.DocumentFieldTermVisitor) error { - // binary search the term locations for the docNum - start, end := di.getDocValueLocs(docNum) - if start == math.MaxUint64 || end == math.MaxUint64 || start == end { - return nil - } - - var uncompressed []byte - var err error - // use the uncompressed copy if available - if len(di.uncompressed) > 0 { - uncompressed = di.uncompressed - } else { - // uncompress the already loaded data - uncompressed, err = snappy.Decode(di.uncompressed[:cap(di.uncompressed)], di.curChunkData) - if err != nil { - return err - } - di.uncompressed = uncompressed - } - - // pick the terms for the given docNum - uncompressed = uncompressed[start:end] - for { - i := bytes.Index(uncompressed, termSeparatorSplitSlice) - if i < 0 { - break - } - - visitor(di.field, uncompressed[0:i]) - uncompressed = uncompressed[i+1:] - } - - return nil -} - -func (di *docValueReader) getDocValueLocs(docNum uint64) (uint64, uint64) { - i := sort.Search(len(di.curChunkHeader), func(i int) bool { - return di.curChunkHeader[i].DocNum >= docNum - }) - if i < len(di.curChunkHeader) && di.curChunkHeader[i].DocNum == docNum { - return ReadDocValueBoundary(i, di.curChunkHeader) - } - return math.MaxUint64, math.MaxUint64 -} - -// VisitDocumentFieldTerms is an implementation of the -// DocumentFieldTermVisitable interface -func (s *SegmentBase) VisitDocumentFieldTerms(localDocNum uint64, fields []string, - visitor index.DocumentFieldTermVisitor, dvsIn segment.DocVisitState) ( - segment.DocVisitState, error) { - dvs, ok := dvsIn.(*docVisitState) - if !ok || dvs == nil { - dvs = &docVisitState{} - } else { - if dvs.segment != s { - dvs.segment = s - dvs.dvrs = nil - } - } - - var fieldIDPlus1 uint16 - if dvs.dvrs == nil { - dvs.dvrs = make(map[uint16]*docValueReader, len(fields)) - for _, field := range fields { - if fieldIDPlus1, ok = s.fieldsMap[field]; !ok { - continue - } - fieldID := fieldIDPlus1 - 1 - if dvIter, exists := s.fieldDvReaders[fieldID]; exists && - dvIter != nil { - dvs.dvrs[fieldID] = dvIter.cloneInto(dvs.dvrs[fieldID]) - } - } - } - - // find the chunkNumber where the docValues are stored - // NOTE: doc values continue to use legacy chunk mode - chunkFactor, err := getChunkSize(LegacyChunkMode, 0, 0) - if err != nil { - return nil, err - } - docInChunk := localDocNum / chunkFactor - var dvr *docValueReader - for _, field := range fields { - if fieldIDPlus1, ok = s.fieldsMap[field]; !ok { - continue - } - fieldID := fieldIDPlus1 - 1 - if dvr, ok = dvs.dvrs[fieldID]; ok && dvr != nil { - // check if the chunk is already loaded - if docInChunk != dvr.curChunkNumber() { - err := dvr.loadDvChunk(docInChunk, s) - if err != nil { - return dvs, err - } - } - - _ = dvr.visitDocValues(localDocNum, visitor) - } - } - return dvs, nil -} - -// VisitableDocValueFields returns the list of fields with -// persisted doc value terms ready to be visitable using the -// VisitDocumentFieldTerms method. -func (s *SegmentBase) VisitableDocValueFields() ([]string, error) { - return s.fieldDvNames, nil -} diff --git a/vendor/github.com/blevesearch/zap/v15/enumerator.go b/vendor/github.com/blevesearch/zap/v15/enumerator.go deleted file mode 100644 index bc5b7e62..00000000 --- a/vendor/github.com/blevesearch/zap/v15/enumerator.go +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright (c) 2018 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "bytes" - - "github.com/couchbase/vellum" -) - -// enumerator provides an ordered traversal of multiple vellum -// iterators. Like JOIN of iterators, the enumerator produces a -// sequence of (key, iteratorIndex, value) tuples, sorted by key ASC, -// then iteratorIndex ASC, where the same key might be seen or -// repeated across multiple child iterators. -type enumerator struct { - itrs []vellum.Iterator - currKs [][]byte - currVs []uint64 - - lowK []byte - lowIdxs []int - lowCurr int -} - -// newEnumerator returns a new enumerator over the vellum Iterators -func newEnumerator(itrs []vellum.Iterator) (*enumerator, error) { - rv := &enumerator{ - itrs: itrs, - currKs: make([][]byte, len(itrs)), - currVs: make([]uint64, len(itrs)), - lowIdxs: make([]int, 0, len(itrs)), - } - for i, itr := range rv.itrs { - rv.currKs[i], rv.currVs[i] = itr.Current() - } - rv.updateMatches(false) - if rv.lowK == nil && len(rv.lowIdxs) == 0 { - return rv, vellum.ErrIteratorDone - } - return rv, nil -} - -// updateMatches maintains the low key matches based on the currKs -func (m *enumerator) updateMatches(skipEmptyKey bool) { - m.lowK = nil - m.lowIdxs = m.lowIdxs[:0] - m.lowCurr = 0 - - for i, key := range m.currKs { - if (key == nil && m.currVs[i] == 0) || // in case of empty iterator - (len(key) == 0 && skipEmptyKey) { // skip empty keys - continue - } - - cmp := bytes.Compare(key, m.lowK) - if cmp < 0 || len(m.lowIdxs) == 0 { - // reached a new low - m.lowK = key - m.lowIdxs = m.lowIdxs[:0] - m.lowIdxs = append(m.lowIdxs, i) - } else if cmp == 0 { - m.lowIdxs = append(m.lowIdxs, i) - } - } -} - -// Current returns the enumerator's current key, iterator-index, and -// value. If the enumerator is not pointing at a valid value (because -// Next returned an error previously), Current will return nil,0,0. -func (m *enumerator) Current() ([]byte, int, uint64) { - var i int - var v uint64 - if m.lowCurr < len(m.lowIdxs) { - i = m.lowIdxs[m.lowCurr] - v = m.currVs[i] - } - return m.lowK, i, v -} - -// GetLowIdxsAndValues will return all of the iterator indices -// which point to the current key, and their corresponding -// values. This can be used by advanced caller which may need -// to peek into these other sets of data before processing. -func (m *enumerator) GetLowIdxsAndValues() ([]int, []uint64) { - values := make([]uint64, 0, len(m.lowIdxs)) - for _, idx := range m.lowIdxs { - values = append(values, m.currVs[idx]) - } - return m.lowIdxs, values -} - -// Next advances the enumerator to the next key/iterator/value result, -// else vellum.ErrIteratorDone is returned. -func (m *enumerator) Next() error { - m.lowCurr += 1 - if m.lowCurr >= len(m.lowIdxs) { - // move all the current low iterators forwards - for _, vi := range m.lowIdxs { - err := m.itrs[vi].Next() - if err != nil && err != vellum.ErrIteratorDone { - return err - } - m.currKs[vi], m.currVs[vi] = m.itrs[vi].Current() - } - // can skip any empty keys encountered at this point - m.updateMatches(true) - } - if m.lowK == nil && len(m.lowIdxs) == 0 { - return vellum.ErrIteratorDone - } - return nil -} - -// Close all the underlying Iterators. The first error, if any, will -// be returned. -func (m *enumerator) Close() error { - var rv error - for _, itr := range m.itrs { - err := itr.Close() - if rv == nil { - rv = err - } - } - return rv -} diff --git a/vendor/github.com/blevesearch/zap/v15/intDecoder.go b/vendor/github.com/blevesearch/zap/v15/intDecoder.go deleted file mode 100644 index ea8021da..00000000 --- a/vendor/github.com/blevesearch/zap/v15/intDecoder.go +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright (c) 2019 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "encoding/binary" - "fmt" - - "github.com/blevesearch/bleve/index/scorch/segment" -) - -type chunkedIntDecoder struct { - startOffset uint64 - dataStartOffset uint64 - chunkOffsets []uint64 - curChunkBytes []byte - data []byte - r *segment.MemUvarintReader -} - -// newChunkedIntDecoder expects an optional or reset chunkedIntDecoder for better reuse. -func newChunkedIntDecoder(buf []byte, offset uint64, rv *chunkedIntDecoder) *chunkedIntDecoder { - if rv == nil { - rv = &chunkedIntDecoder{startOffset: offset, data: buf} - } else { - rv.startOffset = offset - rv.data = buf - } - - var n, numChunks uint64 - var read int - if offset == termNotEncoded { - numChunks = 0 - } else { - numChunks, read = binary.Uvarint(buf[offset+n : offset+n+binary.MaxVarintLen64]) - } - - n += uint64(read) - if cap(rv.chunkOffsets) >= int(numChunks) { - rv.chunkOffsets = rv.chunkOffsets[:int(numChunks)] - } else { - rv.chunkOffsets = make([]uint64, int(numChunks)) - } - for i := 0; i < int(numChunks); i++ { - rv.chunkOffsets[i], read = binary.Uvarint(buf[offset+n : offset+n+binary.MaxVarintLen64]) - n += uint64(read) - } - rv.dataStartOffset = offset + n - return rv -} - -func (d *chunkedIntDecoder) loadChunk(chunk int) error { - if d.startOffset == termNotEncoded { - d.r = segment.NewMemUvarintReader([]byte(nil)) - return nil - } - - if chunk >= len(d.chunkOffsets) { - return fmt.Errorf("tried to load freq chunk that doesn't exist %d/(%d)", - chunk, len(d.chunkOffsets)) - } - - end, start := d.dataStartOffset, d.dataStartOffset - s, e := readChunkBoundary(chunk, d.chunkOffsets) - start += s - end += e - d.curChunkBytes = d.data[start:end] - if d.r == nil { - d.r = segment.NewMemUvarintReader(d.curChunkBytes) - } else { - d.r.Reset(d.curChunkBytes) - } - - return nil -} - -func (d *chunkedIntDecoder) reset() { - d.startOffset = 0 - d.dataStartOffset = 0 - d.chunkOffsets = d.chunkOffsets[:0] - d.curChunkBytes = d.curChunkBytes[:0] - d.data = d.data[:0] - if d.r != nil { - d.r.Reset([]byte(nil)) - } -} - -func (d *chunkedIntDecoder) isNil() bool { - return d.curChunkBytes == nil || len(d.curChunkBytes) == 0 -} - -func (d *chunkedIntDecoder) readUvarint() (uint64, error) { - return d.r.ReadUvarint() -} - -func (d *chunkedIntDecoder) readBytes(start, end int) []byte { - return d.curChunkBytes[start:end] -} - -func (d *chunkedIntDecoder) SkipUvarint() { - d.r.SkipUvarint() -} - -func (d *chunkedIntDecoder) SkipBytes(count int) { - d.r.SkipBytes(count) -} - -func (d *chunkedIntDecoder) Len() int { - return d.r.Len() -} - -func (d *chunkedIntDecoder) remainingLen() int { - return len(d.curChunkBytes) - d.r.Len() -} diff --git a/vendor/github.com/blevesearch/zap/v15/merge.go b/vendor/github.com/blevesearch/zap/v15/merge.go deleted file mode 100644 index 0d73c1ec..00000000 --- a/vendor/github.com/blevesearch/zap/v15/merge.go +++ /dev/null @@ -1,893 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "bufio" - "bytes" - "encoding/binary" - "fmt" - "math" - "os" - "sort" - - "github.com/RoaringBitmap/roaring" - seg "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/couchbase/vellum" - "github.com/golang/snappy" -) - -var DefaultFileMergerBufferSize = 1024 * 1024 - -const docDropped = math.MaxUint64 // sentinel docNum to represent a deleted doc - -// Merge takes a slice of segments and bit masks describing which -// documents may be dropped, and creates a new segment containing the -// remaining data. This new segment is built at the specified path. -func (*ZapPlugin) Merge(segments []seg.Segment, drops []*roaring.Bitmap, path string, - closeCh chan struct{}, s seg.StatsReporter) ( - [][]uint64, uint64, error) { - - segmentBases := make([]*SegmentBase, len(segments)) - for segmenti, segment := range segments { - switch segmentx := segment.(type) { - case *Segment: - segmentBases[segmenti] = &segmentx.SegmentBase - case *SegmentBase: - segmentBases[segmenti] = segmentx - default: - panic(fmt.Sprintf("oops, unexpected segment type: %T", segment)) - } - } - return mergeSegmentBases(segmentBases, drops, path, DefaultChunkMode, closeCh, s) -} - -func mergeSegmentBases(segmentBases []*SegmentBase, drops []*roaring.Bitmap, path string, - chunkMode uint32, closeCh chan struct{}, s seg.StatsReporter) ( - [][]uint64, uint64, error) { - flag := os.O_RDWR | os.O_CREATE - - f, err := os.OpenFile(path, flag, 0600) - if err != nil { - return nil, 0, err - } - - cleanup := func() { - _ = f.Close() - _ = os.Remove(path) - } - - // buffer the output - br := bufio.NewWriterSize(f, DefaultFileMergerBufferSize) - - // wrap it for counting (tracking offsets) - cr := NewCountHashWriterWithStatsReporter(br, s) - - newDocNums, numDocs, storedIndexOffset, fieldsIndexOffset, docValueOffset, _, _, _, err := - MergeToWriter(segmentBases, drops, chunkMode, cr, closeCh) - if err != nil { - cleanup() - return nil, 0, err - } - - err = persistFooter(numDocs, storedIndexOffset, fieldsIndexOffset, - docValueOffset, chunkMode, cr.Sum32(), cr) - if err != nil { - cleanup() - return nil, 0, err - } - - err = br.Flush() - if err != nil { - cleanup() - return nil, 0, err - } - - err = f.Sync() - if err != nil { - cleanup() - return nil, 0, err - } - - err = f.Close() - if err != nil { - cleanup() - return nil, 0, err - } - - return newDocNums, uint64(cr.Count()), nil -} - -func MergeToWriter(segments []*SegmentBase, drops []*roaring.Bitmap, - chunkMode uint32, cr *CountHashWriter, closeCh chan struct{}) ( - newDocNums [][]uint64, - numDocs, storedIndexOffset, fieldsIndexOffset, docValueOffset uint64, - dictLocs []uint64, fieldsInv []string, fieldsMap map[string]uint16, - err error) { - docValueOffset = uint64(fieldNotUninverted) - - var fieldsSame bool - fieldsSame, fieldsInv = mergeFields(segments) - fieldsMap = mapFields(fieldsInv) - - numDocs = computeNewDocCount(segments, drops) - - if isClosed(closeCh) { - return nil, 0, 0, 0, 0, nil, nil, nil, seg.ErrClosed - } - - if numDocs > 0 { - storedIndexOffset, newDocNums, err = mergeStoredAndRemap(segments, drops, - fieldsMap, fieldsInv, fieldsSame, numDocs, cr, closeCh) - if err != nil { - return nil, 0, 0, 0, 0, nil, nil, nil, err - } - - dictLocs, docValueOffset, err = persistMergedRest(segments, drops, - fieldsInv, fieldsMap, fieldsSame, - newDocNums, numDocs, chunkMode, cr, closeCh) - if err != nil { - return nil, 0, 0, 0, 0, nil, nil, nil, err - } - } else { - dictLocs = make([]uint64, len(fieldsInv)) - } - - fieldsIndexOffset, err = persistFields(fieldsInv, cr, dictLocs) - if err != nil { - return nil, 0, 0, 0, 0, nil, nil, nil, err - } - - return newDocNums, numDocs, storedIndexOffset, fieldsIndexOffset, docValueOffset, dictLocs, fieldsInv, fieldsMap, nil -} - -// mapFields takes the fieldsInv list and returns a map of fieldName -// to fieldID+1 -func mapFields(fields []string) map[string]uint16 { - rv := make(map[string]uint16, len(fields)) - for i, fieldName := range fields { - rv[fieldName] = uint16(i) + 1 - } - return rv -} - -// computeNewDocCount determines how many documents will be in the newly -// merged segment when obsoleted docs are dropped -func computeNewDocCount(segments []*SegmentBase, drops []*roaring.Bitmap) uint64 { - var newDocCount uint64 - for segI, segment := range segments { - newDocCount += segment.numDocs - if drops[segI] != nil { - newDocCount -= drops[segI].GetCardinality() - } - } - return newDocCount -} - -func persistMergedRest(segments []*SegmentBase, dropsIn []*roaring.Bitmap, - fieldsInv []string, fieldsMap map[string]uint16, fieldsSame bool, - newDocNumsIn [][]uint64, newSegDocCount uint64, chunkMode uint32, - w *CountHashWriter, closeCh chan struct{}) ([]uint64, uint64, error) { - - var bufMaxVarintLen64 []byte = make([]byte, binary.MaxVarintLen64) - var bufLoc []uint64 - - var postings *PostingsList - var postItr *PostingsIterator - - rv := make([]uint64, len(fieldsInv)) - fieldDvLocsStart := make([]uint64, len(fieldsInv)) - fieldDvLocsEnd := make([]uint64, len(fieldsInv)) - - // these int coders are initialized with chunk size 1024 - // however this will be reset to the correct chunk size - // while processing each individual field-term section - tfEncoder := newChunkedIntCoder(1024, newSegDocCount-1) - locEncoder := newChunkedIntCoder(1024, newSegDocCount-1) - - var vellumBuf bytes.Buffer - newVellum, err := vellum.New(&vellumBuf, nil) - if err != nil { - return nil, 0, err - } - - newRoaring := roaring.NewBitmap() - - // for each field - for fieldID, fieldName := range fieldsInv { - - // collect FST iterators from all active segments for this field - var newDocNums [][]uint64 - var drops []*roaring.Bitmap - var dicts []*Dictionary - var itrs []vellum.Iterator - - var segmentsInFocus []*SegmentBase - - for segmentI, segment := range segments { - - // check for the closure in meantime - if isClosed(closeCh) { - return nil, 0, seg.ErrClosed - } - - dict, err2 := segment.dictionary(fieldName) - if err2 != nil { - return nil, 0, err2 - } - if dict != nil && dict.fst != nil { - itr, err2 := dict.fst.Iterator(nil, nil) - if err2 != nil && err2 != vellum.ErrIteratorDone { - return nil, 0, err2 - } - if itr != nil { - newDocNums = append(newDocNums, newDocNumsIn[segmentI]) - if dropsIn[segmentI] != nil && !dropsIn[segmentI].IsEmpty() { - drops = append(drops, dropsIn[segmentI]) - } else { - drops = append(drops, nil) - } - dicts = append(dicts, dict) - itrs = append(itrs, itr) - segmentsInFocus = append(segmentsInFocus, segment) - } - } - } - - var prevTerm []byte - - newRoaring.Clear() - - var lastDocNum, lastFreq, lastNorm uint64 - - // determines whether to use "1-hit" encoding optimization - // when a term appears in only 1 doc, with no loc info, - // has freq of 1, and the docNum fits into 31-bits - use1HitEncoding := func(termCardinality uint64) (bool, uint64, uint64) { - if termCardinality == uint64(1) && locEncoder.FinalSize() <= 0 { - docNum := uint64(newRoaring.Minimum()) - if under32Bits(docNum) && docNum == lastDocNum && lastFreq == 1 { - return true, docNum, lastNorm - } - } - return false, 0, 0 - } - - finishTerm := func(term []byte) error { - tfEncoder.Close() - locEncoder.Close() - - postingsOffset, err := writePostings(newRoaring, - tfEncoder, locEncoder, use1HitEncoding, w, bufMaxVarintLen64) - if err != nil { - return err - } - - if postingsOffset > 0 { - err = newVellum.Insert(term, postingsOffset) - if err != nil { - return err - } - } - - newRoaring.Clear() - - tfEncoder.Reset() - locEncoder.Reset() - - lastDocNum = 0 - lastFreq = 0 - lastNorm = 0 - - return nil - } - - enumerator, err := newEnumerator(itrs) - - for err == nil { - term, itrI, postingsOffset := enumerator.Current() - - if !bytes.Equal(prevTerm, term) { - // check for the closure in meantime - if isClosed(closeCh) { - return nil, 0, seg.ErrClosed - } - - // if the term changed, write out the info collected - // for the previous term - err = finishTerm(prevTerm) - if err != nil { - return nil, 0, err - } - } - if !bytes.Equal(prevTerm, term) || prevTerm == nil { - // compute cardinality of field-term in new seg - var newCard uint64 - lowItrIdxs, lowItrVals := enumerator.GetLowIdxsAndValues() - for i, idx := range lowItrIdxs { - pl, err := dicts[idx].postingsListFromOffset(lowItrVals[i], drops[idx], nil) - if err != nil { - return nil, 0, err - } - newCard += pl.Count() - } - // compute correct chunk size with this - chunkSize, err := getChunkSize(chunkMode, newCard, newSegDocCount) - if err != nil { - return nil, 0, err - } - // update encoders chunk - tfEncoder.SetChunkSize(chunkSize, newSegDocCount-1) - locEncoder.SetChunkSize(chunkSize, newSegDocCount-1) - } - - postings, err = dicts[itrI].postingsListFromOffset( - postingsOffset, drops[itrI], postings) - if err != nil { - return nil, 0, err - } - - postItr = postings.iterator(true, true, true, postItr) - - if fieldsSame { - // can optimize by copying freq/norm/loc bytes directly - lastDocNum, lastFreq, lastNorm, err = mergeTermFreqNormLocsByCopying( - term, postItr, newDocNums[itrI], newRoaring, - tfEncoder, locEncoder) - } else { - lastDocNum, lastFreq, lastNorm, bufLoc, err = mergeTermFreqNormLocs( - fieldsMap, term, postItr, newDocNums[itrI], newRoaring, - tfEncoder, locEncoder, bufLoc) - } - if err != nil { - return nil, 0, err - } - - prevTerm = prevTerm[:0] // copy to prevTerm in case Next() reuses term mem - prevTerm = append(prevTerm, term...) - - err = enumerator.Next() - } - if err != vellum.ErrIteratorDone { - return nil, 0, err - } - - err = finishTerm(prevTerm) - if err != nil { - return nil, 0, err - } - - dictOffset := uint64(w.Count()) - - err = newVellum.Close() - if err != nil { - return nil, 0, err - } - vellumData := vellumBuf.Bytes() - - // write out the length of the vellum data - n := binary.PutUvarint(bufMaxVarintLen64, uint64(len(vellumData))) - _, err = w.Write(bufMaxVarintLen64[:n]) - if err != nil { - return nil, 0, err - } - - // write this vellum to disk - _, err = w.Write(vellumData) - if err != nil { - return nil, 0, err - } - - rv[fieldID] = dictOffset - - // get the field doc value offset (start) - fieldDvLocsStart[fieldID] = uint64(w.Count()) - - // update the field doc values - // NOTE: doc values continue to use legacy chunk mode - chunkSize, err := getChunkSize(LegacyChunkMode, 0, 0) - if err != nil { - return nil, 0, err - } - fdvEncoder := newChunkedContentCoder(chunkSize, newSegDocCount-1, w, true) - - fdvReadersAvailable := false - var dvIterClone *docValueReader - for segmentI, segment := range segmentsInFocus { - // check for the closure in meantime - if isClosed(closeCh) { - return nil, 0, seg.ErrClosed - } - - fieldIDPlus1 := uint16(segment.fieldsMap[fieldName]) - if dvIter, exists := segment.fieldDvReaders[fieldIDPlus1-1]; exists && - dvIter != nil { - fdvReadersAvailable = true - dvIterClone = dvIter.cloneInto(dvIterClone) - err = dvIterClone.iterateAllDocValues(segment, func(docNum uint64, terms []byte) error { - if newDocNums[segmentI][docNum] == docDropped { - return nil - } - err := fdvEncoder.Add(newDocNums[segmentI][docNum], terms) - if err != nil { - return err - } - return nil - }) - if err != nil { - return nil, 0, err - } - } - } - - if fdvReadersAvailable { - err = fdvEncoder.Close() - if err != nil { - return nil, 0, err - } - - // persist the doc value details for this field - _, err = fdvEncoder.Write() - if err != nil { - return nil, 0, err - } - - // get the field doc value offset (end) - fieldDvLocsEnd[fieldID] = uint64(w.Count()) - } else { - fieldDvLocsStart[fieldID] = fieldNotUninverted - fieldDvLocsEnd[fieldID] = fieldNotUninverted - } - - // reset vellum buffer and vellum builder - vellumBuf.Reset() - err = newVellum.Reset(&vellumBuf) - if err != nil { - return nil, 0, err - } - } - - fieldDvLocsOffset := uint64(w.Count()) - - buf := bufMaxVarintLen64 - for i := 0; i < len(fieldDvLocsStart); i++ { - n := binary.PutUvarint(buf, fieldDvLocsStart[i]) - _, err := w.Write(buf[:n]) - if err != nil { - return nil, 0, err - } - n = binary.PutUvarint(buf, fieldDvLocsEnd[i]) - _, err = w.Write(buf[:n]) - if err != nil { - return nil, 0, err - } - } - - return rv, fieldDvLocsOffset, nil -} - -func mergeTermFreqNormLocsByCopying(term []byte, postItr *PostingsIterator, - newDocNums []uint64, newRoaring *roaring.Bitmap, - tfEncoder *chunkedIntCoder, locEncoder *chunkedIntCoder) ( - lastDocNum uint64, lastFreq uint64, lastNorm uint64, err error) { - nextDocNum, nextFreq, nextNorm, nextFreqNormBytes, nextLocBytes, err := - postItr.nextBytes() - for err == nil && len(nextFreqNormBytes) > 0 { - hitNewDocNum := newDocNums[nextDocNum] - if hitNewDocNum == docDropped { - return 0, 0, 0, fmt.Errorf("see hit with dropped doc num") - } - - newRoaring.Add(uint32(hitNewDocNum)) - err = tfEncoder.AddBytes(hitNewDocNum, nextFreqNormBytes) - if err != nil { - return 0, 0, 0, err - } - - if len(nextLocBytes) > 0 { - err = locEncoder.AddBytes(hitNewDocNum, nextLocBytes) - if err != nil { - return 0, 0, 0, err - } - } - - lastDocNum = hitNewDocNum - lastFreq = nextFreq - lastNorm = nextNorm - - nextDocNum, nextFreq, nextNorm, nextFreqNormBytes, nextLocBytes, err = - postItr.nextBytes() - } - - return lastDocNum, lastFreq, lastNorm, err -} - -func mergeTermFreqNormLocs(fieldsMap map[string]uint16, term []byte, postItr *PostingsIterator, - newDocNums []uint64, newRoaring *roaring.Bitmap, - tfEncoder *chunkedIntCoder, locEncoder *chunkedIntCoder, bufLoc []uint64) ( - lastDocNum uint64, lastFreq uint64, lastNorm uint64, bufLocOut []uint64, err error) { - next, err := postItr.Next() - for next != nil && err == nil { - hitNewDocNum := newDocNums[next.Number()] - if hitNewDocNum == docDropped { - return 0, 0, 0, nil, fmt.Errorf("see hit with dropped docNum") - } - - newRoaring.Add(uint32(hitNewDocNum)) - - nextFreq := next.Frequency() - var nextNorm uint64 - if pi, ok := next.(*Posting); ok { - nextNorm = pi.NormUint64() - } else { - return 0, 0, 0, nil, fmt.Errorf("unexpected posting type %T", next) - } - - locs := next.Locations() - - err = tfEncoder.Add(hitNewDocNum, - encodeFreqHasLocs(nextFreq, len(locs) > 0), nextNorm) - if err != nil { - return 0, 0, 0, nil, err - } - - if len(locs) > 0 { - numBytesLocs := 0 - for _, loc := range locs { - ap := loc.ArrayPositions() - numBytesLocs += totalUvarintBytes(uint64(fieldsMap[loc.Field()]-1), - loc.Pos(), loc.Start(), loc.End(), uint64(len(ap)), ap) - } - - err = locEncoder.Add(hitNewDocNum, uint64(numBytesLocs)) - if err != nil { - return 0, 0, 0, nil, err - } - - for _, loc := range locs { - ap := loc.ArrayPositions() - if cap(bufLoc) < 5+len(ap) { - bufLoc = make([]uint64, 0, 5+len(ap)) - } - args := bufLoc[0:5] - args[0] = uint64(fieldsMap[loc.Field()] - 1) - args[1] = loc.Pos() - args[2] = loc.Start() - args[3] = loc.End() - args[4] = uint64(len(ap)) - args = append(args, ap...) - err = locEncoder.Add(hitNewDocNum, args...) - if err != nil { - return 0, 0, 0, nil, err - } - } - } - - lastDocNum = hitNewDocNum - lastFreq = nextFreq - lastNorm = nextNorm - - next, err = postItr.Next() - } - - return lastDocNum, lastFreq, lastNorm, bufLoc, err -} - -func writePostings(postings *roaring.Bitmap, tfEncoder, locEncoder *chunkedIntCoder, - use1HitEncoding func(uint64) (bool, uint64, uint64), - w *CountHashWriter, bufMaxVarintLen64 []byte) ( - offset uint64, err error) { - termCardinality := postings.GetCardinality() - if termCardinality <= 0 { - return 0, nil - } - - if use1HitEncoding != nil { - encodeAs1Hit, docNum1Hit, normBits1Hit := use1HitEncoding(termCardinality) - if encodeAs1Hit { - return FSTValEncode1Hit(docNum1Hit, normBits1Hit), nil - } - } - - var tfOffset uint64 - tfOffset, _, err = tfEncoder.writeAt(w) - if err != nil { - return 0, err - } - - var locOffset uint64 - locOffset, _, err = locEncoder.writeAt(w) - if err != nil { - return 0, err - } - - postingsOffset := uint64(w.Count()) - - n := binary.PutUvarint(bufMaxVarintLen64, tfOffset) - _, err = w.Write(bufMaxVarintLen64[:n]) - if err != nil { - return 0, err - } - - n = binary.PutUvarint(bufMaxVarintLen64, locOffset) - _, err = w.Write(bufMaxVarintLen64[:n]) - if err != nil { - return 0, err - } - - _, err = writeRoaringWithLen(postings, w, bufMaxVarintLen64) - if err != nil { - return 0, err - } - - return postingsOffset, nil -} - -type varintEncoder func(uint64) (int, error) - -func mergeStoredAndRemap(segments []*SegmentBase, drops []*roaring.Bitmap, - fieldsMap map[string]uint16, fieldsInv []string, fieldsSame bool, newSegDocCount uint64, - w *CountHashWriter, closeCh chan struct{}) (uint64, [][]uint64, error) { - var rv [][]uint64 // The remapped or newDocNums for each segment. - - var newDocNum uint64 - - var curr int - var data, compressed []byte - var metaBuf bytes.Buffer - varBuf := make([]byte, binary.MaxVarintLen64) - metaEncode := func(val uint64) (int, error) { - wb := binary.PutUvarint(varBuf, val) - return metaBuf.Write(varBuf[:wb]) - } - - vals := make([][][]byte, len(fieldsInv)) - typs := make([][]byte, len(fieldsInv)) - poss := make([][][]uint64, len(fieldsInv)) - - var posBuf []uint64 - - docNumOffsets := make([]uint64, newSegDocCount) - - vdc := visitDocumentCtxPool.Get().(*visitDocumentCtx) - defer visitDocumentCtxPool.Put(vdc) - - // for each segment - for segI, segment := range segments { - // check for the closure in meantime - if isClosed(closeCh) { - return 0, nil, seg.ErrClosed - } - - segNewDocNums := make([]uint64, segment.numDocs) - - dropsI := drops[segI] - - // optimize when the field mapping is the same across all - // segments and there are no deletions, via byte-copying - // of stored docs bytes directly to the writer - if fieldsSame && (dropsI == nil || dropsI.GetCardinality() == 0) { - err := segment.copyStoredDocs(newDocNum, docNumOffsets, w) - if err != nil { - return 0, nil, err - } - - for i := uint64(0); i < segment.numDocs; i++ { - segNewDocNums[i] = newDocNum - newDocNum++ - } - rv = append(rv, segNewDocNums) - - continue - } - - // for each doc num - for docNum := uint64(0); docNum < segment.numDocs; docNum++ { - // TODO: roaring's API limits docNums to 32-bits? - if dropsI != nil && dropsI.Contains(uint32(docNum)) { - segNewDocNums[docNum] = docDropped - continue - } - - segNewDocNums[docNum] = newDocNum - - curr = 0 - metaBuf.Reset() - data = data[:0] - - posTemp := posBuf - - // collect all the data - for i := 0; i < len(fieldsInv); i++ { - vals[i] = vals[i][:0] - typs[i] = typs[i][:0] - poss[i] = poss[i][:0] - } - err := segment.visitDocument(vdc, docNum, func(field string, typ byte, value []byte, pos []uint64) bool { - fieldID := int(fieldsMap[field]) - 1 - vals[fieldID] = append(vals[fieldID], value) - typs[fieldID] = append(typs[fieldID], typ) - - // copy array positions to preserve them beyond the scope of this callback - var curPos []uint64 - if len(pos) > 0 { - if cap(posTemp) < len(pos) { - posBuf = make([]uint64, len(pos)*len(fieldsInv)) - posTemp = posBuf - } - curPos = posTemp[0:len(pos)] - copy(curPos, pos) - posTemp = posTemp[len(pos):] - } - poss[fieldID] = append(poss[fieldID], curPos) - - return true - }) - if err != nil { - return 0, nil, err - } - - // _id field special case optimizes ExternalID() lookups - idFieldVal := vals[uint16(0)][0] - _, err = metaEncode(uint64(len(idFieldVal))) - if err != nil { - return 0, nil, err - } - - // now walk the non-"_id" fields in order - for fieldID := 1; fieldID < len(fieldsInv); fieldID++ { - storedFieldValues := vals[fieldID] - - stf := typs[fieldID] - spf := poss[fieldID] - - var err2 error - curr, data, err2 = persistStoredFieldValues(fieldID, - storedFieldValues, stf, spf, curr, metaEncode, data) - if err2 != nil { - return 0, nil, err2 - } - } - - metaBytes := metaBuf.Bytes() - - compressed = snappy.Encode(compressed[:cap(compressed)], data) - - // record where we're about to start writing - docNumOffsets[newDocNum] = uint64(w.Count()) - - // write out the meta len and compressed data len - _, err = writeUvarints(w, - uint64(len(metaBytes)), - uint64(len(idFieldVal)+len(compressed))) - if err != nil { - return 0, nil, err - } - // now write the meta - _, err = w.Write(metaBytes) - if err != nil { - return 0, nil, err - } - // now write the _id field val (counted as part of the 'compressed' data) - _, err = w.Write(idFieldVal) - if err != nil { - return 0, nil, err - } - // now write the compressed data - _, err = w.Write(compressed) - if err != nil { - return 0, nil, err - } - - newDocNum++ - } - - rv = append(rv, segNewDocNums) - } - - // return value is the start of the stored index - storedIndexOffset := uint64(w.Count()) - - // now write out the stored doc index - for _, docNumOffset := range docNumOffsets { - err := binary.Write(w, binary.BigEndian, docNumOffset) - if err != nil { - return 0, nil, err - } - } - - return storedIndexOffset, rv, nil -} - -// copyStoredDocs writes out a segment's stored doc info, optimized by -// using a single Write() call for the entire set of bytes. The -// newDocNumOffsets is filled with the new offsets for each doc. -func (s *SegmentBase) copyStoredDocs(newDocNum uint64, newDocNumOffsets []uint64, - w *CountHashWriter) error { - if s.numDocs <= 0 { - return nil - } - - indexOffset0, storedOffset0, _, _, _ := - s.getDocStoredOffsets(0) // the segment's first doc - - indexOffsetN, storedOffsetN, readN, metaLenN, dataLenN := - s.getDocStoredOffsets(s.numDocs - 1) // the segment's last doc - - storedOffset0New := uint64(w.Count()) - - storedBytes := s.mem[storedOffset0 : storedOffsetN+readN+metaLenN+dataLenN] - _, err := w.Write(storedBytes) - if err != nil { - return err - } - - // remap the storedOffset's for the docs into new offsets relative - // to storedOffset0New, filling the given docNumOffsetsOut array - for indexOffset := indexOffset0; indexOffset <= indexOffsetN; indexOffset += 8 { - storedOffset := binary.BigEndian.Uint64(s.mem[indexOffset : indexOffset+8]) - storedOffsetNew := storedOffset - storedOffset0 + storedOffset0New - newDocNumOffsets[newDocNum] = storedOffsetNew - newDocNum += 1 - } - - return nil -} - -// mergeFields builds a unified list of fields used across all the -// input segments, and computes whether the fields are the same across -// segments (which depends on fields to be sorted in the same way -// across segments) -func mergeFields(segments []*SegmentBase) (bool, []string) { - fieldsSame := true - - var segment0Fields []string - if len(segments) > 0 { - segment0Fields = segments[0].Fields() - } - - fieldsExist := map[string]struct{}{} - for _, segment := range segments { - fields := segment.Fields() - for fieldi, field := range fields { - fieldsExist[field] = struct{}{} - if len(segment0Fields) != len(fields) || segment0Fields[fieldi] != field { - fieldsSame = false - } - } - } - - rv := make([]string, 0, len(fieldsExist)) - // ensure _id stays first - rv = append(rv, "_id") - for k := range fieldsExist { - if k != "_id" { - rv = append(rv, k) - } - } - - sort.Strings(rv[1:]) // leave _id as first - - return fieldsSame, rv -} - -func isClosed(closeCh chan struct{}) bool { - select { - case <-closeCh: - return true - default: - return false - } -} diff --git a/vendor/github.com/blevesearch/zap/v15/new.go b/vendor/github.com/blevesearch/zap/v15/new.go deleted file mode 100644 index c10a6a06..00000000 --- a/vendor/github.com/blevesearch/zap/v15/new.go +++ /dev/null @@ -1,860 +0,0 @@ -// Copyright (c) 2018 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "bytes" - "encoding/binary" - "math" - "sort" - "sync" - - "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/couchbase/vellum" - "github.com/golang/snappy" -) - -var NewSegmentBufferNumResultsBump int = 100 -var NewSegmentBufferNumResultsFactor float64 = 1.0 -var NewSegmentBufferAvgBytesPerDocFactor float64 = 1.0 - -// ValidateDocFields can be set by applications to perform additional checks -// on fields in a document being added to a new segment, by default it does -// nothing. -// This API is experimental and may be removed at any time. -var ValidateDocFields = func(field document.Field) error { - return nil -} - -// AnalysisResultsToSegmentBase produces an in-memory zap-encoded -// SegmentBase from analysis results -func (z *ZapPlugin) New(results []*index.AnalysisResult) ( - segment.Segment, uint64, error) { - return z.newWithChunkMode(results, DefaultChunkMode) -} - -func (*ZapPlugin) newWithChunkMode(results []*index.AnalysisResult, - chunkMode uint32) (segment.Segment, uint64, error) { - s := interimPool.Get().(*interim) - - var br bytes.Buffer - if s.lastNumDocs > 0 { - // use previous results to initialize the buf with an estimate - // size, but note that the interim instance comes from a - // global interimPool, so multiple scorch instances indexing - // different docs can lead to low quality estimates - estimateAvgBytesPerDoc := int(float64(s.lastOutSize/s.lastNumDocs) * - NewSegmentBufferNumResultsFactor) - estimateNumResults := int(float64(len(results)+NewSegmentBufferNumResultsBump) * - NewSegmentBufferAvgBytesPerDocFactor) - br.Grow(estimateAvgBytesPerDoc * estimateNumResults) - } - - s.results = results - s.chunkMode = chunkMode - s.w = NewCountHashWriter(&br) - - storedIndexOffset, fieldsIndexOffset, fdvIndexOffset, dictOffsets, - err := s.convert() - if err != nil { - return nil, uint64(0), err - } - - sb, err := InitSegmentBase(br.Bytes(), s.w.Sum32(), chunkMode, - s.FieldsMap, s.FieldsInv, uint64(len(results)), - storedIndexOffset, fieldsIndexOffset, fdvIndexOffset, dictOffsets) - - if err == nil && s.reset() == nil { - s.lastNumDocs = len(results) - s.lastOutSize = len(br.Bytes()) - interimPool.Put(s) - } - - return sb, uint64(len(br.Bytes())), err -} - -var interimPool = sync.Pool{New: func() interface{} { return &interim{} }} - -// interim holds temporary working data used while converting from -// analysis results to a zap-encoded segment -type interim struct { - results []*index.AnalysisResult - - chunkMode uint32 - - w *CountHashWriter - - // FieldsMap adds 1 to field id to avoid zero value issues - // name -> field id + 1 - FieldsMap map[string]uint16 - - // FieldsInv is the inverse of FieldsMap - // field id -> name - FieldsInv []string - - // Term dictionaries for each field - // field id -> term -> postings list id + 1 - Dicts []map[string]uint64 - - // Terms for each field, where terms are sorted ascending - // field id -> []term - DictKeys [][]string - - // Fields whose IncludeDocValues is true - // field id -> bool - IncludeDocValues []bool - - // postings id -> bitmap of docNums - Postings []*roaring.Bitmap - - // postings id -> freq/norm's, one for each docNum in postings - FreqNorms [][]interimFreqNorm - freqNormsBacking []interimFreqNorm - - // postings id -> locs, one for each freq - Locs [][]interimLoc - locsBacking []interimLoc - - numTermsPerPostingsList []int // key is postings list id - numLocsPerPostingsList []int // key is postings list id - - builder *vellum.Builder - builderBuf bytes.Buffer - - metaBuf bytes.Buffer - - tmp0 []byte - tmp1 []byte - - lastNumDocs int - lastOutSize int -} - -func (s *interim) reset() (err error) { - s.results = nil - s.chunkMode = 0 - s.w = nil - s.FieldsMap = nil - s.FieldsInv = nil - for i := range s.Dicts { - s.Dicts[i] = nil - } - s.Dicts = s.Dicts[:0] - for i := range s.DictKeys { - s.DictKeys[i] = s.DictKeys[i][:0] - } - s.DictKeys = s.DictKeys[:0] - for i := range s.IncludeDocValues { - s.IncludeDocValues[i] = false - } - s.IncludeDocValues = s.IncludeDocValues[:0] - for _, idn := range s.Postings { - idn.Clear() - } - s.Postings = s.Postings[:0] - s.FreqNorms = s.FreqNorms[:0] - for i := range s.freqNormsBacking { - s.freqNormsBacking[i] = interimFreqNorm{} - } - s.freqNormsBacking = s.freqNormsBacking[:0] - s.Locs = s.Locs[:0] - for i := range s.locsBacking { - s.locsBacking[i] = interimLoc{} - } - s.locsBacking = s.locsBacking[:0] - s.numTermsPerPostingsList = s.numTermsPerPostingsList[:0] - s.numLocsPerPostingsList = s.numLocsPerPostingsList[:0] - s.builderBuf.Reset() - if s.builder != nil { - err = s.builder.Reset(&s.builderBuf) - } - s.metaBuf.Reset() - s.tmp0 = s.tmp0[:0] - s.tmp1 = s.tmp1[:0] - s.lastNumDocs = 0 - s.lastOutSize = 0 - - return err -} - -func (s *interim) grabBuf(size int) []byte { - buf := s.tmp0 - if cap(buf) < size { - buf = make([]byte, size) - s.tmp0 = buf - } - return buf[0:size] -} - -type interimStoredField struct { - vals [][]byte - typs []byte - arrayposs [][]uint64 // array positions -} - -type interimFreqNorm struct { - freq uint64 - norm float32 - numLocs int -} - -type interimLoc struct { - fieldID uint16 - pos uint64 - start uint64 - end uint64 - arrayposs []uint64 -} - -func (s *interim) convert() (uint64, uint64, uint64, []uint64, error) { - s.FieldsMap = map[string]uint16{} - - s.getOrDefineField("_id") // _id field is fieldID 0 - - for _, result := range s.results { - for _, field := range result.Document.CompositeFields { - s.getOrDefineField(field.Name()) - } - for _, field := range result.Document.Fields { - s.getOrDefineField(field.Name()) - } - } - - sort.Strings(s.FieldsInv[1:]) // keep _id as first field - - for fieldID, fieldName := range s.FieldsInv { - s.FieldsMap[fieldName] = uint16(fieldID + 1) - } - - if cap(s.IncludeDocValues) >= len(s.FieldsInv) { - s.IncludeDocValues = s.IncludeDocValues[:len(s.FieldsInv)] - } else { - s.IncludeDocValues = make([]bool, len(s.FieldsInv)) - } - - s.prepareDicts() - - for _, dict := range s.DictKeys { - sort.Strings(dict) - } - - s.processDocuments() - - storedIndexOffset, err := s.writeStoredFields() - if err != nil { - return 0, 0, 0, nil, err - } - - var fdvIndexOffset uint64 - var dictOffsets []uint64 - - if len(s.results) > 0 { - fdvIndexOffset, dictOffsets, err = s.writeDicts() - if err != nil { - return 0, 0, 0, nil, err - } - } else { - dictOffsets = make([]uint64, len(s.FieldsInv)) - } - - fieldsIndexOffset, err := persistFields(s.FieldsInv, s.w, dictOffsets) - if err != nil { - return 0, 0, 0, nil, err - } - - return storedIndexOffset, fieldsIndexOffset, fdvIndexOffset, dictOffsets, nil -} - -func (s *interim) getOrDefineField(fieldName string) int { - fieldIDPlus1, exists := s.FieldsMap[fieldName] - if !exists { - fieldIDPlus1 = uint16(len(s.FieldsInv) + 1) - s.FieldsMap[fieldName] = fieldIDPlus1 - s.FieldsInv = append(s.FieldsInv, fieldName) - - s.Dicts = append(s.Dicts, make(map[string]uint64)) - - n := len(s.DictKeys) - if n < cap(s.DictKeys) { - s.DictKeys = s.DictKeys[:n+1] - s.DictKeys[n] = s.DictKeys[n][:0] - } else { - s.DictKeys = append(s.DictKeys, []string(nil)) - } - } - - return int(fieldIDPlus1 - 1) -} - -// fill Dicts and DictKeys from analysis results -func (s *interim) prepareDicts() { - var pidNext int - - var totTFs int - var totLocs int - - visitField := func(fieldID uint16, tfs analysis.TokenFrequencies) { - dict := s.Dicts[fieldID] - dictKeys := s.DictKeys[fieldID] - - for term, tf := range tfs { - pidPlus1, exists := dict[term] - if !exists { - pidNext++ - pidPlus1 = uint64(pidNext) - - dict[term] = pidPlus1 - dictKeys = append(dictKeys, term) - - s.numTermsPerPostingsList = append(s.numTermsPerPostingsList, 0) - s.numLocsPerPostingsList = append(s.numLocsPerPostingsList, 0) - } - - pid := pidPlus1 - 1 - - s.numTermsPerPostingsList[pid] += 1 - s.numLocsPerPostingsList[pid] += len(tf.Locations) - - totLocs += len(tf.Locations) - } - - totTFs += len(tfs) - - s.DictKeys[fieldID] = dictKeys - } - - for _, result := range s.results { - // walk each composite field - for _, field := range result.Document.CompositeFields { - fieldID := uint16(s.getOrDefineField(field.Name())) - _, tf := field.Analyze() - visitField(fieldID, tf) - } - - // walk each field - for i, field := range result.Document.Fields { - fieldID := uint16(s.getOrDefineField(field.Name())) - tf := result.Analyzed[i] - visitField(fieldID, tf) - } - } - - numPostingsLists := pidNext - - if cap(s.Postings) >= numPostingsLists { - s.Postings = s.Postings[:numPostingsLists] - } else { - postings := make([]*roaring.Bitmap, numPostingsLists) - copy(postings, s.Postings[:cap(s.Postings)]) - for i := 0; i < numPostingsLists; i++ { - if postings[i] == nil { - postings[i] = roaring.New() - } - } - s.Postings = postings - } - - if cap(s.FreqNorms) >= numPostingsLists { - s.FreqNorms = s.FreqNorms[:numPostingsLists] - } else { - s.FreqNorms = make([][]interimFreqNorm, numPostingsLists) - } - - if cap(s.freqNormsBacking) >= totTFs { - s.freqNormsBacking = s.freqNormsBacking[:totTFs] - } else { - s.freqNormsBacking = make([]interimFreqNorm, totTFs) - } - - freqNormsBacking := s.freqNormsBacking - for pid, numTerms := range s.numTermsPerPostingsList { - s.FreqNorms[pid] = freqNormsBacking[0:0] - freqNormsBacking = freqNormsBacking[numTerms:] - } - - if cap(s.Locs) >= numPostingsLists { - s.Locs = s.Locs[:numPostingsLists] - } else { - s.Locs = make([][]interimLoc, numPostingsLists) - } - - if cap(s.locsBacking) >= totLocs { - s.locsBacking = s.locsBacking[:totLocs] - } else { - s.locsBacking = make([]interimLoc, totLocs) - } - - locsBacking := s.locsBacking - for pid, numLocs := range s.numLocsPerPostingsList { - s.Locs[pid] = locsBacking[0:0] - locsBacking = locsBacking[numLocs:] - } -} - -func (s *interim) processDocuments() { - numFields := len(s.FieldsInv) - reuseFieldLens := make([]int, numFields) - reuseFieldTFs := make([]analysis.TokenFrequencies, numFields) - - for docNum, result := range s.results { - for i := 0; i < numFields; i++ { // clear these for reuse - reuseFieldLens[i] = 0 - reuseFieldTFs[i] = nil - } - - s.processDocument(uint64(docNum), result, - reuseFieldLens, reuseFieldTFs) - } -} - -func (s *interim) processDocument(docNum uint64, - result *index.AnalysisResult, - fieldLens []int, fieldTFs []analysis.TokenFrequencies) { - visitField := func(fieldID uint16, fieldName string, - ln int, tf analysis.TokenFrequencies) { - fieldLens[fieldID] += ln - - existingFreqs := fieldTFs[fieldID] - if existingFreqs != nil { - existingFreqs.MergeAll(fieldName, tf) - } else { - fieldTFs[fieldID] = tf - } - } - - // walk each composite field - for _, field := range result.Document.CompositeFields { - fieldID := uint16(s.getOrDefineField(field.Name())) - ln, tf := field.Analyze() - visitField(fieldID, field.Name(), ln, tf) - } - - // walk each field - for i, field := range result.Document.Fields { - fieldID := uint16(s.getOrDefineField(field.Name())) - ln := result.Length[i] - tf := result.Analyzed[i] - visitField(fieldID, field.Name(), ln, tf) - } - - // now that it's been rolled up into fieldTFs, walk that - for fieldID, tfs := range fieldTFs { - dict := s.Dicts[fieldID] - norm := math.Float32frombits(uint32(fieldLens[fieldID])) - - for term, tf := range tfs { - pid := dict[term] - 1 - bs := s.Postings[pid] - bs.Add(uint32(docNum)) - - s.FreqNorms[pid] = append(s.FreqNorms[pid], - interimFreqNorm{ - freq: uint64(tf.Frequency()), - norm: norm, - numLocs: len(tf.Locations), - }) - - if len(tf.Locations) > 0 { - locs := s.Locs[pid] - - for _, loc := range tf.Locations { - var locf = uint16(fieldID) - if loc.Field != "" { - locf = uint16(s.getOrDefineField(loc.Field)) - } - var arrayposs []uint64 - if len(loc.ArrayPositions) > 0 { - arrayposs = loc.ArrayPositions - } - locs = append(locs, interimLoc{ - fieldID: locf, - pos: uint64(loc.Position), - start: uint64(loc.Start), - end: uint64(loc.End), - arrayposs: arrayposs, - }) - } - - s.Locs[pid] = locs - } - } - } -} - -func (s *interim) writeStoredFields() ( - storedIndexOffset uint64, err error) { - varBuf := make([]byte, binary.MaxVarintLen64) - metaEncode := func(val uint64) (int, error) { - wb := binary.PutUvarint(varBuf, val) - return s.metaBuf.Write(varBuf[:wb]) - } - - data, compressed := s.tmp0[:0], s.tmp1[:0] - defer func() { s.tmp0, s.tmp1 = data, compressed }() - - // keyed by docNum - docStoredOffsets := make([]uint64, len(s.results)) - - // keyed by fieldID, for the current doc in the loop - docStoredFields := map[uint16]interimStoredField{} - - for docNum, result := range s.results { - for fieldID := range docStoredFields { // reset for next doc - delete(docStoredFields, fieldID) - } - - for _, field := range result.Document.Fields { - fieldID := uint16(s.getOrDefineField(field.Name())) - - opts := field.Options() - - if opts.IsStored() { - isf := docStoredFields[fieldID] - isf.vals = append(isf.vals, field.Value()) - isf.typs = append(isf.typs, encodeFieldType(field)) - isf.arrayposs = append(isf.arrayposs, field.ArrayPositions()) - docStoredFields[fieldID] = isf - } - - if opts.IncludeDocValues() { - s.IncludeDocValues[fieldID] = true - } - - err := ValidateDocFields(field) - if err != nil { - return 0, err - } - } - - var curr int - - s.metaBuf.Reset() - data = data[:0] - - // _id field special case optimizes ExternalID() lookups - idFieldVal := docStoredFields[uint16(0)].vals[0] - _, err = metaEncode(uint64(len(idFieldVal))) - if err != nil { - return 0, err - } - - // handle non-"_id" fields - for fieldID := 1; fieldID < len(s.FieldsInv); fieldID++ { - isf, exists := docStoredFields[uint16(fieldID)] - if exists { - curr, data, err = persistStoredFieldValues( - fieldID, isf.vals, isf.typs, isf.arrayposs, - curr, metaEncode, data) - if err != nil { - return 0, err - } - } - } - - metaBytes := s.metaBuf.Bytes() - - compressed = snappy.Encode(compressed[:cap(compressed)], data) - - docStoredOffsets[docNum] = uint64(s.w.Count()) - - _, err := writeUvarints(s.w, - uint64(len(metaBytes)), - uint64(len(idFieldVal)+len(compressed))) - if err != nil { - return 0, err - } - - _, err = s.w.Write(metaBytes) - if err != nil { - return 0, err - } - - _, err = s.w.Write(idFieldVal) - if err != nil { - return 0, err - } - - _, err = s.w.Write(compressed) - if err != nil { - return 0, err - } - } - - storedIndexOffset = uint64(s.w.Count()) - - for _, docStoredOffset := range docStoredOffsets { - err = binary.Write(s.w, binary.BigEndian, docStoredOffset) - if err != nil { - return 0, err - } - } - - return storedIndexOffset, nil -} - -func (s *interim) writeDicts() (fdvIndexOffset uint64, dictOffsets []uint64, err error) { - dictOffsets = make([]uint64, len(s.FieldsInv)) - - fdvOffsetsStart := make([]uint64, len(s.FieldsInv)) - fdvOffsetsEnd := make([]uint64, len(s.FieldsInv)) - - buf := s.grabBuf(binary.MaxVarintLen64) - - // these int coders are initialized with chunk size 1024 - // however this will be reset to the correct chunk size - // while processing each individual field-term section - tfEncoder := newChunkedIntCoder(1024, uint64(len(s.results)-1)) - locEncoder := newChunkedIntCoder(1024, uint64(len(s.results)-1)) - - var docTermMap [][]byte - - if s.builder == nil { - s.builder, err = vellum.New(&s.builderBuf, nil) - if err != nil { - return 0, nil, err - } - } - - for fieldID, terms := range s.DictKeys { - if cap(docTermMap) < len(s.results) { - docTermMap = make([][]byte, len(s.results)) - } else { - docTermMap = docTermMap[0:len(s.results)] - for docNum := range docTermMap { // reset the docTermMap - docTermMap[docNum] = docTermMap[docNum][:0] - } - } - - dict := s.Dicts[fieldID] - - for _, term := range terms { // terms are already sorted - pid := dict[term] - 1 - - postingsBS := s.Postings[pid] - - freqNorms := s.FreqNorms[pid] - freqNormOffset := 0 - - locs := s.Locs[pid] - locOffset := 0 - - chunkSize, err := getChunkSize(s.chunkMode, postingsBS.GetCardinality(), uint64(len(s.results))) - if err != nil { - return 0, nil, err - } - tfEncoder.SetChunkSize(chunkSize, uint64(len(s.results)-1)) - locEncoder.SetChunkSize(chunkSize, uint64(len(s.results)-1)) - - postingsItr := postingsBS.Iterator() - for postingsItr.HasNext() { - docNum := uint64(postingsItr.Next()) - - freqNorm := freqNorms[freqNormOffset] - - err = tfEncoder.Add(docNum, - encodeFreqHasLocs(freqNorm.freq, freqNorm.numLocs > 0), - uint64(math.Float32bits(freqNorm.norm))) - if err != nil { - return 0, nil, err - } - - if freqNorm.numLocs > 0 { - numBytesLocs := 0 - for _, loc := range locs[locOffset : locOffset+freqNorm.numLocs] { - numBytesLocs += totalUvarintBytes( - uint64(loc.fieldID), loc.pos, loc.start, loc.end, - uint64(len(loc.arrayposs)), loc.arrayposs) - } - - err = locEncoder.Add(docNum, uint64(numBytesLocs)) - if err != nil { - return 0, nil, err - } - - for _, loc := range locs[locOffset : locOffset+freqNorm.numLocs] { - err = locEncoder.Add(docNum, - uint64(loc.fieldID), loc.pos, loc.start, loc.end, - uint64(len(loc.arrayposs))) - if err != nil { - return 0, nil, err - } - - err = locEncoder.Add(docNum, loc.arrayposs...) - if err != nil { - return 0, nil, err - } - } - - locOffset += freqNorm.numLocs - } - - freqNormOffset++ - - docTermMap[docNum] = append( - append(docTermMap[docNum], term...), - termSeparator) - } - - tfEncoder.Close() - locEncoder.Close() - - postingsOffset, err := - writePostings(postingsBS, tfEncoder, locEncoder, nil, s.w, buf) - if err != nil { - return 0, nil, err - } - - if postingsOffset > uint64(0) { - err = s.builder.Insert([]byte(term), postingsOffset) - if err != nil { - return 0, nil, err - } - } - - tfEncoder.Reset() - locEncoder.Reset() - } - - err = s.builder.Close() - if err != nil { - return 0, nil, err - } - - // record where this dictionary starts - dictOffsets[fieldID] = uint64(s.w.Count()) - - vellumData := s.builderBuf.Bytes() - - // write out the length of the vellum data - n := binary.PutUvarint(buf, uint64(len(vellumData))) - _, err = s.w.Write(buf[:n]) - if err != nil { - return 0, nil, err - } - - // write this vellum to disk - _, err = s.w.Write(vellumData) - if err != nil { - return 0, nil, err - } - - // reset vellum for reuse - s.builderBuf.Reset() - - err = s.builder.Reset(&s.builderBuf) - if err != nil { - return 0, nil, err - } - - // write the field doc values - // NOTE: doc values continue to use legacy chunk mode - chunkSize, err := getChunkSize(LegacyChunkMode, 0, 0) - if err != nil { - return 0, nil, err - } - fdvEncoder := newChunkedContentCoder(chunkSize, uint64(len(s.results)-1), s.w, false) - if s.IncludeDocValues[fieldID] { - for docNum, docTerms := range docTermMap { - if len(docTerms) > 0 { - err = fdvEncoder.Add(uint64(docNum), docTerms) - if err != nil { - return 0, nil, err - } - } - } - err = fdvEncoder.Close() - if err != nil { - return 0, nil, err - } - - fdvOffsetsStart[fieldID] = uint64(s.w.Count()) - - _, err = fdvEncoder.Write() - if err != nil { - return 0, nil, err - } - - fdvOffsetsEnd[fieldID] = uint64(s.w.Count()) - - fdvEncoder.Reset() - } else { - fdvOffsetsStart[fieldID] = fieldNotUninverted - fdvOffsetsEnd[fieldID] = fieldNotUninverted - } - } - - fdvIndexOffset = uint64(s.w.Count()) - - for i := 0; i < len(fdvOffsetsStart); i++ { - n := binary.PutUvarint(buf, fdvOffsetsStart[i]) - _, err := s.w.Write(buf[:n]) - if err != nil { - return 0, nil, err - } - n = binary.PutUvarint(buf, fdvOffsetsEnd[i]) - _, err = s.w.Write(buf[:n]) - if err != nil { - return 0, nil, err - } - } - - return fdvIndexOffset, dictOffsets, nil -} - -func encodeFieldType(f document.Field) byte { - fieldType := byte('x') - switch f.(type) { - case *document.TextField: - fieldType = 't' - case *document.NumericField: - fieldType = 'n' - case *document.DateTimeField: - fieldType = 'd' - case *document.BooleanField: - fieldType = 'b' - case *document.GeoPointField: - fieldType = 'g' - case *document.CompositeField: - fieldType = 'c' - } - return fieldType -} - -// returns the total # of bytes needed to encode the given uint64's -// into binary.PutUVarint() encoding -func totalUvarintBytes(a, b, c, d, e uint64, more []uint64) (n int) { - n = numUvarintBytes(a) - n += numUvarintBytes(b) - n += numUvarintBytes(c) - n += numUvarintBytes(d) - n += numUvarintBytes(e) - for _, v := range more { - n += numUvarintBytes(v) - } - return n -} - -// returns # of bytes needed to encode x in binary.PutUvarint() encoding -func numUvarintBytes(x uint64) (n int) { - for x >= 0x80 { - x >>= 7 - n++ - } - return n + 1 -} diff --git a/vendor/github.com/blevesearch/zap/v15/plugin.go b/vendor/github.com/blevesearch/zap/v15/plugin.go deleted file mode 100644 index 38a0638d..00000000 --- a/vendor/github.com/blevesearch/zap/v15/plugin.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (c) 2020 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "github.com/blevesearch/bleve/index/scorch/segment" -) - -// ZapPlugin implements the Plugin interface of -// the blevesearch/bleve/index/scorch/segment pkg -type ZapPlugin struct{} - -func (*ZapPlugin) Type() string { - return Type -} - -func (*ZapPlugin) Version() uint32 { - return Version -} - -// Plugin returns an instance segment.Plugin for use -// by the Scorch indexing scheme -func Plugin() segment.Plugin { - return &ZapPlugin{} -} diff --git a/vendor/github.com/blevesearch/zap/v15/posting.go b/vendor/github.com/blevesearch/zap/v15/posting.go deleted file mode 100644 index 75faa5d7..00000000 --- a/vendor/github.com/blevesearch/zap/v15/posting.go +++ /dev/null @@ -1,853 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "encoding/binary" - "fmt" - "math" - "reflect" - - "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/blevesearch/bleve/size" -) - -var reflectStaticSizePostingsList int -var reflectStaticSizePostingsIterator int -var reflectStaticSizePosting int -var reflectStaticSizeLocation int - -func init() { - var pl PostingsList - reflectStaticSizePostingsList = int(reflect.TypeOf(pl).Size()) - var pi PostingsIterator - reflectStaticSizePostingsIterator = int(reflect.TypeOf(pi).Size()) - var p Posting - reflectStaticSizePosting = int(reflect.TypeOf(p).Size()) - var l Location - reflectStaticSizeLocation = int(reflect.TypeOf(l).Size()) -} - -// FST or vellum value (uint64) encoding is determined by the top two -// highest-order or most significant bits... -// -// encoding : MSB -// name : 63 62 61...to...bit #0 (LSB) -// ----------+---+---+--------------------------------------------------- -// general : 0 | 0 | 62-bits of postingsOffset. -// ~ : 0 | 1 | reserved for future. -// 1-hit : 1 | 0 | 31-bits of positive float31 norm | 31-bits docNum. -// ~ : 1 | 1 | reserved for future. -// -// Encoding "general" is able to handle all cases, where the -// postingsOffset points to more information about the postings for -// the term. -// -// Encoding "1-hit" is used to optimize a commonly seen case when a -// term has only a single hit. For example, a term in the _id field -// will have only 1 hit. The "1-hit" encoding is used for a term -// in a field when... -// -// - term vector info is disabled for that field; -// - and, the term appears in only a single doc for that field; -// - and, the term's freq is exactly 1 in that single doc for that field; -// - and, the docNum must fit into 31-bits; -// -// Otherwise, the "general" encoding is used instead. -// -// In the "1-hit" encoding, the field in that single doc may have -// other terms, which is supported in the "1-hit" encoding by the -// positive float31 norm. - -const FSTValEncodingMask = uint64(0xc000000000000000) -const FSTValEncodingGeneral = uint64(0x0000000000000000) -const FSTValEncoding1Hit = uint64(0x8000000000000000) - -func FSTValEncode1Hit(docNum uint64, normBits uint64) uint64 { - return FSTValEncoding1Hit | ((mask31Bits & normBits) << 31) | (mask31Bits & docNum) -} - -func FSTValDecode1Hit(v uint64) (docNum uint64, normBits uint64) { - return (mask31Bits & v), (mask31Bits & (v >> 31)) -} - -const mask31Bits = uint64(0x000000007fffffff) - -func under32Bits(x uint64) bool { - return x <= mask31Bits -} - -const DocNum1HitFinished = math.MaxUint64 - -var NormBits1Hit = uint64(1) - -// PostingsList is an in-memory representation of a postings list -type PostingsList struct { - sb *SegmentBase - postingsOffset uint64 - freqOffset uint64 - locOffset uint64 - postings *roaring.Bitmap - except *roaring.Bitmap - - // when normBits1Hit != 0, then this postings list came from a - // 1-hit encoding, and only the docNum1Hit & normBits1Hit apply - docNum1Hit uint64 - normBits1Hit uint64 - - chunkSize uint64 -} - -// represents an immutable, empty postings list -var emptyPostingsList = &PostingsList{} - -func (p *PostingsList) Size() int { - sizeInBytes := reflectStaticSizePostingsList + size.SizeOfPtr - - if p.except != nil { - sizeInBytes += int(p.except.GetSizeInBytes()) - } - - return sizeInBytes -} - -func (p *PostingsList) OrInto(receiver *roaring.Bitmap) { - if p.normBits1Hit != 0 { - receiver.Add(uint32(p.docNum1Hit)) - return - } - - if p.postings != nil { - receiver.Or(p.postings) - } -} - -// Iterator returns an iterator for this postings list -func (p *PostingsList) Iterator(includeFreq, includeNorm, includeLocs bool, - prealloc segment.PostingsIterator) segment.PostingsIterator { - if p.normBits1Hit == 0 && p.postings == nil { - return emptyPostingsIterator - } - - var preallocPI *PostingsIterator - pi, ok := prealloc.(*PostingsIterator) - if ok && pi != nil { - preallocPI = pi - } - if preallocPI == emptyPostingsIterator { - preallocPI = nil - } - - return p.iterator(includeFreq, includeNorm, includeLocs, preallocPI) -} - -func (p *PostingsList) iterator(includeFreq, includeNorm, includeLocs bool, - rv *PostingsIterator) *PostingsIterator { - if rv == nil { - rv = &PostingsIterator{} - } else { - freqNormReader := rv.freqNormReader - if freqNormReader != nil { - freqNormReader.reset() - } - - locReader := rv.locReader - if locReader != nil { - locReader.reset() - } - - nextLocs := rv.nextLocs[:0] - nextSegmentLocs := rv.nextSegmentLocs[:0] - - buf := rv.buf - - *rv = PostingsIterator{} // clear the struct - - rv.freqNormReader = freqNormReader - rv.locReader = locReader - - rv.nextLocs = nextLocs - rv.nextSegmentLocs = nextSegmentLocs - - rv.buf = buf - } - - rv.postings = p - rv.includeFreqNorm = includeFreq || includeNorm || includeLocs - rv.includeLocs = includeLocs - - if p.normBits1Hit != 0 { - // "1-hit" encoding - rv.docNum1Hit = p.docNum1Hit - rv.normBits1Hit = p.normBits1Hit - - if p.except != nil && p.except.Contains(uint32(rv.docNum1Hit)) { - rv.docNum1Hit = DocNum1HitFinished - } - - return rv - } - - // "general" encoding, check if empty - if p.postings == nil { - return rv - } - - // initialize freq chunk reader - if rv.includeFreqNorm { - rv.freqNormReader = newChunkedIntDecoder(p.sb.mem, p.freqOffset, rv.freqNormReader) - } - - // initialize the loc chunk reader - if rv.includeLocs { - rv.locReader = newChunkedIntDecoder(p.sb.mem, p.locOffset, rv.locReader) - } - - rv.all = p.postings.Iterator() - if p.except != nil { - rv.ActualBM = roaring.AndNot(p.postings, p.except) - rv.Actual = rv.ActualBM.Iterator() - } else { - rv.ActualBM = p.postings - rv.Actual = rv.all // Optimize to use same iterator for all & Actual. - } - - return rv -} - -// Count returns the number of items on this postings list -func (p *PostingsList) Count() uint64 { - var n, e uint64 - if p.normBits1Hit != 0 { - n = 1 - if p.except != nil && p.except.Contains(uint32(p.docNum1Hit)) { - e = 1 - } - } else if p.postings != nil { - n = p.postings.GetCardinality() - if p.except != nil { - e = p.postings.AndCardinality(p.except) - } - } - return n - e -} - -func (rv *PostingsList) read(postingsOffset uint64, d *Dictionary) error { - rv.postingsOffset = postingsOffset - - // handle "1-hit" encoding special case - if rv.postingsOffset&FSTValEncodingMask == FSTValEncoding1Hit { - return rv.init1Hit(postingsOffset) - } - - // read the location of the freq/norm details - var n uint64 - var read int - - rv.freqOffset, read = binary.Uvarint(d.sb.mem[postingsOffset+n : postingsOffset+binary.MaxVarintLen64]) - n += uint64(read) - - rv.locOffset, read = binary.Uvarint(d.sb.mem[postingsOffset+n : postingsOffset+n+binary.MaxVarintLen64]) - n += uint64(read) - - var postingsLen uint64 - postingsLen, read = binary.Uvarint(d.sb.mem[postingsOffset+n : postingsOffset+n+binary.MaxVarintLen64]) - n += uint64(read) - - roaringBytes := d.sb.mem[postingsOffset+n : postingsOffset+n+postingsLen] - - if rv.postings == nil { - rv.postings = roaring.NewBitmap() - } - _, err := rv.postings.FromBuffer(roaringBytes) - if err != nil { - return fmt.Errorf("error loading roaring bitmap: %v", err) - } - - rv.chunkSize, err = getChunkSize(d.sb.chunkMode, - rv.postings.GetCardinality(), d.sb.numDocs) - if err != nil { - return err - } - - return nil -} - -func (rv *PostingsList) init1Hit(fstVal uint64) error { - docNum, normBits := FSTValDecode1Hit(fstVal) - - rv.docNum1Hit = docNum - rv.normBits1Hit = normBits - - return nil -} - -// PostingsIterator provides a way to iterate through the postings list -type PostingsIterator struct { - postings *PostingsList - all roaring.IntPeekable - Actual roaring.IntPeekable - ActualBM *roaring.Bitmap - - currChunk uint32 - freqNormReader *chunkedIntDecoder - locReader *chunkedIntDecoder - - next Posting // reused across Next() calls - nextLocs []Location // reused across Next() calls - nextSegmentLocs []segment.Location // reused across Next() calls - - docNum1Hit uint64 - normBits1Hit uint64 - - buf []byte - - includeFreqNorm bool - includeLocs bool -} - -var emptyPostingsIterator = &PostingsIterator{} - -func (i *PostingsIterator) Size() int { - sizeInBytes := reflectStaticSizePostingsIterator + size.SizeOfPtr + - i.next.Size() - // account for freqNormReader, locReader if we start using this. - for _, entry := range i.nextLocs { - sizeInBytes += entry.Size() - } - - return sizeInBytes -} - -func (i *PostingsIterator) loadChunk(chunk int) error { - if i.includeFreqNorm { - err := i.freqNormReader.loadChunk(chunk) - if err != nil { - return err - } - } - - if i.includeLocs { - err := i.locReader.loadChunk(chunk) - if err != nil { - return err - } - } - - i.currChunk = uint32(chunk) - return nil -} - -func (i *PostingsIterator) readFreqNormHasLocs() (uint64, uint64, bool, error) { - if i.normBits1Hit != 0 { - return 1, i.normBits1Hit, false, nil - } - - freqHasLocs, err := i.freqNormReader.readUvarint() - if err != nil { - return 0, 0, false, fmt.Errorf("error reading frequency: %v", err) - } - - freq, hasLocs := decodeFreqHasLocs(freqHasLocs) - - normBits, err := i.freqNormReader.readUvarint() - if err != nil { - return 0, 0, false, fmt.Errorf("error reading norm: %v", err) - } - - return freq, normBits, hasLocs, nil -} - -func (i *PostingsIterator) skipFreqNormReadHasLocs() (bool, error) { - if i.normBits1Hit != 0 { - return false, nil - } - - freqHasLocs, err := i.freqNormReader.readUvarint() - if err != nil { - return false, fmt.Errorf("error reading freqHasLocs: %v", err) - } - - i.freqNormReader.SkipUvarint() // Skip normBits. - - return freqHasLocs&0x01 != 0, nil // See decodeFreqHasLocs() / hasLocs. -} - -func encodeFreqHasLocs(freq uint64, hasLocs bool) uint64 { - rv := freq << 1 - if hasLocs { - rv = rv | 0x01 // 0'th LSB encodes whether there are locations - } - return rv -} - -func decodeFreqHasLocs(freqHasLocs uint64) (uint64, bool) { - freq := freqHasLocs >> 1 - hasLocs := freqHasLocs&0x01 != 0 - return freq, hasLocs -} - -// readLocation processes all the integers on the stream representing a single -// location. -func (i *PostingsIterator) readLocation(l *Location) error { - // read off field - fieldID, err := i.locReader.readUvarint() - if err != nil { - return fmt.Errorf("error reading location field: %v", err) - } - // read off pos - pos, err := i.locReader.readUvarint() - if err != nil { - return fmt.Errorf("error reading location pos: %v", err) - } - // read off start - start, err := i.locReader.readUvarint() - if err != nil { - return fmt.Errorf("error reading location start: %v", err) - } - // read off end - end, err := i.locReader.readUvarint() - if err != nil { - return fmt.Errorf("error reading location end: %v", err) - } - // read off num array pos - numArrayPos, err := i.locReader.readUvarint() - if err != nil { - return fmt.Errorf("error reading location num array pos: %v", err) - } - - l.field = i.postings.sb.fieldsInv[fieldID] - l.pos = pos - l.start = start - l.end = end - - if cap(l.ap) < int(numArrayPos) { - l.ap = make([]uint64, int(numArrayPos)) - } else { - l.ap = l.ap[:int(numArrayPos)] - } - - // read off array positions - for k := 0; k < int(numArrayPos); k++ { - ap, err := i.locReader.readUvarint() - if err != nil { - return fmt.Errorf("error reading array position: %v", err) - } - - l.ap[k] = ap - } - - return nil -} - -// Next returns the next posting on the postings list, or nil at the end -func (i *PostingsIterator) Next() (segment.Posting, error) { - return i.nextAtOrAfter(0) -} - -// Advance returns the posting at the specified docNum or it is not present -// the next posting, or if the end is reached, nil -func (i *PostingsIterator) Advance(docNum uint64) (segment.Posting, error) { - return i.nextAtOrAfter(docNum) -} - -// Next returns the next posting on the postings list, or nil at the end -func (i *PostingsIterator) nextAtOrAfter(atOrAfter uint64) (segment.Posting, error) { - docNum, exists, err := i.nextDocNumAtOrAfter(atOrAfter) - if err != nil || !exists { - return nil, err - } - - i.next = Posting{} // clear the struct - rv := &i.next - rv.docNum = docNum - - if !i.includeFreqNorm { - return rv, nil - } - - var normBits uint64 - var hasLocs bool - - rv.freq, normBits, hasLocs, err = i.readFreqNormHasLocs() - if err != nil { - return nil, err - } - - rv.norm = math.Float32frombits(uint32(normBits)) - - if i.includeLocs && hasLocs { - // prepare locations into reused slices, where we assume - // rv.freq >= "number of locs", since in a composite field, - // some component fields might have their IncludeTermVector - // flags disabled while other component fields are enabled - if cap(i.nextLocs) >= int(rv.freq) { - i.nextLocs = i.nextLocs[0:rv.freq] - } else { - i.nextLocs = make([]Location, rv.freq, rv.freq*2) - } - if cap(i.nextSegmentLocs) < int(rv.freq) { - i.nextSegmentLocs = make([]segment.Location, rv.freq, rv.freq*2) - } - rv.locs = i.nextSegmentLocs[:0] - - numLocsBytes, err := i.locReader.readUvarint() - if err != nil { - return nil, fmt.Errorf("error reading location numLocsBytes: %v", err) - } - - j := 0 - startBytesRemaining := i.locReader.Len() // # bytes remaining in the locReader - for startBytesRemaining-i.locReader.Len() < int(numLocsBytes) { - err := i.readLocation(&i.nextLocs[j]) - if err != nil { - return nil, err - } - rv.locs = append(rv.locs, &i.nextLocs[j]) - j++ - } - } - - return rv, nil -} - -// nextDocNum returns the next docNum on the postings list, and also -// sets up the currChunk / loc related fields of the iterator. -func (i *PostingsIterator) nextDocNumAtOrAfter(atOrAfter uint64) (uint64, bool, error) { - if i.normBits1Hit != 0 { - if i.docNum1Hit == DocNum1HitFinished { - return 0, false, nil - } - if i.docNum1Hit < atOrAfter { - // advanced past our 1-hit - i.docNum1Hit = DocNum1HitFinished // consume our 1-hit docNum - return 0, false, nil - } - docNum := i.docNum1Hit - i.docNum1Hit = DocNum1HitFinished // consume our 1-hit docNum - return docNum, true, nil - } - - if i.Actual == nil || !i.Actual.HasNext() { - return 0, false, nil - } - - if i.postings == nil || i.postings.postings == i.ActualBM { - return i.nextDocNumAtOrAfterClean(atOrAfter) - } - - i.Actual.AdvanceIfNeeded(uint32(atOrAfter)) - - if !i.Actual.HasNext() { - // couldn't find anything - return 0, false, nil - } - - n := i.Actual.Next() - allN := i.all.Next() - nChunk := n / uint32(i.postings.chunkSize) - - // when allN becomes >= to here, then allN is in the same chunk as nChunk. - allNReachesNChunk := nChunk * uint32(i.postings.chunkSize) - - // n is the next actual hit (excluding some postings), and - // allN is the next hit in the full postings, and - // if they don't match, move 'all' forwards until they do - for allN != n { - // we've reached same chunk, so move the freq/norm/loc decoders forward - if i.includeFreqNorm && allN >= allNReachesNChunk { - err := i.currChunkNext(nChunk) - if err != nil { - return 0, false, err - } - } - - allN = i.all.Next() - } - - if i.includeFreqNorm && (i.currChunk != nChunk || i.freqNormReader.isNil()) { - err := i.loadChunk(int(nChunk)) - if err != nil { - return 0, false, fmt.Errorf("error loading chunk: %v", err) - } - } - - return uint64(n), true, nil -} - -var freqHasLocs1Hit = encodeFreqHasLocs(1, false) - -// nextBytes returns the docNum and the encoded freq & loc bytes for -// the next posting -func (i *PostingsIterator) nextBytes() ( - docNumOut uint64, freq uint64, normBits uint64, - bytesFreqNorm []byte, bytesLoc []byte, err error) { - docNum, exists, err := i.nextDocNumAtOrAfter(0) - if err != nil || !exists { - return 0, 0, 0, nil, nil, err - } - - if i.normBits1Hit != 0 { - if i.buf == nil { - i.buf = make([]byte, binary.MaxVarintLen64*2) - } - n := binary.PutUvarint(i.buf, freqHasLocs1Hit) - n += binary.PutUvarint(i.buf[n:], i.normBits1Hit) - return docNum, uint64(1), i.normBits1Hit, i.buf[:n], nil, nil - } - - startFreqNorm := i.freqNormReader.remainingLen() - - var hasLocs bool - - freq, normBits, hasLocs, err = i.readFreqNormHasLocs() - if err != nil { - return 0, 0, 0, nil, nil, err - } - - endFreqNorm := i.freqNormReader.remainingLen() - bytesFreqNorm = i.freqNormReader.readBytes(startFreqNorm, endFreqNorm) - - if hasLocs { - startLoc := i.locReader.remainingLen() - - numLocsBytes, err := i.locReader.readUvarint() - if err != nil { - return 0, 0, 0, nil, nil, - fmt.Errorf("error reading location nextBytes numLocs: %v", err) - } - - // skip over all the location bytes - i.locReader.SkipBytes(int(numLocsBytes)) - - endLoc := i.locReader.remainingLen() - bytesLoc = i.locReader.readBytes(startLoc, endLoc) - } - - return docNum, freq, normBits, bytesFreqNorm, bytesLoc, nil -} - -// optimization when the postings list is "clean" (e.g., no updates & -// no deletions) where the all bitmap is the same as the actual bitmap -func (i *PostingsIterator) nextDocNumAtOrAfterClean( - atOrAfter uint64) (uint64, bool, error) { - - if !i.includeFreqNorm { - i.Actual.AdvanceIfNeeded(uint32(atOrAfter)) - - if !i.Actual.HasNext() { - return 0, false, nil // couldn't find anything - } - - return uint64(i.Actual.Next()), true, nil - } - - // freq-norm's needed, so maintain freq-norm chunk reader - sameChunkNexts := 0 // # of times we called Next() in the same chunk - n := i.Actual.Next() - nChunk := n / uint32(i.postings.chunkSize) - - for uint64(n) < atOrAfter && i.Actual.HasNext() { - n = i.Actual.Next() - - nChunkPrev := nChunk - nChunk = n / uint32(i.postings.chunkSize) - - if nChunk != nChunkPrev { - sameChunkNexts = 0 - } else { - sameChunkNexts += 1 - } - } - - if uint64(n) < atOrAfter { - // couldn't find anything - return 0, false, nil - } - - for j := 0; j < sameChunkNexts; j++ { - err := i.currChunkNext(nChunk) - if err != nil { - return 0, false, fmt.Errorf("error optimized currChunkNext: %v", err) - } - } - - if i.currChunk != nChunk || i.freqNormReader.isNil() { - err := i.loadChunk(int(nChunk)) - if err != nil { - return 0, false, fmt.Errorf("error loading chunk: %v", err) - } - } - - return uint64(n), true, nil -} - -func (i *PostingsIterator) currChunkNext(nChunk uint32) error { - if i.currChunk != nChunk || i.freqNormReader.isNil() { - err := i.loadChunk(int(nChunk)) - if err != nil { - return fmt.Errorf("error loading chunk: %v", err) - } - } - - // read off freq/offsets even though we don't care about them - hasLocs, err := i.skipFreqNormReadHasLocs() - if err != nil { - return err - } - - if i.includeLocs && hasLocs { - numLocsBytes, err := i.locReader.readUvarint() - if err != nil { - return fmt.Errorf("error reading location numLocsBytes: %v", err) - } - - // skip over all the location bytes - i.locReader.SkipBytes(int(numLocsBytes)) - } - - return nil -} - -// DocNum1Hit returns the docNum and true if this is "1-hit" optimized -// and the docNum is available. -func (p *PostingsIterator) DocNum1Hit() (uint64, bool) { - if p.normBits1Hit != 0 && p.docNum1Hit != DocNum1HitFinished { - return p.docNum1Hit, true - } - return 0, false -} - -// ActualBitmap returns the underlying actual bitmap -// which can be used up the stack for optimizations -func (p *PostingsIterator) ActualBitmap() *roaring.Bitmap { - return p.ActualBM -} - -// ReplaceActual replaces the ActualBM with the provided -// bitmap -func (p *PostingsIterator) ReplaceActual(abm *roaring.Bitmap) { - p.ActualBM = abm - p.Actual = abm.Iterator() -} - -// PostingsIteratorFromBitmap constructs a PostingsIterator given an -// "actual" bitmap. -func PostingsIteratorFromBitmap(bm *roaring.Bitmap, - includeFreqNorm, includeLocs bool) (segment.PostingsIterator, error) { - return &PostingsIterator{ - ActualBM: bm, - Actual: bm.Iterator(), - includeFreqNorm: includeFreqNorm, - includeLocs: includeLocs, - }, nil -} - -// PostingsIteratorFrom1Hit constructs a PostingsIterator given a -// 1-hit docNum. -func PostingsIteratorFrom1Hit(docNum1Hit uint64, - includeFreqNorm, includeLocs bool) (segment.PostingsIterator, error) { - return &PostingsIterator{ - docNum1Hit: docNum1Hit, - normBits1Hit: NormBits1Hit, - includeFreqNorm: includeFreqNorm, - includeLocs: includeLocs, - }, nil -} - -// Posting is a single entry in a postings list -type Posting struct { - docNum uint64 - freq uint64 - norm float32 - locs []segment.Location -} - -func (p *Posting) Size() int { - sizeInBytes := reflectStaticSizePosting - - for _, entry := range p.locs { - sizeInBytes += entry.Size() - } - - return sizeInBytes -} - -// Number returns the document number of this posting in this segment -func (p *Posting) Number() uint64 { - return p.docNum -} - -// Frequency returns the frequencies of occurrence of this term in this doc/field -func (p *Posting) Frequency() uint64 { - return p.freq -} - -// Norm returns the normalization factor for this posting -func (p *Posting) Norm() float64 { - return float64(float32(1.0 / math.Sqrt(float64(math.Float32bits(p.norm))))) -} - -// Locations returns the location information for each occurrence -func (p *Posting) Locations() []segment.Location { - return p.locs -} - -// NormUint64 returns the norm value as uint64 -func (p *Posting) NormUint64() uint64 { - return uint64(math.Float32bits(p.norm)) -} - -// Location represents the location of a single occurrence -type Location struct { - field string - pos uint64 - start uint64 - end uint64 - ap []uint64 -} - -func (l *Location) Size() int { - return reflectStaticSizeLocation + - len(l.field) + - len(l.ap)*size.SizeOfUint64 -} - -// Field returns the name of the field (useful in composite fields to know -// which original field the value came from) -func (l *Location) Field() string { - return l.field -} - -// Start returns the start byte offset of this occurrence -func (l *Location) Start() uint64 { - return l.start -} - -// End returns the end byte offset of this occurrence -func (l *Location) End() uint64 { - return l.end -} - -// Pos returns the 1-based phrase position of this occurrence -func (l *Location) Pos() uint64 { - return l.pos -} - -// ArrayPositions returns the array position vector associated with this occurrence -func (l *Location) ArrayPositions() []uint64 { - return l.ap -} diff --git a/vendor/github.com/blevesearch/zap/v15/segment.go b/vendor/github.com/blevesearch/zap/v15/segment.go deleted file mode 100644 index 2d158e86..00000000 --- a/vendor/github.com/blevesearch/zap/v15/segment.go +++ /dev/null @@ -1,572 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zap - -import ( - "bytes" - "encoding/binary" - "fmt" - "io" - "os" - "sync" - "unsafe" - - "github.com/RoaringBitmap/roaring" - "github.com/blevesearch/bleve/index/scorch/segment" - "github.com/blevesearch/bleve/size" - "github.com/couchbase/vellum" - mmap "github.com/blevesearch/mmap-go" - "github.com/golang/snappy" -) - -var reflectStaticSizeSegmentBase int - -func init() { - var sb SegmentBase - reflectStaticSizeSegmentBase = int(unsafe.Sizeof(sb)) -} - -// Open returns a zap impl of a segment -func (*ZapPlugin) Open(path string) (segment.Segment, error) { - f, err := os.Open(path) - if err != nil { - return nil, err - } - mm, err := mmap.Map(f, mmap.RDONLY, 0) - if err != nil { - // mmap failed, try to close the file - _ = f.Close() - return nil, err - } - - rv := &Segment{ - SegmentBase: SegmentBase{ - mem: mm[0 : len(mm)-FooterSize], - fieldsMap: make(map[string]uint16), - fieldDvReaders: make(map[uint16]*docValueReader), - fieldFSTs: make(map[uint16]*vellum.FST), - }, - f: f, - mm: mm, - path: path, - refs: 1, - } - rv.SegmentBase.updateSize() - - err = rv.loadConfig() - if err != nil { - _ = rv.Close() - return nil, err - } - - err = rv.loadFields() - if err != nil { - _ = rv.Close() - return nil, err - } - - err = rv.loadDvReaders() - if err != nil { - _ = rv.Close() - return nil, err - } - - return rv, nil -} - -// SegmentBase is a memory only, read-only implementation of the -// segment.Segment interface, using zap's data representation. -type SegmentBase struct { - mem []byte - memCRC uint32 - chunkMode uint32 - fieldsMap map[string]uint16 // fieldName -> fieldID+1 - fieldsInv []string // fieldID -> fieldName - numDocs uint64 - storedIndexOffset uint64 - fieldsIndexOffset uint64 - docValueOffset uint64 - dictLocs []uint64 - fieldDvReaders map[uint16]*docValueReader // naive chunk cache per field - fieldDvNames []string // field names cached in fieldDvReaders - size uint64 - - m sync.Mutex - fieldFSTs map[uint16]*vellum.FST -} - -func (sb *SegmentBase) Size() int { - return int(sb.size) -} - -func (sb *SegmentBase) updateSize() { - sizeInBytes := reflectStaticSizeSegmentBase + - cap(sb.mem) - - // fieldsMap - for k := range sb.fieldsMap { - sizeInBytes += (len(k) + size.SizeOfString) + size.SizeOfUint16 - } - - // fieldsInv, dictLocs - for _, entry := range sb.fieldsInv { - sizeInBytes += len(entry) + size.SizeOfString - } - sizeInBytes += len(sb.dictLocs) * size.SizeOfUint64 - - // fieldDvReaders - for _, v := range sb.fieldDvReaders { - sizeInBytes += size.SizeOfUint16 + size.SizeOfPtr - if v != nil { - sizeInBytes += v.size() - } - } - - sb.size = uint64(sizeInBytes) -} - -func (sb *SegmentBase) AddRef() {} -func (sb *SegmentBase) DecRef() (err error) { return nil } -func (sb *SegmentBase) Close() (err error) { return nil } - -// Segment implements a persisted segment.Segment interface, by -// embedding an mmap()'ed SegmentBase. -type Segment struct { - SegmentBase - - f *os.File - mm mmap.MMap - path string - version uint32 - crc uint32 - - m sync.Mutex // Protects the fields that follow. - refs int64 -} - -func (s *Segment) Size() int { - // 8 /* size of file pointer */ - // 4 /* size of version -> uint32 */ - // 4 /* size of crc -> uint32 */ - sizeOfUints := 16 - - sizeInBytes := (len(s.path) + size.SizeOfString) + sizeOfUints - - // mutex, refs -> int64 - sizeInBytes += 16 - - // do not include the mmap'ed part - return sizeInBytes + s.SegmentBase.Size() - cap(s.mem) -} - -func (s *Segment) AddRef() { - s.m.Lock() - s.refs++ - s.m.Unlock() -} - -func (s *Segment) DecRef() (err error) { - s.m.Lock() - s.refs-- - if s.refs == 0 { - err = s.closeActual() - } - s.m.Unlock() - return err -} - -func (s *Segment) loadConfig() error { - crcOffset := len(s.mm) - 4 - s.crc = binary.BigEndian.Uint32(s.mm[crcOffset : crcOffset+4]) - - verOffset := crcOffset - 4 - s.version = binary.BigEndian.Uint32(s.mm[verOffset : verOffset+4]) - if s.version != Version { - return fmt.Errorf("unsupported version %d != %d", s.version, Version) - } - - chunkOffset := verOffset - 4 - s.chunkMode = binary.BigEndian.Uint32(s.mm[chunkOffset : chunkOffset+4]) - - docValueOffset := chunkOffset - 8 - s.docValueOffset = binary.BigEndian.Uint64(s.mm[docValueOffset : docValueOffset+8]) - - fieldsIndexOffset := docValueOffset - 8 - s.fieldsIndexOffset = binary.BigEndian.Uint64(s.mm[fieldsIndexOffset : fieldsIndexOffset+8]) - - storedIndexOffset := fieldsIndexOffset - 8 - s.storedIndexOffset = binary.BigEndian.Uint64(s.mm[storedIndexOffset : storedIndexOffset+8]) - - numDocsOffset := storedIndexOffset - 8 - s.numDocs = binary.BigEndian.Uint64(s.mm[numDocsOffset : numDocsOffset+8]) - return nil -} - -func (s *SegmentBase) loadFields() error { - // NOTE for now we assume the fields index immediately precedes - // the footer, and if this changes, need to adjust accordingly (or - // store explicit length), where s.mem was sliced from s.mm in Open(). - fieldsIndexEnd := uint64(len(s.mem)) - - // iterate through fields index - var fieldID uint64 - for s.fieldsIndexOffset+(8*fieldID) < fieldsIndexEnd { - addr := binary.BigEndian.Uint64(s.mem[s.fieldsIndexOffset+(8*fieldID) : s.fieldsIndexOffset+(8*fieldID)+8]) - - dictLoc, read := binary.Uvarint(s.mem[addr:fieldsIndexEnd]) - n := uint64(read) - s.dictLocs = append(s.dictLocs, dictLoc) - - var nameLen uint64 - nameLen, read = binary.Uvarint(s.mem[addr+n : fieldsIndexEnd]) - n += uint64(read) - - name := string(s.mem[addr+n : addr+n+nameLen]) - s.fieldsInv = append(s.fieldsInv, name) - s.fieldsMap[name] = uint16(fieldID + 1) - - fieldID++ - } - return nil -} - -// Dictionary returns the term dictionary for the specified field -func (s *SegmentBase) Dictionary(field string) (segment.TermDictionary, error) { - dict, err := s.dictionary(field) - if err == nil && dict == nil { - return &segment.EmptyDictionary{}, nil - } - return dict, err -} - -func (sb *SegmentBase) dictionary(field string) (rv *Dictionary, err error) { - fieldIDPlus1 := sb.fieldsMap[field] - if fieldIDPlus1 > 0 { - rv = &Dictionary{ - sb: sb, - field: field, - fieldID: fieldIDPlus1 - 1, - } - - dictStart := sb.dictLocs[rv.fieldID] - if dictStart > 0 { - var ok bool - sb.m.Lock() - if rv.fst, ok = sb.fieldFSTs[rv.fieldID]; !ok { - // read the length of the vellum data - vellumLen, read := binary.Uvarint(sb.mem[dictStart : dictStart+binary.MaxVarintLen64]) - fstBytes := sb.mem[dictStart+uint64(read) : dictStart+uint64(read)+vellumLen] - rv.fst, err = vellum.Load(fstBytes) - if err != nil { - sb.m.Unlock() - return nil, fmt.Errorf("dictionary field %s vellum err: %v", field, err) - } - - sb.fieldFSTs[rv.fieldID] = rv.fst - } - - sb.m.Unlock() - rv.fstReader, err = rv.fst.Reader() - if err != nil { - return nil, fmt.Errorf("dictionary field %s vellum reader err: %v", field, err) - } - - } - } - - return rv, nil -} - -// visitDocumentCtx holds data structures that are reusable across -// multiple VisitDocument() calls to avoid memory allocations -type visitDocumentCtx struct { - buf []byte - reader bytes.Reader - arrayPos []uint64 -} - -var visitDocumentCtxPool = sync.Pool{ - New: func() interface{} { - reuse := &visitDocumentCtx{} - return reuse - }, -} - -// VisitDocument invokes the DocFieldValueVistor for each stored field -// for the specified doc number -func (s *SegmentBase) VisitDocument(num uint64, visitor segment.DocumentFieldValueVisitor) error { - vdc := visitDocumentCtxPool.Get().(*visitDocumentCtx) - defer visitDocumentCtxPool.Put(vdc) - return s.visitDocument(vdc, num, visitor) -} - -func (s *SegmentBase) visitDocument(vdc *visitDocumentCtx, num uint64, - visitor segment.DocumentFieldValueVisitor) error { - // first make sure this is a valid number in this segment - if num < s.numDocs { - meta, compressed := s.getDocStoredMetaAndCompressed(num) - - vdc.reader.Reset(meta) - - // handle _id field special case - idFieldValLen, err := binary.ReadUvarint(&vdc.reader) - if err != nil { - return err - } - idFieldVal := compressed[:idFieldValLen] - - keepGoing := visitor("_id", byte('t'), idFieldVal, nil) - if !keepGoing { - visitDocumentCtxPool.Put(vdc) - return nil - } - - // handle non-"_id" fields - compressed = compressed[idFieldValLen:] - - uncompressed, err := snappy.Decode(vdc.buf[:cap(vdc.buf)], compressed) - if err != nil { - return err - } - - for keepGoing { - field, err := binary.ReadUvarint(&vdc.reader) - if err == io.EOF { - break - } - if err != nil { - return err - } - typ, err := binary.ReadUvarint(&vdc.reader) - if err != nil { - return err - } - offset, err := binary.ReadUvarint(&vdc.reader) - if err != nil { - return err - } - l, err := binary.ReadUvarint(&vdc.reader) - if err != nil { - return err - } - numap, err := binary.ReadUvarint(&vdc.reader) - if err != nil { - return err - } - var arrayPos []uint64 - if numap > 0 { - if cap(vdc.arrayPos) < int(numap) { - vdc.arrayPos = make([]uint64, numap) - } - arrayPos = vdc.arrayPos[:numap] - for i := 0; i < int(numap); i++ { - ap, err := binary.ReadUvarint(&vdc.reader) - if err != nil { - return err - } - arrayPos[i] = ap - } - } - - value := uncompressed[offset : offset+l] - keepGoing = visitor(s.fieldsInv[field], byte(typ), value, arrayPos) - } - - vdc.buf = uncompressed - } - return nil -} - -// DocID returns the value of the _id field for the given docNum -func (s *SegmentBase) DocID(num uint64) ([]byte, error) { - if num >= s.numDocs { - return nil, nil - } - - vdc := visitDocumentCtxPool.Get().(*visitDocumentCtx) - - meta, compressed := s.getDocStoredMetaAndCompressed(num) - - vdc.reader.Reset(meta) - - // handle _id field special case - idFieldValLen, err := binary.ReadUvarint(&vdc.reader) - if err != nil { - return nil, err - } - idFieldVal := compressed[:idFieldValLen] - - visitDocumentCtxPool.Put(vdc) - - return idFieldVal, nil -} - -// Count returns the number of documents in this segment. -func (s *SegmentBase) Count() uint64 { - return s.numDocs -} - -// DocNumbers returns a bitset corresponding to the doc numbers of all the -// provided _id strings -func (s *SegmentBase) DocNumbers(ids []string) (*roaring.Bitmap, error) { - rv := roaring.New() - - if len(s.fieldsMap) > 0 { - idDict, err := s.dictionary("_id") - if err != nil { - return nil, err - } - - postingsList := emptyPostingsList - - sMax, err := idDict.fst.GetMaxKey() - if err != nil { - return nil, err - } - sMaxStr := string(sMax) - filteredIds := make([]string, 0, len(ids)) - for _, id := range ids { - if id <= sMaxStr { - filteredIds = append(filteredIds, id) - } - } - - for _, id := range filteredIds { - postingsList, err = idDict.postingsList([]byte(id), nil, postingsList) - if err != nil { - return nil, err - } - postingsList.OrInto(rv) - } - } - - return rv, nil -} - -// Fields returns the field names used in this segment -func (s *SegmentBase) Fields() []string { - return s.fieldsInv -} - -// Path returns the path of this segment on disk -func (s *Segment) Path() string { - return s.path -} - -// Close releases all resources associated with this segment -func (s *Segment) Close() (err error) { - return s.DecRef() -} - -func (s *Segment) closeActual() (err error) { - if s.mm != nil { - err = s.mm.Unmap() - } - // try to close file even if unmap failed - if s.f != nil { - err2 := s.f.Close() - if err == nil { - // try to return first error - err = err2 - } - } - return -} - -// some helpers i started adding for the command-line utility - -// Data returns the underlying mmaped data slice -func (s *Segment) Data() []byte { - return s.mm -} - -// CRC returns the CRC value stored in the file footer -func (s *Segment) CRC() uint32 { - return s.crc -} - -// Version returns the file version in the file footer -func (s *Segment) Version() uint32 { - return s.version -} - -// ChunkFactor returns the chunk factor in the file footer -func (s *Segment) ChunkMode() uint32 { - return s.chunkMode -} - -// FieldsIndexOffset returns the fields index offset in the file footer -func (s *Segment) FieldsIndexOffset() uint64 { - return s.fieldsIndexOffset -} - -// StoredIndexOffset returns the stored value index offset in the file footer -func (s *Segment) StoredIndexOffset() uint64 { - return s.storedIndexOffset -} - -// DocValueOffset returns the docValue offset in the file footer -func (s *Segment) DocValueOffset() uint64 { - return s.docValueOffset -} - -// NumDocs returns the number of documents in the file footer -func (s *Segment) NumDocs() uint64 { - return s.numDocs -} - -// DictAddr is a helper function to compute the file offset where the -// dictionary is stored for the specified field. -func (s *Segment) DictAddr(field string) (uint64, error) { - fieldIDPlus1, ok := s.fieldsMap[field] - if !ok { - return 0, fmt.Errorf("no such field '%s'", field) - } - - return s.dictLocs[fieldIDPlus1-1], nil -} - -func (s *SegmentBase) loadDvReaders() error { - if s.docValueOffset == fieldNotUninverted || s.numDocs == 0 { - return nil - } - - var read uint64 - for fieldID, field := range s.fieldsInv { - var fieldLocStart, fieldLocEnd uint64 - var n int - fieldLocStart, n = binary.Uvarint(s.mem[s.docValueOffset+read : s.docValueOffset+read+binary.MaxVarintLen64]) - if n <= 0 { - return fmt.Errorf("loadDvReaders: failed to read the docvalue offset start for field %d", fieldID) - } - read += uint64(n) - fieldLocEnd, n = binary.Uvarint(s.mem[s.docValueOffset+read : s.docValueOffset+read+binary.MaxVarintLen64]) - if n <= 0 { - return fmt.Errorf("loadDvReaders: failed to read the docvalue offset end for field %d", fieldID) - } - read += uint64(n) - - fieldDvReader, err := s.loadFieldDocValueReader(field, fieldLocStart, fieldLocEnd) - if err != nil { - return err - } - if fieldDvReader != nil { - s.fieldDvReaders[uint16(fieldID)] = fieldDvReader - s.fieldDvNames = append(s.fieldDvNames, field) - } - } - - return nil -} diff --git a/vendor/github.com/blevesearch/zap/v11/.gitignore b/vendor/github.com/blevesearch/zapx/v11/.gitignore similarity index 100% rename from vendor/github.com/blevesearch/zap/v11/.gitignore rename to vendor/github.com/blevesearch/zapx/v11/.gitignore diff --git a/vendor/github.com/blevesearch/zapx/v11/.golangci.yml b/vendor/github.com/blevesearch/zapx/v11/.golangci.yml new file mode 100644 index 00000000..1d55bfc0 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v11/.golangci.yml @@ -0,0 +1,28 @@ +linters: + # please, do not use `enable-all`: it's deprecated and will be removed soon. + # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint + disable-all: true + enable: + - bodyclose + - deadcode + - depguard + - dupl + - errcheck + - gofmt + - goimports + - goprintffuncname + - gosec + - gosimple + - govet + - ineffassign + - misspell + - nakedret + - nolintlint + - rowserrcheck + - staticcheck + - structcheck + - typecheck + - unused + - varcheck + - whitespace + diff --git a/vendor/github.com/blevesearch/zap/v15/LICENSE b/vendor/github.com/blevesearch/zapx/v11/LICENSE similarity index 100% rename from vendor/github.com/blevesearch/zap/v15/LICENSE rename to vendor/github.com/blevesearch/zapx/v11/LICENSE diff --git a/vendor/github.com/blevesearch/zapx/v11/README.md b/vendor/github.com/blevesearch/zapx/v11/README.md new file mode 100644 index 00000000..4cbf1a14 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v11/README.md @@ -0,0 +1,163 @@ +# zapx file format + +The zapx module is fork of [zap](https://github.com/blevesearch/zap) module which maintains file format compatibility, but removes dependency on bleve, and instead depends only on the indepenent interface modules: + +- [bleve_index_api](https://github.com/blevesearch/scorch_segment_api) +- [scorch_segment_api](https://github.com/blevesearch/scorch_segment_api) + +Advanced ZAP File Format Documentation is [here](zap.md). + +The file is written in the reverse order that we typically access data. This helps us write in one pass since later sections of the file require file offsets of things we've already written. + +Current usage: + +- mmap the entire file +- crc-32 bytes and version are in fixed position at end of the file +- reading remainder of footer could be version specific +- remainder of footer gives us: + - 3 important offsets (docValue , fields index and stored data index) + - 2 important values (number of docs and chunk factor) +- field data is processed once and memoized onto the heap so that we never have to go back to disk for it +- access to stored data by doc number means first navigating to the stored data index, then accessing a fixed position offset into that slice, which gives us the actual address of the data. the first bytes of that section tell us the size of data so that we know where it ends. +- access to all other indexed data follows the following pattern: + - first know the field name -> convert to id + - next navigate to term dictionary for that field + - some operations stop here and do dictionary ops + - next use dictionary to navigate to posting list for a specific term + - walk posting list + - if necessary, walk posting details as we go + - if location info is desired, consult location bitmap to see if it is there + +## stored fields section + +- for each document + - preparation phase: + - produce a slice of metadata bytes and data bytes + - produce these slices in field id order + - field value is appended to the data slice + - metadata slice is varint encoded with the following values for each field value + - field id (uint16) + - field type (byte) + - field value start offset in uncompressed data slice (uint64) + - field value length (uint64) + - field number of array positions (uint64) + - one additional value for each array position (uint64) + - compress the data slice using snappy + - file writing phase: + - remember the start offset for this document + - write out meta data length (varint uint64) + - write out compressed data length (varint uint64) + - write out the metadata bytes + - write out the compressed data bytes + +## stored fields idx + +- for each document + - write start offset (remembered from previous section) of stored data (big endian uint64) + +With this index and a known document number, we have direct access to all the stored field data. + +## posting details (freq/norm) section + +- for each posting list + - produce a slice containing multiple consecutive chunks (each chunk is varint stream) + - produce a slice remembering offsets of where each chunk starts + - preparation phase: + - for each hit in the posting list + - if this hit is in next chunk close out encoding of last chunk and record offset start of next + - encode term frequency (uint64) + - encode norm factor (float32) + - file writing phase: + - remember start position for this posting list details + - write out number of chunks that follow (varint uint64) + - write out length of each chunk (each a varint uint64) + - write out the byte slice containing all the chunk data + +If you know the doc number you're interested in, this format lets you jump to the correct chunk (docNum/chunkFactor) directly and then seek within that chunk until you find it. + +## posting details (location) section + +- for each posting list + - produce a slice containing multiple consecutive chunks (each chunk is varint stream) + - produce a slice remembering offsets of where each chunk starts + - preparation phase: + - for each hit in the posting list + - if this hit is in next chunk close out encoding of last chunk and record offset start of next + - encode field (uint16) + - encode field pos (uint64) + - encode field start (uint64) + - encode field end (uint64) + - encode number of array positions to follow (uint64) + - encode each array position (each uint64) + - file writing phase: + - remember start position for this posting list details + - write out number of chunks that follow (varint uint64) + - write out length of each chunk (each a varint uint64) + - write out the byte slice containing all the chunk data + +If you know the doc number you're interested in, this format lets you jump to the correct chunk (docNum/chunkFactor) directly and then seek within that chunk until you find it. + +## postings list section + +- for each posting list + - preparation phase: + - encode roaring bitmap posting list to bytes (so we know the length) + - file writing phase: + - remember the start position for this posting list + - write freq/norm details offset (remembered from previous, as varint uint64) + - write location details offset (remembered from previous, as varint uint64) + - write length of encoded roaring bitmap + - write the serialized roaring bitmap data + +## dictionary + +- for each field + - preparation phase: + - encode vellum FST with dictionary data pointing to file offset of posting list (remembered from previous) + - file writing phase: + - remember the start position of this persistDictionary + - write length of vellum data (varint uint64) + - write out vellum data + +## fields section + +- for each field + - file writing phase: + - remember start offset for each field + - write dictionary address (remembered from previous) (varint uint64) + - write length of field name (varint uint64) + - write field name bytes + +## fields idx + +- for each field + - file writing phase: + - write big endian uint64 of start offset for each field + +NOTE: currently we don't know or record the length of this fields index. Instead we rely on the fact that we know it immediately precedes a footer of known size. + +## fields DocValue + +- for each field + - preparation phase: + - produce a slice containing multiple consecutive chunks, where each chunk is composed of a meta section followed by compressed columnar field data + - produce a slice remembering the length of each chunk + - file writing phase: + - remember the start position of this first field DocValue offset in the footer + - write out number of chunks that follow (varint uint64) + - write out length of each chunk (each a varint uint64) + - write out the byte slice containing all the chunk data + +NOTE: currently the meta header inside each chunk gives clue to the location offsets and size of the data pertaining to a given docID and any +read operation leverage that meta information to extract the document specific data from the file. + +## footer + +- file writing phase + - write number of docs (big endian uint64) + - write stored field index location (big endian uint64) + - write field index location (big endian uint64) + - write field docValue location (big endian uint64) + - write out chunk factor (big endian uint32) + - write out version (big endian uint32) + - write out file CRC of everything preceding this (big endian uint32) diff --git a/vendor/github.com/blevesearch/zapx/v11/build.go b/vendor/github.com/blevesearch/zapx/v11/build.go new file mode 100644 index 00000000..3f13a2a6 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v11/build.go @@ -0,0 +1,186 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "bufio" + "fmt" + "io" + "math" + "os" + + "github.com/blevesearch/vellum" +) + +const Version uint32 = 11 + +const Type string = "zap" + +const fieldNotUninverted = math.MaxUint64 + +func (sb *SegmentBase) Persist(path string) error { + return PersistSegmentBase(sb, path) +} + +// WriteTo is an implementation of io.WriterTo interface. +func (sb *SegmentBase) WriteTo(w io.Writer) (int64, error) { + if w == nil { + return 0, fmt.Errorf("invalid writer found") + } + + n, err := persistSegmentBaseToWriter(sb, w) + return int64(n), err +} + +// PersistSegmentBase persists SegmentBase in the zap file format. +func PersistSegmentBase(sb *SegmentBase, path string) error { + flag := os.O_RDWR | os.O_CREATE + + f, err := os.OpenFile(path, flag, 0600) + if err != nil { + return err + } + + cleanup := func() { + _ = f.Close() + _ = os.Remove(path) + } + + _, err = persistSegmentBaseToWriter(sb, f) + if err != nil { + cleanup() + return err + } + + err = f.Sync() + if err != nil { + cleanup() + return err + } + + err = f.Close() + if err != nil { + cleanup() + return err + } + + return err +} + +type bufWriter struct { + w *bufio.Writer + n int +} + +func (br *bufWriter) Write(in []byte) (int, error) { + n, err := br.w.Write(in) + br.n += n + return n, err +} + +func persistSegmentBaseToWriter(sb *SegmentBase, w io.Writer) (int, error) { + br := &bufWriter{w: bufio.NewWriter(w)} + + _, err := br.Write(sb.mem) + if err != nil { + return 0, err + } + + err = persistFooter(sb.numDocs, sb.storedIndexOffset, sb.fieldsIndexOffset, + sb.docValueOffset, sb.chunkFactor, sb.memCRC, br) + if err != nil { + return 0, err + } + + err = br.w.Flush() + if err != nil { + return 0, err + } + + return br.n, nil +} + +func persistStoredFieldValues(fieldID int, + storedFieldValues [][]byte, stf []byte, spf [][]uint64, + curr int, metaEncode varintEncoder, data []byte) ( + int, []byte, error) { + for i := 0; i < len(storedFieldValues); i++ { + // encode field + _, err := metaEncode(uint64(fieldID)) + if err != nil { + return 0, nil, err + } + // encode type + _, err = metaEncode(uint64(stf[i])) + if err != nil { + return 0, nil, err + } + // encode start offset + _, err = metaEncode(uint64(curr)) + if err != nil { + return 0, nil, err + } + // end len + _, err = metaEncode(uint64(len(storedFieldValues[i]))) + if err != nil { + return 0, nil, err + } + // encode number of array pos + _, err = metaEncode(uint64(len(spf[i]))) + if err != nil { + return 0, nil, err + } + // encode all array positions + for _, pos := range spf[i] { + _, err = metaEncode(pos) + if err != nil { + return 0, nil, err + } + } + + data = append(data, storedFieldValues[i]...) + curr += len(storedFieldValues[i]) + } + + return curr, data, nil +} + +func InitSegmentBase(mem []byte, memCRC uint32, chunkFactor uint32, + fieldsMap map[string]uint16, fieldsInv []string, numDocs uint64, + storedIndexOffset uint64, fieldsIndexOffset uint64, docValueOffset uint64, + dictLocs []uint64) (*SegmentBase, error) { + sb := &SegmentBase{ + mem: mem, + memCRC: memCRC, + chunkFactor: chunkFactor, + fieldsMap: fieldsMap, + fieldsInv: fieldsInv, + numDocs: numDocs, + storedIndexOffset: storedIndexOffset, + fieldsIndexOffset: fieldsIndexOffset, + docValueOffset: docValueOffset, + dictLocs: dictLocs, + fieldDvReaders: make(map[uint16]*docValueReader), + fieldFSTs: make(map[uint16]*vellum.FST), + } + sb.updateSize() + + err := sb.loadDvReaders() + if err != nil { + return nil, err + } + + return sb, nil +} diff --git a/vendor/github.com/blevesearch/zap/v11/contentcoder.go b/vendor/github.com/blevesearch/zapx/v11/contentcoder.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v11/contentcoder.go rename to vendor/github.com/blevesearch/zapx/v11/contentcoder.go diff --git a/vendor/github.com/blevesearch/zapx/v11/count.go b/vendor/github.com/blevesearch/zapx/v11/count.go new file mode 100644 index 00000000..b6135359 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v11/count.go @@ -0,0 +1,61 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "hash/crc32" + "io" + + segment "github.com/blevesearch/scorch_segment_api/v2" +) + +// CountHashWriter is a wrapper around a Writer which counts the number of +// bytes which have been written and computes a crc32 hash +type CountHashWriter struct { + w io.Writer + crc uint32 + n int + s segment.StatsReporter +} + +// NewCountHashWriter returns a CountHashWriter which wraps the provided Writer +func NewCountHashWriter(w io.Writer) *CountHashWriter { + return &CountHashWriter{w: w} +} + +func NewCountHashWriterWithStatsReporter(w io.Writer, s segment.StatsReporter) *CountHashWriter { + return &CountHashWriter{w: w, s: s} +} + +// Write writes the provided bytes to the wrapped writer and counts the bytes +func (c *CountHashWriter) Write(b []byte) (int, error) { + n, err := c.w.Write(b) + c.crc = crc32.Update(c.crc, crc32.IEEETable, b[:n]) + c.n += n + if c.s != nil { + c.s.ReportBytesWritten(uint64(n)) + } + return n, err +} + +// Count returns the number of bytes written +func (c *CountHashWriter) Count() int { + return c.n +} + +// Sum32 returns the CRC-32 hash of the content written to this writer +func (c *CountHashWriter) Sum32() uint32 { + return c.crc +} diff --git a/vendor/github.com/blevesearch/zapx/v11/dict.go b/vendor/github.com/blevesearch/zapx/v11/dict.go new file mode 100644 index 00000000..e30bf242 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v11/dict.go @@ -0,0 +1,158 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "fmt" + + "github.com/RoaringBitmap/roaring" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" + "github.com/blevesearch/vellum" +) + +// Dictionary is the zap representation of the term dictionary +type Dictionary struct { + sb *SegmentBase + field string + fieldID uint16 + fst *vellum.FST + fstReader *vellum.Reader +} + +// represents an immutable, empty dictionary +var emptyDictionary = &Dictionary{} + +// PostingsList returns the postings list for the specified term +func (d *Dictionary) PostingsList(term []byte, except *roaring.Bitmap, + prealloc segment.PostingsList) (segment.PostingsList, error) { + var preallocPL *PostingsList + pl, ok := prealloc.(*PostingsList) + if ok && pl != nil { + preallocPL = pl + } + return d.postingsList(term, except, preallocPL) +} + +func (d *Dictionary) postingsList(term []byte, except *roaring.Bitmap, rv *PostingsList) (*PostingsList, error) { + if d.fstReader == nil { + if rv == nil || rv == emptyPostingsList { + return emptyPostingsList, nil + } + return d.postingsListInit(rv, except), nil + } + + postingsOffset, exists, err := d.fstReader.Get(term) + if err != nil { + return nil, fmt.Errorf("vellum err: %v", err) + } + if !exists { + if rv == nil || rv == emptyPostingsList { + return emptyPostingsList, nil + } + return d.postingsListInit(rv, except), nil + } + + return d.postingsListFromOffset(postingsOffset, except, rv) +} + +func (d *Dictionary) postingsListFromOffset(postingsOffset uint64, except *roaring.Bitmap, rv *PostingsList) (*PostingsList, error) { + rv = d.postingsListInit(rv, except) + + err := rv.read(postingsOffset, d) + if err != nil { + return nil, err + } + + return rv, nil +} + +func (d *Dictionary) postingsListInit(rv *PostingsList, except *roaring.Bitmap) *PostingsList { + if rv == nil || rv == emptyPostingsList { + rv = &PostingsList{} + } else { + postings := rv.postings + if postings != nil { + postings.Clear() + } + + *rv = PostingsList{} // clear the struct + + rv.postings = postings + } + rv.sb = d.sb + rv.except = except + return rv +} + +func (d *Dictionary) Contains(key []byte) (bool, error) { + if d.fst != nil { + return d.fst.Contains(key) + } + return false, nil +} + +// AutomatonIterator returns an iterator which only visits terms +// having the the vellum automaton and start/end key range +func (d *Dictionary) AutomatonIterator(a segment.Automaton, + startKeyInclusive, endKeyExclusive []byte) segment.DictionaryIterator { + if d.fst != nil { + rv := &DictionaryIterator{ + d: d, + } + + itr, err := d.fst.Search(a, startKeyInclusive, endKeyExclusive) + if err == nil { + rv.itr = itr + } else if err != vellum.ErrIteratorDone { + rv.err = err + } + + return rv + } + return emptyDictionaryIterator +} + +// DictionaryIterator is an iterator for term dictionary +type DictionaryIterator struct { + d *Dictionary + itr vellum.Iterator + err error + tmp PostingsList + entry index.DictEntry + omitCount bool +} + +var emptyDictionaryIterator = &DictionaryIterator{} + +// Next returns the next entry in the dictionary +func (i *DictionaryIterator) Next() (*index.DictEntry, error) { + if i.err != nil && i.err != vellum.ErrIteratorDone { + return nil, i.err + } else if i.itr == nil || i.err == vellum.ErrIteratorDone { + return nil, nil + } + term, postingsOffset := i.itr.Current() + i.entry.Term = string(term) + if !i.omitCount { + i.err = i.tmp.read(postingsOffset, i.d) + if i.err != nil { + return nil, i.err + } + i.entry.Count = i.tmp.Count() + } + i.err = i.itr.Next() + return &i.entry, nil +} diff --git a/vendor/github.com/blevesearch/zapx/v11/docvalues.go b/vendor/github.com/blevesearch/zapx/v11/docvalues.go new file mode 100644 index 00000000..2f284d07 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v11/docvalues.go @@ -0,0 +1,318 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "bytes" + "encoding/binary" + "fmt" + "math" + "reflect" + "sort" + + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" + "github.com/golang/snappy" +) + +var reflectStaticSizedocValueReader int + +func init() { + var dvi docValueReader + reflectStaticSizedocValueReader = int(reflect.TypeOf(dvi).Size()) +} + +type docNumTermsVisitor func(docNum uint64, terms []byte) error + +type docVisitState struct { + dvrs map[uint16]*docValueReader + segment *SegmentBase +} + +// No-op implementations for DiskStatsReporter interface. +// Supported only in v15 +func (d *docVisitState) BytesRead() uint64 { + return 0 +} + +func (d *docVisitState) BytesWritten() uint64 { + return 0 +} + +func (d *docVisitState) ResetBytesRead(val uint64) {} + +type docValueReader struct { + field string + curChunkNum uint64 + chunkOffsets []uint64 + dvDataLoc uint64 + curChunkHeader []MetaData + curChunkData []byte // compressed data cache + uncompressed []byte // temp buf for snappy decompression +} + +func (di *docValueReader) size() int { + return reflectStaticSizedocValueReader + SizeOfPtr + + len(di.field) + + len(di.chunkOffsets)*SizeOfUint64 + + len(di.curChunkHeader)*reflectStaticSizeMetaData + + len(di.curChunkData) +} + +func (di *docValueReader) cloneInto(rv *docValueReader) *docValueReader { + if rv == nil { + rv = &docValueReader{} + } + + rv.field = di.field + rv.curChunkNum = math.MaxUint64 + rv.chunkOffsets = di.chunkOffsets // immutable, so it's sharable + rv.dvDataLoc = di.dvDataLoc + rv.curChunkHeader = rv.curChunkHeader[:0] + rv.curChunkData = nil + rv.uncompressed = rv.uncompressed[:0] + + return rv +} + +func (di *docValueReader) curChunkNumber() uint64 { + return di.curChunkNum +} + +func (s *SegmentBase) loadFieldDocValueReader(field string, + fieldDvLocStart, fieldDvLocEnd uint64) (*docValueReader, error) { + // get the docValue offset for the given fields + if fieldDvLocStart == fieldNotUninverted { + // no docValues found, nothing to do + return nil, nil + } + + // read the number of chunks, and chunk offsets position + var numChunks, chunkOffsetsPosition uint64 + + if fieldDvLocEnd-fieldDvLocStart > 16 { + numChunks = binary.BigEndian.Uint64(s.mem[fieldDvLocEnd-8 : fieldDvLocEnd]) + // read the length of chunk offsets + chunkOffsetsLen := binary.BigEndian.Uint64(s.mem[fieldDvLocEnd-16 : fieldDvLocEnd-8]) + // acquire position of chunk offsets + chunkOffsetsPosition = (fieldDvLocEnd - 16) - chunkOffsetsLen + } else { + return nil, fmt.Errorf("loadFieldDocValueReader: fieldDvLoc too small: %d-%d", fieldDvLocEnd, fieldDvLocStart) + } + + fdvIter := &docValueReader{ + curChunkNum: math.MaxUint64, + field: field, + chunkOffsets: make([]uint64, int(numChunks)), + } + + // read the chunk offsets + var offset uint64 + for i := 0; i < int(numChunks); i++ { + loc, read := binary.Uvarint(s.mem[chunkOffsetsPosition+offset : chunkOffsetsPosition+offset+binary.MaxVarintLen64]) + if read <= 0 { + return nil, fmt.Errorf("corrupted chunk offset during segment load") + } + fdvIter.chunkOffsets[i] = loc + offset += uint64(read) + } + + // set the data offset + fdvIter.dvDataLoc = fieldDvLocStart + + return fdvIter, nil +} + +func (di *docValueReader) loadDvChunk(chunkNumber uint64, s *SegmentBase) error { + // advance to the chunk where the docValues + // reside for the given docNum + destChunkDataLoc, curChunkEnd := di.dvDataLoc, di.dvDataLoc + start, end := readChunkBoundary(int(chunkNumber), di.chunkOffsets) + if start >= end { + di.curChunkHeader = di.curChunkHeader[:0] + di.curChunkData = nil + di.curChunkNum = chunkNumber + di.uncompressed = di.uncompressed[:0] + return nil + } + + destChunkDataLoc += start + curChunkEnd += end + + // read the number of docs reside in the chunk + numDocs, read := binary.Uvarint(s.mem[destChunkDataLoc : destChunkDataLoc+binary.MaxVarintLen64]) + if read <= 0 { + return fmt.Errorf("failed to read the chunk") + } + chunkMetaLoc := destChunkDataLoc + uint64(read) + + offset := uint64(0) + if cap(di.curChunkHeader) < int(numDocs) { + di.curChunkHeader = make([]MetaData, int(numDocs)) + } else { + di.curChunkHeader = di.curChunkHeader[:int(numDocs)] + } + for i := 0; i < int(numDocs); i++ { + di.curChunkHeader[i].DocNum, read = binary.Uvarint(s.mem[chunkMetaLoc+offset : chunkMetaLoc+offset+binary.MaxVarintLen64]) + offset += uint64(read) + di.curChunkHeader[i].DocDvOffset, read = binary.Uvarint(s.mem[chunkMetaLoc+offset : chunkMetaLoc+offset+binary.MaxVarintLen64]) + offset += uint64(read) + } + + compressedDataLoc := chunkMetaLoc + offset + dataLength := curChunkEnd - compressedDataLoc + di.curChunkData = s.mem[compressedDataLoc : compressedDataLoc+dataLength] + di.curChunkNum = chunkNumber + di.uncompressed = di.uncompressed[:0] + return nil +} + +func (di *docValueReader) iterateAllDocValues(s *SegmentBase, visitor docNumTermsVisitor) error { + for i := 0; i < len(di.chunkOffsets); i++ { + err := di.loadDvChunk(uint64(i), s) + if err != nil { + return err + } + if di.curChunkData == nil || len(di.curChunkHeader) == 0 { + continue + } + + // uncompress the already loaded data + uncompressed, err := snappy.Decode(di.uncompressed[:cap(di.uncompressed)], di.curChunkData) + if err != nil { + return err + } + di.uncompressed = uncompressed + + start := uint64(0) + for _, entry := range di.curChunkHeader { + err = visitor(entry.DocNum, uncompressed[start:entry.DocDvOffset]) + if err != nil { + return err + } + + start = entry.DocDvOffset + } + } + + return nil +} + +func (di *docValueReader) visitDocValues(docNum uint64, + visitor index.DocValueVisitor) error { + // binary search the term locations for the docNum + start, end := di.getDocValueLocs(docNum) + if start == math.MaxUint64 || end == math.MaxUint64 || start == end { + return nil + } + + var uncompressed []byte + var err error + // use the uncompressed copy if available + if len(di.uncompressed) > 0 { + uncompressed = di.uncompressed + } else { + // uncompress the already loaded data + uncompressed, err = snappy.Decode(di.uncompressed[:cap(di.uncompressed)], di.curChunkData) + if err != nil { + return err + } + di.uncompressed = uncompressed + } + + // pick the terms for the given docNum + uncompressed = uncompressed[start:end] + for { + i := bytes.Index(uncompressed, termSeparatorSplitSlice) + if i < 0 { + break + } + + visitor(di.field, uncompressed[0:i]) + uncompressed = uncompressed[i+1:] + } + + return nil +} + +func (di *docValueReader) getDocValueLocs(docNum uint64) (uint64, uint64) { + i := sort.Search(len(di.curChunkHeader), func(i int) bool { + return di.curChunkHeader[i].DocNum >= docNum + }) + if i < len(di.curChunkHeader) && di.curChunkHeader[i].DocNum == docNum { + return ReadDocValueBoundary(i, di.curChunkHeader) + } + return math.MaxUint64, math.MaxUint64 +} + +// VisitDocValues is an implementation of the +// DocValueVisitable interface +func (s *SegmentBase) VisitDocValues(localDocNum uint64, fields []string, + visitor index.DocValueVisitor, dvsIn segment.DocVisitState) ( + segment.DocVisitState, error) { + dvs, ok := dvsIn.(*docVisitState) + if !ok || dvs == nil { + dvs = &docVisitState{} + } else { + if dvs.segment != s { + dvs.segment = s + dvs.dvrs = nil + } + } + + var fieldIDPlus1 uint16 + if dvs.dvrs == nil { + dvs.dvrs = make(map[uint16]*docValueReader, len(fields)) + for _, field := range fields { + if fieldIDPlus1, ok = s.fieldsMap[field]; !ok { + continue + } + fieldID := fieldIDPlus1 - 1 + if dvIter, exists := s.fieldDvReaders[fieldID]; exists && + dvIter != nil { + dvs.dvrs[fieldID] = dvIter.cloneInto(dvs.dvrs[fieldID]) + } + } + } + + // find the chunkNumber where the docValues are stored + docInChunk := localDocNum / uint64(s.chunkFactor) + var dvr *docValueReader + for _, field := range fields { + if fieldIDPlus1, ok = s.fieldsMap[field]; !ok { + continue + } + fieldID := fieldIDPlus1 - 1 + if dvr, ok = dvs.dvrs[fieldID]; ok && dvr != nil { + // check if the chunk is already loaded + if docInChunk != dvr.curChunkNumber() { + err := dvr.loadDvChunk(docInChunk, s) + if err != nil { + return dvs, err + } + } + + _ = dvr.visitDocValues(localDocNum, visitor) + } + } + return dvs, nil +} + +// VisitableDocValueFields returns the list of fields with +// persisted doc value terms ready to be visitable using the +// VisitDocumentFieldTerms method. +func (s *SegmentBase) VisitableDocValueFields() ([]string, error) { + return s.fieldDvNames, nil +} diff --git a/vendor/github.com/blevesearch/zapx/v11/enumerator.go b/vendor/github.com/blevesearch/zapx/v11/enumerator.go new file mode 100644 index 00000000..5531d2cf --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v11/enumerator.go @@ -0,0 +1,126 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "bytes" + + "github.com/blevesearch/vellum" +) + +// enumerator provides an ordered traversal of multiple vellum +// iterators. Like JOIN of iterators, the enumerator produces a +// sequence of (key, iteratorIndex, value) tuples, sorted by key ASC, +// then iteratorIndex ASC, where the same key might be seen or +// repeated across multiple child iterators. +type enumerator struct { + itrs []vellum.Iterator + currKs [][]byte + currVs []uint64 + + lowK []byte + lowIdxs []int + lowCurr int +} + +// newEnumerator returns a new enumerator over the vellum Iterators +func newEnumerator(itrs []vellum.Iterator) (*enumerator, error) { + rv := &enumerator{ + itrs: itrs, + currKs: make([][]byte, len(itrs)), + currVs: make([]uint64, len(itrs)), + lowIdxs: make([]int, 0, len(itrs)), + } + for i, itr := range rv.itrs { + rv.currKs[i], rv.currVs[i] = itr.Current() + } + rv.updateMatches(false) + if rv.lowK == nil && len(rv.lowIdxs) == 0 { + return rv, vellum.ErrIteratorDone + } + return rv, nil +} + +// updateMatches maintains the low key matches based on the currKs +func (m *enumerator) updateMatches(skipEmptyKey bool) { + m.lowK = nil + m.lowIdxs = m.lowIdxs[:0] + m.lowCurr = 0 + + for i, key := range m.currKs { + if (key == nil && m.currVs[i] == 0) || // in case of empty iterator + (len(key) == 0 && skipEmptyKey) { // skip empty keys + continue + } + + cmp := bytes.Compare(key, m.lowK) + if cmp < 0 || len(m.lowIdxs) == 0 { + // reached a new low + m.lowK = key + m.lowIdxs = m.lowIdxs[:0] + m.lowIdxs = append(m.lowIdxs, i) + } else if cmp == 0 { + m.lowIdxs = append(m.lowIdxs, i) + } + } +} + +// Current returns the enumerator's current key, iterator-index, and +// value. If the enumerator is not pointing at a valid value (because +// Next returned an error previously), Current will return nil,0,0. +func (m *enumerator) Current() ([]byte, int, uint64) { + var i int + var v uint64 + if m.lowCurr < len(m.lowIdxs) { + i = m.lowIdxs[m.lowCurr] + v = m.currVs[i] + } + return m.lowK, i, v +} + +// Next advances the enumerator to the next key/iterator/value result, +// else vellum.ErrIteratorDone is returned. +func (m *enumerator) Next() error { + m.lowCurr += 1 + if m.lowCurr >= len(m.lowIdxs) { + // move all the current low iterators forwards + for _, vi := range m.lowIdxs { + err := m.itrs[vi].Next() + if err != nil && err != vellum.ErrIteratorDone { + return err + } + m.currKs[vi], m.currVs[vi] = m.itrs[vi].Current() + } + // can skip any empty keys encountered at this point + m.updateMatches(true) + } + if m.lowK == nil && len(m.lowIdxs) == 0 { + return vellum.ErrIteratorDone + } + return nil +} + +// Close all the underlying Iterators. The first error, if any, will +// be returned. +func (m *enumerator) Close() error { + var rv error + for _, itr := range m.itrs { + err := itr.Close() + if rv == nil { + rv = err + } + } + return rv +} diff --git a/vendor/github.com/blevesearch/zap/v11/intcoder.go b/vendor/github.com/blevesearch/zapx/v11/intcoder.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v11/intcoder.go rename to vendor/github.com/blevesearch/zapx/v11/intcoder.go diff --git a/vendor/github.com/blevesearch/zapx/v11/memuvarint.go b/vendor/github.com/blevesearch/zapx/v11/memuvarint.go new file mode 100644 index 00000000..48a57f9c --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v11/memuvarint.go @@ -0,0 +1,103 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "fmt" +) + +type memUvarintReader struct { + C int // index of next byte to read from S + S []byte +} + +func newMemUvarintReader(s []byte) *memUvarintReader { + return &memUvarintReader{S: s} +} + +// Len returns the number of unread bytes. +func (r *memUvarintReader) Len() int { + n := len(r.S) - r.C + if n < 0 { + return 0 + } + return n +} + +// ReadUvarint reads an encoded uint64. The original code this was +// based on is at encoding/binary/ReadUvarint(). +func (r *memUvarintReader) ReadUvarint() (uint64, error) { + if r.C >= len(r.S) { + // nothing else to read + return 0, nil + } + + var x uint64 + var s uint + var C = r.C + var S = r.S + + for { + b := S[C] + C++ + + if b < 0x80 { + r.C = C + + // why 63? The original code had an 'i += 1' loop var and + // checked for i > 9 || i == 9 ...; but, we no longer + // check for the i var, but instead check here for s, + // which is incremented by 7. So, 7*9 == 63. + // + // why the "extra" >= check? The normal case is that s < + // 63, so we check this single >= guard first so that we + // hit the normal, nil-error return pathway sooner. + if s >= 63 && (s > 63 || b > 1) { + return 0, fmt.Errorf("memUvarintReader overflow") + } + + return x | uint64(b)<= len(r.S) { + return + } + + b := r.S[r.C] + r.C++ + + if b < 0x80 { + return + } + } +} + +// SkipBytes skips a count number of bytes. +func (r *memUvarintReader) SkipBytes(count int) { + r.C = r.C + count +} + +func (r *memUvarintReader) Reset(s []byte) { + r.C = 0 + r.S = s +} diff --git a/vendor/github.com/blevesearch/zapx/v11/merge.go b/vendor/github.com/blevesearch/zapx/v11/merge.go new file mode 100644 index 00000000..f0770e99 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v11/merge.go @@ -0,0 +1,856 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "bufio" + "bytes" + "encoding/binary" + "fmt" + "math" + "os" + "sort" + + "github.com/RoaringBitmap/roaring" + seg "github.com/blevesearch/scorch_segment_api/v2" + "github.com/blevesearch/vellum" + "github.com/golang/snappy" +) + +var DefaultFileMergerBufferSize = 1024 * 1024 + +const docDropped = math.MaxUint64 // sentinel docNum to represent a deleted doc + +// Merge takes a slice of segments and bit masks describing which +// documents may be dropped, and creates a new segment containing the +// remaining data. This new segment is built at the specified path. +func (*ZapPlugin) Merge(segments []seg.Segment, drops []*roaring.Bitmap, path string, + closeCh chan struct{}, s seg.StatsReporter) ( + [][]uint64, uint64, error) { + segmentBases := make([]*SegmentBase, len(segments)) + for segmenti, segment := range segments { + switch segmentx := segment.(type) { + case *Segment: + segmentBases[segmenti] = &segmentx.SegmentBase + case *SegmentBase: + segmentBases[segmenti] = segmentx + default: + panic(fmt.Sprintf("oops, unexpected segment type: %T", segment)) + } + } + return mergeSegmentBases(segmentBases, drops, path, defaultChunkFactor, closeCh, s) +} + +func mergeSegmentBases(segmentBases []*SegmentBase, drops []*roaring.Bitmap, path string, + chunkFactor uint32, closeCh chan struct{}, s seg.StatsReporter) ( + [][]uint64, uint64, error) { + flag := os.O_RDWR | os.O_CREATE + + f, err := os.OpenFile(path, flag, 0600) + if err != nil { + return nil, 0, err + } + + cleanup := func() { + _ = f.Close() + _ = os.Remove(path) + } + + // buffer the output + br := bufio.NewWriterSize(f, DefaultFileMergerBufferSize) + + // wrap it for counting (tracking offsets) + cr := NewCountHashWriterWithStatsReporter(br, s) + + newDocNums, numDocs, storedIndexOffset, fieldsIndexOffset, docValueOffset, _, _, _, err := + MergeToWriter(segmentBases, drops, chunkFactor, cr, closeCh) + if err != nil { + cleanup() + return nil, 0, err + } + + err = persistFooter(numDocs, storedIndexOffset, fieldsIndexOffset, + docValueOffset, chunkFactor, cr.Sum32(), cr) + if err != nil { + cleanup() + return nil, 0, err + } + + err = br.Flush() + if err != nil { + cleanup() + return nil, 0, err + } + + err = f.Sync() + if err != nil { + cleanup() + return nil, 0, err + } + + err = f.Close() + if err != nil { + cleanup() + return nil, 0, err + } + + return newDocNums, uint64(cr.Count()), nil +} + +func MergeToWriter(segments []*SegmentBase, drops []*roaring.Bitmap, + chunkFactor uint32, cr *CountHashWriter, closeCh chan struct{}) ( + newDocNums [][]uint64, + numDocs, storedIndexOffset, fieldsIndexOffset, docValueOffset uint64, + dictLocs []uint64, fieldsInv []string, fieldsMap map[string]uint16, + err error) { + docValueOffset = uint64(fieldNotUninverted) + + var fieldsSame bool + fieldsSame, fieldsInv = mergeFields(segments) + fieldsMap = mapFields(fieldsInv) + + numDocs = computeNewDocCount(segments, drops) + + if isClosed(closeCh) { + return nil, 0, 0, 0, 0, nil, nil, nil, seg.ErrClosed + } + + if numDocs > 0 { + storedIndexOffset, newDocNums, err = mergeStoredAndRemap(segments, drops, + fieldsMap, fieldsInv, fieldsSame, numDocs, cr, closeCh) + if err != nil { + return nil, 0, 0, 0, 0, nil, nil, nil, err + } + + dictLocs, docValueOffset, err = persistMergedRest(segments, drops, + fieldsInv, fieldsMap, fieldsSame, + newDocNums, numDocs, chunkFactor, cr, closeCh) + if err != nil { + return nil, 0, 0, 0, 0, nil, nil, nil, err + } + } else { + dictLocs = make([]uint64, len(fieldsInv)) + } + + fieldsIndexOffset, err = persistFields(fieldsInv, cr, dictLocs) + if err != nil { + return nil, 0, 0, 0, 0, nil, nil, nil, err + } + + return newDocNums, numDocs, storedIndexOffset, fieldsIndexOffset, docValueOffset, dictLocs, fieldsInv, fieldsMap, nil +} + +// mapFields takes the fieldsInv list and returns a map of fieldName +// to fieldID+1 +func mapFields(fields []string) map[string]uint16 { + rv := make(map[string]uint16, len(fields)) + for i, fieldName := range fields { + rv[fieldName] = uint16(i) + 1 + } + return rv +} + +// computeNewDocCount determines how many documents will be in the newly +// merged segment when obsoleted docs are dropped +func computeNewDocCount(segments []*SegmentBase, drops []*roaring.Bitmap) uint64 { + var newDocCount uint64 + for segI, segment := range segments { + newDocCount += segment.numDocs + if drops[segI] != nil { + newDocCount -= drops[segI].GetCardinality() + } + } + return newDocCount +} + +func persistMergedRest(segments []*SegmentBase, dropsIn []*roaring.Bitmap, + fieldsInv []string, fieldsMap map[string]uint16, fieldsSame bool, + newDocNumsIn [][]uint64, newSegDocCount uint64, chunkFactor uint32, + w *CountHashWriter, closeCh chan struct{}) ([]uint64, uint64, error) { + var bufMaxVarintLen64 []byte = make([]byte, binary.MaxVarintLen64) + var bufLoc []uint64 + + var postings *PostingsList + var postItr *PostingsIterator + + rv := make([]uint64, len(fieldsInv)) + fieldDvLocsStart := make([]uint64, len(fieldsInv)) + fieldDvLocsEnd := make([]uint64, len(fieldsInv)) + + tfEncoder := newChunkedIntCoder(uint64(chunkFactor), newSegDocCount-1) + locEncoder := newChunkedIntCoder(uint64(chunkFactor), newSegDocCount-1) + + var vellumBuf bytes.Buffer + newVellum, err := vellum.New(&vellumBuf, nil) + if err != nil { + return nil, 0, err + } + + newRoaring := roaring.NewBitmap() + + // for each field + for fieldID, fieldName := range fieldsInv { + // collect FST iterators from all active segments for this field + var newDocNums [][]uint64 + var drops []*roaring.Bitmap + var dicts []*Dictionary + var itrs []vellum.Iterator + + var segmentsInFocus []*SegmentBase + + for segmentI, segment := range segments { + // check for the closure in meantime + if isClosed(closeCh) { + return nil, 0, seg.ErrClosed + } + + dict, err2 := segment.dictionary(fieldName) + if err2 != nil { + return nil, 0, err2 + } + if dict != nil && dict.fst != nil { + itr, err2 := dict.fst.Iterator(nil, nil) + if err2 != nil && err2 != vellum.ErrIteratorDone { + return nil, 0, err2 + } + if itr != nil { + newDocNums = append(newDocNums, newDocNumsIn[segmentI]) + if dropsIn[segmentI] != nil && !dropsIn[segmentI].IsEmpty() { + drops = append(drops, dropsIn[segmentI]) + } else { + drops = append(drops, nil) + } + dicts = append(dicts, dict) + itrs = append(itrs, itr) + segmentsInFocus = append(segmentsInFocus, segment) + } + } + } + + var prevTerm []byte + + newRoaring.Clear() + + var lastDocNum, lastFreq, lastNorm uint64 + + // determines whether to use "1-hit" encoding optimization + // when a term appears in only 1 doc, with no loc info, + // has freq of 1, and the docNum fits into 31-bits + use1HitEncoding := func(termCardinality uint64) (bool, uint64, uint64) { + if termCardinality == uint64(1) && locEncoder.FinalSize() <= 0 { + docNum := uint64(newRoaring.Minimum()) + if under32Bits(docNum) && docNum == lastDocNum && lastFreq == 1 { + return true, docNum, lastNorm + } + } + return false, 0, 0 + } + + finishTerm := func(term []byte) error { + tfEncoder.Close() + locEncoder.Close() + + postingsOffset, err := writePostings(newRoaring, + tfEncoder, locEncoder, use1HitEncoding, w, bufMaxVarintLen64) + if err != nil { + return err + } + + if postingsOffset > 0 { + err = newVellum.Insert(term, postingsOffset) + if err != nil { + return err + } + } + + newRoaring.Clear() + + tfEncoder.Reset() + locEncoder.Reset() + + lastDocNum = 0 + lastFreq = 0 + lastNorm = 0 + + return nil + } + + enumerator, err := newEnumerator(itrs) + + for err == nil { + term, itrI, postingsOffset := enumerator.Current() + + if !bytes.Equal(prevTerm, term) { + // check for the closure in meantime + if isClosed(closeCh) { + return nil, 0, seg.ErrClosed + } + + // if the term changed, write out the info collected + // for the previous term + err = finishTerm(prevTerm) + if err != nil { + return nil, 0, err + } + } + + postings, err = dicts[itrI].postingsListFromOffset( + postingsOffset, drops[itrI], postings) + if err != nil { + return nil, 0, err + } + + postItr = postings.iterator(true, true, true, postItr) + + if fieldsSame { + // can optimize by copying freq/norm/loc bytes directly + lastDocNum, lastFreq, lastNorm, err = mergeTermFreqNormLocsByCopying( + term, postItr, newDocNums[itrI], newRoaring, + tfEncoder, locEncoder) + } else { + lastDocNum, lastFreq, lastNorm, bufLoc, err = mergeTermFreqNormLocs( + fieldsMap, term, postItr, newDocNums[itrI], newRoaring, + tfEncoder, locEncoder, bufLoc) + } + if err != nil { + return nil, 0, err + } + + prevTerm = prevTerm[:0] // copy to prevTerm in case Next() reuses term mem + prevTerm = append(prevTerm, term...) + + err = enumerator.Next() + } + if err != vellum.ErrIteratorDone { + return nil, 0, err + } + + err = finishTerm(prevTerm) + if err != nil { + return nil, 0, err + } + + dictOffset := uint64(w.Count()) + + err = newVellum.Close() + if err != nil { + return nil, 0, err + } + vellumData := vellumBuf.Bytes() + + // write out the length of the vellum data + n := binary.PutUvarint(bufMaxVarintLen64, uint64(len(vellumData))) + _, err = w.Write(bufMaxVarintLen64[:n]) + if err != nil { + return nil, 0, err + } + + // write this vellum to disk + _, err = w.Write(vellumData) + if err != nil { + return nil, 0, err + } + + rv[fieldID] = dictOffset + + // get the field doc value offset (start) + fieldDvLocsStart[fieldID] = uint64(w.Count()) + + // update the field doc values + fdvEncoder := newChunkedContentCoder(uint64(chunkFactor), newSegDocCount-1, w, true) + + fdvReadersAvailable := false + var dvIterClone *docValueReader + for segmentI, segment := range segmentsInFocus { + // check for the closure in meantime + if isClosed(closeCh) { + return nil, 0, seg.ErrClosed + } + + fieldIDPlus1 := uint16(segment.fieldsMap[fieldName]) + if dvIter, exists := segment.fieldDvReaders[fieldIDPlus1-1]; exists && + dvIter != nil { + fdvReadersAvailable = true + dvIterClone = dvIter.cloneInto(dvIterClone) + err = dvIterClone.iterateAllDocValues(segment, func(docNum uint64, terms []byte) error { + if newDocNums[segmentI][docNum] == docDropped { + return nil + } + err := fdvEncoder.Add(newDocNums[segmentI][docNum], terms) + if err != nil { + return err + } + return nil + }) + if err != nil { + return nil, 0, err + } + } + } + + if fdvReadersAvailable { + err = fdvEncoder.Close() + if err != nil { + return nil, 0, err + } + + // persist the doc value details for this field + _, err = fdvEncoder.Write() + if err != nil { + return nil, 0, err + } + + // get the field doc value offset (end) + fieldDvLocsEnd[fieldID] = uint64(w.Count()) + } else { + fieldDvLocsStart[fieldID] = fieldNotUninverted + fieldDvLocsEnd[fieldID] = fieldNotUninverted + } + + // reset vellum buffer and vellum builder + vellumBuf.Reset() + err = newVellum.Reset(&vellumBuf) + if err != nil { + return nil, 0, err + } + } + + fieldDvLocsOffset := uint64(w.Count()) + + buf := bufMaxVarintLen64 + for i := 0; i < len(fieldDvLocsStart); i++ { + n := binary.PutUvarint(buf, fieldDvLocsStart[i]) + _, err := w.Write(buf[:n]) + if err != nil { + return nil, 0, err + } + n = binary.PutUvarint(buf, fieldDvLocsEnd[i]) + _, err = w.Write(buf[:n]) + if err != nil { + return nil, 0, err + } + } + + return rv, fieldDvLocsOffset, nil +} + +func mergeTermFreqNormLocs(fieldsMap map[string]uint16, term []byte, postItr *PostingsIterator, + newDocNums []uint64, newRoaring *roaring.Bitmap, + tfEncoder *chunkedIntCoder, locEncoder *chunkedIntCoder, bufLoc []uint64) ( + lastDocNum uint64, lastFreq uint64, lastNorm uint64, bufLocOut []uint64, err error) { + next, err := postItr.Next() + for next != nil && err == nil { + hitNewDocNum := newDocNums[next.Number()] + if hitNewDocNum == docDropped { + return 0, 0, 0, nil, fmt.Errorf("see hit with dropped docNum") + } + + newRoaring.Add(uint32(hitNewDocNum)) + + nextFreq := next.Frequency() + nextNorm := uint64(math.Float32bits(float32(next.Norm()))) + + locs := next.Locations() + + err = tfEncoder.Add(hitNewDocNum, + encodeFreqHasLocs(nextFreq, len(locs) > 0), nextNorm) + if err != nil { + return 0, 0, 0, nil, err + } + + if len(locs) > 0 { + numBytesLocs := 0 + for _, loc := range locs { + ap := loc.ArrayPositions() + numBytesLocs += totalUvarintBytes(uint64(fieldsMap[loc.Field()]-1), + loc.Pos(), loc.Start(), loc.End(), uint64(len(ap)), ap) + } + + err = locEncoder.Add(hitNewDocNum, uint64(numBytesLocs)) + if err != nil { + return 0, 0, 0, nil, err + } + + for _, loc := range locs { + ap := loc.ArrayPositions() + if cap(bufLoc) < 5+len(ap) { + bufLoc = make([]uint64, 0, 5+len(ap)) + } + args := bufLoc[0:5] + args[0] = uint64(fieldsMap[loc.Field()] - 1) + args[1] = loc.Pos() + args[2] = loc.Start() + args[3] = loc.End() + args[4] = uint64(len(ap)) + args = append(args, ap...) + err = locEncoder.Add(hitNewDocNum, args...) + if err != nil { + return 0, 0, 0, nil, err + } + } + } + + lastDocNum = hitNewDocNum + lastFreq = nextFreq + lastNorm = nextNorm + + next, err = postItr.Next() + } + + return lastDocNum, lastFreq, lastNorm, bufLoc, err +} + +func mergeTermFreqNormLocsByCopying(term []byte, postItr *PostingsIterator, + newDocNums []uint64, newRoaring *roaring.Bitmap, + tfEncoder *chunkedIntCoder, locEncoder *chunkedIntCoder) ( + lastDocNum uint64, lastFreq uint64, lastNorm uint64, err error) { + nextDocNum, nextFreq, nextNorm, nextFreqNormBytes, nextLocBytes, err := + postItr.nextBytes() + for err == nil && len(nextFreqNormBytes) > 0 { + hitNewDocNum := newDocNums[nextDocNum] + if hitNewDocNum == docDropped { + return 0, 0, 0, fmt.Errorf("see hit with dropped doc num") + } + + newRoaring.Add(uint32(hitNewDocNum)) + err = tfEncoder.AddBytes(hitNewDocNum, nextFreqNormBytes) + if err != nil { + return 0, 0, 0, err + } + + if len(nextLocBytes) > 0 { + err = locEncoder.AddBytes(hitNewDocNum, nextLocBytes) + if err != nil { + return 0, 0, 0, err + } + } + + lastDocNum = hitNewDocNum + lastFreq = nextFreq + lastNorm = nextNorm + + nextDocNum, nextFreq, nextNorm, nextFreqNormBytes, nextLocBytes, err = + postItr.nextBytes() + } + + return lastDocNum, lastFreq, lastNorm, err +} + +func writePostings(postings *roaring.Bitmap, tfEncoder, locEncoder *chunkedIntCoder, + use1HitEncoding func(uint64) (bool, uint64, uint64), + w *CountHashWriter, bufMaxVarintLen64 []byte) ( + offset uint64, err error) { + termCardinality := postings.GetCardinality() + if termCardinality <= 0 { + return 0, nil + } + + if use1HitEncoding != nil { + encodeAs1Hit, docNum1Hit, normBits1Hit := use1HitEncoding(termCardinality) + if encodeAs1Hit { + return FSTValEncode1Hit(docNum1Hit, normBits1Hit), nil + } + } + + tfOffset := uint64(w.Count()) + _, err = tfEncoder.Write(w) + if err != nil { + return 0, err + } + + locOffset := uint64(w.Count()) + _, err = locEncoder.Write(w) + if err != nil { + return 0, err + } + + postingsOffset := uint64(w.Count()) + + n := binary.PutUvarint(bufMaxVarintLen64, tfOffset) + _, err = w.Write(bufMaxVarintLen64[:n]) + if err != nil { + return 0, err + } + + n = binary.PutUvarint(bufMaxVarintLen64, locOffset) + _, err = w.Write(bufMaxVarintLen64[:n]) + if err != nil { + return 0, err + } + + _, err = writeRoaringWithLen(postings, w, bufMaxVarintLen64) + if err != nil { + return 0, err + } + + return postingsOffset, nil +} + +type varintEncoder func(uint64) (int, error) + +func mergeStoredAndRemap(segments []*SegmentBase, drops []*roaring.Bitmap, + fieldsMap map[string]uint16, fieldsInv []string, fieldsSame bool, newSegDocCount uint64, + w *CountHashWriter, closeCh chan struct{}) (uint64, [][]uint64, error) { + var rv [][]uint64 // The remapped or newDocNums for each segment. + + var newDocNum uint64 + + var curr int + var data, compressed []byte + var metaBuf bytes.Buffer + varBuf := make([]byte, binary.MaxVarintLen64) + metaEncode := func(val uint64) (int, error) { + wb := binary.PutUvarint(varBuf, val) + return metaBuf.Write(varBuf[:wb]) + } + + vals := make([][][]byte, len(fieldsInv)) + typs := make([][]byte, len(fieldsInv)) + poss := make([][][]uint64, len(fieldsInv)) + + var posBuf []uint64 + + docNumOffsets := make([]uint64, newSegDocCount) + + vdc := visitDocumentCtxPool.Get().(*visitDocumentCtx) + defer visitDocumentCtxPool.Put(vdc) + + // for each segment + for segI, segment := range segments { + // check for the closure in meantime + if isClosed(closeCh) { + return 0, nil, seg.ErrClosed + } + + segNewDocNums := make([]uint64, segment.numDocs) + + dropsI := drops[segI] + + // optimize when the field mapping is the same across all + // segments and there are no deletions, via byte-copying + // of stored docs bytes directly to the writer + if fieldsSame && (dropsI == nil || dropsI.GetCardinality() == 0) { + err := segment.copyStoredDocs(newDocNum, docNumOffsets, w) + if err != nil { + return 0, nil, err + } + + for i := uint64(0); i < segment.numDocs; i++ { + segNewDocNums[i] = newDocNum + newDocNum++ + } + rv = append(rv, segNewDocNums) + + continue + } + + // for each doc num + for docNum := uint64(0); docNum < segment.numDocs; docNum++ { + // TODO: roaring's API limits docNums to 32-bits? + if dropsI != nil && dropsI.Contains(uint32(docNum)) { + segNewDocNums[docNum] = docDropped + continue + } + + segNewDocNums[docNum] = newDocNum + + curr = 0 + metaBuf.Reset() + data = data[:0] + + posTemp := posBuf + + // collect all the data + for i := 0; i < len(fieldsInv); i++ { + vals[i] = vals[i][:0] + typs[i] = typs[i][:0] + poss[i] = poss[i][:0] + } + err := segment.visitStoredFields(vdc, docNum, func(field string, typ byte, value []byte, pos []uint64) bool { + fieldID := int(fieldsMap[field]) - 1 + vals[fieldID] = append(vals[fieldID], value) + typs[fieldID] = append(typs[fieldID], typ) + + // copy array positions to preserve them beyond the scope of this callback + var curPos []uint64 + if len(pos) > 0 { + if cap(posTemp) < len(pos) { + posBuf = make([]uint64, len(pos)*len(fieldsInv)) + posTemp = posBuf + } + curPos = posTemp[0:len(pos)] + copy(curPos, pos) + posTemp = posTemp[len(pos):] + } + poss[fieldID] = append(poss[fieldID], curPos) + + return true + }) + if err != nil { + return 0, nil, err + } + + // _id field special case optimizes ExternalID() lookups + idFieldVal := vals[uint16(0)][0] + _, err = metaEncode(uint64(len(idFieldVal))) + if err != nil { + return 0, nil, err + } + + // now walk the non-"_id" fields in order + for fieldID := 1; fieldID < len(fieldsInv); fieldID++ { + storedFieldValues := vals[fieldID] + + stf := typs[fieldID] + spf := poss[fieldID] + + var err2 error + curr, data, err2 = persistStoredFieldValues(fieldID, + storedFieldValues, stf, spf, curr, metaEncode, data) + if err2 != nil { + return 0, nil, err2 + } + } + + metaBytes := metaBuf.Bytes() + + compressed = snappy.Encode(compressed[:cap(compressed)], data) + + // record where we're about to start writing + docNumOffsets[newDocNum] = uint64(w.Count()) + + // write out the meta len and compressed data len + _, err = writeUvarints(w, + uint64(len(metaBytes)), + uint64(len(idFieldVal)+len(compressed))) + if err != nil { + return 0, nil, err + } + // now write the meta + _, err = w.Write(metaBytes) + if err != nil { + return 0, nil, err + } + // now write the _id field val (counted as part of the 'compressed' data) + _, err = w.Write(idFieldVal) + if err != nil { + return 0, nil, err + } + // now write the compressed data + _, err = w.Write(compressed) + if err != nil { + return 0, nil, err + } + + newDocNum++ + } + + rv = append(rv, segNewDocNums) + } + + // return value is the start of the stored index + storedIndexOffset := uint64(w.Count()) + + // now write out the stored doc index + for _, docNumOffset := range docNumOffsets { + err := binary.Write(w, binary.BigEndian, docNumOffset) + if err != nil { + return 0, nil, err + } + } + + return storedIndexOffset, rv, nil +} + +// copyStoredDocs writes out a segment's stored doc info, optimized by +// using a single Write() call for the entire set of bytes. The +// newDocNumOffsets is filled with the new offsets for each doc. +func (s *SegmentBase) copyStoredDocs(newDocNum uint64, newDocNumOffsets []uint64, + w *CountHashWriter) error { + if s.numDocs <= 0 { + return nil + } + + indexOffset0, storedOffset0, _, _, _ := + s.getDocStoredOffsets(0) // the segment's first doc + + indexOffsetN, storedOffsetN, readN, metaLenN, dataLenN := + s.getDocStoredOffsets(s.numDocs - 1) // the segment's last doc + + storedOffset0New := uint64(w.Count()) + + storedBytes := s.mem[storedOffset0 : storedOffsetN+readN+metaLenN+dataLenN] + _, err := w.Write(storedBytes) + if err != nil { + return err + } + + // remap the storedOffset's for the docs into new offsets relative + // to storedOffset0New, filling the given docNumOffsetsOut array + for indexOffset := indexOffset0; indexOffset <= indexOffsetN; indexOffset += 8 { + storedOffset := binary.BigEndian.Uint64(s.mem[indexOffset : indexOffset+8]) + storedOffsetNew := storedOffset - storedOffset0 + storedOffset0New + newDocNumOffsets[newDocNum] = storedOffsetNew + newDocNum += 1 + } + + return nil +} + +// mergeFields builds a unified list of fields used across all the +// input segments, and computes whether the fields are the same across +// segments (which depends on fields to be sorted in the same way +// across segments) +func mergeFields(segments []*SegmentBase) (bool, []string) { + fieldsSame := true + + var segment0Fields []string + if len(segments) > 0 { + segment0Fields = segments[0].Fields() + } + + fieldsExist := map[string]struct{}{} + for _, segment := range segments { + fields := segment.Fields() + for fieldi, field := range fields { + fieldsExist[field] = struct{}{} + if len(segment0Fields) != len(fields) || segment0Fields[fieldi] != field { + fieldsSame = false + } + } + } + + rv := make([]string, 0, len(fieldsExist)) + // ensure _id stays first + rv = append(rv, "_id") + for k := range fieldsExist { + if k != "_id" { + rv = append(rv, k) + } + } + + sort.Strings(rv[1:]) // leave _id as first + + return fieldsSame, rv +} + +func isClosed(closeCh chan struct{}) bool { + select { + case <-closeCh: + return true + default: + return false + } +} diff --git a/vendor/github.com/blevesearch/zapx/v11/new.go b/vendor/github.com/blevesearch/zapx/v11/new.go new file mode 100644 index 00000000..4491422a --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v11/new.go @@ -0,0 +1,817 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "bytes" + "encoding/binary" + "math" + "sort" + "sync" + + "github.com/RoaringBitmap/roaring" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" + "github.com/blevesearch/vellum" + "github.com/golang/snappy" +) + +var NewSegmentBufferNumResultsBump int = 100 +var NewSegmentBufferNumResultsFactor float64 = 1.0 +var NewSegmentBufferAvgBytesPerDocFactor float64 = 1.0 + +// ValidateDocFields can be set by applications to perform additional checks +// on fields in a document being added to a new segment, by default it does +// nothing. +// This API is experimental and may be removed at any time. +var ValidateDocFields = func(field index.Field) error { + return nil +} + +var defaultChunkFactor uint32 = 1024 + +// New creates an in-memory zap-encoded SegmentBase from a set of Documents +func (z *ZapPlugin) New(results []index.Document) ( + segment.Segment, uint64, error) { + return z.newWithChunkFactor(results, defaultChunkFactor) +} + +func (*ZapPlugin) newWithChunkFactor(results []index.Document, + chunkFactor uint32) (segment.Segment, uint64, error) { + s := interimPool.Get().(*interim) + + var br bytes.Buffer + if s.lastNumDocs > 0 { + // use previous results to initialize the buf with an estimate + // size, but note that the interim instance comes from a + // global interimPool, so multiple scorch instances indexing + // different docs can lead to low quality estimates + estimateAvgBytesPerDoc := int(float64(s.lastOutSize/s.lastNumDocs) * + NewSegmentBufferNumResultsFactor) + estimateNumResults := int(float64(len(results)+NewSegmentBufferNumResultsBump) * + NewSegmentBufferAvgBytesPerDocFactor) + br.Grow(estimateAvgBytesPerDoc * estimateNumResults) + } + + s.results = results + s.chunkFactor = chunkFactor + s.w = NewCountHashWriter(&br) + + storedIndexOffset, fieldsIndexOffset, fdvIndexOffset, dictOffsets, + err := s.convert() + if err != nil { + return nil, uint64(0), err + } + + sb, err := InitSegmentBase(br.Bytes(), s.w.Sum32(), chunkFactor, + s.FieldsMap, s.FieldsInv, uint64(len(results)), + storedIndexOffset, fieldsIndexOffset, fdvIndexOffset, dictOffsets) + + if err == nil && s.reset() == nil { + s.lastNumDocs = len(results) + s.lastOutSize = len(br.Bytes()) + interimPool.Put(s) + } + + return sb, uint64(len(br.Bytes())), err +} + +var interimPool = sync.Pool{New: func() interface{} { return &interim{} }} + +// interim holds temporary working data used while converting from +// analysis results to a zap-encoded segment +type interim struct { + results []index.Document + + chunkFactor uint32 + + w *CountHashWriter + + // FieldsMap adds 1 to field id to avoid zero value issues + // name -> field id + 1 + FieldsMap map[string]uint16 + + // FieldsInv is the inverse of FieldsMap + // field id -> name + FieldsInv []string + + // Term dictionaries for each field + // field id -> term -> postings list id + 1 + Dicts []map[string]uint64 + + // Terms for each field, where terms are sorted ascending + // field id -> []term + DictKeys [][]string + + // Fields whose IncludeDocValues is true + // field id -> bool + IncludeDocValues []bool + + // postings id -> bitmap of docNums + Postings []*roaring.Bitmap + + // postings id -> freq/norm's, one for each docNum in postings + FreqNorms [][]interimFreqNorm + freqNormsBacking []interimFreqNorm + + // postings id -> locs, one for each freq + Locs [][]interimLoc + locsBacking []interimLoc + + numTermsPerPostingsList []int // key is postings list id + numLocsPerPostingsList []int // key is postings list id + + builder *vellum.Builder + builderBuf bytes.Buffer + + metaBuf bytes.Buffer + + tmp0 []byte + tmp1 []byte + + lastNumDocs int + lastOutSize int +} + +func (s *interim) reset() (err error) { + s.results = nil + s.chunkFactor = 0 + s.w = nil + s.FieldsMap = nil + s.FieldsInv = nil + for i := range s.Dicts { + s.Dicts[i] = nil + } + s.Dicts = s.Dicts[:0] + for i := range s.DictKeys { + s.DictKeys[i] = s.DictKeys[i][:0] + } + s.DictKeys = s.DictKeys[:0] + for i := range s.IncludeDocValues { + s.IncludeDocValues[i] = false + } + s.IncludeDocValues = s.IncludeDocValues[:0] + for _, idn := range s.Postings { + idn.Clear() + } + s.Postings = s.Postings[:0] + s.FreqNorms = s.FreqNorms[:0] + for i := range s.freqNormsBacking { + s.freqNormsBacking[i] = interimFreqNorm{} + } + s.freqNormsBacking = s.freqNormsBacking[:0] + s.Locs = s.Locs[:0] + for i := range s.locsBacking { + s.locsBacking[i] = interimLoc{} + } + s.locsBacking = s.locsBacking[:0] + s.numTermsPerPostingsList = s.numTermsPerPostingsList[:0] + s.numLocsPerPostingsList = s.numLocsPerPostingsList[:0] + s.builderBuf.Reset() + if s.builder != nil { + err = s.builder.Reset(&s.builderBuf) + } + s.metaBuf.Reset() + s.tmp0 = s.tmp0[:0] + s.tmp1 = s.tmp1[:0] + s.lastNumDocs = 0 + s.lastOutSize = 0 + + return err +} + +func (s *interim) grabBuf(size int) []byte { + buf := s.tmp0 + if cap(buf) < size { + buf = make([]byte, size) + s.tmp0 = buf + } + return buf[0:size] +} + +type interimStoredField struct { + vals [][]byte + typs []byte + arrayposs [][]uint64 // array positions +} + +type interimFreqNorm struct { + freq uint64 + norm float32 + numLocs int +} + +type interimLoc struct { + fieldID uint16 + pos uint64 + start uint64 + end uint64 + arrayposs []uint64 +} + +func (s *interim) convert() (uint64, uint64, uint64, []uint64, error) { + s.FieldsMap = map[string]uint16{} + + s.getOrDefineField("_id") // _id field is fieldID 0 + + for _, result := range s.results { + result.VisitComposite(func(field index.CompositeField) { + s.getOrDefineField(field.Name()) + }) + result.VisitFields(func(field index.Field) { + s.getOrDefineField(field.Name()) + }) + } + + sort.Strings(s.FieldsInv[1:]) // keep _id as first field + + for fieldID, fieldName := range s.FieldsInv { + s.FieldsMap[fieldName] = uint16(fieldID + 1) + } + + if cap(s.IncludeDocValues) >= len(s.FieldsInv) { + s.IncludeDocValues = s.IncludeDocValues[:len(s.FieldsInv)] + } else { + s.IncludeDocValues = make([]bool, len(s.FieldsInv)) + } + + s.prepareDicts() + + for _, dict := range s.DictKeys { + sort.Strings(dict) + } + + s.processDocuments() + + storedIndexOffset, err := s.writeStoredFields() + if err != nil { + return 0, 0, 0, nil, err + } + + var fdvIndexOffset uint64 + var dictOffsets []uint64 + + if len(s.results) > 0 { + fdvIndexOffset, dictOffsets, err = s.writeDicts() + if err != nil { + return 0, 0, 0, nil, err + } + } else { + dictOffsets = make([]uint64, len(s.FieldsInv)) + } + + fieldsIndexOffset, err := persistFields(s.FieldsInv, s.w, dictOffsets) + if err != nil { + return 0, 0, 0, nil, err + } + + return storedIndexOffset, fieldsIndexOffset, fdvIndexOffset, dictOffsets, nil +} + +func (s *interim) getOrDefineField(fieldName string) int { + fieldIDPlus1, exists := s.FieldsMap[fieldName] + if !exists { + fieldIDPlus1 = uint16(len(s.FieldsInv) + 1) + s.FieldsMap[fieldName] = fieldIDPlus1 + s.FieldsInv = append(s.FieldsInv, fieldName) + + s.Dicts = append(s.Dicts, make(map[string]uint64)) + + n := len(s.DictKeys) + if n < cap(s.DictKeys) { + s.DictKeys = s.DictKeys[:n+1] + s.DictKeys[n] = s.DictKeys[n][:0] + } else { + s.DictKeys = append(s.DictKeys, []string(nil)) + } + } + + return int(fieldIDPlus1 - 1) +} + +// fill Dicts and DictKeys from analysis results +func (s *interim) prepareDicts() { + var pidNext int + + var totTFs int + var totLocs int + + visitField := func(field index.Field) { + fieldID := uint16(s.getOrDefineField(field.Name())) + + dict := s.Dicts[fieldID] + dictKeys := s.DictKeys[fieldID] + + tfs := field.AnalyzedTokenFrequencies() + for term, tf := range tfs { + pidPlus1, exists := dict[term] + if !exists { + pidNext++ + pidPlus1 = uint64(pidNext) + + dict[term] = pidPlus1 + dictKeys = append(dictKeys, term) + + s.numTermsPerPostingsList = append(s.numTermsPerPostingsList, 0) + s.numLocsPerPostingsList = append(s.numLocsPerPostingsList, 0) + } + + pid := pidPlus1 - 1 + + s.numTermsPerPostingsList[pid] += 1 + s.numLocsPerPostingsList[pid] += len(tf.Locations) + + totLocs += len(tf.Locations) + } + + totTFs += len(tfs) + + s.DictKeys[fieldID] = dictKeys + } + + for _, result := range s.results { + // walk each composite field + result.VisitComposite(func(field index.CompositeField) { + visitField(field) + }) + + // walk each field + result.VisitFields(visitField) + } + + numPostingsLists := pidNext + + if cap(s.Postings) >= numPostingsLists { + s.Postings = s.Postings[:numPostingsLists] + } else { + postings := make([]*roaring.Bitmap, numPostingsLists) + copy(postings, s.Postings[:cap(s.Postings)]) + for i := 0; i < numPostingsLists; i++ { + if postings[i] == nil { + postings[i] = roaring.New() + } + } + s.Postings = postings + } + + if cap(s.FreqNorms) >= numPostingsLists { + s.FreqNorms = s.FreqNorms[:numPostingsLists] + } else { + s.FreqNorms = make([][]interimFreqNorm, numPostingsLists) + } + + if cap(s.freqNormsBacking) >= totTFs { + s.freqNormsBacking = s.freqNormsBacking[:totTFs] + } else { + s.freqNormsBacking = make([]interimFreqNorm, totTFs) + } + + freqNormsBacking := s.freqNormsBacking + for pid, numTerms := range s.numTermsPerPostingsList { + s.FreqNorms[pid] = freqNormsBacking[0:0] + freqNormsBacking = freqNormsBacking[numTerms:] + } + + if cap(s.Locs) >= numPostingsLists { + s.Locs = s.Locs[:numPostingsLists] + } else { + s.Locs = make([][]interimLoc, numPostingsLists) + } + + if cap(s.locsBacking) >= totLocs { + s.locsBacking = s.locsBacking[:totLocs] + } else { + s.locsBacking = make([]interimLoc, totLocs) + } + + locsBacking := s.locsBacking + for pid, numLocs := range s.numLocsPerPostingsList { + s.Locs[pid] = locsBacking[0:0] + locsBacking = locsBacking[numLocs:] + } +} + +func (s *interim) processDocuments() { + numFields := len(s.FieldsInv) + reuseFieldLens := make([]int, numFields) + reuseFieldTFs := make([]index.TokenFrequencies, numFields) + + for docNum, result := range s.results { + for i := 0; i < numFields; i++ { // clear these for reuse + reuseFieldLens[i] = 0 + reuseFieldTFs[i] = nil + } + + s.processDocument(uint64(docNum), result, + reuseFieldLens, reuseFieldTFs) + } +} + +func (s *interim) processDocument(docNum uint64, + result index.Document, + fieldLens []int, fieldTFs []index.TokenFrequencies) { + visitField := func(field index.Field) { + fieldID := uint16(s.getOrDefineField(field.Name())) + fieldLens[fieldID] += field.AnalyzedLength() + + existingFreqs := fieldTFs[fieldID] + if existingFreqs != nil { + existingFreqs.MergeAll(field.Name(), field.AnalyzedTokenFrequencies()) + } else { + fieldTFs[fieldID] = field.AnalyzedTokenFrequencies() + } + } + + // walk each composite field + result.VisitComposite(func(field index.CompositeField) { + visitField(field) + }) + + // walk each field + result.VisitFields(visitField) + + // now that it's been rolled up into fieldTFs, walk that + for fieldID, tfs := range fieldTFs { + dict := s.Dicts[fieldID] + norm := float32(1.0 / math.Sqrt(float64(fieldLens[fieldID]))) + + for term, tf := range tfs { + pid := dict[term] - 1 + bs := s.Postings[pid] + bs.Add(uint32(docNum)) + + s.FreqNorms[pid] = append(s.FreqNorms[pid], + interimFreqNorm{ + freq: uint64(tf.Frequency()), + norm: norm, + numLocs: len(tf.Locations), + }) + + if len(tf.Locations) > 0 { + locs := s.Locs[pid] + + for _, loc := range tf.Locations { + var locf = uint16(fieldID) + if loc.Field != "" { + locf = uint16(s.getOrDefineField(loc.Field)) + } + var arrayposs []uint64 + if len(loc.ArrayPositions) > 0 { + arrayposs = loc.ArrayPositions + } + locs = append(locs, interimLoc{ + fieldID: locf, + pos: uint64(loc.Position), + start: uint64(loc.Start), + end: uint64(loc.End), + arrayposs: arrayposs, + }) + } + + s.Locs[pid] = locs + } + } + } +} + +func (s *interim) writeStoredFields() ( + storedIndexOffset uint64, err error) { + varBuf := make([]byte, binary.MaxVarintLen64) + metaEncode := func(val uint64) (int, error) { + wb := binary.PutUvarint(varBuf, val) + return s.metaBuf.Write(varBuf[:wb]) + } + + data, compressed := s.tmp0[:0], s.tmp1[:0] + defer func() { s.tmp0, s.tmp1 = data, compressed }() + + // keyed by docNum + docStoredOffsets := make([]uint64, len(s.results)) + + // keyed by fieldID, for the current doc in the loop + docStoredFields := map[uint16]interimStoredField{} + + for docNum, result := range s.results { + for fieldID := range docStoredFields { // reset for next doc + delete(docStoredFields, fieldID) + } + + var validationErr error + result.VisitFields(func(field index.Field) { + fieldID := uint16(s.getOrDefineField(field.Name())) + + if field.Options().IsStored() { + isf := docStoredFields[fieldID] + isf.vals = append(isf.vals, field.Value()) + isf.typs = append(isf.typs, field.EncodedFieldType()) + isf.arrayposs = append(isf.arrayposs, field.ArrayPositions()) + docStoredFields[fieldID] = isf + } + + if field.Options().IncludeDocValues() { + s.IncludeDocValues[fieldID] = true + } + + err := ValidateDocFields(field) + if err != nil && validationErr == nil { + validationErr = err + } + }) + if validationErr != nil { + return 0, validationErr + } + + var curr int + + s.metaBuf.Reset() + data = data[:0] + + // _id field special case optimizes ExternalID() lookups + idFieldVal := docStoredFields[uint16(0)].vals[0] + _, err = metaEncode(uint64(len(idFieldVal))) + if err != nil { + return 0, err + } + + // handle non-"_id" fields + for fieldID := 1; fieldID < len(s.FieldsInv); fieldID++ { + isf, exists := docStoredFields[uint16(fieldID)] + if exists { + curr, data, err = persistStoredFieldValues( + fieldID, isf.vals, isf.typs, isf.arrayposs, + curr, metaEncode, data) + if err != nil { + return 0, err + } + } + } + + metaBytes := s.metaBuf.Bytes() + + compressed = snappy.Encode(compressed[:cap(compressed)], data) + + docStoredOffsets[docNum] = uint64(s.w.Count()) + + _, err := writeUvarints(s.w, + uint64(len(metaBytes)), + uint64(len(idFieldVal)+len(compressed))) + if err != nil { + return 0, err + } + + _, err = s.w.Write(metaBytes) + if err != nil { + return 0, err + } + + _, err = s.w.Write(idFieldVal) + if err != nil { + return 0, err + } + + _, err = s.w.Write(compressed) + if err != nil { + return 0, err + } + } + + storedIndexOffset = uint64(s.w.Count()) + + for _, docStoredOffset := range docStoredOffsets { + err = binary.Write(s.w, binary.BigEndian, docStoredOffset) + if err != nil { + return 0, err + } + } + + return storedIndexOffset, nil +} + +func (s *interim) writeDicts() (fdvIndexOffset uint64, dictOffsets []uint64, err error) { + dictOffsets = make([]uint64, len(s.FieldsInv)) + + fdvOffsetsStart := make([]uint64, len(s.FieldsInv)) + fdvOffsetsEnd := make([]uint64, len(s.FieldsInv)) + + buf := s.grabBuf(binary.MaxVarintLen64) + + tfEncoder := newChunkedIntCoder(uint64(s.chunkFactor), uint64(len(s.results)-1)) + locEncoder := newChunkedIntCoder(uint64(s.chunkFactor), uint64(len(s.results)-1)) + fdvEncoder := newChunkedContentCoder(uint64(s.chunkFactor), uint64(len(s.results)-1), s.w, false) + + var docTermMap [][]byte + + if s.builder == nil { + s.builder, err = vellum.New(&s.builderBuf, nil) + if err != nil { + return 0, nil, err + } + } + + for fieldID, terms := range s.DictKeys { + if cap(docTermMap) < len(s.results) { + docTermMap = make([][]byte, len(s.results)) + } else { + docTermMap = docTermMap[0:len(s.results)] + for docNum := range docTermMap { // reset the docTermMap + docTermMap[docNum] = docTermMap[docNum][:0] + } + } + + dict := s.Dicts[fieldID] + + for _, term := range terms { // terms are already sorted + pid := dict[term] - 1 + + postingsBS := s.Postings[pid] + + freqNorms := s.FreqNorms[pid] + freqNormOffset := 0 + + locs := s.Locs[pid] + locOffset := 0 + + postingsItr := postingsBS.Iterator() + for postingsItr.HasNext() { + docNum := uint64(postingsItr.Next()) + + freqNorm := freqNorms[freqNormOffset] + + err = tfEncoder.Add(docNum, + encodeFreqHasLocs(freqNorm.freq, freqNorm.numLocs > 0), + uint64(math.Float32bits(freqNorm.norm))) + if err != nil { + return 0, nil, err + } + + if freqNorm.numLocs > 0 { + numBytesLocs := 0 + for _, loc := range locs[locOffset : locOffset+freqNorm.numLocs] { + numBytesLocs += totalUvarintBytes( + uint64(loc.fieldID), loc.pos, loc.start, loc.end, + uint64(len(loc.arrayposs)), loc.arrayposs) + } + + err = locEncoder.Add(docNum, uint64(numBytesLocs)) + if err != nil { + return 0, nil, err + } + + for _, loc := range locs[locOffset : locOffset+freqNorm.numLocs] { + err = locEncoder.Add(docNum, + uint64(loc.fieldID), loc.pos, loc.start, loc.end, + uint64(len(loc.arrayposs))) + if err != nil { + return 0, nil, err + } + + err = locEncoder.Add(docNum, loc.arrayposs...) + if err != nil { + return 0, nil, err + } + } + + locOffset += freqNorm.numLocs + } + + freqNormOffset++ + + docTermMap[docNum] = append( + append(docTermMap[docNum], term...), + termSeparator) + } + + tfEncoder.Close() + locEncoder.Close() + + postingsOffset, err := + writePostings(postingsBS, tfEncoder, locEncoder, nil, s.w, buf) + if err != nil { + return 0, nil, err + } + + if postingsOffset > uint64(0) { + err = s.builder.Insert([]byte(term), postingsOffset) + if err != nil { + return 0, nil, err + } + } + + tfEncoder.Reset() + locEncoder.Reset() + } + + err = s.builder.Close() + if err != nil { + return 0, nil, err + } + + // record where this dictionary starts + dictOffsets[fieldID] = uint64(s.w.Count()) + + vellumData := s.builderBuf.Bytes() + + // write out the length of the vellum data + n := binary.PutUvarint(buf, uint64(len(vellumData))) + _, err = s.w.Write(buf[:n]) + if err != nil { + return 0, nil, err + } + + // write this vellum to disk + _, err = s.w.Write(vellumData) + if err != nil { + return 0, nil, err + } + + // reset vellum for reuse + s.builderBuf.Reset() + + err = s.builder.Reset(&s.builderBuf) + if err != nil { + return 0, nil, err + } + + // write the field doc values + if s.IncludeDocValues[fieldID] { + for docNum, docTerms := range docTermMap { + if len(docTerms) > 0 { + err = fdvEncoder.Add(uint64(docNum), docTerms) + if err != nil { + return 0, nil, err + } + } + } + err = fdvEncoder.Close() + if err != nil { + return 0, nil, err + } + + fdvOffsetsStart[fieldID] = uint64(s.w.Count()) + + _, err = fdvEncoder.Write() + if err != nil { + return 0, nil, err + } + + fdvOffsetsEnd[fieldID] = uint64(s.w.Count()) + + fdvEncoder.Reset() + } else { + fdvOffsetsStart[fieldID] = fieldNotUninverted + fdvOffsetsEnd[fieldID] = fieldNotUninverted + } + } + + fdvIndexOffset = uint64(s.w.Count()) + + for i := 0; i < len(fdvOffsetsStart); i++ { + n := binary.PutUvarint(buf, fdvOffsetsStart[i]) + _, err := s.w.Write(buf[:n]) + if err != nil { + return 0, nil, err + } + n = binary.PutUvarint(buf, fdvOffsetsEnd[i]) + _, err = s.w.Write(buf[:n]) + if err != nil { + return 0, nil, err + } + } + + return fdvIndexOffset, dictOffsets, nil +} + +// returns the total # of bytes needed to encode the given uint64's +// into binary.PutUVarint() encoding +func totalUvarintBytes(a, b, c, d, e uint64, more []uint64) (n int) { + n = numUvarintBytes(a) + n += numUvarintBytes(b) + n += numUvarintBytes(c) + n += numUvarintBytes(d) + n += numUvarintBytes(e) + for _, v := range more { + n += numUvarintBytes(v) + } + return n +} + +// returns # of bytes needed to encode x in binary.PutUvarint() encoding +func numUvarintBytes(x uint64) (n int) { + for x >= 0x80 { + x >>= 7 + n++ + } + return n + 1 +} diff --git a/vendor/github.com/blevesearch/zapx/v11/plugin.go b/vendor/github.com/blevesearch/zapx/v11/plugin.go new file mode 100644 index 00000000..f67297ec --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v11/plugin.go @@ -0,0 +1,27 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +// ZapPlugin implements the Plugin interface of +// the blevesearch/scorch_segment_api pkg +type ZapPlugin struct{} + +func (*ZapPlugin) Type() string { + return Type +} + +func (*ZapPlugin) Version() uint32 { + return Version +} diff --git a/vendor/github.com/blevesearch/zapx/v11/posting.go b/vendor/github.com/blevesearch/zapx/v11/posting.go new file mode 100644 index 00000000..71b8e52b --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v11/posting.go @@ -0,0 +1,949 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "encoding/binary" + "fmt" + "math" + "reflect" + + "github.com/RoaringBitmap/roaring" + segment "github.com/blevesearch/scorch_segment_api/v2" +) + +var reflectStaticSizePostingsList int +var reflectStaticSizePostingsIterator int +var reflectStaticSizePosting int +var reflectStaticSizeLocation int + +func init() { + var pl PostingsList + reflectStaticSizePostingsList = int(reflect.TypeOf(pl).Size()) + var pi PostingsIterator + reflectStaticSizePostingsIterator = int(reflect.TypeOf(pi).Size()) + var p Posting + reflectStaticSizePosting = int(reflect.TypeOf(p).Size()) + var l Location + reflectStaticSizeLocation = int(reflect.TypeOf(l).Size()) +} + +// FST or vellum value (uint64) encoding is determined by the top two +// highest-order or most significant bits... +// +// encoding : MSB +// name : 63 62 61...to...bit #0 (LSB) +// ----------+---+---+--------------------------------------------------- +// general : 0 | 0 | 62-bits of postingsOffset. +// ~ : 0 | 1 | reserved for future. +// 1-hit : 1 | 0 | 31-bits of positive float31 norm | 31-bits docNum. +// ~ : 1 | 1 | reserved for future. +// +// Encoding "general" is able to handle all cases, where the +// postingsOffset points to more information about the postings for +// the term. +// +// Encoding "1-hit" is used to optimize a commonly seen case when a +// term has only a single hit. For example, a term in the _id field +// will have only 1 hit. The "1-hit" encoding is used for a term +// in a field when... +// +// - term vector info is disabled for that field; +// - and, the term appears in only a single doc for that field; +// - and, the term's freq is exactly 1 in that single doc for that field; +// - and, the docNum must fit into 31-bits; +// +// Otherwise, the "general" encoding is used instead. +// +// In the "1-hit" encoding, the field in that single doc may have +// other terms, which is supported in the "1-hit" encoding by the +// positive float31 norm. + +const FSTValEncodingMask = uint64(0xc000000000000000) +const FSTValEncodingGeneral = uint64(0x0000000000000000) +const FSTValEncoding1Hit = uint64(0x8000000000000000) + +func FSTValEncode1Hit(docNum uint64, normBits uint64) uint64 { + return FSTValEncoding1Hit | ((mask31Bits & normBits) << 31) | (mask31Bits & docNum) +} + +func FSTValDecode1Hit(v uint64) (docNum uint64, normBits uint64) { + return (mask31Bits & v), (mask31Bits & (v >> 31)) +} + +const mask31Bits = uint64(0x000000007fffffff) + +func under32Bits(x uint64) bool { + return x <= mask31Bits +} + +const DocNum1HitFinished = math.MaxUint64 + +var NormBits1Hit = uint64(math.Float32bits(float32(1))) + +// PostingsList is an in-memory representation of a postings list +type PostingsList struct { + sb *SegmentBase + postingsOffset uint64 + freqOffset uint64 + locOffset uint64 + postings *roaring.Bitmap + except *roaring.Bitmap + + // when normBits1Hit != 0, then this postings list came from a + // 1-hit encoding, and only the docNum1Hit & normBits1Hit apply + docNum1Hit uint64 + normBits1Hit uint64 +} + +// represents an immutable, empty postings list +var emptyPostingsList = &PostingsList{} + +// Following methods are dummy implementations for the interface +// DiskStatsReporter (for backward compatibility). +// The working implementations are supported only in zap v15.x +// and not in the earlier versions of zap. +func (i *PostingsList) ResetBytesRead(uint64) {} + +func (i *PostingsList) BytesRead() uint64 { + return 0 +} + +func (i *PostingsList) incrementBytesRead(uint64) {} + +func (i *PostingsList) BytesWritten() uint64 { + return 0 +} + +func (p *PostingsList) Size() int { + sizeInBytes := reflectStaticSizePostingsList + SizeOfPtr + + if p.except != nil { + sizeInBytes += int(p.except.GetSizeInBytes()) + } + + return sizeInBytes +} + +func (p *PostingsList) OrInto(receiver *roaring.Bitmap) { + if p.normBits1Hit != 0 { + receiver.Add(uint32(p.docNum1Hit)) + return + } + + if p.postings != nil { + receiver.Or(p.postings) + } +} + +// Iterator returns an iterator for this postings list +func (p *PostingsList) Iterator(includeFreq, includeNorm, includeLocs bool, + prealloc segment.PostingsIterator) segment.PostingsIterator { + if p.normBits1Hit == 0 && p.postings == nil { + return emptyPostingsIterator + } + + var preallocPI *PostingsIterator + pi, ok := prealloc.(*PostingsIterator) + if ok && pi != nil { + preallocPI = pi + } + if preallocPI == emptyPostingsIterator { + preallocPI = nil + } + + return p.iterator(includeFreq, includeNorm, includeLocs, preallocPI) +} + +func (p *PostingsList) iterator(includeFreq, includeNorm, includeLocs bool, + rv *PostingsIterator) *PostingsIterator { + if rv == nil { + rv = &PostingsIterator{} + } else { + freqNormReader := rv.freqNormReader + if freqNormReader != nil { + freqNormReader.Reset([]byte(nil)) + } + + locReader := rv.locReader + if locReader != nil { + locReader.Reset([]byte(nil)) + } + + freqChunkOffsets := rv.freqChunkOffsets[:0] + locChunkOffsets := rv.locChunkOffsets[:0] + + nextLocs := rv.nextLocs[:0] + nextSegmentLocs := rv.nextSegmentLocs[:0] + + buf := rv.buf + + *rv = PostingsIterator{} // clear the struct + + rv.freqNormReader = freqNormReader + rv.locReader = locReader + + rv.freqChunkOffsets = freqChunkOffsets + rv.locChunkOffsets = locChunkOffsets + + rv.nextLocs = nextLocs + rv.nextSegmentLocs = nextSegmentLocs + + rv.buf = buf + } + + rv.postings = p + rv.includeFreqNorm = includeFreq || includeNorm || includeLocs + rv.includeLocs = includeLocs + + if p.normBits1Hit != 0 { + // "1-hit" encoding + rv.docNum1Hit = p.docNum1Hit + rv.normBits1Hit = p.normBits1Hit + + if p.except != nil && p.except.Contains(uint32(rv.docNum1Hit)) { + rv.docNum1Hit = DocNum1HitFinished + } + + return rv + } + + // "general" encoding, check if empty + if p.postings == nil { + return rv + } + + var n uint64 + var read int + + // prepare the freq chunk details + if rv.includeFreqNorm { + var numFreqChunks uint64 + numFreqChunks, read = binary.Uvarint(p.sb.mem[p.freqOffset+n : p.freqOffset+n+binary.MaxVarintLen64]) + n += uint64(read) + if cap(rv.freqChunkOffsets) >= int(numFreqChunks) { + rv.freqChunkOffsets = rv.freqChunkOffsets[:int(numFreqChunks)] + } else { + rv.freqChunkOffsets = make([]uint64, int(numFreqChunks)) + } + for i := 0; i < int(numFreqChunks); i++ { + rv.freqChunkOffsets[i], read = binary.Uvarint(p.sb.mem[p.freqOffset+n : p.freqOffset+n+binary.MaxVarintLen64]) + n += uint64(read) + } + rv.freqChunkStart = p.freqOffset + n + } + + // prepare the loc chunk details + if rv.includeLocs { + n = 0 + var numLocChunks uint64 + numLocChunks, read = binary.Uvarint(p.sb.mem[p.locOffset+n : p.locOffset+n+binary.MaxVarintLen64]) + n += uint64(read) + if cap(rv.locChunkOffsets) >= int(numLocChunks) { + rv.locChunkOffsets = rv.locChunkOffsets[:int(numLocChunks)] + } else { + rv.locChunkOffsets = make([]uint64, int(numLocChunks)) + } + for i := 0; i < int(numLocChunks); i++ { + rv.locChunkOffsets[i], read = binary.Uvarint(p.sb.mem[p.locOffset+n : p.locOffset+n+binary.MaxVarintLen64]) + n += uint64(read) + } + rv.locChunkStart = p.locOffset + n + } + + rv.all = p.postings.Iterator() + if p.except != nil { + rv.ActualBM = roaring.AndNot(p.postings, p.except) + rv.Actual = rv.ActualBM.Iterator() + } else { + rv.ActualBM = p.postings + rv.Actual = rv.all // Optimize to use same iterator for all & Actual. + } + + return rv +} + +// Count returns the number of items on this postings list +func (p *PostingsList) Count() uint64 { + var n, e uint64 + if p.normBits1Hit != 0 { + n = 1 + if p.except != nil && p.except.Contains(uint32(p.docNum1Hit)) { + e = 1 + } + } else if p.postings != nil { + n = p.postings.GetCardinality() + if p.except != nil { + e = p.postings.AndCardinality(p.except) + } + } + return n - e +} + +func (rv *PostingsList) read(postingsOffset uint64, d *Dictionary) error { + rv.postingsOffset = postingsOffset + + // handle "1-hit" encoding special case + if rv.postingsOffset&FSTValEncodingMask == FSTValEncoding1Hit { + return rv.init1Hit(postingsOffset) + } + + // read the location of the freq/norm details + var n uint64 + var read int + + rv.freqOffset, read = binary.Uvarint(d.sb.mem[postingsOffset+n : postingsOffset+binary.MaxVarintLen64]) + n += uint64(read) + + rv.locOffset, read = binary.Uvarint(d.sb.mem[postingsOffset+n : postingsOffset+n+binary.MaxVarintLen64]) + n += uint64(read) + + var postingsLen uint64 + postingsLen, read = binary.Uvarint(d.sb.mem[postingsOffset+n : postingsOffset+n+binary.MaxVarintLen64]) + n += uint64(read) + + roaringBytes := d.sb.mem[postingsOffset+n : postingsOffset+n+postingsLen] + + if rv.postings == nil { + rv.postings = roaring.NewBitmap() + } + _, err := rv.postings.FromBuffer(roaringBytes) + if err != nil { + return fmt.Errorf("error loading roaring bitmap: %v", err) + } + + return nil +} + +func (rv *PostingsList) init1Hit(fstVal uint64) error { + docNum, normBits := FSTValDecode1Hit(fstVal) + + rv.docNum1Hit = docNum + rv.normBits1Hit = normBits + + return nil +} + +// PostingsIterator provides a way to iterate through the postings list +type PostingsIterator struct { + postings *PostingsList + all roaring.IntPeekable + Actual roaring.IntPeekable + ActualBM *roaring.Bitmap + + currChunk uint32 + currChunkFreqNorm []byte + currChunkLoc []byte + + freqNormReader *memUvarintReader + locReader *memUvarintReader + + freqChunkOffsets []uint64 + freqChunkStart uint64 + + locChunkOffsets []uint64 + locChunkStart uint64 + + next Posting // reused across Next() calls + nextLocs []Location // reused across Next() calls + nextSegmentLocs []segment.Location // reused across Next() calls + + docNum1Hit uint64 + normBits1Hit uint64 + + buf []byte + + includeFreqNorm bool + includeLocs bool +} + +var emptyPostingsIterator = &PostingsIterator{} + +// Following methods are dummy implementations for the interface +// DiskStatsReporter (for backward compatibility). +// The working implementations are supported only in zap v15.x +// and not in the earlier versions of zap. +func (i *PostingsIterator) ResetBytesRead(uint64) {} + +func (i *PostingsIterator) BytesRead() uint64 { + return 0 +} + +func (i *PostingsIterator) incrementBytesRead(uint64) {} + +func (i *PostingsIterator) BytesWritten() uint64 { + return 0 +} + +func (i *PostingsIterator) Size() int { + sizeInBytes := reflectStaticSizePostingsIterator + SizeOfPtr + + len(i.currChunkFreqNorm) + + len(i.currChunkLoc) + + len(i.freqChunkOffsets)*SizeOfUint64 + + len(i.locChunkOffsets)*SizeOfUint64 + + i.next.Size() + + for _, entry := range i.nextLocs { + sizeInBytes += entry.Size() + } + + return sizeInBytes +} + +func (i *PostingsIterator) loadChunk(chunk int) error { + if i.includeFreqNorm { + if chunk >= len(i.freqChunkOffsets) { + return fmt.Errorf("tried to load freq chunk that doesn't exist %d/(%d)", + chunk, len(i.freqChunkOffsets)) + } + + end, start := i.freqChunkStart, i.freqChunkStart + s, e := readChunkBoundary(chunk, i.freqChunkOffsets) + start += s + end += e + i.currChunkFreqNorm = i.postings.sb.mem[start:end] + if i.freqNormReader == nil { + i.freqNormReader = newMemUvarintReader(i.currChunkFreqNorm) + } else { + i.freqNormReader.Reset(i.currChunkFreqNorm) + } + } + + if i.includeLocs { + if chunk >= len(i.locChunkOffsets) { + return fmt.Errorf("tried to load loc chunk that doesn't exist %d/(%d)", + chunk, len(i.locChunkOffsets)) + } + + end, start := i.locChunkStart, i.locChunkStart + s, e := readChunkBoundary(chunk, i.locChunkOffsets) + start += s + end += e + i.currChunkLoc = i.postings.sb.mem[start:end] + if i.locReader == nil { + i.locReader = newMemUvarintReader(i.currChunkLoc) + } else { + i.locReader.Reset(i.currChunkLoc) + } + } + + i.currChunk = uint32(chunk) + return nil +} + +func (i *PostingsIterator) readFreqNormHasLocs() (uint64, uint64, bool, error) { + if i.normBits1Hit != 0 { + return 1, i.normBits1Hit, false, nil + } + + freqHasLocs, err := i.freqNormReader.ReadUvarint() + if err != nil { + return 0, 0, false, fmt.Errorf("error reading frequency: %v", err) + } + + freq, hasLocs := decodeFreqHasLocs(freqHasLocs) + + normBits, err := i.freqNormReader.ReadUvarint() + if err != nil { + return 0, 0, false, fmt.Errorf("error reading norm: %v", err) + } + + return freq, normBits, hasLocs, nil +} + +func (i *PostingsIterator) skipFreqNormReadHasLocs() (bool, error) { + if i.normBits1Hit != 0 { + return false, nil + } + + freqHasLocs, err := i.freqNormReader.ReadUvarint() + if err != nil { + return false, fmt.Errorf("error reading freqHasLocs: %v", err) + } + + i.freqNormReader.SkipUvarint() // Skip normBits. + + return freqHasLocs&0x01 != 0, nil // See decodeFreqHasLocs() / hasLocs. +} + +func encodeFreqHasLocs(freq uint64, hasLocs bool) uint64 { + rv := freq << 1 + if hasLocs { + rv = rv | 0x01 // 0'th LSB encodes whether there are locations + } + return rv +} + +func decodeFreqHasLocs(freqHasLocs uint64) (uint64, bool) { + freq := freqHasLocs >> 1 + hasLocs := freqHasLocs&0x01 != 0 + return freq, hasLocs +} + +// readLocation processes all the integers on the stream representing a single +// location. +func (i *PostingsIterator) readLocation(l *Location) error { + // read off field + fieldID, err := i.locReader.ReadUvarint() + if err != nil { + return fmt.Errorf("error reading location field: %v", err) + } + // read off pos + pos, err := i.locReader.ReadUvarint() + if err != nil { + return fmt.Errorf("error reading location pos: %v", err) + } + // read off start + start, err := i.locReader.ReadUvarint() + if err != nil { + return fmt.Errorf("error reading location start: %v", err) + } + // read off end + end, err := i.locReader.ReadUvarint() + if err != nil { + return fmt.Errorf("error reading location end: %v", err) + } + // read off num array pos + numArrayPos, err := i.locReader.ReadUvarint() + if err != nil { + return fmt.Errorf("error reading location num array pos: %v", err) + } + + l.field = i.postings.sb.fieldsInv[fieldID] + l.pos = pos + l.start = start + l.end = end + + if cap(l.ap) < int(numArrayPos) { + l.ap = make([]uint64, int(numArrayPos)) + } else { + l.ap = l.ap[:int(numArrayPos)] + } + + // read off array positions + for k := 0; k < int(numArrayPos); k++ { + ap, err := i.locReader.ReadUvarint() + if err != nil { + return fmt.Errorf("error reading array position: %v", err) + } + + l.ap[k] = ap + } + + return nil +} + +// Next returns the next posting on the postings list, or nil at the end +func (i *PostingsIterator) Next() (segment.Posting, error) { + return i.nextAtOrAfter(0) +} + +// Advance returns the posting at the specified docNum or it is not present +// the next posting, or if the end is reached, nil +func (i *PostingsIterator) Advance(docNum uint64) (segment.Posting, error) { + return i.nextAtOrAfter(docNum) +} + +// Next returns the next posting on the postings list, or nil at the end +func (i *PostingsIterator) nextAtOrAfter(atOrAfter uint64) (segment.Posting, error) { + docNum, exists, err := i.nextDocNumAtOrAfter(atOrAfter) + if err != nil || !exists { + return nil, err + } + + i.next = Posting{} // clear the struct + rv := &i.next + rv.docNum = docNum + + if !i.includeFreqNorm { + return rv, nil + } + + var normBits uint64 + var hasLocs bool + + rv.freq, normBits, hasLocs, err = i.readFreqNormHasLocs() + if err != nil { + return nil, err + } + + rv.norm = math.Float32frombits(uint32(normBits)) + + if i.includeLocs && hasLocs { + // prepare locations into reused slices, where we assume + // rv.freq >= "number of locs", since in a composite field, + // some component fields might have their IncludeTermVector + // flags disabled while other component fields are enabled + if cap(i.nextLocs) >= int(rv.freq) { + i.nextLocs = i.nextLocs[0:rv.freq] + } else { + i.nextLocs = make([]Location, rv.freq, rv.freq*2) + } + if cap(i.nextSegmentLocs) < int(rv.freq) { + i.nextSegmentLocs = make([]segment.Location, rv.freq, rv.freq*2) + } + rv.locs = i.nextSegmentLocs[:0] + + numLocsBytes, err := i.locReader.ReadUvarint() + if err != nil { + return nil, fmt.Errorf("error reading location numLocsBytes: %v", err) + } + + j := 0 + startBytesRemaining := i.locReader.Len() // # bytes remaining in the locReader + for startBytesRemaining-i.locReader.Len() < int(numLocsBytes) { + err := i.readLocation(&i.nextLocs[j]) + if err != nil { + return nil, err + } + rv.locs = append(rv.locs, &i.nextLocs[j]) + j++ + } + } + + return rv, nil +} + +var freqHasLocs1Hit = encodeFreqHasLocs(1, false) + +// nextBytes returns the docNum and the encoded freq & loc bytes for +// the next posting +func (i *PostingsIterator) nextBytes() ( + docNumOut uint64, freq uint64, normBits uint64, + bytesFreqNorm []byte, bytesLoc []byte, err error) { + docNum, exists, err := i.nextDocNumAtOrAfter(0) + if err != nil || !exists { + return 0, 0, 0, nil, nil, err + } + + if i.normBits1Hit != 0 { + if i.buf == nil { + i.buf = make([]byte, binary.MaxVarintLen64*2) + } + n := binary.PutUvarint(i.buf, freqHasLocs1Hit) + n += binary.PutUvarint(i.buf[n:], i.normBits1Hit) + return docNum, uint64(1), i.normBits1Hit, i.buf[:n], nil, nil + } + + startFreqNorm := len(i.currChunkFreqNorm) - i.freqNormReader.Len() + + var hasLocs bool + + freq, normBits, hasLocs, err = i.readFreqNormHasLocs() + if err != nil { + return 0, 0, 0, nil, nil, err + } + + endFreqNorm := len(i.currChunkFreqNorm) - i.freqNormReader.Len() + bytesFreqNorm = i.currChunkFreqNorm[startFreqNorm:endFreqNorm] + + if hasLocs { + startLoc := len(i.currChunkLoc) - i.locReader.Len() + + numLocsBytes, err := i.locReader.ReadUvarint() + if err != nil { + return 0, 0, 0, nil, nil, + fmt.Errorf("error reading location nextBytes numLocs: %v", err) + } + + // skip over all the location bytes + i.locReader.SkipBytes(int(numLocsBytes)) + + endLoc := len(i.currChunkLoc) - i.locReader.Len() + bytesLoc = i.currChunkLoc[startLoc:endLoc] + } + + return docNum, freq, normBits, bytesFreqNorm, bytesLoc, nil +} + +// nextDocNum returns the next docNum on the postings list, and also +// sets up the currChunk / loc related fields of the iterator. +func (i *PostingsIterator) nextDocNumAtOrAfter(atOrAfter uint64) (uint64, bool, error) { + if i.normBits1Hit != 0 { + if i.docNum1Hit == DocNum1HitFinished { + return 0, false, nil + } + if i.docNum1Hit < atOrAfter { + // advanced past our 1-hit + i.docNum1Hit = DocNum1HitFinished // consume our 1-hit docNum + return 0, false, nil + } + docNum := i.docNum1Hit + i.docNum1Hit = DocNum1HitFinished // consume our 1-hit docNum + return docNum, true, nil + } + + if i.Actual == nil || !i.Actual.HasNext() { + return 0, false, nil + } + + if i.postings == nil || i.postings == emptyPostingsList { + // couldn't find anything + return 0, false, nil + } + + if i.postings.postings == i.ActualBM { + return i.nextDocNumAtOrAfterClean(atOrAfter) + } + + i.Actual.AdvanceIfNeeded(uint32(atOrAfter)) + + if !i.Actual.HasNext() || !i.all.HasNext() { + // couldn't find anything + return 0, false, nil + } + + n := i.Actual.Next() + allN := i.all.Next() + + nChunk := n / i.postings.sb.chunkFactor + + // when allN becomes >= to here, then allN is in the same chunk as nChunk. + allNReachesNChunk := nChunk * i.postings.sb.chunkFactor + + // n is the next actual hit (excluding some postings), and + // allN is the next hit in the full postings, and + // if they don't match, move 'all' forwards until they do + for allN != n { + // we've reached same chunk, so move the freq/norm/loc decoders forward + if i.includeFreqNorm && allN >= allNReachesNChunk { + err := i.currChunkNext(nChunk) + if err != nil { + return 0, false, err + } + } + + if !i.all.HasNext() { + return 0, false, nil + } + + allN = i.all.Next() + } + + if i.includeFreqNorm && (i.currChunk != nChunk || i.currChunkFreqNorm == nil) { + err := i.loadChunk(int(nChunk)) + if err != nil { + return 0, false, fmt.Errorf("error loading chunk: %v", err) + } + } + + return uint64(n), true, nil +} + +// optimization when the postings list is "clean" (e.g., no updates & +// no deletions) where the all bitmap is the same as the actual bitmap +func (i *PostingsIterator) nextDocNumAtOrAfterClean( + atOrAfter uint64) (uint64, bool, error) { + if !i.includeFreqNorm { + i.Actual.AdvanceIfNeeded(uint32(atOrAfter)) + + if !i.Actual.HasNext() { + return 0, false, nil // couldn't find anything + } + + return uint64(i.Actual.Next()), true, nil + } + + // freq-norm's needed, so maintain freq-norm chunk reader + sameChunkNexts := 0 // # of times we called Next() in the same chunk + n := i.Actual.Next() + nChunk := n / i.postings.sb.chunkFactor + + for uint64(n) < atOrAfter && i.Actual.HasNext() { + n = i.Actual.Next() + + nChunkPrev := nChunk + nChunk = n / i.postings.sb.chunkFactor + + if nChunk != nChunkPrev { + sameChunkNexts = 0 + } else { + sameChunkNexts += 1 + } + } + + if uint64(n) < atOrAfter { + // couldn't find anything + return 0, false, nil + } + + for j := 0; j < sameChunkNexts; j++ { + err := i.currChunkNext(nChunk) + if err != nil { + return 0, false, fmt.Errorf("error optimized currChunkNext: %v", err) + } + } + + if i.currChunk != nChunk || i.currChunkFreqNorm == nil { + err := i.loadChunk(int(nChunk)) + if err != nil { + return 0, false, fmt.Errorf("error loading chunk: %v", err) + } + } + + return uint64(n), true, nil +} + +func (i *PostingsIterator) currChunkNext(nChunk uint32) error { + if i.currChunk != nChunk || i.currChunkFreqNorm == nil { + err := i.loadChunk(int(nChunk)) + if err != nil { + return fmt.Errorf("error loading chunk: %v", err) + } + } + + // read off freq/offsets even though we don't care about them + hasLocs, err := i.skipFreqNormReadHasLocs() + if err != nil { + return err + } + + if i.includeLocs && hasLocs { + numLocsBytes, err := i.locReader.ReadUvarint() + if err != nil { + return fmt.Errorf("error reading location numLocsBytes: %v", err) + } + + // skip over all the location bytes + i.locReader.SkipBytes(int(numLocsBytes)) + } + + return nil +} + +// DocNum1Hit returns the docNum and true if this is "1-hit" optimized +// and the docNum is available. +func (p *PostingsIterator) DocNum1Hit() (uint64, bool) { + if p.normBits1Hit != 0 && p.docNum1Hit != DocNum1HitFinished { + return p.docNum1Hit, true + } + return 0, false +} + +// ActualBitmap returns the underlying actual bitmap +// which can be used up the stack for optimizations +func (p *PostingsIterator) ActualBitmap() *roaring.Bitmap { + return p.ActualBM +} + +// ReplaceActual replaces the ActualBM with the provided +// bitmap +func (p *PostingsIterator) ReplaceActual(abm *roaring.Bitmap) { + p.ActualBM = abm + p.Actual = abm.Iterator() +} + +// PostingsIteratorFromBitmap constructs a PostingsIterator given an +// "actual" bitmap. +func PostingsIteratorFromBitmap(bm *roaring.Bitmap, + includeFreqNorm, includeLocs bool) (segment.PostingsIterator, error) { + return &PostingsIterator{ + ActualBM: bm, + Actual: bm.Iterator(), + includeFreqNorm: includeFreqNorm, + includeLocs: includeLocs, + }, nil +} + +// PostingsIteratorFrom1Hit constructs a PostingsIterator given a +// 1-hit docNum. +func PostingsIteratorFrom1Hit(docNum1Hit uint64, + includeFreqNorm, includeLocs bool) (segment.PostingsIterator, error) { + return &PostingsIterator{ + docNum1Hit: docNum1Hit, + normBits1Hit: NormBits1Hit, + includeFreqNorm: includeFreqNorm, + includeLocs: includeLocs, + }, nil +} + +// Posting is a single entry in a postings list +type Posting struct { + docNum uint64 + freq uint64 + norm float32 + locs []segment.Location +} + +func (p *Posting) Size() int { + sizeInBytes := reflectStaticSizePosting + + for _, entry := range p.locs { + sizeInBytes += entry.Size() + } + + return sizeInBytes +} + +// Number returns the document number of this posting in this segment +func (p *Posting) Number() uint64 { + return p.docNum +} + +// Frequency returns the frequencies of occurrence of this term in this doc/field +func (p *Posting) Frequency() uint64 { + return p.freq +} + +// Norm returns the normalization factor for this posting +func (p *Posting) Norm() float64 { + return float64(p.norm) +} + +// Locations returns the location information for each occurrence +func (p *Posting) Locations() []segment.Location { + return p.locs +} + +// Location represents the location of a single occurrence +type Location struct { + field string + pos uint64 + start uint64 + end uint64 + ap []uint64 +} + +func (l *Location) Size() int { + return reflectStaticSizeLocation + + len(l.field) + + len(l.ap)*SizeOfUint64 +} + +// Field returns the name of the field (useful in composite fields to know +// which original field the value came from) +func (l *Location) Field() string { + return l.field +} + +// Start returns the start byte offset of this occurrence +func (l *Location) Start() uint64 { + return l.start +} + +// End returns the end byte offset of this occurrence +func (l *Location) End() uint64 { + return l.end +} + +// Pos returns the 1-based phrase position of this occurrence +func (l *Location) Pos() uint64 { + return l.pos +} + +// ArrayPositions returns the array position vector associated with this occurrence +func (l *Location) ArrayPositions() []uint64 { + return l.ap +} diff --git a/vendor/github.com/blevesearch/zap/v11/read.go b/vendor/github.com/blevesearch/zapx/v11/read.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v11/read.go rename to vendor/github.com/blevesearch/zapx/v11/read.go diff --git a/vendor/github.com/blevesearch/zapx/v11/segment.go b/vendor/github.com/blevesearch/zapx/v11/segment.go new file mode 100644 index 00000000..0b0b192f --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v11/segment.go @@ -0,0 +1,600 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "os" + "sync" + "unsafe" + + "github.com/RoaringBitmap/roaring" + mmap "github.com/blevesearch/mmap-go" + segment "github.com/blevesearch/scorch_segment_api/v2" + "github.com/blevesearch/vellum" + "github.com/golang/snappy" +) + +var reflectStaticSizeSegmentBase int + +func init() { + var sb SegmentBase + reflectStaticSizeSegmentBase = int(unsafe.Sizeof(sb)) +} + +// Open returns a zap impl of a segment +func (*ZapPlugin) Open(path string) (segment.Segment, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + mm, err := mmap.Map(f, mmap.RDONLY, 0) + if err != nil { + // mmap failed, try to close the file + _ = f.Close() + return nil, err + } + + rv := &Segment{ + SegmentBase: SegmentBase{ + mem: mm[0 : len(mm)-FooterSize], + fieldsMap: make(map[string]uint16), + fieldDvReaders: make(map[uint16]*docValueReader), + fieldFSTs: make(map[uint16]*vellum.FST), + }, + f: f, + mm: mm, + path: path, + refs: 1, + } + rv.SegmentBase.updateSize() + + err = rv.loadConfig() + if err != nil { + _ = rv.Close() + return nil, err + } + + err = rv.loadFields() + if err != nil { + _ = rv.Close() + return nil, err + } + + err = rv.loadDvReaders() + if err != nil { + _ = rv.Close() + return nil, err + } + + return rv, nil +} + +// SegmentBase is a memory only, read-only implementation of the +// segment.Segment interface, using zap's data representation. +type SegmentBase struct { + mem []byte + memCRC uint32 + chunkFactor uint32 + fieldsMap map[string]uint16 // fieldName -> fieldID+1 + fieldsInv []string // fieldID -> fieldName + numDocs uint64 + storedIndexOffset uint64 + fieldsIndexOffset uint64 + docValueOffset uint64 + dictLocs []uint64 + fieldDvReaders map[uint16]*docValueReader // naive chunk cache per field + fieldDvNames []string // field names cached in fieldDvReaders + size uint64 + + m sync.Mutex + fieldFSTs map[uint16]*vellum.FST +} + +func (sb *SegmentBase) Size() int { + return int(sb.size) +} + +func (sb *SegmentBase) updateSize() { + sizeInBytes := reflectStaticSizeSegmentBase + + cap(sb.mem) + + // fieldsMap + for k := range sb.fieldsMap { + sizeInBytes += (len(k) + SizeOfString) + SizeOfUint16 + } + + // fieldsInv, dictLocs + for _, entry := range sb.fieldsInv { + sizeInBytes += len(entry) + SizeOfString + } + sizeInBytes += len(sb.dictLocs) * SizeOfUint64 + + // fieldDvReaders + for _, v := range sb.fieldDvReaders { + sizeInBytes += SizeOfUint16 + SizeOfPtr + if v != nil { + sizeInBytes += v.size() + } + } + + sb.size = uint64(sizeInBytes) +} + +func (sb *SegmentBase) AddRef() {} +func (sb *SegmentBase) DecRef() (err error) { return nil } +func (sb *SegmentBase) Close() (err error) { return nil } + +// Segment implements a persisted segment.Segment interface, by +// embedding an mmap()'ed SegmentBase. +type Segment struct { + SegmentBase + + f *os.File + mm mmap.MMap + path string + version uint32 + crc uint32 + + m sync.Mutex // Protects the fields that follow. + refs int64 +} + +func (s *Segment) Size() int { + // 8 /* size of file pointer */ + // 4 /* size of version -> uint32 */ + // 4 /* size of crc -> uint32 */ + sizeOfUints := 16 + + sizeInBytes := (len(s.path) + SizeOfString) + sizeOfUints + + // mutex, refs -> int64 + sizeInBytes += 16 + + // do not include the mmap'ed part + return sizeInBytes + s.SegmentBase.Size() - cap(s.mem) +} + +func (s *Segment) AddRef() { + s.m.Lock() + s.refs++ + s.m.Unlock() +} + +func (s *Segment) DecRef() (err error) { + s.m.Lock() + s.refs-- + if s.refs == 0 { + err = s.closeActual() + } + s.m.Unlock() + return err +} + +func (s *Segment) loadConfig() error { + crcOffset := len(s.mm) - 4 + s.crc = binary.BigEndian.Uint32(s.mm[crcOffset : crcOffset+4]) + + verOffset := crcOffset - 4 + s.version = binary.BigEndian.Uint32(s.mm[verOffset : verOffset+4]) + if s.version != Version { + return fmt.Errorf("unsupported version %d", s.version) + } + + chunkOffset := verOffset - 4 + s.chunkFactor = binary.BigEndian.Uint32(s.mm[chunkOffset : chunkOffset+4]) + + docValueOffset := chunkOffset - 8 + s.docValueOffset = binary.BigEndian.Uint64(s.mm[docValueOffset : docValueOffset+8]) + + fieldsIndexOffset := docValueOffset - 8 + s.fieldsIndexOffset = binary.BigEndian.Uint64(s.mm[fieldsIndexOffset : fieldsIndexOffset+8]) + + storedIndexOffset := fieldsIndexOffset - 8 + s.storedIndexOffset = binary.BigEndian.Uint64(s.mm[storedIndexOffset : storedIndexOffset+8]) + + numDocsOffset := storedIndexOffset - 8 + s.numDocs = binary.BigEndian.Uint64(s.mm[numDocsOffset : numDocsOffset+8]) + return nil +} + +// Following methods are dummy implementations for the interface +// DiskStatsReporter (for backward compatibility). +// The working implementations are supported only in zap v15.x +// and not in the earlier versions of zap. +func (s *Segment) ResetBytesRead(uint64) {} + +func (s *Segment) BytesRead() uint64 { + return 0 +} + +func (s *Segment) BytesWritten() uint64 { + return 0 +} + +func (s *Segment) incrementBytesRead(uint64) {} + +func (s *SegmentBase) BytesWritten() uint64 { + return 0 +} + +func (s *SegmentBase) setBytesWritten(uint64) {} + +func (s *SegmentBase) BytesRead() uint64 { + return 0 +} + +func (s *SegmentBase) ResetBytesRead(uint64) {} + +func (s *SegmentBase) incrementBytesRead(uint64) {} + +func (s *SegmentBase) loadFields() error { + // NOTE for now we assume the fields index immediately precedes + // the footer, and if this changes, need to adjust accordingly (or + // store explicit length), where s.mem was sliced from s.mm in Open(). + fieldsIndexEnd := uint64(len(s.mem)) + + // iterate through fields index + var fieldID uint64 + for s.fieldsIndexOffset+(8*fieldID) < fieldsIndexEnd { + addr := binary.BigEndian.Uint64(s.mem[s.fieldsIndexOffset+(8*fieldID) : s.fieldsIndexOffset+(8*fieldID)+8]) + + dictLoc, read := binary.Uvarint(s.mem[addr:fieldsIndexEnd]) + n := uint64(read) + s.dictLocs = append(s.dictLocs, dictLoc) + + var nameLen uint64 + nameLen, read = binary.Uvarint(s.mem[addr+n : fieldsIndexEnd]) + n += uint64(read) + + name := string(s.mem[addr+n : addr+n+nameLen]) + s.fieldsInv = append(s.fieldsInv, name) + s.fieldsMap[name] = uint16(fieldID + 1) + + fieldID++ + } + return nil +} + +// Dictionary returns the term dictionary for the specified field +func (s *SegmentBase) Dictionary(field string) (segment.TermDictionary, error) { + dict, err := s.dictionary(field) + if err == nil && dict == nil { + return emptyDictionary, nil + } + return dict, err +} + +func (sb *SegmentBase) dictionary(field string) (rv *Dictionary, err error) { + fieldIDPlus1 := sb.fieldsMap[field] + if fieldIDPlus1 > 0 { + rv = &Dictionary{ + sb: sb, + field: field, + fieldID: fieldIDPlus1 - 1, + } + + dictStart := sb.dictLocs[rv.fieldID] + if dictStart > 0 { + var ok bool + sb.m.Lock() + if rv.fst, ok = sb.fieldFSTs[rv.fieldID]; !ok { + // read the length of the vellum data + vellumLen, read := binary.Uvarint(sb.mem[dictStart : dictStart+binary.MaxVarintLen64]) + fstBytes := sb.mem[dictStart+uint64(read) : dictStart+uint64(read)+vellumLen] + rv.fst, err = vellum.Load(fstBytes) + if err != nil { + sb.m.Unlock() + return nil, fmt.Errorf("dictionary field %s vellum err: %v", field, err) + } + + sb.fieldFSTs[rv.fieldID] = rv.fst + } + + sb.m.Unlock() + rv.fstReader, err = rv.fst.Reader() + if err != nil { + return nil, fmt.Errorf("dictionary field %s vellum reader err: %v", field, err) + } + } + } + + return rv, nil +} + +// visitDocumentCtx holds data structures that are reusable across +// multiple VisitDocument() calls to avoid memory allocations +type visitDocumentCtx struct { + buf []byte + reader bytes.Reader + arrayPos []uint64 +} + +var visitDocumentCtxPool = sync.Pool{ + New: func() interface{} { + reuse := &visitDocumentCtx{} + return reuse + }, +} + +// VisitStoredFields invokes the StoredFieldValueVisitor for each stored field +// for the specified doc number +func (s *SegmentBase) VisitStoredFields(num uint64, visitor segment.StoredFieldValueVisitor) error { + vdc := visitDocumentCtxPool.Get().(*visitDocumentCtx) + defer visitDocumentCtxPool.Put(vdc) + return s.visitStoredFields(vdc, num, visitor) +} + +func (s *SegmentBase) visitStoredFields(vdc *visitDocumentCtx, num uint64, + visitor segment.StoredFieldValueVisitor) error { + // first make sure this is a valid number in this segment + if num < s.numDocs { + meta, compressed := s.getDocStoredMetaAndCompressed(num) + + vdc.reader.Reset(meta) + + // handle _id field special case + idFieldValLen, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return err + } + idFieldVal := compressed[:idFieldValLen] + + keepGoing := visitor("_id", byte('t'), idFieldVal, nil) + if !keepGoing { + visitDocumentCtxPool.Put(vdc) + return nil + } + + // handle non-"_id" fields + compressed = compressed[idFieldValLen:] + + uncompressed, err := snappy.Decode(vdc.buf[:cap(vdc.buf)], compressed) + if err != nil { + return err + } + + for keepGoing { + field, err := binary.ReadUvarint(&vdc.reader) + if err == io.EOF { + break + } + if err != nil { + return err + } + typ, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return err + } + offset, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return err + } + l, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return err + } + numap, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return err + } + var arrayPos []uint64 + if numap > 0 { + if cap(vdc.arrayPos) < int(numap) { + vdc.arrayPos = make([]uint64, numap) + } + arrayPos = vdc.arrayPos[:numap] + for i := 0; i < int(numap); i++ { + ap, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return err + } + arrayPos[i] = ap + } + } + + value := uncompressed[offset : offset+l] + keepGoing = visitor(s.fieldsInv[field], byte(typ), value, arrayPos) + } + + vdc.buf = uncompressed + } + return nil +} + +// DocID returns the value of the _id field for the given docNum +func (s *SegmentBase) DocID(num uint64) ([]byte, error) { + if num >= s.numDocs { + return nil, nil + } + + vdc := visitDocumentCtxPool.Get().(*visitDocumentCtx) + + meta, compressed := s.getDocStoredMetaAndCompressed(num) + + vdc.reader.Reset(meta) + + // handle _id field special case + idFieldValLen, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return nil, err + } + idFieldVal := compressed[:idFieldValLen] + + visitDocumentCtxPool.Put(vdc) + + return idFieldVal, nil +} + +// Count returns the number of documents in this segment. +func (s *SegmentBase) Count() uint64 { + return s.numDocs +} + +// DocNumbers returns a bitset corresponding to the doc numbers of all the +// provided _id strings +func (s *SegmentBase) DocNumbers(ids []string) (*roaring.Bitmap, error) { + rv := roaring.New() + + if len(s.fieldsMap) > 0 { + idDict, err := s.dictionary("_id") + if err != nil { + return nil, err + } + + postingsList := emptyPostingsList + + sMax, err := idDict.fst.GetMaxKey() + if err != nil { + return nil, err + } + sMaxStr := string(sMax) + filteredIds := make([]string, 0, len(ids)) + for _, id := range ids { + if id <= sMaxStr { + filteredIds = append(filteredIds, id) + } + } + + for _, id := range filteredIds { + postingsList, err = idDict.postingsList([]byte(id), nil, postingsList) + if err != nil { + return nil, err + } + postingsList.OrInto(rv) + } + } + + return rv, nil +} + +// Fields returns the field names used in this segment +func (s *SegmentBase) Fields() []string { + return s.fieldsInv +} + +// Path returns the path of this segment on disk +func (s *Segment) Path() string { + return s.path +} + +// Close releases all resources associated with this segment +func (s *Segment) Close() (err error) { + return s.DecRef() +} + +func (s *Segment) closeActual() (err error) { + if s.mm != nil { + err = s.mm.Unmap() + } + // try to close file even if unmap failed + if s.f != nil { + err2 := s.f.Close() + if err == nil { + // try to return first error + err = err2 + } + } + return +} + +// some helpers i started adding for the command-line utility + +// Data returns the underlying mmaped data slice +func (s *Segment) Data() []byte { + return s.mm +} + +// CRC returns the CRC value stored in the file footer +func (s *Segment) CRC() uint32 { + return s.crc +} + +// Version returns the file version in the file footer +func (s *Segment) Version() uint32 { + return s.version +} + +// ChunkFactor returns the chunk factor in the file footer +func (s *Segment) ChunkFactor() uint32 { + return s.chunkFactor +} + +// FieldsIndexOffset returns the fields index offset in the file footer +func (s *Segment) FieldsIndexOffset() uint64 { + return s.fieldsIndexOffset +} + +// StoredIndexOffset returns the stored value index offset in the file footer +func (s *Segment) StoredIndexOffset() uint64 { + return s.storedIndexOffset +} + +// DocValueOffset returns the docValue offset in the file footer +func (s *Segment) DocValueOffset() uint64 { + return s.docValueOffset +} + +// NumDocs returns the number of documents in the file footer +func (s *Segment) NumDocs() uint64 { + return s.numDocs +} + +// DictAddr is a helper function to compute the file offset where the +// dictionary is stored for the specified field. +func (s *Segment) DictAddr(field string) (uint64, error) { + fieldIDPlus1, ok := s.fieldsMap[field] + if !ok { + return 0, fmt.Errorf("no such field '%s'", field) + } + + return s.dictLocs[fieldIDPlus1-1], nil +} + +func (s *SegmentBase) loadDvReaders() error { + if s.docValueOffset == fieldNotUninverted || s.numDocs == 0 { + return nil + } + + var read uint64 + for fieldID, field := range s.fieldsInv { + var fieldLocStart, fieldLocEnd uint64 + var n int + fieldLocStart, n = binary.Uvarint(s.mem[s.docValueOffset+read : s.docValueOffset+read+binary.MaxVarintLen64]) + if n <= 0 { + return fmt.Errorf("loadDvReaders: failed to read the docvalue offset start for field %d", fieldID) + } + read += uint64(n) + fieldLocEnd, n = binary.Uvarint(s.mem[s.docValueOffset+read : s.docValueOffset+read+binary.MaxVarintLen64]) + if n <= 0 { + return fmt.Errorf("loadDvReaders: failed to read the docvalue offset end for field %d", fieldID) + } + read += uint64(n) + + fieldDvReader, err := s.loadFieldDocValueReader(field, fieldLocStart, fieldLocEnd) + if err != nil { + return err + } + if fieldDvReader != nil { + s.fieldDvReaders[uint16(fieldID)] = fieldDvReader + s.fieldDvNames = append(s.fieldDvNames, field) + } + } + + return nil +} diff --git a/vendor/github.com/blevesearch/zapx/v11/sizes.go b/vendor/github.com/blevesearch/zapx/v11/sizes.go new file mode 100644 index 00000000..34166ea3 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v11/sizes.go @@ -0,0 +1,59 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "reflect" +) + +func init() { + var b bool + SizeOfBool = int(reflect.TypeOf(b).Size()) + var f32 float32 + SizeOfFloat32 = int(reflect.TypeOf(f32).Size()) + var f64 float64 + SizeOfFloat64 = int(reflect.TypeOf(f64).Size()) + var i int + SizeOfInt = int(reflect.TypeOf(i).Size()) + var m map[int]int + SizeOfMap = int(reflect.TypeOf(m).Size()) + var ptr *int + SizeOfPtr = int(reflect.TypeOf(ptr).Size()) + var slice []int + SizeOfSlice = int(reflect.TypeOf(slice).Size()) + var str string + SizeOfString = int(reflect.TypeOf(str).Size()) + var u8 uint8 + SizeOfUint8 = int(reflect.TypeOf(u8).Size()) + var u16 uint16 + SizeOfUint16 = int(reflect.TypeOf(u16).Size()) + var u32 uint32 + SizeOfUint32 = int(reflect.TypeOf(u32).Size()) + var u64 uint64 + SizeOfUint64 = int(reflect.TypeOf(u64).Size()) +} + +var SizeOfBool int +var SizeOfFloat32 int +var SizeOfFloat64 int +var SizeOfInt int +var SizeOfMap int +var SizeOfPtr int +var SizeOfSlice int +var SizeOfString int +var SizeOfUint8 int +var SizeOfUint16 int +var SizeOfUint32 int +var SizeOfUint64 int diff --git a/vendor/github.com/blevesearch/zap/v11/write.go b/vendor/github.com/blevesearch/zapx/v11/write.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v11/write.go rename to vendor/github.com/blevesearch/zapx/v11/write.go diff --git a/vendor/github.com/blevesearch/zap/v11/zap.md b/vendor/github.com/blevesearch/zapx/v11/zap.md similarity index 100% rename from vendor/github.com/blevesearch/zap/v11/zap.md rename to vendor/github.com/blevesearch/zapx/v11/zap.md diff --git a/vendor/github.com/blevesearch/zap/v12/.gitignore b/vendor/github.com/blevesearch/zapx/v12/.gitignore similarity index 100% rename from vendor/github.com/blevesearch/zap/v12/.gitignore rename to vendor/github.com/blevesearch/zapx/v12/.gitignore diff --git a/vendor/github.com/blevesearch/zapx/v12/.golangci.yml b/vendor/github.com/blevesearch/zapx/v12/.golangci.yml new file mode 100644 index 00000000..1d55bfc0 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v12/.golangci.yml @@ -0,0 +1,28 @@ +linters: + # please, do not use `enable-all`: it's deprecated and will be removed soon. + # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint + disable-all: true + enable: + - bodyclose + - deadcode + - depguard + - dupl + - errcheck + - gofmt + - goimports + - goprintffuncname + - gosec + - gosimple + - govet + - ineffassign + - misspell + - nakedret + - nolintlint + - rowserrcheck + - staticcheck + - structcheck + - typecheck + - unused + - varcheck + - whitespace + diff --git a/vendor/github.com/couchbase/vellum/LICENSE b/vendor/github.com/blevesearch/zapx/v12/LICENSE similarity index 100% rename from vendor/github.com/couchbase/vellum/LICENSE rename to vendor/github.com/blevesearch/zapx/v12/LICENSE diff --git a/vendor/github.com/blevesearch/zapx/v12/README.md b/vendor/github.com/blevesearch/zapx/v12/README.md new file mode 100644 index 00000000..4cbf1a14 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v12/README.md @@ -0,0 +1,163 @@ +# zapx file format + +The zapx module is fork of [zap](https://github.com/blevesearch/zap) module which maintains file format compatibility, but removes dependency on bleve, and instead depends only on the indepenent interface modules: + +- [bleve_index_api](https://github.com/blevesearch/scorch_segment_api) +- [scorch_segment_api](https://github.com/blevesearch/scorch_segment_api) + +Advanced ZAP File Format Documentation is [here](zap.md). + +The file is written in the reverse order that we typically access data. This helps us write in one pass since later sections of the file require file offsets of things we've already written. + +Current usage: + +- mmap the entire file +- crc-32 bytes and version are in fixed position at end of the file +- reading remainder of footer could be version specific +- remainder of footer gives us: + - 3 important offsets (docValue , fields index and stored data index) + - 2 important values (number of docs and chunk factor) +- field data is processed once and memoized onto the heap so that we never have to go back to disk for it +- access to stored data by doc number means first navigating to the stored data index, then accessing a fixed position offset into that slice, which gives us the actual address of the data. the first bytes of that section tell us the size of data so that we know where it ends. +- access to all other indexed data follows the following pattern: + - first know the field name -> convert to id + - next navigate to term dictionary for that field + - some operations stop here and do dictionary ops + - next use dictionary to navigate to posting list for a specific term + - walk posting list + - if necessary, walk posting details as we go + - if location info is desired, consult location bitmap to see if it is there + +## stored fields section + +- for each document + - preparation phase: + - produce a slice of metadata bytes and data bytes + - produce these slices in field id order + - field value is appended to the data slice + - metadata slice is varint encoded with the following values for each field value + - field id (uint16) + - field type (byte) + - field value start offset in uncompressed data slice (uint64) + - field value length (uint64) + - field number of array positions (uint64) + - one additional value for each array position (uint64) + - compress the data slice using snappy + - file writing phase: + - remember the start offset for this document + - write out meta data length (varint uint64) + - write out compressed data length (varint uint64) + - write out the metadata bytes + - write out the compressed data bytes + +## stored fields idx + +- for each document + - write start offset (remembered from previous section) of stored data (big endian uint64) + +With this index and a known document number, we have direct access to all the stored field data. + +## posting details (freq/norm) section + +- for each posting list + - produce a slice containing multiple consecutive chunks (each chunk is varint stream) + - produce a slice remembering offsets of where each chunk starts + - preparation phase: + - for each hit in the posting list + - if this hit is in next chunk close out encoding of last chunk and record offset start of next + - encode term frequency (uint64) + - encode norm factor (float32) + - file writing phase: + - remember start position for this posting list details + - write out number of chunks that follow (varint uint64) + - write out length of each chunk (each a varint uint64) + - write out the byte slice containing all the chunk data + +If you know the doc number you're interested in, this format lets you jump to the correct chunk (docNum/chunkFactor) directly and then seek within that chunk until you find it. + +## posting details (location) section + +- for each posting list + - produce a slice containing multiple consecutive chunks (each chunk is varint stream) + - produce a slice remembering offsets of where each chunk starts + - preparation phase: + - for each hit in the posting list + - if this hit is in next chunk close out encoding of last chunk and record offset start of next + - encode field (uint16) + - encode field pos (uint64) + - encode field start (uint64) + - encode field end (uint64) + - encode number of array positions to follow (uint64) + - encode each array position (each uint64) + - file writing phase: + - remember start position for this posting list details + - write out number of chunks that follow (varint uint64) + - write out length of each chunk (each a varint uint64) + - write out the byte slice containing all the chunk data + +If you know the doc number you're interested in, this format lets you jump to the correct chunk (docNum/chunkFactor) directly and then seek within that chunk until you find it. + +## postings list section + +- for each posting list + - preparation phase: + - encode roaring bitmap posting list to bytes (so we know the length) + - file writing phase: + - remember the start position for this posting list + - write freq/norm details offset (remembered from previous, as varint uint64) + - write location details offset (remembered from previous, as varint uint64) + - write length of encoded roaring bitmap + - write the serialized roaring bitmap data + +## dictionary + +- for each field + - preparation phase: + - encode vellum FST with dictionary data pointing to file offset of posting list (remembered from previous) + - file writing phase: + - remember the start position of this persistDictionary + - write length of vellum data (varint uint64) + - write out vellum data + +## fields section + +- for each field + - file writing phase: + - remember start offset for each field + - write dictionary address (remembered from previous) (varint uint64) + - write length of field name (varint uint64) + - write field name bytes + +## fields idx + +- for each field + - file writing phase: + - write big endian uint64 of start offset for each field + +NOTE: currently we don't know or record the length of this fields index. Instead we rely on the fact that we know it immediately precedes a footer of known size. + +## fields DocValue + +- for each field + - preparation phase: + - produce a slice containing multiple consecutive chunks, where each chunk is composed of a meta section followed by compressed columnar field data + - produce a slice remembering the length of each chunk + - file writing phase: + - remember the start position of this first field DocValue offset in the footer + - write out number of chunks that follow (varint uint64) + - write out length of each chunk (each a varint uint64) + - write out the byte slice containing all the chunk data + +NOTE: currently the meta header inside each chunk gives clue to the location offsets and size of the data pertaining to a given docID and any +read operation leverage that meta information to extract the document specific data from the file. + +## footer + +- file writing phase + - write number of docs (big endian uint64) + - write stored field index location (big endian uint64) + - write field index location (big endian uint64) + - write field docValue location (big endian uint64) + - write out chunk factor (big endian uint32) + - write out version (big endian uint32) + - write out file CRC of everything preceding this (big endian uint32) diff --git a/vendor/github.com/blevesearch/zapx/v12/build.go b/vendor/github.com/blevesearch/zapx/v12/build.go new file mode 100644 index 00000000..de8265c1 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v12/build.go @@ -0,0 +1,186 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "bufio" + "fmt" + "io" + "math" + "os" + + "github.com/blevesearch/vellum" +) + +const Version uint32 = 12 + +const Type string = "zap" + +const fieldNotUninverted = math.MaxUint64 + +func (sb *SegmentBase) Persist(path string) error { + return PersistSegmentBase(sb, path) +} + +// WriteTo is an implementation of io.WriterTo interface. +func (sb *SegmentBase) WriteTo(w io.Writer) (int64, error) { + if w == nil { + return 0, fmt.Errorf("invalid writer found") + } + + n, err := persistSegmentBaseToWriter(sb, w) + return int64(n), err +} + +// PersistSegmentBase persists SegmentBase in the zap file format. +func PersistSegmentBase(sb *SegmentBase, path string) error { + flag := os.O_RDWR | os.O_CREATE + + f, err := os.OpenFile(path, flag, 0600) + if err != nil { + return err + } + + cleanup := func() { + _ = f.Close() + _ = os.Remove(path) + } + + _, err = persistSegmentBaseToWriter(sb, f) + if err != nil { + cleanup() + return err + } + + err = f.Sync() + if err != nil { + cleanup() + return err + } + + err = f.Close() + if err != nil { + cleanup() + return err + } + + return err +} + +type bufWriter struct { + w *bufio.Writer + n int +} + +func (br *bufWriter) Write(in []byte) (int, error) { + n, err := br.w.Write(in) + br.n += n + return n, err +} + +func persistSegmentBaseToWriter(sb *SegmentBase, w io.Writer) (int, error) { + br := &bufWriter{w: bufio.NewWriter(w)} + + _, err := br.Write(sb.mem) + if err != nil { + return 0, err + } + + err = persistFooter(sb.numDocs, sb.storedIndexOffset, sb.fieldsIndexOffset, + sb.docValueOffset, sb.chunkMode, sb.memCRC, br) + if err != nil { + return 0, err + } + + err = br.w.Flush() + if err != nil { + return 0, err + } + + return br.n, nil +} + +func persistStoredFieldValues(fieldID int, + storedFieldValues [][]byte, stf []byte, spf [][]uint64, + curr int, metaEncode varintEncoder, data []byte) ( + int, []byte, error) { + for i := 0; i < len(storedFieldValues); i++ { + // encode field + _, err := metaEncode(uint64(fieldID)) + if err != nil { + return 0, nil, err + } + // encode type + _, err = metaEncode(uint64(stf[i])) + if err != nil { + return 0, nil, err + } + // encode start offset + _, err = metaEncode(uint64(curr)) + if err != nil { + return 0, nil, err + } + // end len + _, err = metaEncode(uint64(len(storedFieldValues[i]))) + if err != nil { + return 0, nil, err + } + // encode number of array pos + _, err = metaEncode(uint64(len(spf[i]))) + if err != nil { + return 0, nil, err + } + // encode all array positions + for _, pos := range spf[i] { + _, err = metaEncode(pos) + if err != nil { + return 0, nil, err + } + } + + data = append(data, storedFieldValues[i]...) + curr += len(storedFieldValues[i]) + } + + return curr, data, nil +} + +func InitSegmentBase(mem []byte, memCRC uint32, chunkMode uint32, + fieldsMap map[string]uint16, fieldsInv []string, numDocs uint64, + storedIndexOffset uint64, fieldsIndexOffset uint64, docValueOffset uint64, + dictLocs []uint64) (*SegmentBase, error) { + sb := &SegmentBase{ + mem: mem, + memCRC: memCRC, + chunkMode: chunkMode, + fieldsMap: fieldsMap, + fieldsInv: fieldsInv, + numDocs: numDocs, + storedIndexOffset: storedIndexOffset, + fieldsIndexOffset: fieldsIndexOffset, + docValueOffset: docValueOffset, + dictLocs: dictLocs, + fieldDvReaders: make(map[uint16]*docValueReader), + fieldFSTs: make(map[uint16]*vellum.FST), + } + sb.updateSize() + + err := sb.loadDvReaders() + if err != nil { + return nil, err + } + + return sb, nil +} diff --git a/vendor/github.com/blevesearch/zap/v12/chunk.go b/vendor/github.com/blevesearch/zapx/v12/chunk.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v12/chunk.go rename to vendor/github.com/blevesearch/zapx/v12/chunk.go diff --git a/vendor/github.com/blevesearch/zap/v12/contentcoder.go b/vendor/github.com/blevesearch/zapx/v12/contentcoder.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v12/contentcoder.go rename to vendor/github.com/blevesearch/zapx/v12/contentcoder.go diff --git a/vendor/github.com/blevesearch/zapx/v12/count.go b/vendor/github.com/blevesearch/zapx/v12/count.go new file mode 100644 index 00000000..b6135359 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v12/count.go @@ -0,0 +1,61 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "hash/crc32" + "io" + + segment "github.com/blevesearch/scorch_segment_api/v2" +) + +// CountHashWriter is a wrapper around a Writer which counts the number of +// bytes which have been written and computes a crc32 hash +type CountHashWriter struct { + w io.Writer + crc uint32 + n int + s segment.StatsReporter +} + +// NewCountHashWriter returns a CountHashWriter which wraps the provided Writer +func NewCountHashWriter(w io.Writer) *CountHashWriter { + return &CountHashWriter{w: w} +} + +func NewCountHashWriterWithStatsReporter(w io.Writer, s segment.StatsReporter) *CountHashWriter { + return &CountHashWriter{w: w, s: s} +} + +// Write writes the provided bytes to the wrapped writer and counts the bytes +func (c *CountHashWriter) Write(b []byte) (int, error) { + n, err := c.w.Write(b) + c.crc = crc32.Update(c.crc, crc32.IEEETable, b[:n]) + c.n += n + if c.s != nil { + c.s.ReportBytesWritten(uint64(n)) + } + return n, err +} + +// Count returns the number of bytes written +func (c *CountHashWriter) Count() int { + return c.n +} + +// Sum32 returns the CRC-32 hash of the content written to this writer +func (c *CountHashWriter) Sum32() uint32 { + return c.crc +} diff --git a/vendor/github.com/blevesearch/zapx/v12/dict.go b/vendor/github.com/blevesearch/zapx/v12/dict.go new file mode 100644 index 00000000..e30bf242 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v12/dict.go @@ -0,0 +1,158 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "fmt" + + "github.com/RoaringBitmap/roaring" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" + "github.com/blevesearch/vellum" +) + +// Dictionary is the zap representation of the term dictionary +type Dictionary struct { + sb *SegmentBase + field string + fieldID uint16 + fst *vellum.FST + fstReader *vellum.Reader +} + +// represents an immutable, empty dictionary +var emptyDictionary = &Dictionary{} + +// PostingsList returns the postings list for the specified term +func (d *Dictionary) PostingsList(term []byte, except *roaring.Bitmap, + prealloc segment.PostingsList) (segment.PostingsList, error) { + var preallocPL *PostingsList + pl, ok := prealloc.(*PostingsList) + if ok && pl != nil { + preallocPL = pl + } + return d.postingsList(term, except, preallocPL) +} + +func (d *Dictionary) postingsList(term []byte, except *roaring.Bitmap, rv *PostingsList) (*PostingsList, error) { + if d.fstReader == nil { + if rv == nil || rv == emptyPostingsList { + return emptyPostingsList, nil + } + return d.postingsListInit(rv, except), nil + } + + postingsOffset, exists, err := d.fstReader.Get(term) + if err != nil { + return nil, fmt.Errorf("vellum err: %v", err) + } + if !exists { + if rv == nil || rv == emptyPostingsList { + return emptyPostingsList, nil + } + return d.postingsListInit(rv, except), nil + } + + return d.postingsListFromOffset(postingsOffset, except, rv) +} + +func (d *Dictionary) postingsListFromOffset(postingsOffset uint64, except *roaring.Bitmap, rv *PostingsList) (*PostingsList, error) { + rv = d.postingsListInit(rv, except) + + err := rv.read(postingsOffset, d) + if err != nil { + return nil, err + } + + return rv, nil +} + +func (d *Dictionary) postingsListInit(rv *PostingsList, except *roaring.Bitmap) *PostingsList { + if rv == nil || rv == emptyPostingsList { + rv = &PostingsList{} + } else { + postings := rv.postings + if postings != nil { + postings.Clear() + } + + *rv = PostingsList{} // clear the struct + + rv.postings = postings + } + rv.sb = d.sb + rv.except = except + return rv +} + +func (d *Dictionary) Contains(key []byte) (bool, error) { + if d.fst != nil { + return d.fst.Contains(key) + } + return false, nil +} + +// AutomatonIterator returns an iterator which only visits terms +// having the the vellum automaton and start/end key range +func (d *Dictionary) AutomatonIterator(a segment.Automaton, + startKeyInclusive, endKeyExclusive []byte) segment.DictionaryIterator { + if d.fst != nil { + rv := &DictionaryIterator{ + d: d, + } + + itr, err := d.fst.Search(a, startKeyInclusive, endKeyExclusive) + if err == nil { + rv.itr = itr + } else if err != vellum.ErrIteratorDone { + rv.err = err + } + + return rv + } + return emptyDictionaryIterator +} + +// DictionaryIterator is an iterator for term dictionary +type DictionaryIterator struct { + d *Dictionary + itr vellum.Iterator + err error + tmp PostingsList + entry index.DictEntry + omitCount bool +} + +var emptyDictionaryIterator = &DictionaryIterator{} + +// Next returns the next entry in the dictionary +func (i *DictionaryIterator) Next() (*index.DictEntry, error) { + if i.err != nil && i.err != vellum.ErrIteratorDone { + return nil, i.err + } else if i.itr == nil || i.err == vellum.ErrIteratorDone { + return nil, nil + } + term, postingsOffset := i.itr.Current() + i.entry.Term = string(term) + if !i.omitCount { + i.err = i.tmp.read(postingsOffset, i.d) + if i.err != nil { + return nil, i.err + } + i.entry.Count = i.tmp.Count() + } + i.err = i.itr.Next() + return &i.entry, nil +} diff --git a/vendor/github.com/blevesearch/zapx/v12/docvalues.go b/vendor/github.com/blevesearch/zapx/v12/docvalues.go new file mode 100644 index 00000000..a36485c8 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v12/docvalues.go @@ -0,0 +1,323 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "bytes" + "encoding/binary" + "fmt" + "math" + "reflect" + "sort" + + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" + "github.com/golang/snappy" +) + +var reflectStaticSizedocValueReader int + +func init() { + var dvi docValueReader + reflectStaticSizedocValueReader = int(reflect.TypeOf(dvi).Size()) +} + +type docNumTermsVisitor func(docNum uint64, terms []byte) error + +type docVisitState struct { + dvrs map[uint16]*docValueReader + segment *SegmentBase +} + +// No-op implementations for DiskStatsReporter interface. +// Supported only in v15 +func (d *docVisitState) BytesRead() uint64 { + return 0 +} + +func (d *docVisitState) BytesWritten() uint64 { + return 0 +} + +func (d *docVisitState) ResetBytesRead(val uint64) {} + +type docValueReader struct { + field string + curChunkNum uint64 + chunkOffsets []uint64 + dvDataLoc uint64 + curChunkHeader []MetaData + curChunkData []byte // compressed data cache + uncompressed []byte // temp buf for snappy decompression +} + +func (di *docValueReader) size() int { + return reflectStaticSizedocValueReader + SizeOfPtr + + len(di.field) + + len(di.chunkOffsets)*SizeOfUint64 + + len(di.curChunkHeader)*reflectStaticSizeMetaData + + len(di.curChunkData) +} + +func (di *docValueReader) cloneInto(rv *docValueReader) *docValueReader { + if rv == nil { + rv = &docValueReader{} + } + + rv.field = di.field + rv.curChunkNum = math.MaxUint64 + rv.chunkOffsets = di.chunkOffsets // immutable, so it's sharable + rv.dvDataLoc = di.dvDataLoc + rv.curChunkHeader = rv.curChunkHeader[:0] + rv.curChunkData = nil + rv.uncompressed = rv.uncompressed[:0] + + return rv +} + +func (di *docValueReader) curChunkNumber() uint64 { + return di.curChunkNum +} + +func (s *SegmentBase) loadFieldDocValueReader(field string, + fieldDvLocStart, fieldDvLocEnd uint64) (*docValueReader, error) { + // get the docValue offset for the given fields + if fieldDvLocStart == fieldNotUninverted { + // no docValues found, nothing to do + return nil, nil + } + + // read the number of chunks, and chunk offsets position + var numChunks, chunkOffsetsPosition uint64 + + if fieldDvLocEnd-fieldDvLocStart > 16 { + numChunks = binary.BigEndian.Uint64(s.mem[fieldDvLocEnd-8 : fieldDvLocEnd]) + // read the length of chunk offsets + chunkOffsetsLen := binary.BigEndian.Uint64(s.mem[fieldDvLocEnd-16 : fieldDvLocEnd-8]) + // acquire position of chunk offsets + chunkOffsetsPosition = (fieldDvLocEnd - 16) - chunkOffsetsLen + } else { + return nil, fmt.Errorf("loadFieldDocValueReader: fieldDvLoc too small: %d-%d", fieldDvLocEnd, fieldDvLocStart) + } + + fdvIter := &docValueReader{ + curChunkNum: math.MaxUint64, + field: field, + chunkOffsets: make([]uint64, int(numChunks)), + } + + // read the chunk offsets + var offset uint64 + for i := 0; i < int(numChunks); i++ { + loc, read := binary.Uvarint(s.mem[chunkOffsetsPosition+offset : chunkOffsetsPosition+offset+binary.MaxVarintLen64]) + if read <= 0 { + return nil, fmt.Errorf("corrupted chunk offset during segment load") + } + fdvIter.chunkOffsets[i] = loc + offset += uint64(read) + } + + // set the data offset + fdvIter.dvDataLoc = fieldDvLocStart + + return fdvIter, nil +} + +func (di *docValueReader) loadDvChunk(chunkNumber uint64, s *SegmentBase) error { + // advance to the chunk where the docValues + // reside for the given docNum + destChunkDataLoc, curChunkEnd := di.dvDataLoc, di.dvDataLoc + start, end := readChunkBoundary(int(chunkNumber), di.chunkOffsets) + if start >= end { + di.curChunkHeader = di.curChunkHeader[:0] + di.curChunkData = nil + di.curChunkNum = chunkNumber + di.uncompressed = di.uncompressed[:0] + return nil + } + + destChunkDataLoc += start + curChunkEnd += end + + // read the number of docs reside in the chunk + numDocs, read := binary.Uvarint(s.mem[destChunkDataLoc : destChunkDataLoc+binary.MaxVarintLen64]) + if read <= 0 { + return fmt.Errorf("failed to read the chunk") + } + chunkMetaLoc := destChunkDataLoc + uint64(read) + + offset := uint64(0) + if cap(di.curChunkHeader) < int(numDocs) { + di.curChunkHeader = make([]MetaData, int(numDocs)) + } else { + di.curChunkHeader = di.curChunkHeader[:int(numDocs)] + } + for i := 0; i < int(numDocs); i++ { + di.curChunkHeader[i].DocNum, read = binary.Uvarint(s.mem[chunkMetaLoc+offset : chunkMetaLoc+offset+binary.MaxVarintLen64]) + offset += uint64(read) + di.curChunkHeader[i].DocDvOffset, read = binary.Uvarint(s.mem[chunkMetaLoc+offset : chunkMetaLoc+offset+binary.MaxVarintLen64]) + offset += uint64(read) + } + + compressedDataLoc := chunkMetaLoc + offset + dataLength := curChunkEnd - compressedDataLoc + di.curChunkData = s.mem[compressedDataLoc : compressedDataLoc+dataLength] + di.curChunkNum = chunkNumber + di.uncompressed = di.uncompressed[:0] + return nil +} + +func (di *docValueReader) iterateAllDocValues(s *SegmentBase, visitor docNumTermsVisitor) error { + for i := 0; i < len(di.chunkOffsets); i++ { + err := di.loadDvChunk(uint64(i), s) + if err != nil { + return err + } + if di.curChunkData == nil || len(di.curChunkHeader) == 0 { + continue + } + + // uncompress the already loaded data + uncompressed, err := snappy.Decode(di.uncompressed[:cap(di.uncompressed)], di.curChunkData) + if err != nil { + return err + } + di.uncompressed = uncompressed + + start := uint64(0) + for _, entry := range di.curChunkHeader { + err = visitor(entry.DocNum, uncompressed[start:entry.DocDvOffset]) + if err != nil { + return err + } + + start = entry.DocDvOffset + } + } + + return nil +} + +func (di *docValueReader) visitDocValues(docNum uint64, + visitor index.DocValueVisitor) error { + // binary search the term locations for the docNum + start, end := di.getDocValueLocs(docNum) + if start == math.MaxUint64 || end == math.MaxUint64 || start == end { + return nil + } + + var uncompressed []byte + var err error + // use the uncompressed copy if available + if len(di.uncompressed) > 0 { + uncompressed = di.uncompressed + } else { + // uncompress the already loaded data + uncompressed, err = snappy.Decode(di.uncompressed[:cap(di.uncompressed)], di.curChunkData) + if err != nil { + return err + } + di.uncompressed = uncompressed + } + + // pick the terms for the given docNum + uncompressed = uncompressed[start:end] + for { + i := bytes.Index(uncompressed, termSeparatorSplitSlice) + if i < 0 { + break + } + + visitor(di.field, uncompressed[0:i]) + uncompressed = uncompressed[i+1:] + } + + return nil +} + +func (di *docValueReader) getDocValueLocs(docNum uint64) (uint64, uint64) { + i := sort.Search(len(di.curChunkHeader), func(i int) bool { + return di.curChunkHeader[i].DocNum >= docNum + }) + if i < len(di.curChunkHeader) && di.curChunkHeader[i].DocNum == docNum { + return ReadDocValueBoundary(i, di.curChunkHeader) + } + return math.MaxUint64, math.MaxUint64 +} + +// VisitDocValues is an implementation of the +// DocValueVisitable interface +func (s *SegmentBase) VisitDocValues(localDocNum uint64, fields []string, + visitor index.DocValueVisitor, dvsIn segment.DocVisitState) ( + segment.DocVisitState, error) { + dvs, ok := dvsIn.(*docVisitState) + if !ok || dvs == nil { + dvs = &docVisitState{} + } else { + if dvs.segment != s { + dvs.segment = s + dvs.dvrs = nil + } + } + + var fieldIDPlus1 uint16 + if dvs.dvrs == nil { + dvs.dvrs = make(map[uint16]*docValueReader, len(fields)) + for _, field := range fields { + if fieldIDPlus1, ok = s.fieldsMap[field]; !ok { + continue + } + fieldID := fieldIDPlus1 - 1 + if dvIter, exists := s.fieldDvReaders[fieldID]; exists && + dvIter != nil { + dvs.dvrs[fieldID] = dvIter.cloneInto(dvs.dvrs[fieldID]) + } + } + } + + // find the chunkNumber where the docValues are stored + // NOTE: doc values continue to use legacy chunk mode + chunkFactor, err := getChunkSize(LegacyChunkMode, 0, 0) + if err != nil { + return nil, err + } + docInChunk := localDocNum / chunkFactor + var dvr *docValueReader + for _, field := range fields { + if fieldIDPlus1, ok = s.fieldsMap[field]; !ok { + continue + } + fieldID := fieldIDPlus1 - 1 + if dvr, ok = dvs.dvrs[fieldID]; ok && dvr != nil { + // check if the chunk is already loaded + if docInChunk != dvr.curChunkNumber() { + err := dvr.loadDvChunk(docInChunk, s) + if err != nil { + return dvs, err + } + } + + _ = dvr.visitDocValues(localDocNum, visitor) + } + } + return dvs, nil +} + +// VisitableDocValueFields returns the list of fields with +// persisted doc value terms ready to be visitable using the +// VisitDocumentFieldTerms method. +func (s *SegmentBase) VisitableDocValueFields() ([]string, error) { + return s.fieldDvNames, nil +} diff --git a/vendor/github.com/blevesearch/zapx/v12/enumerator.go b/vendor/github.com/blevesearch/zapx/v12/enumerator.go new file mode 100644 index 00000000..972a2241 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v12/enumerator.go @@ -0,0 +1,138 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "bytes" + + "github.com/blevesearch/vellum" +) + +// enumerator provides an ordered traversal of multiple vellum +// iterators. Like JOIN of iterators, the enumerator produces a +// sequence of (key, iteratorIndex, value) tuples, sorted by key ASC, +// then iteratorIndex ASC, where the same key might be seen or +// repeated across multiple child iterators. +type enumerator struct { + itrs []vellum.Iterator + currKs [][]byte + currVs []uint64 + + lowK []byte + lowIdxs []int + lowCurr int +} + +// newEnumerator returns a new enumerator over the vellum Iterators +func newEnumerator(itrs []vellum.Iterator) (*enumerator, error) { + rv := &enumerator{ + itrs: itrs, + currKs: make([][]byte, len(itrs)), + currVs: make([]uint64, len(itrs)), + lowIdxs: make([]int, 0, len(itrs)), + } + for i, itr := range rv.itrs { + rv.currKs[i], rv.currVs[i] = itr.Current() + } + rv.updateMatches(false) + if rv.lowK == nil && len(rv.lowIdxs) == 0 { + return rv, vellum.ErrIteratorDone + } + return rv, nil +} + +// updateMatches maintains the low key matches based on the currKs +func (m *enumerator) updateMatches(skipEmptyKey bool) { + m.lowK = nil + m.lowIdxs = m.lowIdxs[:0] + m.lowCurr = 0 + + for i, key := range m.currKs { + if (key == nil && m.currVs[i] == 0) || // in case of empty iterator + (len(key) == 0 && skipEmptyKey) { // skip empty keys + continue + } + + cmp := bytes.Compare(key, m.lowK) + if cmp < 0 || len(m.lowIdxs) == 0 { + // reached a new low + m.lowK = key + m.lowIdxs = m.lowIdxs[:0] + m.lowIdxs = append(m.lowIdxs, i) + } else if cmp == 0 { + m.lowIdxs = append(m.lowIdxs, i) + } + } +} + +// Current returns the enumerator's current key, iterator-index, and +// value. If the enumerator is not pointing at a valid value (because +// Next returned an error previously), Current will return nil,0,0. +func (m *enumerator) Current() ([]byte, int, uint64) { + var i int + var v uint64 + if m.lowCurr < len(m.lowIdxs) { + i = m.lowIdxs[m.lowCurr] + v = m.currVs[i] + } + return m.lowK, i, v +} + +// GetLowIdxsAndValues will return all of the iterator indices +// which point to the current key, and their corresponding +// values. This can be used by advanced caller which may need +// to peek into these other sets of data before processing. +func (m *enumerator) GetLowIdxsAndValues() ([]int, []uint64) { + values := make([]uint64, 0, len(m.lowIdxs)) + for _, idx := range m.lowIdxs { + values = append(values, m.currVs[idx]) + } + return m.lowIdxs, values +} + +// Next advances the enumerator to the next key/iterator/value result, +// else vellum.ErrIteratorDone is returned. +func (m *enumerator) Next() error { + m.lowCurr += 1 + if m.lowCurr >= len(m.lowIdxs) { + // move all the current low iterators forwards + for _, vi := range m.lowIdxs { + err := m.itrs[vi].Next() + if err != nil && err != vellum.ErrIteratorDone { + return err + } + m.currKs[vi], m.currVs[vi] = m.itrs[vi].Current() + } + // can skip any empty keys encountered at this point + m.updateMatches(true) + } + if m.lowK == nil && len(m.lowIdxs) == 0 { + return vellum.ErrIteratorDone + } + return nil +} + +// Close all the underlying Iterators. The first error, if any, will +// be returned. +func (m *enumerator) Close() error { + var rv error + for _, itr := range m.itrs { + err := itr.Close() + if rv == nil { + rv = err + } + } + return rv +} diff --git a/vendor/github.com/blevesearch/zapx/v12/intDecoder.go b/vendor/github.com/blevesearch/zapx/v12/intDecoder.go new file mode 100644 index 00000000..e9680931 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v12/intDecoder.go @@ -0,0 +1,109 @@ +// Copyright (c) 2019 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "encoding/binary" + "fmt" +) + +type chunkedIntDecoder struct { + startOffset uint64 + dataStartOffset uint64 + chunkOffsets []uint64 + curChunkBytes []byte + data []byte + r *memUvarintReader +} + +func newChunkedIntDecoder(buf []byte, offset uint64) *chunkedIntDecoder { + rv := &chunkedIntDecoder{startOffset: offset, data: buf} + var n, numChunks uint64 + var read int + if offset == termNotEncoded { + numChunks = 0 + } else { + numChunks, read = binary.Uvarint(buf[offset+n : offset+n+binary.MaxVarintLen64]) + } + + n += uint64(read) + if cap(rv.chunkOffsets) >= int(numChunks) { + rv.chunkOffsets = rv.chunkOffsets[:int(numChunks)] + } else { + rv.chunkOffsets = make([]uint64, int(numChunks)) + } + for i := 0; i < int(numChunks); i++ { + rv.chunkOffsets[i], read = binary.Uvarint(buf[offset+n : offset+n+binary.MaxVarintLen64]) + n += uint64(read) + } + rv.dataStartOffset = offset + n + return rv +} + +func (d *chunkedIntDecoder) loadChunk(chunk int) error { + if d.startOffset == termNotEncoded { + d.r = newMemUvarintReader([]byte(nil)) + return nil + } + + if chunk >= len(d.chunkOffsets) { + return fmt.Errorf("tried to load freq chunk that doesn't exist %d/(%d)", + chunk, len(d.chunkOffsets)) + } + + end, start := d.dataStartOffset, d.dataStartOffset + s, e := readChunkBoundary(chunk, d.chunkOffsets) + start += s + end += e + d.curChunkBytes = d.data[start:end] + if d.r == nil { + d.r = newMemUvarintReader(d.curChunkBytes) + } else { + d.r.Reset(d.curChunkBytes) + } + + return nil +} + +func (d *chunkedIntDecoder) reset() { + d.startOffset = 0 + d.dataStartOffset = 0 + d.chunkOffsets = d.chunkOffsets[:0] + d.curChunkBytes = d.curChunkBytes[:0] + d.data = d.data[:0] + if d.r != nil { + d.r.Reset([]byte(nil)) + } +} + +func (d *chunkedIntDecoder) isNil() bool { + return d.curChunkBytes == nil +} + +func (d *chunkedIntDecoder) readUvarint() (uint64, error) { + return d.r.ReadUvarint() +} + +func (d *chunkedIntDecoder) SkipUvarint() { + d.r.SkipUvarint() +} + +func (d *chunkedIntDecoder) SkipBytes(count int) { + d.r.SkipBytes(count) +} + +func (d *chunkedIntDecoder) Len() int { + return d.r.Len() +} diff --git a/vendor/github.com/blevesearch/zap/v12/intcoder.go b/vendor/github.com/blevesearch/zapx/v12/intcoder.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v12/intcoder.go rename to vendor/github.com/blevesearch/zapx/v12/intcoder.go diff --git a/vendor/github.com/blevesearch/zapx/v12/memuvarint.go b/vendor/github.com/blevesearch/zapx/v12/memuvarint.go new file mode 100644 index 00000000..48a57f9c --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v12/memuvarint.go @@ -0,0 +1,103 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "fmt" +) + +type memUvarintReader struct { + C int // index of next byte to read from S + S []byte +} + +func newMemUvarintReader(s []byte) *memUvarintReader { + return &memUvarintReader{S: s} +} + +// Len returns the number of unread bytes. +func (r *memUvarintReader) Len() int { + n := len(r.S) - r.C + if n < 0 { + return 0 + } + return n +} + +// ReadUvarint reads an encoded uint64. The original code this was +// based on is at encoding/binary/ReadUvarint(). +func (r *memUvarintReader) ReadUvarint() (uint64, error) { + if r.C >= len(r.S) { + // nothing else to read + return 0, nil + } + + var x uint64 + var s uint + var C = r.C + var S = r.S + + for { + b := S[C] + C++ + + if b < 0x80 { + r.C = C + + // why 63? The original code had an 'i += 1' loop var and + // checked for i > 9 || i == 9 ...; but, we no longer + // check for the i var, but instead check here for s, + // which is incremented by 7. So, 7*9 == 63. + // + // why the "extra" >= check? The normal case is that s < + // 63, so we check this single >= guard first so that we + // hit the normal, nil-error return pathway sooner. + if s >= 63 && (s > 63 || b > 1) { + return 0, fmt.Errorf("memUvarintReader overflow") + } + + return x | uint64(b)<= len(r.S) { + return + } + + b := r.S[r.C] + r.C++ + + if b < 0x80 { + return + } + } +} + +// SkipBytes skips a count number of bytes. +func (r *memUvarintReader) SkipBytes(count int) { + r.C = r.C + count +} + +func (r *memUvarintReader) Reset(s []byte) { + r.C = 0 + r.S = s +} diff --git a/vendor/github.com/blevesearch/zapx/v12/merge.go b/vendor/github.com/blevesearch/zapx/v12/merge.go new file mode 100644 index 00000000..6a853a16 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v12/merge.go @@ -0,0 +1,843 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "bufio" + "bytes" + "encoding/binary" + "fmt" + "math" + "os" + "sort" + + "github.com/RoaringBitmap/roaring" + seg "github.com/blevesearch/scorch_segment_api/v2" + "github.com/blevesearch/vellum" + "github.com/golang/snappy" +) + +var DefaultFileMergerBufferSize = 1024 * 1024 + +const docDropped = math.MaxUint64 // sentinel docNum to represent a deleted doc + +// Merge takes a slice of segments and bit masks describing which +// documents may be dropped, and creates a new segment containing the +// remaining data. This new segment is built at the specified path. +func (*ZapPlugin) Merge(segments []seg.Segment, drops []*roaring.Bitmap, path string, + closeCh chan struct{}, s seg.StatsReporter) ( + [][]uint64, uint64, error) { + segmentBases := make([]*SegmentBase, len(segments)) + for segmenti, segment := range segments { + switch segmentx := segment.(type) { + case *Segment: + segmentBases[segmenti] = &segmentx.SegmentBase + case *SegmentBase: + segmentBases[segmenti] = segmentx + default: + panic(fmt.Sprintf("oops, unexpected segment type: %T", segment)) + } + } + return mergeSegmentBases(segmentBases, drops, path, DefaultChunkMode, closeCh, s) +} + +func mergeSegmentBases(segmentBases []*SegmentBase, drops []*roaring.Bitmap, path string, + chunkMode uint32, closeCh chan struct{}, s seg.StatsReporter) ( + [][]uint64, uint64, error) { + flag := os.O_RDWR | os.O_CREATE + + f, err := os.OpenFile(path, flag, 0600) + if err != nil { + return nil, 0, err + } + + cleanup := func() { + _ = f.Close() + _ = os.Remove(path) + } + + // buffer the output + br := bufio.NewWriterSize(f, DefaultFileMergerBufferSize) + + // wrap it for counting (tracking offsets) + cr := NewCountHashWriterWithStatsReporter(br, s) + + newDocNums, numDocs, storedIndexOffset, fieldsIndexOffset, docValueOffset, _, _, _, err := + MergeToWriter(segmentBases, drops, chunkMode, cr, closeCh) + if err != nil { + cleanup() + return nil, 0, err + } + + err = persistFooter(numDocs, storedIndexOffset, fieldsIndexOffset, + docValueOffset, chunkMode, cr.Sum32(), cr) + if err != nil { + cleanup() + return nil, 0, err + } + + err = br.Flush() + if err != nil { + cleanup() + return nil, 0, err + } + + err = f.Sync() + if err != nil { + cleanup() + return nil, 0, err + } + + err = f.Close() + if err != nil { + cleanup() + return nil, 0, err + } + + return newDocNums, uint64(cr.Count()), nil +} + +func MergeToWriter(segments []*SegmentBase, drops []*roaring.Bitmap, + chunkMode uint32, cr *CountHashWriter, closeCh chan struct{}) ( + newDocNums [][]uint64, + numDocs, storedIndexOffset, fieldsIndexOffset, docValueOffset uint64, + dictLocs []uint64, fieldsInv []string, fieldsMap map[string]uint16, + err error) { + docValueOffset = uint64(fieldNotUninverted) + + var fieldsSame bool + fieldsSame, fieldsInv = mergeFields(segments) + fieldsMap = mapFields(fieldsInv) + + numDocs = computeNewDocCount(segments, drops) + + if isClosed(closeCh) { + return nil, 0, 0, 0, 0, nil, nil, nil, seg.ErrClosed + } + + if numDocs > 0 { + storedIndexOffset, newDocNums, err = mergeStoredAndRemap(segments, drops, + fieldsMap, fieldsInv, fieldsSame, numDocs, cr, closeCh) + if err != nil { + return nil, 0, 0, 0, 0, nil, nil, nil, err + } + + dictLocs, docValueOffset, err = persistMergedRest(segments, drops, + fieldsInv, fieldsMap, fieldsSame, + newDocNums, numDocs, chunkMode, cr, closeCh) + if err != nil { + return nil, 0, 0, 0, 0, nil, nil, nil, err + } + } else { + dictLocs = make([]uint64, len(fieldsInv)) + } + + fieldsIndexOffset, err = persistFields(fieldsInv, cr, dictLocs) + if err != nil { + return nil, 0, 0, 0, 0, nil, nil, nil, err + } + + return newDocNums, numDocs, storedIndexOffset, fieldsIndexOffset, docValueOffset, dictLocs, fieldsInv, fieldsMap, nil +} + +// mapFields takes the fieldsInv list and returns a map of fieldName +// to fieldID+1 +func mapFields(fields []string) map[string]uint16 { + rv := make(map[string]uint16, len(fields)) + for i, fieldName := range fields { + rv[fieldName] = uint16(i) + 1 + } + return rv +} + +// computeNewDocCount determines how many documents will be in the newly +// merged segment when obsoleted docs are dropped +func computeNewDocCount(segments []*SegmentBase, drops []*roaring.Bitmap) uint64 { + var newDocCount uint64 + for segI, segment := range segments { + newDocCount += segment.numDocs + if drops[segI] != nil { + newDocCount -= drops[segI].GetCardinality() + } + } + return newDocCount +} + +func persistMergedRest(segments []*SegmentBase, dropsIn []*roaring.Bitmap, + fieldsInv []string, fieldsMap map[string]uint16, fieldsSame bool, + newDocNumsIn [][]uint64, newSegDocCount uint64, chunkMode uint32, + w *CountHashWriter, closeCh chan struct{}) ([]uint64, uint64, error) { + var bufMaxVarintLen64 []byte = make([]byte, binary.MaxVarintLen64) + var bufLoc []uint64 + + var postings *PostingsList + var postItr *PostingsIterator + + rv := make([]uint64, len(fieldsInv)) + fieldDvLocsStart := make([]uint64, len(fieldsInv)) + fieldDvLocsEnd := make([]uint64, len(fieldsInv)) + + // these int coders are initialized with chunk size 1024 + // however this will be reset to the correct chunk size + // while processing each individual field-term section + tfEncoder := newChunkedIntCoder(1024, newSegDocCount-1) + locEncoder := newChunkedIntCoder(1024, newSegDocCount-1) + + var vellumBuf bytes.Buffer + newVellum, err := vellum.New(&vellumBuf, nil) + if err != nil { + return nil, 0, err + } + + newRoaring := roaring.NewBitmap() + + // for each field + for fieldID, fieldName := range fieldsInv { + // collect FST iterators from all active segments for this field + var newDocNums [][]uint64 + var drops []*roaring.Bitmap + var dicts []*Dictionary + var itrs []vellum.Iterator + + var segmentsInFocus []*SegmentBase + + for segmentI, segment := range segments { + // check for the closure in meantime + if isClosed(closeCh) { + return nil, 0, seg.ErrClosed + } + + dict, err2 := segment.dictionary(fieldName) + if err2 != nil { + return nil, 0, err2 + } + if dict != nil && dict.fst != nil { + itr, err2 := dict.fst.Iterator(nil, nil) + if err2 != nil && err2 != vellum.ErrIteratorDone { + return nil, 0, err2 + } + if itr != nil { + newDocNums = append(newDocNums, newDocNumsIn[segmentI]) + if dropsIn[segmentI] != nil && !dropsIn[segmentI].IsEmpty() { + drops = append(drops, dropsIn[segmentI]) + } else { + drops = append(drops, nil) + } + dicts = append(dicts, dict) + itrs = append(itrs, itr) + segmentsInFocus = append(segmentsInFocus, segment) + } + } + } + + var prevTerm []byte + + newRoaring.Clear() + + var lastDocNum, lastFreq, lastNorm uint64 + + // determines whether to use "1-hit" encoding optimization + // when a term appears in only 1 doc, with no loc info, + // has freq of 1, and the docNum fits into 31-bits + use1HitEncoding := func(termCardinality uint64) (bool, uint64, uint64) { + if termCardinality == uint64(1) && locEncoder.FinalSize() <= 0 { + docNum := uint64(newRoaring.Minimum()) + if under32Bits(docNum) && docNum == lastDocNum && lastFreq == 1 { + return true, docNum, lastNorm + } + } + return false, 0, 0 + } + + finishTerm := func(term []byte) error { + tfEncoder.Close() + locEncoder.Close() + + postingsOffset, err := writePostings(newRoaring, + tfEncoder, locEncoder, use1HitEncoding, w, bufMaxVarintLen64) + if err != nil { + return err + } + + if postingsOffset > 0 { + err = newVellum.Insert(term, postingsOffset) + if err != nil { + return err + } + } + + newRoaring.Clear() + + tfEncoder.Reset() + locEncoder.Reset() + + lastDocNum = 0 + lastFreq = 0 + lastNorm = 0 + + return nil + } + + enumerator, err := newEnumerator(itrs) + + for err == nil { + term, itrI, postingsOffset := enumerator.Current() + + if !bytes.Equal(prevTerm, term) { + // check for the closure in meantime + if isClosed(closeCh) { + return nil, 0, seg.ErrClosed + } + + // if the term changed, write out the info collected + // for the previous term + err = finishTerm(prevTerm) + if err != nil { + return nil, 0, err + } + } + if !bytes.Equal(prevTerm, term) || prevTerm == nil { + // compute cardinality of field-term in new seg + var newCard uint64 + lowItrIdxs, lowItrVals := enumerator.GetLowIdxsAndValues() + for i, idx := range lowItrIdxs { + pl, err := dicts[idx].postingsListFromOffset(lowItrVals[i], drops[idx], nil) + if err != nil { + return nil, 0, err + } + newCard += pl.Count() + } + // compute correct chunk size with this + chunkSize, err := getChunkSize(chunkMode, newCard, newSegDocCount) + if err != nil { + return nil, 0, err + } + // update encoders chunk + tfEncoder.SetChunkSize(chunkSize, newSegDocCount-1) + locEncoder.SetChunkSize(chunkSize, newSegDocCount-1) + } + + postings, err = dicts[itrI].postingsListFromOffset( + postingsOffset, drops[itrI], postings) + if err != nil { + return nil, 0, err + } + + postItr = postings.iterator(true, true, true, postItr) + + // can no longer optimize by copying, since chunk factor could have changed + lastDocNum, lastFreq, lastNorm, bufLoc, err = mergeTermFreqNormLocs( + fieldsMap, term, postItr, newDocNums[itrI], newRoaring, + tfEncoder, locEncoder, bufLoc) + + if err != nil { + return nil, 0, err + } + + prevTerm = prevTerm[:0] // copy to prevTerm in case Next() reuses term mem + prevTerm = append(prevTerm, term...) + + err = enumerator.Next() + } + if err != vellum.ErrIteratorDone { + return nil, 0, err + } + + err = finishTerm(prevTerm) + if err != nil { + return nil, 0, err + } + + dictOffset := uint64(w.Count()) + + err = newVellum.Close() + if err != nil { + return nil, 0, err + } + vellumData := vellumBuf.Bytes() + + // write out the length of the vellum data + n := binary.PutUvarint(bufMaxVarintLen64, uint64(len(vellumData))) + _, err = w.Write(bufMaxVarintLen64[:n]) + if err != nil { + return nil, 0, err + } + + // write this vellum to disk + _, err = w.Write(vellumData) + if err != nil { + return nil, 0, err + } + + rv[fieldID] = dictOffset + + // get the field doc value offset (start) + fieldDvLocsStart[fieldID] = uint64(w.Count()) + + // update the field doc values + // NOTE: doc values continue to use legacy chunk mode + chunkSize, err := getChunkSize(LegacyChunkMode, 0, 0) + if err != nil { + return nil, 0, err + } + fdvEncoder := newChunkedContentCoder(chunkSize, newSegDocCount-1, w, true) + + fdvReadersAvailable := false + var dvIterClone *docValueReader + for segmentI, segment := range segmentsInFocus { + // check for the closure in meantime + if isClosed(closeCh) { + return nil, 0, seg.ErrClosed + } + + fieldIDPlus1 := uint16(segment.fieldsMap[fieldName]) + if dvIter, exists := segment.fieldDvReaders[fieldIDPlus1-1]; exists && + dvIter != nil { + fdvReadersAvailable = true + dvIterClone = dvIter.cloneInto(dvIterClone) + err = dvIterClone.iterateAllDocValues(segment, func(docNum uint64, terms []byte) error { + if newDocNums[segmentI][docNum] == docDropped { + return nil + } + err := fdvEncoder.Add(newDocNums[segmentI][docNum], terms) + if err != nil { + return err + } + return nil + }) + if err != nil { + return nil, 0, err + } + } + } + + if fdvReadersAvailable { + err = fdvEncoder.Close() + if err != nil { + return nil, 0, err + } + + // persist the doc value details for this field + _, err = fdvEncoder.Write() + if err != nil { + return nil, 0, err + } + + // get the field doc value offset (end) + fieldDvLocsEnd[fieldID] = uint64(w.Count()) + } else { + fieldDvLocsStart[fieldID] = fieldNotUninverted + fieldDvLocsEnd[fieldID] = fieldNotUninverted + } + + // reset vellum buffer and vellum builder + vellumBuf.Reset() + err = newVellum.Reset(&vellumBuf) + if err != nil { + return nil, 0, err + } + } + + fieldDvLocsOffset := uint64(w.Count()) + + buf := bufMaxVarintLen64 + for i := 0; i < len(fieldDvLocsStart); i++ { + n := binary.PutUvarint(buf, fieldDvLocsStart[i]) + _, err := w.Write(buf[:n]) + if err != nil { + return nil, 0, err + } + n = binary.PutUvarint(buf, fieldDvLocsEnd[i]) + _, err = w.Write(buf[:n]) + if err != nil { + return nil, 0, err + } + } + + return rv, fieldDvLocsOffset, nil +} + +func mergeTermFreqNormLocs(fieldsMap map[string]uint16, term []byte, postItr *PostingsIterator, + newDocNums []uint64, newRoaring *roaring.Bitmap, + tfEncoder *chunkedIntCoder, locEncoder *chunkedIntCoder, bufLoc []uint64) ( + lastDocNum uint64, lastFreq uint64, lastNorm uint64, bufLocOut []uint64, err error) { + next, err := postItr.Next() + for next != nil && err == nil { + hitNewDocNum := newDocNums[next.Number()] + if hitNewDocNum == docDropped { + return 0, 0, 0, nil, fmt.Errorf("see hit with dropped docNum") + } + + newRoaring.Add(uint32(hitNewDocNum)) + + nextFreq := next.Frequency() + nextNorm := uint64(math.Float32bits(float32(next.Norm()))) + + locs := next.Locations() + + err = tfEncoder.Add(hitNewDocNum, + encodeFreqHasLocs(nextFreq, len(locs) > 0), nextNorm) + if err != nil { + return 0, 0, 0, nil, err + } + + if len(locs) > 0 { + numBytesLocs := 0 + for _, loc := range locs { + ap := loc.ArrayPositions() + numBytesLocs += totalUvarintBytes(uint64(fieldsMap[loc.Field()]-1), + loc.Pos(), loc.Start(), loc.End(), uint64(len(ap)), ap) + } + + err = locEncoder.Add(hitNewDocNum, uint64(numBytesLocs)) + if err != nil { + return 0, 0, 0, nil, err + } + + for _, loc := range locs { + ap := loc.ArrayPositions() + if cap(bufLoc) < 5+len(ap) { + bufLoc = make([]uint64, 0, 5+len(ap)) + } + args := bufLoc[0:5] + args[0] = uint64(fieldsMap[loc.Field()] - 1) + args[1] = loc.Pos() + args[2] = loc.Start() + args[3] = loc.End() + args[4] = uint64(len(ap)) + args = append(args, ap...) + err = locEncoder.Add(hitNewDocNum, args...) + if err != nil { + return 0, 0, 0, nil, err + } + } + } + + lastDocNum = hitNewDocNum + lastFreq = nextFreq + lastNorm = nextNorm + + next, err = postItr.Next() + } + + return lastDocNum, lastFreq, lastNorm, bufLoc, err +} + +func writePostings(postings *roaring.Bitmap, tfEncoder, locEncoder *chunkedIntCoder, + use1HitEncoding func(uint64) (bool, uint64, uint64), + w *CountHashWriter, bufMaxVarintLen64 []byte) ( + offset uint64, err error) { + termCardinality := postings.GetCardinality() + if termCardinality <= 0 { + return 0, nil + } + + if use1HitEncoding != nil { + encodeAs1Hit, docNum1Hit, normBits1Hit := use1HitEncoding(termCardinality) + if encodeAs1Hit { + return FSTValEncode1Hit(docNum1Hit, normBits1Hit), nil + } + } + + var tfOffset uint64 + tfOffset, _, err = tfEncoder.writeAt(w) + if err != nil { + return 0, err + } + + var locOffset uint64 + locOffset, _, err = locEncoder.writeAt(w) + if err != nil { + return 0, err + } + + postingsOffset := uint64(w.Count()) + + n := binary.PutUvarint(bufMaxVarintLen64, tfOffset) + _, err = w.Write(bufMaxVarintLen64[:n]) + if err != nil { + return 0, err + } + + n = binary.PutUvarint(bufMaxVarintLen64, locOffset) + _, err = w.Write(bufMaxVarintLen64[:n]) + if err != nil { + return 0, err + } + + _, err = writeRoaringWithLen(postings, w, bufMaxVarintLen64) + if err != nil { + return 0, err + } + + return postingsOffset, nil +} + +type varintEncoder func(uint64) (int, error) + +func mergeStoredAndRemap(segments []*SegmentBase, drops []*roaring.Bitmap, + fieldsMap map[string]uint16, fieldsInv []string, fieldsSame bool, newSegDocCount uint64, + w *CountHashWriter, closeCh chan struct{}) (uint64, [][]uint64, error) { + var rv [][]uint64 // The remapped or newDocNums for each segment. + + var newDocNum uint64 + + var curr int + var data, compressed []byte + var metaBuf bytes.Buffer + varBuf := make([]byte, binary.MaxVarintLen64) + metaEncode := func(val uint64) (int, error) { + wb := binary.PutUvarint(varBuf, val) + return metaBuf.Write(varBuf[:wb]) + } + + vals := make([][][]byte, len(fieldsInv)) + typs := make([][]byte, len(fieldsInv)) + poss := make([][][]uint64, len(fieldsInv)) + + var posBuf []uint64 + + docNumOffsets := make([]uint64, newSegDocCount) + + vdc := visitDocumentCtxPool.Get().(*visitDocumentCtx) + defer visitDocumentCtxPool.Put(vdc) + + // for each segment + for segI, segment := range segments { + // check for the closure in meantime + if isClosed(closeCh) { + return 0, nil, seg.ErrClosed + } + + segNewDocNums := make([]uint64, segment.numDocs) + + dropsI := drops[segI] + + // optimize when the field mapping is the same across all + // segments and there are no deletions, via byte-copying + // of stored docs bytes directly to the writer + if fieldsSame && (dropsI == nil || dropsI.GetCardinality() == 0) { + err := segment.copyStoredDocs(newDocNum, docNumOffsets, w) + if err != nil { + return 0, nil, err + } + + for i := uint64(0); i < segment.numDocs; i++ { + segNewDocNums[i] = newDocNum + newDocNum++ + } + rv = append(rv, segNewDocNums) + + continue + } + + // for each doc num + for docNum := uint64(0); docNum < segment.numDocs; docNum++ { + // TODO: roaring's API limits docNums to 32-bits? + if dropsI != nil && dropsI.Contains(uint32(docNum)) { + segNewDocNums[docNum] = docDropped + continue + } + + segNewDocNums[docNum] = newDocNum + + curr = 0 + metaBuf.Reset() + data = data[:0] + + posTemp := posBuf + + // collect all the data + for i := 0; i < len(fieldsInv); i++ { + vals[i] = vals[i][:0] + typs[i] = typs[i][:0] + poss[i] = poss[i][:0] + } + err := segment.visitStoredFields(vdc, docNum, func(field string, typ byte, value []byte, pos []uint64) bool { + fieldID := int(fieldsMap[field]) - 1 + vals[fieldID] = append(vals[fieldID], value) + typs[fieldID] = append(typs[fieldID], typ) + + // copy array positions to preserve them beyond the scope of this callback + var curPos []uint64 + if len(pos) > 0 { + if cap(posTemp) < len(pos) { + posBuf = make([]uint64, len(pos)*len(fieldsInv)) + posTemp = posBuf + } + curPos = posTemp[0:len(pos)] + copy(curPos, pos) + posTemp = posTemp[len(pos):] + } + poss[fieldID] = append(poss[fieldID], curPos) + + return true + }) + if err != nil { + return 0, nil, err + } + + // _id field special case optimizes ExternalID() lookups + idFieldVal := vals[uint16(0)][0] + _, err = metaEncode(uint64(len(idFieldVal))) + if err != nil { + return 0, nil, err + } + + // now walk the non-"_id" fields in order + for fieldID := 1; fieldID < len(fieldsInv); fieldID++ { + storedFieldValues := vals[fieldID] + + stf := typs[fieldID] + spf := poss[fieldID] + + var err2 error + curr, data, err2 = persistStoredFieldValues(fieldID, + storedFieldValues, stf, spf, curr, metaEncode, data) + if err2 != nil { + return 0, nil, err2 + } + } + + metaBytes := metaBuf.Bytes() + + compressed = snappy.Encode(compressed[:cap(compressed)], data) + + // record where we're about to start writing + docNumOffsets[newDocNum] = uint64(w.Count()) + + // write out the meta len and compressed data len + _, err = writeUvarints(w, + uint64(len(metaBytes)), + uint64(len(idFieldVal)+len(compressed))) + if err != nil { + return 0, nil, err + } + // now write the meta + _, err = w.Write(metaBytes) + if err != nil { + return 0, nil, err + } + // now write the _id field val (counted as part of the 'compressed' data) + _, err = w.Write(idFieldVal) + if err != nil { + return 0, nil, err + } + // now write the compressed data + _, err = w.Write(compressed) + if err != nil { + return 0, nil, err + } + + newDocNum++ + } + + rv = append(rv, segNewDocNums) + } + + // return value is the start of the stored index + storedIndexOffset := uint64(w.Count()) + + // now write out the stored doc index + for _, docNumOffset := range docNumOffsets { + err := binary.Write(w, binary.BigEndian, docNumOffset) + if err != nil { + return 0, nil, err + } + } + + return storedIndexOffset, rv, nil +} + +// copyStoredDocs writes out a segment's stored doc info, optimized by +// using a single Write() call for the entire set of bytes. The +// newDocNumOffsets is filled with the new offsets for each doc. +func (s *SegmentBase) copyStoredDocs(newDocNum uint64, newDocNumOffsets []uint64, + w *CountHashWriter) error { + if s.numDocs <= 0 { + return nil + } + + indexOffset0, storedOffset0, _, _, _ := + s.getDocStoredOffsets(0) // the segment's first doc + + indexOffsetN, storedOffsetN, readN, metaLenN, dataLenN := + s.getDocStoredOffsets(s.numDocs - 1) // the segment's last doc + + storedOffset0New := uint64(w.Count()) + + storedBytes := s.mem[storedOffset0 : storedOffsetN+readN+metaLenN+dataLenN] + _, err := w.Write(storedBytes) + if err != nil { + return err + } + + // remap the storedOffset's for the docs into new offsets relative + // to storedOffset0New, filling the given docNumOffsetsOut array + for indexOffset := indexOffset0; indexOffset <= indexOffsetN; indexOffset += 8 { + storedOffset := binary.BigEndian.Uint64(s.mem[indexOffset : indexOffset+8]) + storedOffsetNew := storedOffset - storedOffset0 + storedOffset0New + newDocNumOffsets[newDocNum] = storedOffsetNew + newDocNum += 1 + } + + return nil +} + +// mergeFields builds a unified list of fields used across all the +// input segments, and computes whether the fields are the same across +// segments (which depends on fields to be sorted in the same way +// across segments) +func mergeFields(segments []*SegmentBase) (bool, []string) { + fieldsSame := true + + var segment0Fields []string + if len(segments) > 0 { + segment0Fields = segments[0].Fields() + } + + fieldsExist := map[string]struct{}{} + for _, segment := range segments { + fields := segment.Fields() + for fieldi, field := range fields { + fieldsExist[field] = struct{}{} + if len(segment0Fields) != len(fields) || segment0Fields[fieldi] != field { + fieldsSame = false + } + } + } + + rv := make([]string, 0, len(fieldsExist)) + // ensure _id stays first + rv = append(rv, "_id") + for k := range fieldsExist { + if k != "_id" { + rv = append(rv, k) + } + } + + sort.Strings(rv[1:]) // leave _id as first + + return fieldsSame, rv +} + +func isClosed(closeCh chan struct{}) bool { + select { + case <-closeCh: + return true + default: + return false + } +} diff --git a/vendor/github.com/blevesearch/zapx/v12/new.go b/vendor/github.com/blevesearch/zapx/v12/new.go new file mode 100644 index 00000000..b4e0d034 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v12/new.go @@ -0,0 +1,830 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "bytes" + "encoding/binary" + "math" + "sort" + "sync" + + "github.com/RoaringBitmap/roaring" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" + "github.com/blevesearch/vellum" + "github.com/golang/snappy" +) + +var NewSegmentBufferNumResultsBump int = 100 +var NewSegmentBufferNumResultsFactor float64 = 1.0 +var NewSegmentBufferAvgBytesPerDocFactor float64 = 1.0 + +// ValidateDocFields can be set by applications to perform additional checks +// on fields in a document being added to a new segment, by default it does +// nothing. +// This API is experimental and may be removed at any time. +var ValidateDocFields = func(field index.Field) error { + return nil +} + +// New creates an in-memory zap-encoded SegmentBase from a set of Documents +func (z *ZapPlugin) New(results []index.Document) ( + segment.Segment, uint64, error) { + return z.newWithChunkMode(results, DefaultChunkMode) +} + +func (*ZapPlugin) newWithChunkMode(results []index.Document, + chunkMode uint32) (segment.Segment, uint64, error) { + s := interimPool.Get().(*interim) + + var br bytes.Buffer + if s.lastNumDocs > 0 { + // use previous results to initialize the buf with an estimate + // size, but note that the interim instance comes from a + // global interimPool, so multiple scorch instances indexing + // different docs can lead to low quality estimates + estimateAvgBytesPerDoc := int(float64(s.lastOutSize/s.lastNumDocs) * + NewSegmentBufferNumResultsFactor) + estimateNumResults := int(float64(len(results)+NewSegmentBufferNumResultsBump) * + NewSegmentBufferAvgBytesPerDocFactor) + br.Grow(estimateAvgBytesPerDoc * estimateNumResults) + } + + s.results = results + s.chunkMode = chunkMode + s.w = NewCountHashWriter(&br) + + storedIndexOffset, fieldsIndexOffset, fdvIndexOffset, dictOffsets, + err := s.convert() + if err != nil { + return nil, uint64(0), err + } + + sb, err := InitSegmentBase(br.Bytes(), s.w.Sum32(), chunkMode, + s.FieldsMap, s.FieldsInv, uint64(len(results)), + storedIndexOffset, fieldsIndexOffset, fdvIndexOffset, dictOffsets) + + if err == nil && s.reset() == nil { + s.lastNumDocs = len(results) + s.lastOutSize = len(br.Bytes()) + interimPool.Put(s) + } + + return sb, uint64(len(br.Bytes())), err +} + +var interimPool = sync.Pool{New: func() interface{} { return &interim{} }} + +// interim holds temporary working data used while converting from +// analysis results to a zap-encoded segment +type interim struct { + results []index.Document + + chunkMode uint32 + + w *CountHashWriter + + // FieldsMap adds 1 to field id to avoid zero value issues + // name -> field id + 1 + FieldsMap map[string]uint16 + + // FieldsInv is the inverse of FieldsMap + // field id -> name + FieldsInv []string + + // Term dictionaries for each field + // field id -> term -> postings list id + 1 + Dicts []map[string]uint64 + + // Terms for each field, where terms are sorted ascending + // field id -> []term + DictKeys [][]string + + // Fields whose IncludeDocValues is true + // field id -> bool + IncludeDocValues []bool + + // postings id -> bitmap of docNums + Postings []*roaring.Bitmap + + // postings id -> freq/norm's, one for each docNum in postings + FreqNorms [][]interimFreqNorm + freqNormsBacking []interimFreqNorm + + // postings id -> locs, one for each freq + Locs [][]interimLoc + locsBacking []interimLoc + + numTermsPerPostingsList []int // key is postings list id + numLocsPerPostingsList []int // key is postings list id + + builder *vellum.Builder + builderBuf bytes.Buffer + + metaBuf bytes.Buffer + + tmp0 []byte + tmp1 []byte + + lastNumDocs int + lastOutSize int +} + +func (s *interim) reset() (err error) { + s.results = nil + s.chunkMode = 0 + s.w = nil + s.FieldsMap = nil + s.FieldsInv = nil + for i := range s.Dicts { + s.Dicts[i] = nil + } + s.Dicts = s.Dicts[:0] + for i := range s.DictKeys { + s.DictKeys[i] = s.DictKeys[i][:0] + } + s.DictKeys = s.DictKeys[:0] + for i := range s.IncludeDocValues { + s.IncludeDocValues[i] = false + } + s.IncludeDocValues = s.IncludeDocValues[:0] + for _, idn := range s.Postings { + idn.Clear() + } + s.Postings = s.Postings[:0] + s.FreqNorms = s.FreqNorms[:0] + for i := range s.freqNormsBacking { + s.freqNormsBacking[i] = interimFreqNorm{} + } + s.freqNormsBacking = s.freqNormsBacking[:0] + s.Locs = s.Locs[:0] + for i := range s.locsBacking { + s.locsBacking[i] = interimLoc{} + } + s.locsBacking = s.locsBacking[:0] + s.numTermsPerPostingsList = s.numTermsPerPostingsList[:0] + s.numLocsPerPostingsList = s.numLocsPerPostingsList[:0] + s.builderBuf.Reset() + if s.builder != nil { + err = s.builder.Reset(&s.builderBuf) + } + s.metaBuf.Reset() + s.tmp0 = s.tmp0[:0] + s.tmp1 = s.tmp1[:0] + s.lastNumDocs = 0 + s.lastOutSize = 0 + + return err +} + +func (s *interim) grabBuf(size int) []byte { + buf := s.tmp0 + if cap(buf) < size { + buf = make([]byte, size) + s.tmp0 = buf + } + return buf[0:size] +} + +type interimStoredField struct { + vals [][]byte + typs []byte + arrayposs [][]uint64 // array positions +} + +type interimFreqNorm struct { + freq uint64 + norm float32 + numLocs int +} + +type interimLoc struct { + fieldID uint16 + pos uint64 + start uint64 + end uint64 + arrayposs []uint64 +} + +func (s *interim) convert() (uint64, uint64, uint64, []uint64, error) { + s.FieldsMap = map[string]uint16{} + + s.getOrDefineField("_id") // _id field is fieldID 0 + + for _, result := range s.results { + result.VisitComposite(func(field index.CompositeField) { + s.getOrDefineField(field.Name()) + }) + result.VisitFields(func(field index.Field) { + s.getOrDefineField(field.Name()) + }) + } + + sort.Strings(s.FieldsInv[1:]) // keep _id as first field + + for fieldID, fieldName := range s.FieldsInv { + s.FieldsMap[fieldName] = uint16(fieldID + 1) + } + + if cap(s.IncludeDocValues) >= len(s.FieldsInv) { + s.IncludeDocValues = s.IncludeDocValues[:len(s.FieldsInv)] + } else { + s.IncludeDocValues = make([]bool, len(s.FieldsInv)) + } + + s.prepareDicts() + + for _, dict := range s.DictKeys { + sort.Strings(dict) + } + + s.processDocuments() + + storedIndexOffset, err := s.writeStoredFields() + if err != nil { + return 0, 0, 0, nil, err + } + + var fdvIndexOffset uint64 + var dictOffsets []uint64 + + if len(s.results) > 0 { + fdvIndexOffset, dictOffsets, err = s.writeDicts() + if err != nil { + return 0, 0, 0, nil, err + } + } else { + dictOffsets = make([]uint64, len(s.FieldsInv)) + } + + fieldsIndexOffset, err := persistFields(s.FieldsInv, s.w, dictOffsets) + if err != nil { + return 0, 0, 0, nil, err + } + + return storedIndexOffset, fieldsIndexOffset, fdvIndexOffset, dictOffsets, nil +} + +func (s *interim) getOrDefineField(fieldName string) int { + fieldIDPlus1, exists := s.FieldsMap[fieldName] + if !exists { + fieldIDPlus1 = uint16(len(s.FieldsInv) + 1) + s.FieldsMap[fieldName] = fieldIDPlus1 + s.FieldsInv = append(s.FieldsInv, fieldName) + + s.Dicts = append(s.Dicts, make(map[string]uint64)) + + n := len(s.DictKeys) + if n < cap(s.DictKeys) { + s.DictKeys = s.DictKeys[:n+1] + s.DictKeys[n] = s.DictKeys[n][:0] + } else { + s.DictKeys = append(s.DictKeys, []string(nil)) + } + } + + return int(fieldIDPlus1 - 1) +} + +// fill Dicts and DictKeys from analysis results +func (s *interim) prepareDicts() { + var pidNext int + + var totTFs int + var totLocs int + + visitField := func(field index.Field) { + fieldID := uint16(s.getOrDefineField(field.Name())) + + dict := s.Dicts[fieldID] + dictKeys := s.DictKeys[fieldID] + + tfs := field.AnalyzedTokenFrequencies() + for term, tf := range tfs { + pidPlus1, exists := dict[term] + if !exists { + pidNext++ + pidPlus1 = uint64(pidNext) + + dict[term] = pidPlus1 + dictKeys = append(dictKeys, term) + + s.numTermsPerPostingsList = append(s.numTermsPerPostingsList, 0) + s.numLocsPerPostingsList = append(s.numLocsPerPostingsList, 0) + } + + pid := pidPlus1 - 1 + + s.numTermsPerPostingsList[pid] += 1 + s.numLocsPerPostingsList[pid] += len(tf.Locations) + + totLocs += len(tf.Locations) + } + + totTFs += len(tfs) + + s.DictKeys[fieldID] = dictKeys + } + + for _, result := range s.results { + // walk each composite field + result.VisitComposite(func(field index.CompositeField) { + visitField(field) + }) + + // walk each field + result.VisitFields(visitField) + } + + numPostingsLists := pidNext + + if cap(s.Postings) >= numPostingsLists { + s.Postings = s.Postings[:numPostingsLists] + } else { + postings := make([]*roaring.Bitmap, numPostingsLists) + copy(postings, s.Postings[:cap(s.Postings)]) + for i := 0; i < numPostingsLists; i++ { + if postings[i] == nil { + postings[i] = roaring.New() + } + } + s.Postings = postings + } + + if cap(s.FreqNorms) >= numPostingsLists { + s.FreqNorms = s.FreqNorms[:numPostingsLists] + } else { + s.FreqNorms = make([][]interimFreqNorm, numPostingsLists) + } + + if cap(s.freqNormsBacking) >= totTFs { + s.freqNormsBacking = s.freqNormsBacking[:totTFs] + } else { + s.freqNormsBacking = make([]interimFreqNorm, totTFs) + } + + freqNormsBacking := s.freqNormsBacking + for pid, numTerms := range s.numTermsPerPostingsList { + s.FreqNorms[pid] = freqNormsBacking[0:0] + freqNormsBacking = freqNormsBacking[numTerms:] + } + + if cap(s.Locs) >= numPostingsLists { + s.Locs = s.Locs[:numPostingsLists] + } else { + s.Locs = make([][]interimLoc, numPostingsLists) + } + + if cap(s.locsBacking) >= totLocs { + s.locsBacking = s.locsBacking[:totLocs] + } else { + s.locsBacking = make([]interimLoc, totLocs) + } + + locsBacking := s.locsBacking + for pid, numLocs := range s.numLocsPerPostingsList { + s.Locs[pid] = locsBacking[0:0] + locsBacking = locsBacking[numLocs:] + } +} + +func (s *interim) processDocuments() { + numFields := len(s.FieldsInv) + reuseFieldLens := make([]int, numFields) + reuseFieldTFs := make([]index.TokenFrequencies, numFields) + + for docNum, result := range s.results { + for i := 0; i < numFields; i++ { // clear these for reuse + reuseFieldLens[i] = 0 + reuseFieldTFs[i] = nil + } + + s.processDocument(uint64(docNum), result, + reuseFieldLens, reuseFieldTFs) + } +} + +func (s *interim) processDocument(docNum uint64, + result index.Document, + fieldLens []int, fieldTFs []index.TokenFrequencies) { + visitField := func(field index.Field) { + fieldID := uint16(s.getOrDefineField(field.Name())) + fieldLens[fieldID] += field.AnalyzedLength() + + existingFreqs := fieldTFs[fieldID] + if existingFreqs != nil { + existingFreqs.MergeAll(field.Name(), field.AnalyzedTokenFrequencies()) + } else { + fieldTFs[fieldID] = field.AnalyzedTokenFrequencies() + } + } + + // walk each composite field + result.VisitComposite(func(field index.CompositeField) { + visitField(field) + }) + + // walk each field + result.VisitFields(visitField) + + // now that it's been rolled up into fieldTFs, walk that + for fieldID, tfs := range fieldTFs { + dict := s.Dicts[fieldID] + norm := float32(1.0 / math.Sqrt(float64(fieldLens[fieldID]))) + + for term, tf := range tfs { + pid := dict[term] - 1 + bs := s.Postings[pid] + bs.Add(uint32(docNum)) + + s.FreqNorms[pid] = append(s.FreqNorms[pid], + interimFreqNorm{ + freq: uint64(tf.Frequency()), + norm: norm, + numLocs: len(tf.Locations), + }) + + if len(tf.Locations) > 0 { + locs := s.Locs[pid] + + for _, loc := range tf.Locations { + var locf = uint16(fieldID) + if loc.Field != "" { + locf = uint16(s.getOrDefineField(loc.Field)) + } + var arrayposs []uint64 + if len(loc.ArrayPositions) > 0 { + arrayposs = loc.ArrayPositions + } + locs = append(locs, interimLoc{ + fieldID: locf, + pos: uint64(loc.Position), + start: uint64(loc.Start), + end: uint64(loc.End), + arrayposs: arrayposs, + }) + } + + s.Locs[pid] = locs + } + } + } +} + +func (s *interim) writeStoredFields() ( + storedIndexOffset uint64, err error) { + varBuf := make([]byte, binary.MaxVarintLen64) + metaEncode := func(val uint64) (int, error) { + wb := binary.PutUvarint(varBuf, val) + return s.metaBuf.Write(varBuf[:wb]) + } + + data, compressed := s.tmp0[:0], s.tmp1[:0] + defer func() { s.tmp0, s.tmp1 = data, compressed }() + + // keyed by docNum + docStoredOffsets := make([]uint64, len(s.results)) + + // keyed by fieldID, for the current doc in the loop + docStoredFields := map[uint16]interimStoredField{} + + for docNum, result := range s.results { + for fieldID := range docStoredFields { // reset for next doc + delete(docStoredFields, fieldID) + } + + var validationErr error + result.VisitFields(func(field index.Field) { + fieldID := uint16(s.getOrDefineField(field.Name())) + + if field.Options().IsStored() { + isf := docStoredFields[fieldID] + isf.vals = append(isf.vals, field.Value()) + isf.typs = append(isf.typs, field.EncodedFieldType()) + isf.arrayposs = append(isf.arrayposs, field.ArrayPositions()) + docStoredFields[fieldID] = isf + } + + if field.Options().IncludeDocValues() { + s.IncludeDocValues[fieldID] = true + } + + err := ValidateDocFields(field) + if err != nil && validationErr == nil { + validationErr = err + } + }) + if validationErr != nil { + return 0, validationErr + } + + var curr int + + s.metaBuf.Reset() + data = data[:0] + + // _id field special case optimizes ExternalID() lookups + idFieldVal := docStoredFields[uint16(0)].vals[0] + _, err = metaEncode(uint64(len(idFieldVal))) + if err != nil { + return 0, err + } + + // handle non-"_id" fields + for fieldID := 1; fieldID < len(s.FieldsInv); fieldID++ { + isf, exists := docStoredFields[uint16(fieldID)] + if exists { + curr, data, err = persistStoredFieldValues( + fieldID, isf.vals, isf.typs, isf.arrayposs, + curr, metaEncode, data) + if err != nil { + return 0, err + } + } + } + + metaBytes := s.metaBuf.Bytes() + + compressed = snappy.Encode(compressed[:cap(compressed)], data) + + docStoredOffsets[docNum] = uint64(s.w.Count()) + + _, err := writeUvarints(s.w, + uint64(len(metaBytes)), + uint64(len(idFieldVal)+len(compressed))) + if err != nil { + return 0, err + } + + _, err = s.w.Write(metaBytes) + if err != nil { + return 0, err + } + + _, err = s.w.Write(idFieldVal) + if err != nil { + return 0, err + } + + _, err = s.w.Write(compressed) + if err != nil { + return 0, err + } + } + + storedIndexOffset = uint64(s.w.Count()) + + for _, docStoredOffset := range docStoredOffsets { + err = binary.Write(s.w, binary.BigEndian, docStoredOffset) + if err != nil { + return 0, err + } + } + + return storedIndexOffset, nil +} + +func (s *interim) writeDicts() (fdvIndexOffset uint64, dictOffsets []uint64, err error) { + dictOffsets = make([]uint64, len(s.FieldsInv)) + + fdvOffsetsStart := make([]uint64, len(s.FieldsInv)) + fdvOffsetsEnd := make([]uint64, len(s.FieldsInv)) + + buf := s.grabBuf(binary.MaxVarintLen64) + + // these int coders are initialized with chunk size 1024 + // however this will be reset to the correct chunk size + // while processing each individual field-term section + tfEncoder := newChunkedIntCoder(1024, uint64(len(s.results)-1)) + locEncoder := newChunkedIntCoder(1024, uint64(len(s.results)-1)) + + var docTermMap [][]byte + + if s.builder == nil { + s.builder, err = vellum.New(&s.builderBuf, nil) + if err != nil { + return 0, nil, err + } + } + + for fieldID, terms := range s.DictKeys { + if cap(docTermMap) < len(s.results) { + docTermMap = make([][]byte, len(s.results)) + } else { + docTermMap = docTermMap[0:len(s.results)] + for docNum := range docTermMap { // reset the docTermMap + docTermMap[docNum] = docTermMap[docNum][:0] + } + } + + dict := s.Dicts[fieldID] + + for _, term := range terms { // terms are already sorted + pid := dict[term] - 1 + + postingsBS := s.Postings[pid] + + freqNorms := s.FreqNorms[pid] + freqNormOffset := 0 + + locs := s.Locs[pid] + locOffset := 0 + + chunkSize, err := getChunkSize(s.chunkMode, postingsBS.GetCardinality(), uint64(len(s.results))) + if err != nil { + return 0, nil, err + } + tfEncoder.SetChunkSize(chunkSize, uint64(len(s.results)-1)) + locEncoder.SetChunkSize(chunkSize, uint64(len(s.results)-1)) + + postingsItr := postingsBS.Iterator() + for postingsItr.HasNext() { + docNum := uint64(postingsItr.Next()) + + freqNorm := freqNorms[freqNormOffset] + + err = tfEncoder.Add(docNum, + encodeFreqHasLocs(freqNorm.freq, freqNorm.numLocs > 0), + uint64(math.Float32bits(freqNorm.norm))) + if err != nil { + return 0, nil, err + } + + if freqNorm.numLocs > 0 { + numBytesLocs := 0 + for _, loc := range locs[locOffset : locOffset+freqNorm.numLocs] { + numBytesLocs += totalUvarintBytes( + uint64(loc.fieldID), loc.pos, loc.start, loc.end, + uint64(len(loc.arrayposs)), loc.arrayposs) + } + + err = locEncoder.Add(docNum, uint64(numBytesLocs)) + if err != nil { + return 0, nil, err + } + + for _, loc := range locs[locOffset : locOffset+freqNorm.numLocs] { + err = locEncoder.Add(docNum, + uint64(loc.fieldID), loc.pos, loc.start, loc.end, + uint64(len(loc.arrayposs))) + if err != nil { + return 0, nil, err + } + + err = locEncoder.Add(docNum, loc.arrayposs...) + if err != nil { + return 0, nil, err + } + } + + locOffset += freqNorm.numLocs + } + + freqNormOffset++ + + docTermMap[docNum] = append( + append(docTermMap[docNum], term...), + termSeparator) + } + + tfEncoder.Close() + locEncoder.Close() + + postingsOffset, err := + writePostings(postingsBS, tfEncoder, locEncoder, nil, s.w, buf) + if err != nil { + return 0, nil, err + } + + if postingsOffset > uint64(0) { + err = s.builder.Insert([]byte(term), postingsOffset) + if err != nil { + return 0, nil, err + } + } + + tfEncoder.Reset() + locEncoder.Reset() + } + + err = s.builder.Close() + if err != nil { + return 0, nil, err + } + + // record where this dictionary starts + dictOffsets[fieldID] = uint64(s.w.Count()) + + vellumData := s.builderBuf.Bytes() + + // write out the length of the vellum data + n := binary.PutUvarint(buf, uint64(len(vellumData))) + _, err = s.w.Write(buf[:n]) + if err != nil { + return 0, nil, err + } + + // write this vellum to disk + _, err = s.w.Write(vellumData) + if err != nil { + return 0, nil, err + } + + // reset vellum for reuse + s.builderBuf.Reset() + + err = s.builder.Reset(&s.builderBuf) + if err != nil { + return 0, nil, err + } + + // write the field doc values + // NOTE: doc values continue to use legacy chunk mode + chunkSize, err := getChunkSize(LegacyChunkMode, 0, 0) + if err != nil { + return 0, nil, err + } + fdvEncoder := newChunkedContentCoder(chunkSize, uint64(len(s.results)-1), s.w, false) + if s.IncludeDocValues[fieldID] { + for docNum, docTerms := range docTermMap { + if len(docTerms) > 0 { + err = fdvEncoder.Add(uint64(docNum), docTerms) + if err != nil { + return 0, nil, err + } + } + } + err = fdvEncoder.Close() + if err != nil { + return 0, nil, err + } + + fdvOffsetsStart[fieldID] = uint64(s.w.Count()) + + _, err = fdvEncoder.Write() + if err != nil { + return 0, nil, err + } + + fdvOffsetsEnd[fieldID] = uint64(s.w.Count()) + + fdvEncoder.Reset() + } else { + fdvOffsetsStart[fieldID] = fieldNotUninverted + fdvOffsetsEnd[fieldID] = fieldNotUninverted + } + } + + fdvIndexOffset = uint64(s.w.Count()) + + for i := 0; i < len(fdvOffsetsStart); i++ { + n := binary.PutUvarint(buf, fdvOffsetsStart[i]) + _, err := s.w.Write(buf[:n]) + if err != nil { + return 0, nil, err + } + n = binary.PutUvarint(buf, fdvOffsetsEnd[i]) + _, err = s.w.Write(buf[:n]) + if err != nil { + return 0, nil, err + } + } + + return fdvIndexOffset, dictOffsets, nil +} + +// returns the total # of bytes needed to encode the given uint64's +// into binary.PutUVarint() encoding +func totalUvarintBytes(a, b, c, d, e uint64, more []uint64) (n int) { + n = numUvarintBytes(a) + n += numUvarintBytes(b) + n += numUvarintBytes(c) + n += numUvarintBytes(d) + n += numUvarintBytes(e) + for _, v := range more { + n += numUvarintBytes(v) + } + return n +} + +// returns # of bytes needed to encode x in binary.PutUvarint() encoding +func numUvarintBytes(x uint64) (n int) { + for x >= 0x80 { + x >>= 7 + n++ + } + return n + 1 +} diff --git a/vendor/github.com/blevesearch/zapx/v12/plugin.go b/vendor/github.com/blevesearch/zapx/v12/plugin.go new file mode 100644 index 00000000..f67297ec --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v12/plugin.go @@ -0,0 +1,27 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +// ZapPlugin implements the Plugin interface of +// the blevesearch/scorch_segment_api pkg +type ZapPlugin struct{} + +func (*ZapPlugin) Type() string { + return Type +} + +func (*ZapPlugin) Version() uint32 { + return Version +} diff --git a/vendor/github.com/blevesearch/zapx/v12/posting.go b/vendor/github.com/blevesearch/zapx/v12/posting.go new file mode 100644 index 00000000..627a19b1 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v12/posting.go @@ -0,0 +1,837 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "encoding/binary" + "fmt" + "math" + "reflect" + + "github.com/RoaringBitmap/roaring" + segment "github.com/blevesearch/scorch_segment_api/v2" +) + +var reflectStaticSizePostingsList int +var reflectStaticSizePostingsIterator int +var reflectStaticSizePosting int +var reflectStaticSizeLocation int + +func init() { + var pl PostingsList + reflectStaticSizePostingsList = int(reflect.TypeOf(pl).Size()) + var pi PostingsIterator + reflectStaticSizePostingsIterator = int(reflect.TypeOf(pi).Size()) + var p Posting + reflectStaticSizePosting = int(reflect.TypeOf(p).Size()) + var l Location + reflectStaticSizeLocation = int(reflect.TypeOf(l).Size()) +} + +// FST or vellum value (uint64) encoding is determined by the top two +// highest-order or most significant bits... +// +// encoding : MSB +// name : 63 62 61...to...bit #0 (LSB) +// ----------+---+---+--------------------------------------------------- +// general : 0 | 0 | 62-bits of postingsOffset. +// ~ : 0 | 1 | reserved for future. +// 1-hit : 1 | 0 | 31-bits of positive float31 norm | 31-bits docNum. +// ~ : 1 | 1 | reserved for future. +// +// Encoding "general" is able to handle all cases, where the +// postingsOffset points to more information about the postings for +// the term. +// +// Encoding "1-hit" is used to optimize a commonly seen case when a +// term has only a single hit. For example, a term in the _id field +// will have only 1 hit. The "1-hit" encoding is used for a term +// in a field when... +// +// - term vector info is disabled for that field; +// - and, the term appears in only a single doc for that field; +// - and, the term's freq is exactly 1 in that single doc for that field; +// - and, the docNum must fit into 31-bits; +// +// Otherwise, the "general" encoding is used instead. +// +// In the "1-hit" encoding, the field in that single doc may have +// other terms, which is supported in the "1-hit" encoding by the +// positive float31 norm. + +const FSTValEncodingMask = uint64(0xc000000000000000) +const FSTValEncodingGeneral = uint64(0x0000000000000000) +const FSTValEncoding1Hit = uint64(0x8000000000000000) + +func FSTValEncode1Hit(docNum uint64, normBits uint64) uint64 { + return FSTValEncoding1Hit | ((mask31Bits & normBits) << 31) | (mask31Bits & docNum) +} + +func FSTValDecode1Hit(v uint64) (docNum uint64, normBits uint64) { + return (mask31Bits & v), (mask31Bits & (v >> 31)) +} + +const mask31Bits = uint64(0x000000007fffffff) + +func under32Bits(x uint64) bool { + return x <= mask31Bits +} + +const DocNum1HitFinished = math.MaxUint64 + +var NormBits1Hit = uint64(math.Float32bits(float32(1))) + +// PostingsList is an in-memory representation of a postings list +type PostingsList struct { + sb *SegmentBase + postingsOffset uint64 + freqOffset uint64 + locOffset uint64 + postings *roaring.Bitmap + except *roaring.Bitmap + + // when normBits1Hit != 0, then this postings list came from a + // 1-hit encoding, and only the docNum1Hit & normBits1Hit apply + docNum1Hit uint64 + normBits1Hit uint64 +} + +// Following methods are dummy implementations for the interface +// DiskStatsReporter (for backward compatibility). +// The working implementations are supported only in zap v15.x +// and not in the earlier versions of zap. +func (i *PostingsList) ResetBytesRead(uint64) {} + +func (i *PostingsList) BytesRead() uint64 { + return 0 +} + +func (i *PostingsList) incrementBytesRead(uint64) {} + +func (i *PostingsList) BytesWritten() uint64 { + return 0 +} + +// represents an immutable, empty postings list +var emptyPostingsList = &PostingsList{} + +func (p *PostingsList) Size() int { + sizeInBytes := reflectStaticSizePostingsList + SizeOfPtr + + if p.except != nil { + sizeInBytes += int(p.except.GetSizeInBytes()) + } + + return sizeInBytes +} + +func (p *PostingsList) OrInto(receiver *roaring.Bitmap) { + if p.normBits1Hit != 0 { + receiver.Add(uint32(p.docNum1Hit)) + return + } + + if p.postings != nil { + receiver.Or(p.postings) + } +} + +// Iterator returns an iterator for this postings list +func (p *PostingsList) Iterator(includeFreq, includeNorm, includeLocs bool, + prealloc segment.PostingsIterator) segment.PostingsIterator { + if p.normBits1Hit == 0 && p.postings == nil { + return emptyPostingsIterator + } + + var preallocPI *PostingsIterator + pi, ok := prealloc.(*PostingsIterator) + if ok && pi != nil { + preallocPI = pi + } + if preallocPI == emptyPostingsIterator { + preallocPI = nil + } + + return p.iterator(includeFreq, includeNorm, includeLocs, preallocPI) +} + +func (p *PostingsList) iterator(includeFreq, includeNorm, includeLocs bool, + rv *PostingsIterator) *PostingsIterator { + if rv == nil { + rv = &PostingsIterator{} + } else { + freqNormReader := rv.freqNormReader + if freqNormReader != nil { + freqNormReader.reset() + } + + locReader := rv.locReader + if locReader != nil { + locReader.reset() + } + + nextLocs := rv.nextLocs[:0] + nextSegmentLocs := rv.nextSegmentLocs[:0] + + buf := rv.buf + + *rv = PostingsIterator{} // clear the struct + + rv.freqNormReader = freqNormReader + rv.locReader = locReader + + rv.nextLocs = nextLocs + rv.nextSegmentLocs = nextSegmentLocs + + rv.buf = buf + } + + rv.postings = p + rv.includeFreqNorm = includeFreq || includeNorm || includeLocs + rv.includeLocs = includeLocs + + if p.normBits1Hit != 0 { + // "1-hit" encoding + rv.docNum1Hit = p.docNum1Hit + rv.normBits1Hit = p.normBits1Hit + + if p.except != nil && p.except.Contains(uint32(rv.docNum1Hit)) { + rv.docNum1Hit = DocNum1HitFinished + } + + return rv + } + + // "general" encoding, check if empty + if p.postings == nil { + return rv + } + + // initialize freq chunk reader + if rv.includeFreqNorm { + rv.freqNormReader = newChunkedIntDecoder(p.sb.mem, p.freqOffset) + } + + // initialize the loc chunk reader + if rv.includeLocs { + rv.locReader = newChunkedIntDecoder(p.sb.mem, p.locOffset) + } + + rv.all = p.postings.Iterator() + if p.except != nil { + rv.ActualBM = roaring.AndNot(p.postings, p.except) + rv.Actual = rv.ActualBM.Iterator() + } else { + rv.ActualBM = p.postings + rv.Actual = rv.all // Optimize to use same iterator for all & Actual. + } + + return rv +} + +// Count returns the number of items on this postings list +func (p *PostingsList) Count() uint64 { + var n, e uint64 + if p.normBits1Hit != 0 { + n = 1 + if p.except != nil && p.except.Contains(uint32(p.docNum1Hit)) { + e = 1 + } + } else if p.postings != nil { + n = p.postings.GetCardinality() + if p.except != nil { + e = p.postings.AndCardinality(p.except) + } + } + return n - e +} + +func (rv *PostingsList) read(postingsOffset uint64, d *Dictionary) error { + rv.postingsOffset = postingsOffset + + // handle "1-hit" encoding special case + if rv.postingsOffset&FSTValEncodingMask == FSTValEncoding1Hit { + return rv.init1Hit(postingsOffset) + } + + // read the location of the freq/norm details + var n uint64 + var read int + + rv.freqOffset, read = binary.Uvarint(d.sb.mem[postingsOffset+n : postingsOffset+binary.MaxVarintLen64]) + n += uint64(read) + + rv.locOffset, read = binary.Uvarint(d.sb.mem[postingsOffset+n : postingsOffset+n+binary.MaxVarintLen64]) + n += uint64(read) + + var postingsLen uint64 + postingsLen, read = binary.Uvarint(d.sb.mem[postingsOffset+n : postingsOffset+n+binary.MaxVarintLen64]) + n += uint64(read) + + roaringBytes := d.sb.mem[postingsOffset+n : postingsOffset+n+postingsLen] + + if rv.postings == nil { + rv.postings = roaring.NewBitmap() + } + _, err := rv.postings.FromBuffer(roaringBytes) + if err != nil { + return fmt.Errorf("error loading roaring bitmap: %v", err) + } + + return nil +} + +func (rv *PostingsList) init1Hit(fstVal uint64) error { + docNum, normBits := FSTValDecode1Hit(fstVal) + + rv.docNum1Hit = docNum + rv.normBits1Hit = normBits + + return nil +} + +// PostingsIterator provides a way to iterate through the postings list +type PostingsIterator struct { + postings *PostingsList + all roaring.IntPeekable + Actual roaring.IntPeekable + ActualBM *roaring.Bitmap + + currChunk uint32 + freqNormReader *chunkedIntDecoder + locReader *chunkedIntDecoder + + next Posting // reused across Next() calls + nextLocs []Location // reused across Next() calls + nextSegmentLocs []segment.Location // reused across Next() calls + + docNum1Hit uint64 + normBits1Hit uint64 + + buf []byte + + includeFreqNorm bool + includeLocs bool +} + +var emptyPostingsIterator = &PostingsIterator{} + +// Following methods are dummy implementations for the interface +// DiskStatsReporter (for backward compatibility). +// The working implementations are supported only in zap v15.x +// and not in the earlier versions of zap. +func (i *PostingsIterator) ResetBytesRead(uint64) {} + +func (i *PostingsIterator) BytesRead() uint64 { + return 0 +} + +func (i *PostingsIterator) incrementBytesRead(uint64) {} + +func (i *PostingsIterator) BytesWritten() uint64 { + return 0 +} + +func (i *PostingsIterator) Size() int { + sizeInBytes := reflectStaticSizePostingsIterator + SizeOfPtr + + i.next.Size() + // account for freqNormReader, locReader if we start using this. + for _, entry := range i.nextLocs { + sizeInBytes += entry.Size() + } + + return sizeInBytes +} + +func (i *PostingsIterator) loadChunk(chunk int) error { + if i.includeFreqNorm { + err := i.freqNormReader.loadChunk(chunk) + if err != nil { + return err + } + } + + if i.includeLocs { + err := i.locReader.loadChunk(chunk) + if err != nil { + return err + } + } + + i.currChunk = uint32(chunk) + return nil +} + +func (i *PostingsIterator) readFreqNormHasLocs() (uint64, uint64, bool, error) { + if i.normBits1Hit != 0 { + return 1, i.normBits1Hit, false, nil + } + + freqHasLocs, err := i.freqNormReader.readUvarint() + if err != nil { + return 0, 0, false, fmt.Errorf("error reading frequency: %v", err) + } + + freq, hasLocs := decodeFreqHasLocs(freqHasLocs) + + normBits, err := i.freqNormReader.readUvarint() + if err != nil { + return 0, 0, false, fmt.Errorf("error reading norm: %v", err) + } + + return freq, normBits, hasLocs, nil +} + +func (i *PostingsIterator) skipFreqNormReadHasLocs() (bool, error) { + if i.normBits1Hit != 0 { + return false, nil + } + + freqHasLocs, err := i.freqNormReader.readUvarint() + if err != nil { + return false, fmt.Errorf("error reading freqHasLocs: %v", err) + } + + i.freqNormReader.SkipUvarint() // Skip normBits. + + return freqHasLocs&0x01 != 0, nil // See decodeFreqHasLocs() / hasLocs. +} + +func encodeFreqHasLocs(freq uint64, hasLocs bool) uint64 { + rv := freq << 1 + if hasLocs { + rv = rv | 0x01 // 0'th LSB encodes whether there are locations + } + return rv +} + +func decodeFreqHasLocs(freqHasLocs uint64) (uint64, bool) { + freq := freqHasLocs >> 1 + hasLocs := freqHasLocs&0x01 != 0 + return freq, hasLocs +} + +// readLocation processes all the integers on the stream representing a single +// location. +func (i *PostingsIterator) readLocation(l *Location) error { + // read off field + fieldID, err := i.locReader.readUvarint() + if err != nil { + return fmt.Errorf("error reading location field: %v", err) + } + // read off pos + pos, err := i.locReader.readUvarint() + if err != nil { + return fmt.Errorf("error reading location pos: %v", err) + } + // read off start + start, err := i.locReader.readUvarint() + if err != nil { + return fmt.Errorf("error reading location start: %v", err) + } + // read off end + end, err := i.locReader.readUvarint() + if err != nil { + return fmt.Errorf("error reading location end: %v", err) + } + // read off num array pos + numArrayPos, err := i.locReader.readUvarint() + if err != nil { + return fmt.Errorf("error reading location num array pos: %v", err) + } + + l.field = i.postings.sb.fieldsInv[fieldID] + l.pos = pos + l.start = start + l.end = end + + if cap(l.ap) < int(numArrayPos) { + l.ap = make([]uint64, int(numArrayPos)) + } else { + l.ap = l.ap[:int(numArrayPos)] + } + + // read off array positions + for k := 0; k < int(numArrayPos); k++ { + ap, err := i.locReader.readUvarint() + if err != nil { + return fmt.Errorf("error reading array position: %v", err) + } + + l.ap[k] = ap + } + + return nil +} + +// Next returns the next posting on the postings list, or nil at the end +func (i *PostingsIterator) Next() (segment.Posting, error) { + return i.nextAtOrAfter(0) +} + +// Advance returns the posting at the specified docNum or it is not present +// the next posting, or if the end is reached, nil +func (i *PostingsIterator) Advance(docNum uint64) (segment.Posting, error) { + return i.nextAtOrAfter(docNum) +} + +// Next returns the next posting on the postings list, or nil at the end +func (i *PostingsIterator) nextAtOrAfter(atOrAfter uint64) (segment.Posting, error) { + docNum, exists, err := i.nextDocNumAtOrAfter(atOrAfter) + if err != nil || !exists { + return nil, err + } + + i.next = Posting{} // clear the struct + rv := &i.next + rv.docNum = docNum + + if !i.includeFreqNorm { + return rv, nil + } + + var normBits uint64 + var hasLocs bool + + rv.freq, normBits, hasLocs, err = i.readFreqNormHasLocs() + if err != nil { + return nil, err + } + + rv.norm = math.Float32frombits(uint32(normBits)) + + if i.includeLocs && hasLocs { + // prepare locations into reused slices, where we assume + // rv.freq >= "number of locs", since in a composite field, + // some component fields might have their IncludeTermVector + // flags disabled while other component fields are enabled + if cap(i.nextLocs) >= int(rv.freq) { + i.nextLocs = i.nextLocs[0:rv.freq] + } else { + i.nextLocs = make([]Location, rv.freq, rv.freq*2) + } + if cap(i.nextSegmentLocs) < int(rv.freq) { + i.nextSegmentLocs = make([]segment.Location, rv.freq, rv.freq*2) + } + rv.locs = i.nextSegmentLocs[:0] + + numLocsBytes, err := i.locReader.readUvarint() + if err != nil { + return nil, fmt.Errorf("error reading location numLocsBytes: %v", err) + } + + j := 0 + startBytesRemaining := i.locReader.Len() // # bytes remaining in the locReader + for startBytesRemaining-i.locReader.Len() < int(numLocsBytes) { + err := i.readLocation(&i.nextLocs[j]) + if err != nil { + return nil, err + } + rv.locs = append(rv.locs, &i.nextLocs[j]) + j++ + } + } + + return rv, nil +} + +// nextDocNum returns the next docNum on the postings list, and also +// sets up the currChunk / loc related fields of the iterator. +func (i *PostingsIterator) nextDocNumAtOrAfter(atOrAfter uint64) (uint64, bool, error) { + if i.normBits1Hit != 0 { + if i.docNum1Hit == DocNum1HitFinished { + return 0, false, nil + } + if i.docNum1Hit < atOrAfter { + // advanced past our 1-hit + i.docNum1Hit = DocNum1HitFinished // consume our 1-hit docNum + return 0, false, nil + } + docNum := i.docNum1Hit + i.docNum1Hit = DocNum1HitFinished // consume our 1-hit docNum + return docNum, true, nil + } + + if i.Actual == nil || !i.Actual.HasNext() { + return 0, false, nil + } + + if i.postings == nil || i.postings == emptyPostingsList { + // couldn't find anything + return 0, false, nil + } + + if i.postings.postings == i.ActualBM { + return i.nextDocNumAtOrAfterClean(atOrAfter) + } + + i.Actual.AdvanceIfNeeded(uint32(atOrAfter)) + + if !i.Actual.HasNext() || !i.all.HasNext() { + // couldn't find anything + return 0, false, nil + } + + n := i.Actual.Next() + allN := i.all.Next() + + chunkSize, err := getChunkSize(i.postings.sb.chunkMode, i.postings.postings.GetCardinality(), i.postings.sb.numDocs) + if err != nil { + return 0, false, err + } + nChunk := n / uint32(chunkSize) + + // when allN becomes >= to here, then allN is in the same chunk as nChunk. + allNReachesNChunk := nChunk * uint32(chunkSize) + + // n is the next actual hit (excluding some postings), and + // allN is the next hit in the full postings, and + // if they don't match, move 'all' forwards until they do + for allN != n { + // we've reached same chunk, so move the freq/norm/loc decoders forward + if i.includeFreqNorm && allN >= allNReachesNChunk { + err := i.currChunkNext(nChunk) + if err != nil { + return 0, false, err + } + } + + if !i.all.HasNext() { + return 0, false, nil + } + + allN = i.all.Next() + } + + if i.includeFreqNorm && (i.currChunk != nChunk || i.freqNormReader.isNil()) { + err := i.loadChunk(int(nChunk)) + if err != nil { + return 0, false, fmt.Errorf("error loading chunk: %v", err) + } + } + + return uint64(n), true, nil +} + +// optimization when the postings list is "clean" (e.g., no updates & +// no deletions) where the all bitmap is the same as the actual bitmap +func (i *PostingsIterator) nextDocNumAtOrAfterClean( + atOrAfter uint64) (uint64, bool, error) { + if !i.includeFreqNorm { + i.Actual.AdvanceIfNeeded(uint32(atOrAfter)) + + if !i.Actual.HasNext() { + return 0, false, nil // couldn't find anything + } + + return uint64(i.Actual.Next()), true, nil + } + + chunkSize, err := getChunkSize(i.postings.sb.chunkMode, i.postings.postings.GetCardinality(), i.postings.sb.numDocs) + if err != nil { + return 0, false, err + } + + // freq-norm's needed, so maintain freq-norm chunk reader + sameChunkNexts := 0 // # of times we called Next() in the same chunk + n := i.Actual.Next() + nChunk := n / uint32(chunkSize) + + for uint64(n) < atOrAfter && i.Actual.HasNext() { + n = i.Actual.Next() + + nChunkPrev := nChunk + nChunk = n / uint32(chunkSize) + + if nChunk != nChunkPrev { + sameChunkNexts = 0 + } else { + sameChunkNexts += 1 + } + } + + if uint64(n) < atOrAfter { + // couldn't find anything + return 0, false, nil + } + + for j := 0; j < sameChunkNexts; j++ { + err := i.currChunkNext(nChunk) + if err != nil { + return 0, false, fmt.Errorf("error optimized currChunkNext: %v", err) + } + } + + if i.currChunk != nChunk || i.freqNormReader.isNil() { + err := i.loadChunk(int(nChunk)) + if err != nil { + return 0, false, fmt.Errorf("error loading chunk: %v", err) + } + } + + return uint64(n), true, nil +} + +func (i *PostingsIterator) currChunkNext(nChunk uint32) error { + if i.currChunk != nChunk || i.freqNormReader.isNil() { + err := i.loadChunk(int(nChunk)) + if err != nil { + return fmt.Errorf("error loading chunk: %v", err) + } + } + + // read off freq/offsets even though we don't care about them + hasLocs, err := i.skipFreqNormReadHasLocs() + if err != nil { + return err + } + + if i.includeLocs && hasLocs { + numLocsBytes, err := i.locReader.readUvarint() + if err != nil { + return fmt.Errorf("error reading location numLocsBytes: %v", err) + } + + // skip over all the location bytes + i.locReader.SkipBytes(int(numLocsBytes)) + } + + return nil +} + +// DocNum1Hit returns the docNum and true if this is "1-hit" optimized +// and the docNum is available. +func (p *PostingsIterator) DocNum1Hit() (uint64, bool) { + if p.normBits1Hit != 0 && p.docNum1Hit != DocNum1HitFinished { + return p.docNum1Hit, true + } + return 0, false +} + +// ActualBitmap returns the underlying actual bitmap +// which can be used up the stack for optimizations +func (p *PostingsIterator) ActualBitmap() *roaring.Bitmap { + return p.ActualBM +} + +// ReplaceActual replaces the ActualBM with the provided +// bitmap +func (p *PostingsIterator) ReplaceActual(abm *roaring.Bitmap) { + p.ActualBM = abm + p.Actual = abm.Iterator() +} + +// PostingsIteratorFromBitmap constructs a PostingsIterator given an +// "actual" bitmap. +func PostingsIteratorFromBitmap(bm *roaring.Bitmap, + includeFreqNorm, includeLocs bool) (segment.PostingsIterator, error) { + return &PostingsIterator{ + ActualBM: bm, + Actual: bm.Iterator(), + includeFreqNorm: includeFreqNorm, + includeLocs: includeLocs, + }, nil +} + +// PostingsIteratorFrom1Hit constructs a PostingsIterator given a +// 1-hit docNum. +func PostingsIteratorFrom1Hit(docNum1Hit uint64, + includeFreqNorm, includeLocs bool) (segment.PostingsIterator, error) { + return &PostingsIterator{ + docNum1Hit: docNum1Hit, + normBits1Hit: NormBits1Hit, + includeFreqNorm: includeFreqNorm, + includeLocs: includeLocs, + }, nil +} + +// Posting is a single entry in a postings list +type Posting struct { + docNum uint64 + freq uint64 + norm float32 + locs []segment.Location +} + +func (p *Posting) Size() int { + sizeInBytes := reflectStaticSizePosting + + for _, entry := range p.locs { + sizeInBytes += entry.Size() + } + + return sizeInBytes +} + +// Number returns the document number of this posting in this segment +func (p *Posting) Number() uint64 { + return p.docNum +} + +// Frequency returns the frequencies of occurrence of this term in this doc/field +func (p *Posting) Frequency() uint64 { + return p.freq +} + +// Norm returns the normalization factor for this posting +func (p *Posting) Norm() float64 { + return float64(p.norm) +} + +// Locations returns the location information for each occurrence +func (p *Posting) Locations() []segment.Location { + return p.locs +} + +// Location represents the location of a single occurrence +type Location struct { + field string + pos uint64 + start uint64 + end uint64 + ap []uint64 +} + +func (l *Location) Size() int { + return reflectStaticSizeLocation + + len(l.field) + + len(l.ap)*SizeOfUint64 +} + +// Field returns the name of the field (useful in composite fields to know +// which original field the value came from) +func (l *Location) Field() string { + return l.field +} + +// Start returns the start byte offset of this occurrence +func (l *Location) Start() uint64 { + return l.start +} + +// End returns the end byte offset of this occurrence +func (l *Location) End() uint64 { + return l.end +} + +// Pos returns the 1-based phrase position of this occurrence +func (l *Location) Pos() uint64 { + return l.pos +} + +// ArrayPositions returns the array position vector associated with this occurrence +func (l *Location) ArrayPositions() []uint64 { + return l.ap +} diff --git a/vendor/github.com/blevesearch/zap/v12/read.go b/vendor/github.com/blevesearch/zapx/v12/read.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v12/read.go rename to vendor/github.com/blevesearch/zapx/v12/read.go diff --git a/vendor/github.com/blevesearch/zapx/v12/segment.go b/vendor/github.com/blevesearch/zapx/v12/segment.go new file mode 100644 index 00000000..1fbf7848 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v12/segment.go @@ -0,0 +1,600 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "os" + "sync" + "unsafe" + + "github.com/RoaringBitmap/roaring" + mmap "github.com/blevesearch/mmap-go" + segment "github.com/blevesearch/scorch_segment_api/v2" + "github.com/blevesearch/vellum" + "github.com/golang/snappy" +) + +var reflectStaticSizeSegmentBase int + +func init() { + var sb SegmentBase + reflectStaticSizeSegmentBase = int(unsafe.Sizeof(sb)) +} + +// Open returns a zap impl of a segment +func (*ZapPlugin) Open(path string) (segment.Segment, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + mm, err := mmap.Map(f, mmap.RDONLY, 0) + if err != nil { + // mmap failed, try to close the file + _ = f.Close() + return nil, err + } + + rv := &Segment{ + SegmentBase: SegmentBase{ + mem: mm[0 : len(mm)-FooterSize], + fieldsMap: make(map[string]uint16), + fieldDvReaders: make(map[uint16]*docValueReader), + fieldFSTs: make(map[uint16]*vellum.FST), + }, + f: f, + mm: mm, + path: path, + refs: 1, + } + rv.SegmentBase.updateSize() + + err = rv.loadConfig() + if err != nil { + _ = rv.Close() + return nil, err + } + + err = rv.loadFields() + if err != nil { + _ = rv.Close() + return nil, err + } + + err = rv.loadDvReaders() + if err != nil { + _ = rv.Close() + return nil, err + } + + return rv, nil +} + +// SegmentBase is a memory only, read-only implementation of the +// segment.Segment interface, using zap's data representation. +type SegmentBase struct { + mem []byte + memCRC uint32 + chunkMode uint32 + fieldsMap map[string]uint16 // fieldName -> fieldID+1 + fieldsInv []string // fieldID -> fieldName + numDocs uint64 + storedIndexOffset uint64 + fieldsIndexOffset uint64 + docValueOffset uint64 + dictLocs []uint64 + fieldDvReaders map[uint16]*docValueReader // naive chunk cache per field + fieldDvNames []string // field names cached in fieldDvReaders + size uint64 + + m sync.Mutex + fieldFSTs map[uint16]*vellum.FST +} + +func (sb *SegmentBase) Size() int { + return int(sb.size) +} + +func (sb *SegmentBase) updateSize() { + sizeInBytes := reflectStaticSizeSegmentBase + + cap(sb.mem) + + // fieldsMap + for k := range sb.fieldsMap { + sizeInBytes += (len(k) + SizeOfString) + SizeOfUint16 + } + + // fieldsInv, dictLocs + for _, entry := range sb.fieldsInv { + sizeInBytes += len(entry) + SizeOfString + } + sizeInBytes += len(sb.dictLocs) * SizeOfUint64 + + // fieldDvReaders + for _, v := range sb.fieldDvReaders { + sizeInBytes += SizeOfUint16 + SizeOfPtr + if v != nil { + sizeInBytes += v.size() + } + } + + sb.size = uint64(sizeInBytes) +} + +func (sb *SegmentBase) AddRef() {} +func (sb *SegmentBase) DecRef() (err error) { return nil } +func (sb *SegmentBase) Close() (err error) { return nil } + +// Segment implements a persisted segment.Segment interface, by +// embedding an mmap()'ed SegmentBase. +type Segment struct { + SegmentBase + + f *os.File + mm mmap.MMap + path string + version uint32 + crc uint32 + + m sync.Mutex // Protects the fields that follow. + refs int64 +} + +func (s *Segment) Size() int { + // 8 /* size of file pointer */ + // 4 /* size of version -> uint32 */ + // 4 /* size of crc -> uint32 */ + sizeOfUints := 16 + + sizeInBytes := (len(s.path) + SizeOfString) + sizeOfUints + + // mutex, refs -> int64 + sizeInBytes += 16 + + // do not include the mmap'ed part + return sizeInBytes + s.SegmentBase.Size() - cap(s.mem) +} + +func (s *Segment) AddRef() { + s.m.Lock() + s.refs++ + s.m.Unlock() +} + +func (s *Segment) DecRef() (err error) { + s.m.Lock() + s.refs-- + if s.refs == 0 { + err = s.closeActual() + } + s.m.Unlock() + return err +} + +func (s *Segment) loadConfig() error { + crcOffset := len(s.mm) - 4 + s.crc = binary.BigEndian.Uint32(s.mm[crcOffset : crcOffset+4]) + + verOffset := crcOffset - 4 + s.version = binary.BigEndian.Uint32(s.mm[verOffset : verOffset+4]) + if s.version != Version { + return fmt.Errorf("unsupported version %d", s.version) + } + + chunkOffset := verOffset - 4 + s.chunkMode = binary.BigEndian.Uint32(s.mm[chunkOffset : chunkOffset+4]) + + docValueOffset := chunkOffset - 8 + s.docValueOffset = binary.BigEndian.Uint64(s.mm[docValueOffset : docValueOffset+8]) + + fieldsIndexOffset := docValueOffset - 8 + s.fieldsIndexOffset = binary.BigEndian.Uint64(s.mm[fieldsIndexOffset : fieldsIndexOffset+8]) + + storedIndexOffset := fieldsIndexOffset - 8 + s.storedIndexOffset = binary.BigEndian.Uint64(s.mm[storedIndexOffset : storedIndexOffset+8]) + + numDocsOffset := storedIndexOffset - 8 + s.numDocs = binary.BigEndian.Uint64(s.mm[numDocsOffset : numDocsOffset+8]) + return nil +} + +// Following methods are dummy implementations for the interface +// DiskStatsReporter (for backward compatibility). +// The working implementations are supported only in zap v15.x +// and not in the earlier versions of zap. +func (s *Segment) ResetBytesRead(uint64) {} + +func (s *Segment) BytesRead() uint64 { + return 0 +} + +func (s *Segment) BytesWritten() uint64 { + return 0 +} + +func (s *Segment) incrementBytesRead(uint64) {} + +func (s *SegmentBase) BytesWritten() uint64 { + return 0 +} + +func (s *SegmentBase) setBytesWritten(uint64) {} + +func (s *SegmentBase) BytesRead() uint64 { + return 0 +} + +func (s *SegmentBase) ResetBytesRead(uint64) {} + +func (s *SegmentBase) incrementBytesRead(uint64) {} + +func (s *SegmentBase) loadFields() error { + // NOTE for now we assume the fields index immediately precedes + // the footer, and if this changes, need to adjust accordingly (or + // store explicit length), where s.mem was sliced from s.mm in Open(). + fieldsIndexEnd := uint64(len(s.mem)) + + // iterate through fields index + var fieldID uint64 + for s.fieldsIndexOffset+(8*fieldID) < fieldsIndexEnd { + addr := binary.BigEndian.Uint64(s.mem[s.fieldsIndexOffset+(8*fieldID) : s.fieldsIndexOffset+(8*fieldID)+8]) + + dictLoc, read := binary.Uvarint(s.mem[addr:fieldsIndexEnd]) + n := uint64(read) + s.dictLocs = append(s.dictLocs, dictLoc) + + var nameLen uint64 + nameLen, read = binary.Uvarint(s.mem[addr+n : fieldsIndexEnd]) + n += uint64(read) + + name := string(s.mem[addr+n : addr+n+nameLen]) + s.fieldsInv = append(s.fieldsInv, name) + s.fieldsMap[name] = uint16(fieldID + 1) + + fieldID++ + } + return nil +} + +// Dictionary returns the term dictionary for the specified field +func (s *SegmentBase) Dictionary(field string) (segment.TermDictionary, error) { + dict, err := s.dictionary(field) + if err == nil && dict == nil { + return emptyDictionary, nil + } + return dict, err +} + +func (sb *SegmentBase) dictionary(field string) (rv *Dictionary, err error) { + fieldIDPlus1 := sb.fieldsMap[field] + if fieldIDPlus1 > 0 { + rv = &Dictionary{ + sb: sb, + field: field, + fieldID: fieldIDPlus1 - 1, + } + + dictStart := sb.dictLocs[rv.fieldID] + if dictStart > 0 { + var ok bool + sb.m.Lock() + if rv.fst, ok = sb.fieldFSTs[rv.fieldID]; !ok { + // read the length of the vellum data + vellumLen, read := binary.Uvarint(sb.mem[dictStart : dictStart+binary.MaxVarintLen64]) + fstBytes := sb.mem[dictStart+uint64(read) : dictStart+uint64(read)+vellumLen] + rv.fst, err = vellum.Load(fstBytes) + if err != nil { + sb.m.Unlock() + return nil, fmt.Errorf("dictionary field %s vellum err: %v", field, err) + } + + sb.fieldFSTs[rv.fieldID] = rv.fst + } + + sb.m.Unlock() + rv.fstReader, err = rv.fst.Reader() + if err != nil { + return nil, fmt.Errorf("dictionary field %s vellum reader err: %v", field, err) + } + } + } + + return rv, nil +} + +// visitDocumentCtx holds data structures that are reusable across +// multiple VisitDocument() calls to avoid memory allocations +type visitDocumentCtx struct { + buf []byte + reader bytes.Reader + arrayPos []uint64 +} + +var visitDocumentCtxPool = sync.Pool{ + New: func() interface{} { + reuse := &visitDocumentCtx{} + return reuse + }, +} + +// VisitStoredFields invokes the StoredFieldValueVisitor for each stored field +// for the specified doc number +func (s *SegmentBase) VisitStoredFields(num uint64, visitor segment.StoredFieldValueVisitor) error { + vdc := visitDocumentCtxPool.Get().(*visitDocumentCtx) + defer visitDocumentCtxPool.Put(vdc) + return s.visitStoredFields(vdc, num, visitor) +} + +func (s *SegmentBase) visitStoredFields(vdc *visitDocumentCtx, num uint64, + visitor segment.StoredFieldValueVisitor) error { + // first make sure this is a valid number in this segment + if num < s.numDocs { + meta, compressed := s.getDocStoredMetaAndCompressed(num) + + vdc.reader.Reset(meta) + + // handle _id field special case + idFieldValLen, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return err + } + idFieldVal := compressed[:idFieldValLen] + + keepGoing := visitor("_id", byte('t'), idFieldVal, nil) + if !keepGoing { + visitDocumentCtxPool.Put(vdc) + return nil + } + + // handle non-"_id" fields + compressed = compressed[idFieldValLen:] + + uncompressed, err := snappy.Decode(vdc.buf[:cap(vdc.buf)], compressed) + if err != nil { + return err + } + + for keepGoing { + field, err := binary.ReadUvarint(&vdc.reader) + if err == io.EOF { + break + } + if err != nil { + return err + } + typ, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return err + } + offset, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return err + } + l, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return err + } + numap, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return err + } + var arrayPos []uint64 + if numap > 0 { + if cap(vdc.arrayPos) < int(numap) { + vdc.arrayPos = make([]uint64, numap) + } + arrayPos = vdc.arrayPos[:numap] + for i := 0; i < int(numap); i++ { + ap, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return err + } + arrayPos[i] = ap + } + } + + value := uncompressed[offset : offset+l] + keepGoing = visitor(s.fieldsInv[field], byte(typ), value, arrayPos) + } + + vdc.buf = uncompressed + } + return nil +} + +// DocID returns the value of the _id field for the given docNum +func (s *SegmentBase) DocID(num uint64) ([]byte, error) { + if num >= s.numDocs { + return nil, nil + } + + vdc := visitDocumentCtxPool.Get().(*visitDocumentCtx) + + meta, compressed := s.getDocStoredMetaAndCompressed(num) + + vdc.reader.Reset(meta) + + // handle _id field special case + idFieldValLen, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return nil, err + } + idFieldVal := compressed[:idFieldValLen] + + visitDocumentCtxPool.Put(vdc) + + return idFieldVal, nil +} + +// Count returns the number of documents in this segment. +func (s *SegmentBase) Count() uint64 { + return s.numDocs +} + +// DocNumbers returns a bitset corresponding to the doc numbers of all the +// provided _id strings +func (s *SegmentBase) DocNumbers(ids []string) (*roaring.Bitmap, error) { + rv := roaring.New() + + if len(s.fieldsMap) > 0 { + idDict, err := s.dictionary("_id") + if err != nil { + return nil, err + } + + postingsList := emptyPostingsList + + sMax, err := idDict.fst.GetMaxKey() + if err != nil { + return nil, err + } + sMaxStr := string(sMax) + filteredIds := make([]string, 0, len(ids)) + for _, id := range ids { + if id <= sMaxStr { + filteredIds = append(filteredIds, id) + } + } + + for _, id := range filteredIds { + postingsList, err = idDict.postingsList([]byte(id), nil, postingsList) + if err != nil { + return nil, err + } + postingsList.OrInto(rv) + } + } + + return rv, nil +} + +// Fields returns the field names used in this segment +func (s *SegmentBase) Fields() []string { + return s.fieldsInv +} + +// Path returns the path of this segment on disk +func (s *Segment) Path() string { + return s.path +} + +// Close releases all resources associated with this segment +func (s *Segment) Close() (err error) { + return s.DecRef() +} + +func (s *Segment) closeActual() (err error) { + if s.mm != nil { + err = s.mm.Unmap() + } + // try to close file even if unmap failed + if s.f != nil { + err2 := s.f.Close() + if err == nil { + // try to return first error + err = err2 + } + } + return +} + +// some helpers i started adding for the command-line utility + +// Data returns the underlying mmaped data slice +func (s *Segment) Data() []byte { + return s.mm +} + +// CRC returns the CRC value stored in the file footer +func (s *Segment) CRC() uint32 { + return s.crc +} + +// Version returns the file version in the file footer +func (s *Segment) Version() uint32 { + return s.version +} + +// ChunkFactor returns the chunk factor in the file footer +func (s *Segment) ChunkMode() uint32 { + return s.chunkMode +} + +// FieldsIndexOffset returns the fields index offset in the file footer +func (s *Segment) FieldsIndexOffset() uint64 { + return s.fieldsIndexOffset +} + +// StoredIndexOffset returns the stored value index offset in the file footer +func (s *Segment) StoredIndexOffset() uint64 { + return s.storedIndexOffset +} + +// DocValueOffset returns the docValue offset in the file footer +func (s *Segment) DocValueOffset() uint64 { + return s.docValueOffset +} + +// NumDocs returns the number of documents in the file footer +func (s *Segment) NumDocs() uint64 { + return s.numDocs +} + +// DictAddr is a helper function to compute the file offset where the +// dictionary is stored for the specified field. +func (s *Segment) DictAddr(field string) (uint64, error) { + fieldIDPlus1, ok := s.fieldsMap[field] + if !ok { + return 0, fmt.Errorf("no such field '%s'", field) + } + + return s.dictLocs[fieldIDPlus1-1], nil +} + +func (s *SegmentBase) loadDvReaders() error { + if s.docValueOffset == fieldNotUninverted || s.numDocs == 0 { + return nil + } + + var read uint64 + for fieldID, field := range s.fieldsInv { + var fieldLocStart, fieldLocEnd uint64 + var n int + fieldLocStart, n = binary.Uvarint(s.mem[s.docValueOffset+read : s.docValueOffset+read+binary.MaxVarintLen64]) + if n <= 0 { + return fmt.Errorf("loadDvReaders: failed to read the docvalue offset start for field %d", fieldID) + } + read += uint64(n) + fieldLocEnd, n = binary.Uvarint(s.mem[s.docValueOffset+read : s.docValueOffset+read+binary.MaxVarintLen64]) + if n <= 0 { + return fmt.Errorf("loadDvReaders: failed to read the docvalue offset end for field %d", fieldID) + } + read += uint64(n) + + fieldDvReader, err := s.loadFieldDocValueReader(field, fieldLocStart, fieldLocEnd) + if err != nil { + return err + } + if fieldDvReader != nil { + s.fieldDvReaders[uint16(fieldID)] = fieldDvReader + s.fieldDvNames = append(s.fieldDvNames, field) + } + } + + return nil +} diff --git a/vendor/github.com/blevesearch/zapx/v12/sizes.go b/vendor/github.com/blevesearch/zapx/v12/sizes.go new file mode 100644 index 00000000..34166ea3 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v12/sizes.go @@ -0,0 +1,59 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "reflect" +) + +func init() { + var b bool + SizeOfBool = int(reflect.TypeOf(b).Size()) + var f32 float32 + SizeOfFloat32 = int(reflect.TypeOf(f32).Size()) + var f64 float64 + SizeOfFloat64 = int(reflect.TypeOf(f64).Size()) + var i int + SizeOfInt = int(reflect.TypeOf(i).Size()) + var m map[int]int + SizeOfMap = int(reflect.TypeOf(m).Size()) + var ptr *int + SizeOfPtr = int(reflect.TypeOf(ptr).Size()) + var slice []int + SizeOfSlice = int(reflect.TypeOf(slice).Size()) + var str string + SizeOfString = int(reflect.TypeOf(str).Size()) + var u8 uint8 + SizeOfUint8 = int(reflect.TypeOf(u8).Size()) + var u16 uint16 + SizeOfUint16 = int(reflect.TypeOf(u16).Size()) + var u32 uint32 + SizeOfUint32 = int(reflect.TypeOf(u32).Size()) + var u64 uint64 + SizeOfUint64 = int(reflect.TypeOf(u64).Size()) +} + +var SizeOfBool int +var SizeOfFloat32 int +var SizeOfFloat64 int +var SizeOfInt int +var SizeOfMap int +var SizeOfPtr int +var SizeOfSlice int +var SizeOfString int +var SizeOfUint8 int +var SizeOfUint16 int +var SizeOfUint32 int +var SizeOfUint64 int diff --git a/vendor/github.com/blevesearch/zap/v12/write.go b/vendor/github.com/blevesearch/zapx/v12/write.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v12/write.go rename to vendor/github.com/blevesearch/zapx/v12/write.go diff --git a/vendor/github.com/blevesearch/zap/v12/zap.md b/vendor/github.com/blevesearch/zapx/v12/zap.md similarity index 100% rename from vendor/github.com/blevesearch/zap/v12/zap.md rename to vendor/github.com/blevesearch/zapx/v12/zap.md diff --git a/vendor/github.com/blevesearch/zap/v13/.gitignore b/vendor/github.com/blevesearch/zapx/v13/.gitignore similarity index 100% rename from vendor/github.com/blevesearch/zap/v13/.gitignore rename to vendor/github.com/blevesearch/zapx/v13/.gitignore diff --git a/vendor/github.com/blevesearch/zapx/v13/.golangci.yml b/vendor/github.com/blevesearch/zapx/v13/.golangci.yml new file mode 100644 index 00000000..1d55bfc0 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v13/.golangci.yml @@ -0,0 +1,28 @@ +linters: + # please, do not use `enable-all`: it's deprecated and will be removed soon. + # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint + disable-all: true + enable: + - bodyclose + - deadcode + - depguard + - dupl + - errcheck + - gofmt + - goimports + - goprintffuncname + - gosec + - gosimple + - govet + - ineffassign + - misspell + - nakedret + - nolintlint + - rowserrcheck + - staticcheck + - structcheck + - typecheck + - unused + - varcheck + - whitespace + diff --git a/vendor/github.com/blevesearch/zapx/v13/LICENSE b/vendor/github.com/blevesearch/zapx/v13/LICENSE new file mode 100644 index 00000000..7a4a3ea2 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v13/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/vendor/github.com/blevesearch/zapx/v13/README.md b/vendor/github.com/blevesearch/zapx/v13/README.md new file mode 100644 index 00000000..4cbf1a14 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v13/README.md @@ -0,0 +1,163 @@ +# zapx file format + +The zapx module is fork of [zap](https://github.com/blevesearch/zap) module which maintains file format compatibility, but removes dependency on bleve, and instead depends only on the indepenent interface modules: + +- [bleve_index_api](https://github.com/blevesearch/scorch_segment_api) +- [scorch_segment_api](https://github.com/blevesearch/scorch_segment_api) + +Advanced ZAP File Format Documentation is [here](zap.md). + +The file is written in the reverse order that we typically access data. This helps us write in one pass since later sections of the file require file offsets of things we've already written. + +Current usage: + +- mmap the entire file +- crc-32 bytes and version are in fixed position at end of the file +- reading remainder of footer could be version specific +- remainder of footer gives us: + - 3 important offsets (docValue , fields index and stored data index) + - 2 important values (number of docs and chunk factor) +- field data is processed once and memoized onto the heap so that we never have to go back to disk for it +- access to stored data by doc number means first navigating to the stored data index, then accessing a fixed position offset into that slice, which gives us the actual address of the data. the first bytes of that section tell us the size of data so that we know where it ends. +- access to all other indexed data follows the following pattern: + - first know the field name -> convert to id + - next navigate to term dictionary for that field + - some operations stop here and do dictionary ops + - next use dictionary to navigate to posting list for a specific term + - walk posting list + - if necessary, walk posting details as we go + - if location info is desired, consult location bitmap to see if it is there + +## stored fields section + +- for each document + - preparation phase: + - produce a slice of metadata bytes and data bytes + - produce these slices in field id order + - field value is appended to the data slice + - metadata slice is varint encoded with the following values for each field value + - field id (uint16) + - field type (byte) + - field value start offset in uncompressed data slice (uint64) + - field value length (uint64) + - field number of array positions (uint64) + - one additional value for each array position (uint64) + - compress the data slice using snappy + - file writing phase: + - remember the start offset for this document + - write out meta data length (varint uint64) + - write out compressed data length (varint uint64) + - write out the metadata bytes + - write out the compressed data bytes + +## stored fields idx + +- for each document + - write start offset (remembered from previous section) of stored data (big endian uint64) + +With this index and a known document number, we have direct access to all the stored field data. + +## posting details (freq/norm) section + +- for each posting list + - produce a slice containing multiple consecutive chunks (each chunk is varint stream) + - produce a slice remembering offsets of where each chunk starts + - preparation phase: + - for each hit in the posting list + - if this hit is in next chunk close out encoding of last chunk and record offset start of next + - encode term frequency (uint64) + - encode norm factor (float32) + - file writing phase: + - remember start position for this posting list details + - write out number of chunks that follow (varint uint64) + - write out length of each chunk (each a varint uint64) + - write out the byte slice containing all the chunk data + +If you know the doc number you're interested in, this format lets you jump to the correct chunk (docNum/chunkFactor) directly and then seek within that chunk until you find it. + +## posting details (location) section + +- for each posting list + - produce a slice containing multiple consecutive chunks (each chunk is varint stream) + - produce a slice remembering offsets of where each chunk starts + - preparation phase: + - for each hit in the posting list + - if this hit is in next chunk close out encoding of last chunk and record offset start of next + - encode field (uint16) + - encode field pos (uint64) + - encode field start (uint64) + - encode field end (uint64) + - encode number of array positions to follow (uint64) + - encode each array position (each uint64) + - file writing phase: + - remember start position for this posting list details + - write out number of chunks that follow (varint uint64) + - write out length of each chunk (each a varint uint64) + - write out the byte slice containing all the chunk data + +If you know the doc number you're interested in, this format lets you jump to the correct chunk (docNum/chunkFactor) directly and then seek within that chunk until you find it. + +## postings list section + +- for each posting list + - preparation phase: + - encode roaring bitmap posting list to bytes (so we know the length) + - file writing phase: + - remember the start position for this posting list + - write freq/norm details offset (remembered from previous, as varint uint64) + - write location details offset (remembered from previous, as varint uint64) + - write length of encoded roaring bitmap + - write the serialized roaring bitmap data + +## dictionary + +- for each field + - preparation phase: + - encode vellum FST with dictionary data pointing to file offset of posting list (remembered from previous) + - file writing phase: + - remember the start position of this persistDictionary + - write length of vellum data (varint uint64) + - write out vellum data + +## fields section + +- for each field + - file writing phase: + - remember start offset for each field + - write dictionary address (remembered from previous) (varint uint64) + - write length of field name (varint uint64) + - write field name bytes + +## fields idx + +- for each field + - file writing phase: + - write big endian uint64 of start offset for each field + +NOTE: currently we don't know or record the length of this fields index. Instead we rely on the fact that we know it immediately precedes a footer of known size. + +## fields DocValue + +- for each field + - preparation phase: + - produce a slice containing multiple consecutive chunks, where each chunk is composed of a meta section followed by compressed columnar field data + - produce a slice remembering the length of each chunk + - file writing phase: + - remember the start position of this first field DocValue offset in the footer + - write out number of chunks that follow (varint uint64) + - write out length of each chunk (each a varint uint64) + - write out the byte slice containing all the chunk data + +NOTE: currently the meta header inside each chunk gives clue to the location offsets and size of the data pertaining to a given docID and any +read operation leverage that meta information to extract the document specific data from the file. + +## footer + +- file writing phase + - write number of docs (big endian uint64) + - write stored field index location (big endian uint64) + - write field index location (big endian uint64) + - write field docValue location (big endian uint64) + - write out chunk factor (big endian uint32) + - write out version (big endian uint32) + - write out file CRC of everything preceding this (big endian uint32) diff --git a/vendor/github.com/blevesearch/zapx/v13/build.go b/vendor/github.com/blevesearch/zapx/v13/build.go new file mode 100644 index 00000000..827e5c47 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v13/build.go @@ -0,0 +1,186 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "bufio" + "fmt" + "io" + "math" + "os" + + "github.com/blevesearch/vellum" +) + +const Version uint32 = 13 + +const Type string = "zap" + +const fieldNotUninverted = math.MaxUint64 + +func (sb *SegmentBase) Persist(path string) error { + return PersistSegmentBase(sb, path) +} + +// WriteTo is an implementation of io.WriterTo interface. +func (sb *SegmentBase) WriteTo(w io.Writer) (int64, error) { + if w == nil { + return 0, fmt.Errorf("invalid writer found") + } + + n, err := persistSegmentBaseToWriter(sb, w) + return int64(n), err +} + +// PersistSegmentBase persists SegmentBase in the zap file format. +func PersistSegmentBase(sb *SegmentBase, path string) error { + flag := os.O_RDWR | os.O_CREATE + + f, err := os.OpenFile(path, flag, 0600) + if err != nil { + return err + } + + cleanup := func() { + _ = f.Close() + _ = os.Remove(path) + } + + _, err = persistSegmentBaseToWriter(sb, f) + if err != nil { + cleanup() + return err + } + + err = f.Sync() + if err != nil { + cleanup() + return err + } + + err = f.Close() + if err != nil { + cleanup() + return err + } + + return err +} + +type bufWriter struct { + w *bufio.Writer + n int +} + +func (br *bufWriter) Write(in []byte) (int, error) { + n, err := br.w.Write(in) + br.n += n + return n, err +} + +func persistSegmentBaseToWriter(sb *SegmentBase, w io.Writer) (int, error) { + br := &bufWriter{w: bufio.NewWriter(w)} + + _, err := br.Write(sb.mem) + if err != nil { + return 0, err + } + + err = persistFooter(sb.numDocs, sb.storedIndexOffset, sb.fieldsIndexOffset, + sb.docValueOffset, sb.chunkMode, sb.memCRC, br) + if err != nil { + return 0, err + } + + err = br.w.Flush() + if err != nil { + return 0, err + } + + return br.n, nil +} + +func persistStoredFieldValues(fieldID int, + storedFieldValues [][]byte, stf []byte, spf [][]uint64, + curr int, metaEncode varintEncoder, data []byte) ( + int, []byte, error) { + for i := 0; i < len(storedFieldValues); i++ { + // encode field + _, err := metaEncode(uint64(fieldID)) + if err != nil { + return 0, nil, err + } + // encode type + _, err = metaEncode(uint64(stf[i])) + if err != nil { + return 0, nil, err + } + // encode start offset + _, err = metaEncode(uint64(curr)) + if err != nil { + return 0, nil, err + } + // end len + _, err = metaEncode(uint64(len(storedFieldValues[i]))) + if err != nil { + return 0, nil, err + } + // encode number of array pos + _, err = metaEncode(uint64(len(spf[i]))) + if err != nil { + return 0, nil, err + } + // encode all array positions + for _, pos := range spf[i] { + _, err = metaEncode(pos) + if err != nil { + return 0, nil, err + } + } + + data = append(data, storedFieldValues[i]...) + curr += len(storedFieldValues[i]) + } + + return curr, data, nil +} + +func InitSegmentBase(mem []byte, memCRC uint32, chunkMode uint32, + fieldsMap map[string]uint16, fieldsInv []string, numDocs uint64, + storedIndexOffset uint64, fieldsIndexOffset uint64, docValueOffset uint64, + dictLocs []uint64) (*SegmentBase, error) { + sb := &SegmentBase{ + mem: mem, + memCRC: memCRC, + chunkMode: chunkMode, + fieldsMap: fieldsMap, + fieldsInv: fieldsInv, + numDocs: numDocs, + storedIndexOffset: storedIndexOffset, + fieldsIndexOffset: fieldsIndexOffset, + docValueOffset: docValueOffset, + dictLocs: dictLocs, + fieldDvReaders: make(map[uint16]*docValueReader), + fieldFSTs: make(map[uint16]*vellum.FST), + } + sb.updateSize() + + err := sb.loadDvReaders() + if err != nil { + return nil, err + } + + return sb, nil +} diff --git a/vendor/github.com/blevesearch/zap/v13/chunk.go b/vendor/github.com/blevesearch/zapx/v13/chunk.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v13/chunk.go rename to vendor/github.com/blevesearch/zapx/v13/chunk.go diff --git a/vendor/github.com/blevesearch/zap/v13/contentcoder.go b/vendor/github.com/blevesearch/zapx/v13/contentcoder.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v13/contentcoder.go rename to vendor/github.com/blevesearch/zapx/v13/contentcoder.go diff --git a/vendor/github.com/blevesearch/zapx/v13/count.go b/vendor/github.com/blevesearch/zapx/v13/count.go new file mode 100644 index 00000000..b6135359 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v13/count.go @@ -0,0 +1,61 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "hash/crc32" + "io" + + segment "github.com/blevesearch/scorch_segment_api/v2" +) + +// CountHashWriter is a wrapper around a Writer which counts the number of +// bytes which have been written and computes a crc32 hash +type CountHashWriter struct { + w io.Writer + crc uint32 + n int + s segment.StatsReporter +} + +// NewCountHashWriter returns a CountHashWriter which wraps the provided Writer +func NewCountHashWriter(w io.Writer) *CountHashWriter { + return &CountHashWriter{w: w} +} + +func NewCountHashWriterWithStatsReporter(w io.Writer, s segment.StatsReporter) *CountHashWriter { + return &CountHashWriter{w: w, s: s} +} + +// Write writes the provided bytes to the wrapped writer and counts the bytes +func (c *CountHashWriter) Write(b []byte) (int, error) { + n, err := c.w.Write(b) + c.crc = crc32.Update(c.crc, crc32.IEEETable, b[:n]) + c.n += n + if c.s != nil { + c.s.ReportBytesWritten(uint64(n)) + } + return n, err +} + +// Count returns the number of bytes written +func (c *CountHashWriter) Count() int { + return c.n +} + +// Sum32 returns the CRC-32 hash of the content written to this writer +func (c *CountHashWriter) Sum32() uint32 { + return c.crc +} diff --git a/vendor/github.com/blevesearch/zapx/v13/dict.go b/vendor/github.com/blevesearch/zapx/v13/dict.go new file mode 100644 index 00000000..e30bf242 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v13/dict.go @@ -0,0 +1,158 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "fmt" + + "github.com/RoaringBitmap/roaring" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" + "github.com/blevesearch/vellum" +) + +// Dictionary is the zap representation of the term dictionary +type Dictionary struct { + sb *SegmentBase + field string + fieldID uint16 + fst *vellum.FST + fstReader *vellum.Reader +} + +// represents an immutable, empty dictionary +var emptyDictionary = &Dictionary{} + +// PostingsList returns the postings list for the specified term +func (d *Dictionary) PostingsList(term []byte, except *roaring.Bitmap, + prealloc segment.PostingsList) (segment.PostingsList, error) { + var preallocPL *PostingsList + pl, ok := prealloc.(*PostingsList) + if ok && pl != nil { + preallocPL = pl + } + return d.postingsList(term, except, preallocPL) +} + +func (d *Dictionary) postingsList(term []byte, except *roaring.Bitmap, rv *PostingsList) (*PostingsList, error) { + if d.fstReader == nil { + if rv == nil || rv == emptyPostingsList { + return emptyPostingsList, nil + } + return d.postingsListInit(rv, except), nil + } + + postingsOffset, exists, err := d.fstReader.Get(term) + if err != nil { + return nil, fmt.Errorf("vellum err: %v", err) + } + if !exists { + if rv == nil || rv == emptyPostingsList { + return emptyPostingsList, nil + } + return d.postingsListInit(rv, except), nil + } + + return d.postingsListFromOffset(postingsOffset, except, rv) +} + +func (d *Dictionary) postingsListFromOffset(postingsOffset uint64, except *roaring.Bitmap, rv *PostingsList) (*PostingsList, error) { + rv = d.postingsListInit(rv, except) + + err := rv.read(postingsOffset, d) + if err != nil { + return nil, err + } + + return rv, nil +} + +func (d *Dictionary) postingsListInit(rv *PostingsList, except *roaring.Bitmap) *PostingsList { + if rv == nil || rv == emptyPostingsList { + rv = &PostingsList{} + } else { + postings := rv.postings + if postings != nil { + postings.Clear() + } + + *rv = PostingsList{} // clear the struct + + rv.postings = postings + } + rv.sb = d.sb + rv.except = except + return rv +} + +func (d *Dictionary) Contains(key []byte) (bool, error) { + if d.fst != nil { + return d.fst.Contains(key) + } + return false, nil +} + +// AutomatonIterator returns an iterator which only visits terms +// having the the vellum automaton and start/end key range +func (d *Dictionary) AutomatonIterator(a segment.Automaton, + startKeyInclusive, endKeyExclusive []byte) segment.DictionaryIterator { + if d.fst != nil { + rv := &DictionaryIterator{ + d: d, + } + + itr, err := d.fst.Search(a, startKeyInclusive, endKeyExclusive) + if err == nil { + rv.itr = itr + } else if err != vellum.ErrIteratorDone { + rv.err = err + } + + return rv + } + return emptyDictionaryIterator +} + +// DictionaryIterator is an iterator for term dictionary +type DictionaryIterator struct { + d *Dictionary + itr vellum.Iterator + err error + tmp PostingsList + entry index.DictEntry + omitCount bool +} + +var emptyDictionaryIterator = &DictionaryIterator{} + +// Next returns the next entry in the dictionary +func (i *DictionaryIterator) Next() (*index.DictEntry, error) { + if i.err != nil && i.err != vellum.ErrIteratorDone { + return nil, i.err + } else if i.itr == nil || i.err == vellum.ErrIteratorDone { + return nil, nil + } + term, postingsOffset := i.itr.Current() + i.entry.Term = string(term) + if !i.omitCount { + i.err = i.tmp.read(postingsOffset, i.d) + if i.err != nil { + return nil, i.err + } + i.entry.Count = i.tmp.Count() + } + i.err = i.itr.Next() + return &i.entry, nil +} diff --git a/vendor/github.com/blevesearch/zapx/v13/docvalues.go b/vendor/github.com/blevesearch/zapx/v13/docvalues.go new file mode 100644 index 00000000..a36485c8 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v13/docvalues.go @@ -0,0 +1,323 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "bytes" + "encoding/binary" + "fmt" + "math" + "reflect" + "sort" + + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" + "github.com/golang/snappy" +) + +var reflectStaticSizedocValueReader int + +func init() { + var dvi docValueReader + reflectStaticSizedocValueReader = int(reflect.TypeOf(dvi).Size()) +} + +type docNumTermsVisitor func(docNum uint64, terms []byte) error + +type docVisitState struct { + dvrs map[uint16]*docValueReader + segment *SegmentBase +} + +// No-op implementations for DiskStatsReporter interface. +// Supported only in v15 +func (d *docVisitState) BytesRead() uint64 { + return 0 +} + +func (d *docVisitState) BytesWritten() uint64 { + return 0 +} + +func (d *docVisitState) ResetBytesRead(val uint64) {} + +type docValueReader struct { + field string + curChunkNum uint64 + chunkOffsets []uint64 + dvDataLoc uint64 + curChunkHeader []MetaData + curChunkData []byte // compressed data cache + uncompressed []byte // temp buf for snappy decompression +} + +func (di *docValueReader) size() int { + return reflectStaticSizedocValueReader + SizeOfPtr + + len(di.field) + + len(di.chunkOffsets)*SizeOfUint64 + + len(di.curChunkHeader)*reflectStaticSizeMetaData + + len(di.curChunkData) +} + +func (di *docValueReader) cloneInto(rv *docValueReader) *docValueReader { + if rv == nil { + rv = &docValueReader{} + } + + rv.field = di.field + rv.curChunkNum = math.MaxUint64 + rv.chunkOffsets = di.chunkOffsets // immutable, so it's sharable + rv.dvDataLoc = di.dvDataLoc + rv.curChunkHeader = rv.curChunkHeader[:0] + rv.curChunkData = nil + rv.uncompressed = rv.uncompressed[:0] + + return rv +} + +func (di *docValueReader) curChunkNumber() uint64 { + return di.curChunkNum +} + +func (s *SegmentBase) loadFieldDocValueReader(field string, + fieldDvLocStart, fieldDvLocEnd uint64) (*docValueReader, error) { + // get the docValue offset for the given fields + if fieldDvLocStart == fieldNotUninverted { + // no docValues found, nothing to do + return nil, nil + } + + // read the number of chunks, and chunk offsets position + var numChunks, chunkOffsetsPosition uint64 + + if fieldDvLocEnd-fieldDvLocStart > 16 { + numChunks = binary.BigEndian.Uint64(s.mem[fieldDvLocEnd-8 : fieldDvLocEnd]) + // read the length of chunk offsets + chunkOffsetsLen := binary.BigEndian.Uint64(s.mem[fieldDvLocEnd-16 : fieldDvLocEnd-8]) + // acquire position of chunk offsets + chunkOffsetsPosition = (fieldDvLocEnd - 16) - chunkOffsetsLen + } else { + return nil, fmt.Errorf("loadFieldDocValueReader: fieldDvLoc too small: %d-%d", fieldDvLocEnd, fieldDvLocStart) + } + + fdvIter := &docValueReader{ + curChunkNum: math.MaxUint64, + field: field, + chunkOffsets: make([]uint64, int(numChunks)), + } + + // read the chunk offsets + var offset uint64 + for i := 0; i < int(numChunks); i++ { + loc, read := binary.Uvarint(s.mem[chunkOffsetsPosition+offset : chunkOffsetsPosition+offset+binary.MaxVarintLen64]) + if read <= 0 { + return nil, fmt.Errorf("corrupted chunk offset during segment load") + } + fdvIter.chunkOffsets[i] = loc + offset += uint64(read) + } + + // set the data offset + fdvIter.dvDataLoc = fieldDvLocStart + + return fdvIter, nil +} + +func (di *docValueReader) loadDvChunk(chunkNumber uint64, s *SegmentBase) error { + // advance to the chunk where the docValues + // reside for the given docNum + destChunkDataLoc, curChunkEnd := di.dvDataLoc, di.dvDataLoc + start, end := readChunkBoundary(int(chunkNumber), di.chunkOffsets) + if start >= end { + di.curChunkHeader = di.curChunkHeader[:0] + di.curChunkData = nil + di.curChunkNum = chunkNumber + di.uncompressed = di.uncompressed[:0] + return nil + } + + destChunkDataLoc += start + curChunkEnd += end + + // read the number of docs reside in the chunk + numDocs, read := binary.Uvarint(s.mem[destChunkDataLoc : destChunkDataLoc+binary.MaxVarintLen64]) + if read <= 0 { + return fmt.Errorf("failed to read the chunk") + } + chunkMetaLoc := destChunkDataLoc + uint64(read) + + offset := uint64(0) + if cap(di.curChunkHeader) < int(numDocs) { + di.curChunkHeader = make([]MetaData, int(numDocs)) + } else { + di.curChunkHeader = di.curChunkHeader[:int(numDocs)] + } + for i := 0; i < int(numDocs); i++ { + di.curChunkHeader[i].DocNum, read = binary.Uvarint(s.mem[chunkMetaLoc+offset : chunkMetaLoc+offset+binary.MaxVarintLen64]) + offset += uint64(read) + di.curChunkHeader[i].DocDvOffset, read = binary.Uvarint(s.mem[chunkMetaLoc+offset : chunkMetaLoc+offset+binary.MaxVarintLen64]) + offset += uint64(read) + } + + compressedDataLoc := chunkMetaLoc + offset + dataLength := curChunkEnd - compressedDataLoc + di.curChunkData = s.mem[compressedDataLoc : compressedDataLoc+dataLength] + di.curChunkNum = chunkNumber + di.uncompressed = di.uncompressed[:0] + return nil +} + +func (di *docValueReader) iterateAllDocValues(s *SegmentBase, visitor docNumTermsVisitor) error { + for i := 0; i < len(di.chunkOffsets); i++ { + err := di.loadDvChunk(uint64(i), s) + if err != nil { + return err + } + if di.curChunkData == nil || len(di.curChunkHeader) == 0 { + continue + } + + // uncompress the already loaded data + uncompressed, err := snappy.Decode(di.uncompressed[:cap(di.uncompressed)], di.curChunkData) + if err != nil { + return err + } + di.uncompressed = uncompressed + + start := uint64(0) + for _, entry := range di.curChunkHeader { + err = visitor(entry.DocNum, uncompressed[start:entry.DocDvOffset]) + if err != nil { + return err + } + + start = entry.DocDvOffset + } + } + + return nil +} + +func (di *docValueReader) visitDocValues(docNum uint64, + visitor index.DocValueVisitor) error { + // binary search the term locations for the docNum + start, end := di.getDocValueLocs(docNum) + if start == math.MaxUint64 || end == math.MaxUint64 || start == end { + return nil + } + + var uncompressed []byte + var err error + // use the uncompressed copy if available + if len(di.uncompressed) > 0 { + uncompressed = di.uncompressed + } else { + // uncompress the already loaded data + uncompressed, err = snappy.Decode(di.uncompressed[:cap(di.uncompressed)], di.curChunkData) + if err != nil { + return err + } + di.uncompressed = uncompressed + } + + // pick the terms for the given docNum + uncompressed = uncompressed[start:end] + for { + i := bytes.Index(uncompressed, termSeparatorSplitSlice) + if i < 0 { + break + } + + visitor(di.field, uncompressed[0:i]) + uncompressed = uncompressed[i+1:] + } + + return nil +} + +func (di *docValueReader) getDocValueLocs(docNum uint64) (uint64, uint64) { + i := sort.Search(len(di.curChunkHeader), func(i int) bool { + return di.curChunkHeader[i].DocNum >= docNum + }) + if i < len(di.curChunkHeader) && di.curChunkHeader[i].DocNum == docNum { + return ReadDocValueBoundary(i, di.curChunkHeader) + } + return math.MaxUint64, math.MaxUint64 +} + +// VisitDocValues is an implementation of the +// DocValueVisitable interface +func (s *SegmentBase) VisitDocValues(localDocNum uint64, fields []string, + visitor index.DocValueVisitor, dvsIn segment.DocVisitState) ( + segment.DocVisitState, error) { + dvs, ok := dvsIn.(*docVisitState) + if !ok || dvs == nil { + dvs = &docVisitState{} + } else { + if dvs.segment != s { + dvs.segment = s + dvs.dvrs = nil + } + } + + var fieldIDPlus1 uint16 + if dvs.dvrs == nil { + dvs.dvrs = make(map[uint16]*docValueReader, len(fields)) + for _, field := range fields { + if fieldIDPlus1, ok = s.fieldsMap[field]; !ok { + continue + } + fieldID := fieldIDPlus1 - 1 + if dvIter, exists := s.fieldDvReaders[fieldID]; exists && + dvIter != nil { + dvs.dvrs[fieldID] = dvIter.cloneInto(dvs.dvrs[fieldID]) + } + } + } + + // find the chunkNumber where the docValues are stored + // NOTE: doc values continue to use legacy chunk mode + chunkFactor, err := getChunkSize(LegacyChunkMode, 0, 0) + if err != nil { + return nil, err + } + docInChunk := localDocNum / chunkFactor + var dvr *docValueReader + for _, field := range fields { + if fieldIDPlus1, ok = s.fieldsMap[field]; !ok { + continue + } + fieldID := fieldIDPlus1 - 1 + if dvr, ok = dvs.dvrs[fieldID]; ok && dvr != nil { + // check if the chunk is already loaded + if docInChunk != dvr.curChunkNumber() { + err := dvr.loadDvChunk(docInChunk, s) + if err != nil { + return dvs, err + } + } + + _ = dvr.visitDocValues(localDocNum, visitor) + } + } + return dvs, nil +} + +// VisitableDocValueFields returns the list of fields with +// persisted doc value terms ready to be visitable using the +// VisitDocumentFieldTerms method. +func (s *SegmentBase) VisitableDocValueFields() ([]string, error) { + return s.fieldDvNames, nil +} diff --git a/vendor/github.com/blevesearch/zapx/v13/enumerator.go b/vendor/github.com/blevesearch/zapx/v13/enumerator.go new file mode 100644 index 00000000..972a2241 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v13/enumerator.go @@ -0,0 +1,138 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "bytes" + + "github.com/blevesearch/vellum" +) + +// enumerator provides an ordered traversal of multiple vellum +// iterators. Like JOIN of iterators, the enumerator produces a +// sequence of (key, iteratorIndex, value) tuples, sorted by key ASC, +// then iteratorIndex ASC, where the same key might be seen or +// repeated across multiple child iterators. +type enumerator struct { + itrs []vellum.Iterator + currKs [][]byte + currVs []uint64 + + lowK []byte + lowIdxs []int + lowCurr int +} + +// newEnumerator returns a new enumerator over the vellum Iterators +func newEnumerator(itrs []vellum.Iterator) (*enumerator, error) { + rv := &enumerator{ + itrs: itrs, + currKs: make([][]byte, len(itrs)), + currVs: make([]uint64, len(itrs)), + lowIdxs: make([]int, 0, len(itrs)), + } + for i, itr := range rv.itrs { + rv.currKs[i], rv.currVs[i] = itr.Current() + } + rv.updateMatches(false) + if rv.lowK == nil && len(rv.lowIdxs) == 0 { + return rv, vellum.ErrIteratorDone + } + return rv, nil +} + +// updateMatches maintains the low key matches based on the currKs +func (m *enumerator) updateMatches(skipEmptyKey bool) { + m.lowK = nil + m.lowIdxs = m.lowIdxs[:0] + m.lowCurr = 0 + + for i, key := range m.currKs { + if (key == nil && m.currVs[i] == 0) || // in case of empty iterator + (len(key) == 0 && skipEmptyKey) { // skip empty keys + continue + } + + cmp := bytes.Compare(key, m.lowK) + if cmp < 0 || len(m.lowIdxs) == 0 { + // reached a new low + m.lowK = key + m.lowIdxs = m.lowIdxs[:0] + m.lowIdxs = append(m.lowIdxs, i) + } else if cmp == 0 { + m.lowIdxs = append(m.lowIdxs, i) + } + } +} + +// Current returns the enumerator's current key, iterator-index, and +// value. If the enumerator is not pointing at a valid value (because +// Next returned an error previously), Current will return nil,0,0. +func (m *enumerator) Current() ([]byte, int, uint64) { + var i int + var v uint64 + if m.lowCurr < len(m.lowIdxs) { + i = m.lowIdxs[m.lowCurr] + v = m.currVs[i] + } + return m.lowK, i, v +} + +// GetLowIdxsAndValues will return all of the iterator indices +// which point to the current key, and their corresponding +// values. This can be used by advanced caller which may need +// to peek into these other sets of data before processing. +func (m *enumerator) GetLowIdxsAndValues() ([]int, []uint64) { + values := make([]uint64, 0, len(m.lowIdxs)) + for _, idx := range m.lowIdxs { + values = append(values, m.currVs[idx]) + } + return m.lowIdxs, values +} + +// Next advances the enumerator to the next key/iterator/value result, +// else vellum.ErrIteratorDone is returned. +func (m *enumerator) Next() error { + m.lowCurr += 1 + if m.lowCurr >= len(m.lowIdxs) { + // move all the current low iterators forwards + for _, vi := range m.lowIdxs { + err := m.itrs[vi].Next() + if err != nil && err != vellum.ErrIteratorDone { + return err + } + m.currKs[vi], m.currVs[vi] = m.itrs[vi].Current() + } + // can skip any empty keys encountered at this point + m.updateMatches(true) + } + if m.lowK == nil && len(m.lowIdxs) == 0 { + return vellum.ErrIteratorDone + } + return nil +} + +// Close all the underlying Iterators. The first error, if any, will +// be returned. +func (m *enumerator) Close() error { + var rv error + for _, itr := range m.itrs { + err := itr.Close() + if rv == nil { + rv = err + } + } + return rv +} diff --git a/vendor/github.com/blevesearch/zapx/v13/intDecoder.go b/vendor/github.com/blevesearch/zapx/v13/intDecoder.go new file mode 100644 index 00000000..e9680931 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v13/intDecoder.go @@ -0,0 +1,109 @@ +// Copyright (c) 2019 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "encoding/binary" + "fmt" +) + +type chunkedIntDecoder struct { + startOffset uint64 + dataStartOffset uint64 + chunkOffsets []uint64 + curChunkBytes []byte + data []byte + r *memUvarintReader +} + +func newChunkedIntDecoder(buf []byte, offset uint64) *chunkedIntDecoder { + rv := &chunkedIntDecoder{startOffset: offset, data: buf} + var n, numChunks uint64 + var read int + if offset == termNotEncoded { + numChunks = 0 + } else { + numChunks, read = binary.Uvarint(buf[offset+n : offset+n+binary.MaxVarintLen64]) + } + + n += uint64(read) + if cap(rv.chunkOffsets) >= int(numChunks) { + rv.chunkOffsets = rv.chunkOffsets[:int(numChunks)] + } else { + rv.chunkOffsets = make([]uint64, int(numChunks)) + } + for i := 0; i < int(numChunks); i++ { + rv.chunkOffsets[i], read = binary.Uvarint(buf[offset+n : offset+n+binary.MaxVarintLen64]) + n += uint64(read) + } + rv.dataStartOffset = offset + n + return rv +} + +func (d *chunkedIntDecoder) loadChunk(chunk int) error { + if d.startOffset == termNotEncoded { + d.r = newMemUvarintReader([]byte(nil)) + return nil + } + + if chunk >= len(d.chunkOffsets) { + return fmt.Errorf("tried to load freq chunk that doesn't exist %d/(%d)", + chunk, len(d.chunkOffsets)) + } + + end, start := d.dataStartOffset, d.dataStartOffset + s, e := readChunkBoundary(chunk, d.chunkOffsets) + start += s + end += e + d.curChunkBytes = d.data[start:end] + if d.r == nil { + d.r = newMemUvarintReader(d.curChunkBytes) + } else { + d.r.Reset(d.curChunkBytes) + } + + return nil +} + +func (d *chunkedIntDecoder) reset() { + d.startOffset = 0 + d.dataStartOffset = 0 + d.chunkOffsets = d.chunkOffsets[:0] + d.curChunkBytes = d.curChunkBytes[:0] + d.data = d.data[:0] + if d.r != nil { + d.r.Reset([]byte(nil)) + } +} + +func (d *chunkedIntDecoder) isNil() bool { + return d.curChunkBytes == nil +} + +func (d *chunkedIntDecoder) readUvarint() (uint64, error) { + return d.r.ReadUvarint() +} + +func (d *chunkedIntDecoder) SkipUvarint() { + d.r.SkipUvarint() +} + +func (d *chunkedIntDecoder) SkipBytes(count int) { + d.r.SkipBytes(count) +} + +func (d *chunkedIntDecoder) Len() int { + return d.r.Len() +} diff --git a/vendor/github.com/blevesearch/zap/v13/intcoder.go b/vendor/github.com/blevesearch/zapx/v13/intcoder.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v13/intcoder.go rename to vendor/github.com/blevesearch/zapx/v13/intcoder.go diff --git a/vendor/github.com/blevesearch/zapx/v13/memuvarint.go b/vendor/github.com/blevesearch/zapx/v13/memuvarint.go new file mode 100644 index 00000000..48a57f9c --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v13/memuvarint.go @@ -0,0 +1,103 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "fmt" +) + +type memUvarintReader struct { + C int // index of next byte to read from S + S []byte +} + +func newMemUvarintReader(s []byte) *memUvarintReader { + return &memUvarintReader{S: s} +} + +// Len returns the number of unread bytes. +func (r *memUvarintReader) Len() int { + n := len(r.S) - r.C + if n < 0 { + return 0 + } + return n +} + +// ReadUvarint reads an encoded uint64. The original code this was +// based on is at encoding/binary/ReadUvarint(). +func (r *memUvarintReader) ReadUvarint() (uint64, error) { + if r.C >= len(r.S) { + // nothing else to read + return 0, nil + } + + var x uint64 + var s uint + var C = r.C + var S = r.S + + for { + b := S[C] + C++ + + if b < 0x80 { + r.C = C + + // why 63? The original code had an 'i += 1' loop var and + // checked for i > 9 || i == 9 ...; but, we no longer + // check for the i var, but instead check here for s, + // which is incremented by 7. So, 7*9 == 63. + // + // why the "extra" >= check? The normal case is that s < + // 63, so we check this single >= guard first so that we + // hit the normal, nil-error return pathway sooner. + if s >= 63 && (s > 63 || b > 1) { + return 0, fmt.Errorf("memUvarintReader overflow") + } + + return x | uint64(b)<= len(r.S) { + return + } + + b := r.S[r.C] + r.C++ + + if b < 0x80 { + return + } + } +} + +// SkipBytes skips a count number of bytes. +func (r *memUvarintReader) SkipBytes(count int) { + r.C = r.C + count +} + +func (r *memUvarintReader) Reset(s []byte) { + r.C = 0 + r.S = s +} diff --git a/vendor/github.com/blevesearch/zapx/v13/merge.go b/vendor/github.com/blevesearch/zapx/v13/merge.go new file mode 100644 index 00000000..6a853a16 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v13/merge.go @@ -0,0 +1,843 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "bufio" + "bytes" + "encoding/binary" + "fmt" + "math" + "os" + "sort" + + "github.com/RoaringBitmap/roaring" + seg "github.com/blevesearch/scorch_segment_api/v2" + "github.com/blevesearch/vellum" + "github.com/golang/snappy" +) + +var DefaultFileMergerBufferSize = 1024 * 1024 + +const docDropped = math.MaxUint64 // sentinel docNum to represent a deleted doc + +// Merge takes a slice of segments and bit masks describing which +// documents may be dropped, and creates a new segment containing the +// remaining data. This new segment is built at the specified path. +func (*ZapPlugin) Merge(segments []seg.Segment, drops []*roaring.Bitmap, path string, + closeCh chan struct{}, s seg.StatsReporter) ( + [][]uint64, uint64, error) { + segmentBases := make([]*SegmentBase, len(segments)) + for segmenti, segment := range segments { + switch segmentx := segment.(type) { + case *Segment: + segmentBases[segmenti] = &segmentx.SegmentBase + case *SegmentBase: + segmentBases[segmenti] = segmentx + default: + panic(fmt.Sprintf("oops, unexpected segment type: %T", segment)) + } + } + return mergeSegmentBases(segmentBases, drops, path, DefaultChunkMode, closeCh, s) +} + +func mergeSegmentBases(segmentBases []*SegmentBase, drops []*roaring.Bitmap, path string, + chunkMode uint32, closeCh chan struct{}, s seg.StatsReporter) ( + [][]uint64, uint64, error) { + flag := os.O_RDWR | os.O_CREATE + + f, err := os.OpenFile(path, flag, 0600) + if err != nil { + return nil, 0, err + } + + cleanup := func() { + _ = f.Close() + _ = os.Remove(path) + } + + // buffer the output + br := bufio.NewWriterSize(f, DefaultFileMergerBufferSize) + + // wrap it for counting (tracking offsets) + cr := NewCountHashWriterWithStatsReporter(br, s) + + newDocNums, numDocs, storedIndexOffset, fieldsIndexOffset, docValueOffset, _, _, _, err := + MergeToWriter(segmentBases, drops, chunkMode, cr, closeCh) + if err != nil { + cleanup() + return nil, 0, err + } + + err = persistFooter(numDocs, storedIndexOffset, fieldsIndexOffset, + docValueOffset, chunkMode, cr.Sum32(), cr) + if err != nil { + cleanup() + return nil, 0, err + } + + err = br.Flush() + if err != nil { + cleanup() + return nil, 0, err + } + + err = f.Sync() + if err != nil { + cleanup() + return nil, 0, err + } + + err = f.Close() + if err != nil { + cleanup() + return nil, 0, err + } + + return newDocNums, uint64(cr.Count()), nil +} + +func MergeToWriter(segments []*SegmentBase, drops []*roaring.Bitmap, + chunkMode uint32, cr *CountHashWriter, closeCh chan struct{}) ( + newDocNums [][]uint64, + numDocs, storedIndexOffset, fieldsIndexOffset, docValueOffset uint64, + dictLocs []uint64, fieldsInv []string, fieldsMap map[string]uint16, + err error) { + docValueOffset = uint64(fieldNotUninverted) + + var fieldsSame bool + fieldsSame, fieldsInv = mergeFields(segments) + fieldsMap = mapFields(fieldsInv) + + numDocs = computeNewDocCount(segments, drops) + + if isClosed(closeCh) { + return nil, 0, 0, 0, 0, nil, nil, nil, seg.ErrClosed + } + + if numDocs > 0 { + storedIndexOffset, newDocNums, err = mergeStoredAndRemap(segments, drops, + fieldsMap, fieldsInv, fieldsSame, numDocs, cr, closeCh) + if err != nil { + return nil, 0, 0, 0, 0, nil, nil, nil, err + } + + dictLocs, docValueOffset, err = persistMergedRest(segments, drops, + fieldsInv, fieldsMap, fieldsSame, + newDocNums, numDocs, chunkMode, cr, closeCh) + if err != nil { + return nil, 0, 0, 0, 0, nil, nil, nil, err + } + } else { + dictLocs = make([]uint64, len(fieldsInv)) + } + + fieldsIndexOffset, err = persistFields(fieldsInv, cr, dictLocs) + if err != nil { + return nil, 0, 0, 0, 0, nil, nil, nil, err + } + + return newDocNums, numDocs, storedIndexOffset, fieldsIndexOffset, docValueOffset, dictLocs, fieldsInv, fieldsMap, nil +} + +// mapFields takes the fieldsInv list and returns a map of fieldName +// to fieldID+1 +func mapFields(fields []string) map[string]uint16 { + rv := make(map[string]uint16, len(fields)) + for i, fieldName := range fields { + rv[fieldName] = uint16(i) + 1 + } + return rv +} + +// computeNewDocCount determines how many documents will be in the newly +// merged segment when obsoleted docs are dropped +func computeNewDocCount(segments []*SegmentBase, drops []*roaring.Bitmap) uint64 { + var newDocCount uint64 + for segI, segment := range segments { + newDocCount += segment.numDocs + if drops[segI] != nil { + newDocCount -= drops[segI].GetCardinality() + } + } + return newDocCount +} + +func persistMergedRest(segments []*SegmentBase, dropsIn []*roaring.Bitmap, + fieldsInv []string, fieldsMap map[string]uint16, fieldsSame bool, + newDocNumsIn [][]uint64, newSegDocCount uint64, chunkMode uint32, + w *CountHashWriter, closeCh chan struct{}) ([]uint64, uint64, error) { + var bufMaxVarintLen64 []byte = make([]byte, binary.MaxVarintLen64) + var bufLoc []uint64 + + var postings *PostingsList + var postItr *PostingsIterator + + rv := make([]uint64, len(fieldsInv)) + fieldDvLocsStart := make([]uint64, len(fieldsInv)) + fieldDvLocsEnd := make([]uint64, len(fieldsInv)) + + // these int coders are initialized with chunk size 1024 + // however this will be reset to the correct chunk size + // while processing each individual field-term section + tfEncoder := newChunkedIntCoder(1024, newSegDocCount-1) + locEncoder := newChunkedIntCoder(1024, newSegDocCount-1) + + var vellumBuf bytes.Buffer + newVellum, err := vellum.New(&vellumBuf, nil) + if err != nil { + return nil, 0, err + } + + newRoaring := roaring.NewBitmap() + + // for each field + for fieldID, fieldName := range fieldsInv { + // collect FST iterators from all active segments for this field + var newDocNums [][]uint64 + var drops []*roaring.Bitmap + var dicts []*Dictionary + var itrs []vellum.Iterator + + var segmentsInFocus []*SegmentBase + + for segmentI, segment := range segments { + // check for the closure in meantime + if isClosed(closeCh) { + return nil, 0, seg.ErrClosed + } + + dict, err2 := segment.dictionary(fieldName) + if err2 != nil { + return nil, 0, err2 + } + if dict != nil && dict.fst != nil { + itr, err2 := dict.fst.Iterator(nil, nil) + if err2 != nil && err2 != vellum.ErrIteratorDone { + return nil, 0, err2 + } + if itr != nil { + newDocNums = append(newDocNums, newDocNumsIn[segmentI]) + if dropsIn[segmentI] != nil && !dropsIn[segmentI].IsEmpty() { + drops = append(drops, dropsIn[segmentI]) + } else { + drops = append(drops, nil) + } + dicts = append(dicts, dict) + itrs = append(itrs, itr) + segmentsInFocus = append(segmentsInFocus, segment) + } + } + } + + var prevTerm []byte + + newRoaring.Clear() + + var lastDocNum, lastFreq, lastNorm uint64 + + // determines whether to use "1-hit" encoding optimization + // when a term appears in only 1 doc, with no loc info, + // has freq of 1, and the docNum fits into 31-bits + use1HitEncoding := func(termCardinality uint64) (bool, uint64, uint64) { + if termCardinality == uint64(1) && locEncoder.FinalSize() <= 0 { + docNum := uint64(newRoaring.Minimum()) + if under32Bits(docNum) && docNum == lastDocNum && lastFreq == 1 { + return true, docNum, lastNorm + } + } + return false, 0, 0 + } + + finishTerm := func(term []byte) error { + tfEncoder.Close() + locEncoder.Close() + + postingsOffset, err := writePostings(newRoaring, + tfEncoder, locEncoder, use1HitEncoding, w, bufMaxVarintLen64) + if err != nil { + return err + } + + if postingsOffset > 0 { + err = newVellum.Insert(term, postingsOffset) + if err != nil { + return err + } + } + + newRoaring.Clear() + + tfEncoder.Reset() + locEncoder.Reset() + + lastDocNum = 0 + lastFreq = 0 + lastNorm = 0 + + return nil + } + + enumerator, err := newEnumerator(itrs) + + for err == nil { + term, itrI, postingsOffset := enumerator.Current() + + if !bytes.Equal(prevTerm, term) { + // check for the closure in meantime + if isClosed(closeCh) { + return nil, 0, seg.ErrClosed + } + + // if the term changed, write out the info collected + // for the previous term + err = finishTerm(prevTerm) + if err != nil { + return nil, 0, err + } + } + if !bytes.Equal(prevTerm, term) || prevTerm == nil { + // compute cardinality of field-term in new seg + var newCard uint64 + lowItrIdxs, lowItrVals := enumerator.GetLowIdxsAndValues() + for i, idx := range lowItrIdxs { + pl, err := dicts[idx].postingsListFromOffset(lowItrVals[i], drops[idx], nil) + if err != nil { + return nil, 0, err + } + newCard += pl.Count() + } + // compute correct chunk size with this + chunkSize, err := getChunkSize(chunkMode, newCard, newSegDocCount) + if err != nil { + return nil, 0, err + } + // update encoders chunk + tfEncoder.SetChunkSize(chunkSize, newSegDocCount-1) + locEncoder.SetChunkSize(chunkSize, newSegDocCount-1) + } + + postings, err = dicts[itrI].postingsListFromOffset( + postingsOffset, drops[itrI], postings) + if err != nil { + return nil, 0, err + } + + postItr = postings.iterator(true, true, true, postItr) + + // can no longer optimize by copying, since chunk factor could have changed + lastDocNum, lastFreq, lastNorm, bufLoc, err = mergeTermFreqNormLocs( + fieldsMap, term, postItr, newDocNums[itrI], newRoaring, + tfEncoder, locEncoder, bufLoc) + + if err != nil { + return nil, 0, err + } + + prevTerm = prevTerm[:0] // copy to prevTerm in case Next() reuses term mem + prevTerm = append(prevTerm, term...) + + err = enumerator.Next() + } + if err != vellum.ErrIteratorDone { + return nil, 0, err + } + + err = finishTerm(prevTerm) + if err != nil { + return nil, 0, err + } + + dictOffset := uint64(w.Count()) + + err = newVellum.Close() + if err != nil { + return nil, 0, err + } + vellumData := vellumBuf.Bytes() + + // write out the length of the vellum data + n := binary.PutUvarint(bufMaxVarintLen64, uint64(len(vellumData))) + _, err = w.Write(bufMaxVarintLen64[:n]) + if err != nil { + return nil, 0, err + } + + // write this vellum to disk + _, err = w.Write(vellumData) + if err != nil { + return nil, 0, err + } + + rv[fieldID] = dictOffset + + // get the field doc value offset (start) + fieldDvLocsStart[fieldID] = uint64(w.Count()) + + // update the field doc values + // NOTE: doc values continue to use legacy chunk mode + chunkSize, err := getChunkSize(LegacyChunkMode, 0, 0) + if err != nil { + return nil, 0, err + } + fdvEncoder := newChunkedContentCoder(chunkSize, newSegDocCount-1, w, true) + + fdvReadersAvailable := false + var dvIterClone *docValueReader + for segmentI, segment := range segmentsInFocus { + // check for the closure in meantime + if isClosed(closeCh) { + return nil, 0, seg.ErrClosed + } + + fieldIDPlus1 := uint16(segment.fieldsMap[fieldName]) + if dvIter, exists := segment.fieldDvReaders[fieldIDPlus1-1]; exists && + dvIter != nil { + fdvReadersAvailable = true + dvIterClone = dvIter.cloneInto(dvIterClone) + err = dvIterClone.iterateAllDocValues(segment, func(docNum uint64, terms []byte) error { + if newDocNums[segmentI][docNum] == docDropped { + return nil + } + err := fdvEncoder.Add(newDocNums[segmentI][docNum], terms) + if err != nil { + return err + } + return nil + }) + if err != nil { + return nil, 0, err + } + } + } + + if fdvReadersAvailable { + err = fdvEncoder.Close() + if err != nil { + return nil, 0, err + } + + // persist the doc value details for this field + _, err = fdvEncoder.Write() + if err != nil { + return nil, 0, err + } + + // get the field doc value offset (end) + fieldDvLocsEnd[fieldID] = uint64(w.Count()) + } else { + fieldDvLocsStart[fieldID] = fieldNotUninverted + fieldDvLocsEnd[fieldID] = fieldNotUninverted + } + + // reset vellum buffer and vellum builder + vellumBuf.Reset() + err = newVellum.Reset(&vellumBuf) + if err != nil { + return nil, 0, err + } + } + + fieldDvLocsOffset := uint64(w.Count()) + + buf := bufMaxVarintLen64 + for i := 0; i < len(fieldDvLocsStart); i++ { + n := binary.PutUvarint(buf, fieldDvLocsStart[i]) + _, err := w.Write(buf[:n]) + if err != nil { + return nil, 0, err + } + n = binary.PutUvarint(buf, fieldDvLocsEnd[i]) + _, err = w.Write(buf[:n]) + if err != nil { + return nil, 0, err + } + } + + return rv, fieldDvLocsOffset, nil +} + +func mergeTermFreqNormLocs(fieldsMap map[string]uint16, term []byte, postItr *PostingsIterator, + newDocNums []uint64, newRoaring *roaring.Bitmap, + tfEncoder *chunkedIntCoder, locEncoder *chunkedIntCoder, bufLoc []uint64) ( + lastDocNum uint64, lastFreq uint64, lastNorm uint64, bufLocOut []uint64, err error) { + next, err := postItr.Next() + for next != nil && err == nil { + hitNewDocNum := newDocNums[next.Number()] + if hitNewDocNum == docDropped { + return 0, 0, 0, nil, fmt.Errorf("see hit with dropped docNum") + } + + newRoaring.Add(uint32(hitNewDocNum)) + + nextFreq := next.Frequency() + nextNorm := uint64(math.Float32bits(float32(next.Norm()))) + + locs := next.Locations() + + err = tfEncoder.Add(hitNewDocNum, + encodeFreqHasLocs(nextFreq, len(locs) > 0), nextNorm) + if err != nil { + return 0, 0, 0, nil, err + } + + if len(locs) > 0 { + numBytesLocs := 0 + for _, loc := range locs { + ap := loc.ArrayPositions() + numBytesLocs += totalUvarintBytes(uint64(fieldsMap[loc.Field()]-1), + loc.Pos(), loc.Start(), loc.End(), uint64(len(ap)), ap) + } + + err = locEncoder.Add(hitNewDocNum, uint64(numBytesLocs)) + if err != nil { + return 0, 0, 0, nil, err + } + + for _, loc := range locs { + ap := loc.ArrayPositions() + if cap(bufLoc) < 5+len(ap) { + bufLoc = make([]uint64, 0, 5+len(ap)) + } + args := bufLoc[0:5] + args[0] = uint64(fieldsMap[loc.Field()] - 1) + args[1] = loc.Pos() + args[2] = loc.Start() + args[3] = loc.End() + args[4] = uint64(len(ap)) + args = append(args, ap...) + err = locEncoder.Add(hitNewDocNum, args...) + if err != nil { + return 0, 0, 0, nil, err + } + } + } + + lastDocNum = hitNewDocNum + lastFreq = nextFreq + lastNorm = nextNorm + + next, err = postItr.Next() + } + + return lastDocNum, lastFreq, lastNorm, bufLoc, err +} + +func writePostings(postings *roaring.Bitmap, tfEncoder, locEncoder *chunkedIntCoder, + use1HitEncoding func(uint64) (bool, uint64, uint64), + w *CountHashWriter, bufMaxVarintLen64 []byte) ( + offset uint64, err error) { + termCardinality := postings.GetCardinality() + if termCardinality <= 0 { + return 0, nil + } + + if use1HitEncoding != nil { + encodeAs1Hit, docNum1Hit, normBits1Hit := use1HitEncoding(termCardinality) + if encodeAs1Hit { + return FSTValEncode1Hit(docNum1Hit, normBits1Hit), nil + } + } + + var tfOffset uint64 + tfOffset, _, err = tfEncoder.writeAt(w) + if err != nil { + return 0, err + } + + var locOffset uint64 + locOffset, _, err = locEncoder.writeAt(w) + if err != nil { + return 0, err + } + + postingsOffset := uint64(w.Count()) + + n := binary.PutUvarint(bufMaxVarintLen64, tfOffset) + _, err = w.Write(bufMaxVarintLen64[:n]) + if err != nil { + return 0, err + } + + n = binary.PutUvarint(bufMaxVarintLen64, locOffset) + _, err = w.Write(bufMaxVarintLen64[:n]) + if err != nil { + return 0, err + } + + _, err = writeRoaringWithLen(postings, w, bufMaxVarintLen64) + if err != nil { + return 0, err + } + + return postingsOffset, nil +} + +type varintEncoder func(uint64) (int, error) + +func mergeStoredAndRemap(segments []*SegmentBase, drops []*roaring.Bitmap, + fieldsMap map[string]uint16, fieldsInv []string, fieldsSame bool, newSegDocCount uint64, + w *CountHashWriter, closeCh chan struct{}) (uint64, [][]uint64, error) { + var rv [][]uint64 // The remapped or newDocNums for each segment. + + var newDocNum uint64 + + var curr int + var data, compressed []byte + var metaBuf bytes.Buffer + varBuf := make([]byte, binary.MaxVarintLen64) + metaEncode := func(val uint64) (int, error) { + wb := binary.PutUvarint(varBuf, val) + return metaBuf.Write(varBuf[:wb]) + } + + vals := make([][][]byte, len(fieldsInv)) + typs := make([][]byte, len(fieldsInv)) + poss := make([][][]uint64, len(fieldsInv)) + + var posBuf []uint64 + + docNumOffsets := make([]uint64, newSegDocCount) + + vdc := visitDocumentCtxPool.Get().(*visitDocumentCtx) + defer visitDocumentCtxPool.Put(vdc) + + // for each segment + for segI, segment := range segments { + // check for the closure in meantime + if isClosed(closeCh) { + return 0, nil, seg.ErrClosed + } + + segNewDocNums := make([]uint64, segment.numDocs) + + dropsI := drops[segI] + + // optimize when the field mapping is the same across all + // segments and there are no deletions, via byte-copying + // of stored docs bytes directly to the writer + if fieldsSame && (dropsI == nil || dropsI.GetCardinality() == 0) { + err := segment.copyStoredDocs(newDocNum, docNumOffsets, w) + if err != nil { + return 0, nil, err + } + + for i := uint64(0); i < segment.numDocs; i++ { + segNewDocNums[i] = newDocNum + newDocNum++ + } + rv = append(rv, segNewDocNums) + + continue + } + + // for each doc num + for docNum := uint64(0); docNum < segment.numDocs; docNum++ { + // TODO: roaring's API limits docNums to 32-bits? + if dropsI != nil && dropsI.Contains(uint32(docNum)) { + segNewDocNums[docNum] = docDropped + continue + } + + segNewDocNums[docNum] = newDocNum + + curr = 0 + metaBuf.Reset() + data = data[:0] + + posTemp := posBuf + + // collect all the data + for i := 0; i < len(fieldsInv); i++ { + vals[i] = vals[i][:0] + typs[i] = typs[i][:0] + poss[i] = poss[i][:0] + } + err := segment.visitStoredFields(vdc, docNum, func(field string, typ byte, value []byte, pos []uint64) bool { + fieldID := int(fieldsMap[field]) - 1 + vals[fieldID] = append(vals[fieldID], value) + typs[fieldID] = append(typs[fieldID], typ) + + // copy array positions to preserve them beyond the scope of this callback + var curPos []uint64 + if len(pos) > 0 { + if cap(posTemp) < len(pos) { + posBuf = make([]uint64, len(pos)*len(fieldsInv)) + posTemp = posBuf + } + curPos = posTemp[0:len(pos)] + copy(curPos, pos) + posTemp = posTemp[len(pos):] + } + poss[fieldID] = append(poss[fieldID], curPos) + + return true + }) + if err != nil { + return 0, nil, err + } + + // _id field special case optimizes ExternalID() lookups + idFieldVal := vals[uint16(0)][0] + _, err = metaEncode(uint64(len(idFieldVal))) + if err != nil { + return 0, nil, err + } + + // now walk the non-"_id" fields in order + for fieldID := 1; fieldID < len(fieldsInv); fieldID++ { + storedFieldValues := vals[fieldID] + + stf := typs[fieldID] + spf := poss[fieldID] + + var err2 error + curr, data, err2 = persistStoredFieldValues(fieldID, + storedFieldValues, stf, spf, curr, metaEncode, data) + if err2 != nil { + return 0, nil, err2 + } + } + + metaBytes := metaBuf.Bytes() + + compressed = snappy.Encode(compressed[:cap(compressed)], data) + + // record where we're about to start writing + docNumOffsets[newDocNum] = uint64(w.Count()) + + // write out the meta len and compressed data len + _, err = writeUvarints(w, + uint64(len(metaBytes)), + uint64(len(idFieldVal)+len(compressed))) + if err != nil { + return 0, nil, err + } + // now write the meta + _, err = w.Write(metaBytes) + if err != nil { + return 0, nil, err + } + // now write the _id field val (counted as part of the 'compressed' data) + _, err = w.Write(idFieldVal) + if err != nil { + return 0, nil, err + } + // now write the compressed data + _, err = w.Write(compressed) + if err != nil { + return 0, nil, err + } + + newDocNum++ + } + + rv = append(rv, segNewDocNums) + } + + // return value is the start of the stored index + storedIndexOffset := uint64(w.Count()) + + // now write out the stored doc index + for _, docNumOffset := range docNumOffsets { + err := binary.Write(w, binary.BigEndian, docNumOffset) + if err != nil { + return 0, nil, err + } + } + + return storedIndexOffset, rv, nil +} + +// copyStoredDocs writes out a segment's stored doc info, optimized by +// using a single Write() call for the entire set of bytes. The +// newDocNumOffsets is filled with the new offsets for each doc. +func (s *SegmentBase) copyStoredDocs(newDocNum uint64, newDocNumOffsets []uint64, + w *CountHashWriter) error { + if s.numDocs <= 0 { + return nil + } + + indexOffset0, storedOffset0, _, _, _ := + s.getDocStoredOffsets(0) // the segment's first doc + + indexOffsetN, storedOffsetN, readN, metaLenN, dataLenN := + s.getDocStoredOffsets(s.numDocs - 1) // the segment's last doc + + storedOffset0New := uint64(w.Count()) + + storedBytes := s.mem[storedOffset0 : storedOffsetN+readN+metaLenN+dataLenN] + _, err := w.Write(storedBytes) + if err != nil { + return err + } + + // remap the storedOffset's for the docs into new offsets relative + // to storedOffset0New, filling the given docNumOffsetsOut array + for indexOffset := indexOffset0; indexOffset <= indexOffsetN; indexOffset += 8 { + storedOffset := binary.BigEndian.Uint64(s.mem[indexOffset : indexOffset+8]) + storedOffsetNew := storedOffset - storedOffset0 + storedOffset0New + newDocNumOffsets[newDocNum] = storedOffsetNew + newDocNum += 1 + } + + return nil +} + +// mergeFields builds a unified list of fields used across all the +// input segments, and computes whether the fields are the same across +// segments (which depends on fields to be sorted in the same way +// across segments) +func mergeFields(segments []*SegmentBase) (bool, []string) { + fieldsSame := true + + var segment0Fields []string + if len(segments) > 0 { + segment0Fields = segments[0].Fields() + } + + fieldsExist := map[string]struct{}{} + for _, segment := range segments { + fields := segment.Fields() + for fieldi, field := range fields { + fieldsExist[field] = struct{}{} + if len(segment0Fields) != len(fields) || segment0Fields[fieldi] != field { + fieldsSame = false + } + } + } + + rv := make([]string, 0, len(fieldsExist)) + // ensure _id stays first + rv = append(rv, "_id") + for k := range fieldsExist { + if k != "_id" { + rv = append(rv, k) + } + } + + sort.Strings(rv[1:]) // leave _id as first + + return fieldsSame, rv +} + +func isClosed(closeCh chan struct{}) bool { + select { + case <-closeCh: + return true + default: + return false + } +} diff --git a/vendor/github.com/blevesearch/zapx/v13/new.go b/vendor/github.com/blevesearch/zapx/v13/new.go new file mode 100644 index 00000000..b4e0d034 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v13/new.go @@ -0,0 +1,830 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "bytes" + "encoding/binary" + "math" + "sort" + "sync" + + "github.com/RoaringBitmap/roaring" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" + "github.com/blevesearch/vellum" + "github.com/golang/snappy" +) + +var NewSegmentBufferNumResultsBump int = 100 +var NewSegmentBufferNumResultsFactor float64 = 1.0 +var NewSegmentBufferAvgBytesPerDocFactor float64 = 1.0 + +// ValidateDocFields can be set by applications to perform additional checks +// on fields in a document being added to a new segment, by default it does +// nothing. +// This API is experimental and may be removed at any time. +var ValidateDocFields = func(field index.Field) error { + return nil +} + +// New creates an in-memory zap-encoded SegmentBase from a set of Documents +func (z *ZapPlugin) New(results []index.Document) ( + segment.Segment, uint64, error) { + return z.newWithChunkMode(results, DefaultChunkMode) +} + +func (*ZapPlugin) newWithChunkMode(results []index.Document, + chunkMode uint32) (segment.Segment, uint64, error) { + s := interimPool.Get().(*interim) + + var br bytes.Buffer + if s.lastNumDocs > 0 { + // use previous results to initialize the buf with an estimate + // size, but note that the interim instance comes from a + // global interimPool, so multiple scorch instances indexing + // different docs can lead to low quality estimates + estimateAvgBytesPerDoc := int(float64(s.lastOutSize/s.lastNumDocs) * + NewSegmentBufferNumResultsFactor) + estimateNumResults := int(float64(len(results)+NewSegmentBufferNumResultsBump) * + NewSegmentBufferAvgBytesPerDocFactor) + br.Grow(estimateAvgBytesPerDoc * estimateNumResults) + } + + s.results = results + s.chunkMode = chunkMode + s.w = NewCountHashWriter(&br) + + storedIndexOffset, fieldsIndexOffset, fdvIndexOffset, dictOffsets, + err := s.convert() + if err != nil { + return nil, uint64(0), err + } + + sb, err := InitSegmentBase(br.Bytes(), s.w.Sum32(), chunkMode, + s.FieldsMap, s.FieldsInv, uint64(len(results)), + storedIndexOffset, fieldsIndexOffset, fdvIndexOffset, dictOffsets) + + if err == nil && s.reset() == nil { + s.lastNumDocs = len(results) + s.lastOutSize = len(br.Bytes()) + interimPool.Put(s) + } + + return sb, uint64(len(br.Bytes())), err +} + +var interimPool = sync.Pool{New: func() interface{} { return &interim{} }} + +// interim holds temporary working data used while converting from +// analysis results to a zap-encoded segment +type interim struct { + results []index.Document + + chunkMode uint32 + + w *CountHashWriter + + // FieldsMap adds 1 to field id to avoid zero value issues + // name -> field id + 1 + FieldsMap map[string]uint16 + + // FieldsInv is the inverse of FieldsMap + // field id -> name + FieldsInv []string + + // Term dictionaries for each field + // field id -> term -> postings list id + 1 + Dicts []map[string]uint64 + + // Terms for each field, where terms are sorted ascending + // field id -> []term + DictKeys [][]string + + // Fields whose IncludeDocValues is true + // field id -> bool + IncludeDocValues []bool + + // postings id -> bitmap of docNums + Postings []*roaring.Bitmap + + // postings id -> freq/norm's, one for each docNum in postings + FreqNorms [][]interimFreqNorm + freqNormsBacking []interimFreqNorm + + // postings id -> locs, one for each freq + Locs [][]interimLoc + locsBacking []interimLoc + + numTermsPerPostingsList []int // key is postings list id + numLocsPerPostingsList []int // key is postings list id + + builder *vellum.Builder + builderBuf bytes.Buffer + + metaBuf bytes.Buffer + + tmp0 []byte + tmp1 []byte + + lastNumDocs int + lastOutSize int +} + +func (s *interim) reset() (err error) { + s.results = nil + s.chunkMode = 0 + s.w = nil + s.FieldsMap = nil + s.FieldsInv = nil + for i := range s.Dicts { + s.Dicts[i] = nil + } + s.Dicts = s.Dicts[:0] + for i := range s.DictKeys { + s.DictKeys[i] = s.DictKeys[i][:0] + } + s.DictKeys = s.DictKeys[:0] + for i := range s.IncludeDocValues { + s.IncludeDocValues[i] = false + } + s.IncludeDocValues = s.IncludeDocValues[:0] + for _, idn := range s.Postings { + idn.Clear() + } + s.Postings = s.Postings[:0] + s.FreqNorms = s.FreqNorms[:0] + for i := range s.freqNormsBacking { + s.freqNormsBacking[i] = interimFreqNorm{} + } + s.freqNormsBacking = s.freqNormsBacking[:0] + s.Locs = s.Locs[:0] + for i := range s.locsBacking { + s.locsBacking[i] = interimLoc{} + } + s.locsBacking = s.locsBacking[:0] + s.numTermsPerPostingsList = s.numTermsPerPostingsList[:0] + s.numLocsPerPostingsList = s.numLocsPerPostingsList[:0] + s.builderBuf.Reset() + if s.builder != nil { + err = s.builder.Reset(&s.builderBuf) + } + s.metaBuf.Reset() + s.tmp0 = s.tmp0[:0] + s.tmp1 = s.tmp1[:0] + s.lastNumDocs = 0 + s.lastOutSize = 0 + + return err +} + +func (s *interim) grabBuf(size int) []byte { + buf := s.tmp0 + if cap(buf) < size { + buf = make([]byte, size) + s.tmp0 = buf + } + return buf[0:size] +} + +type interimStoredField struct { + vals [][]byte + typs []byte + arrayposs [][]uint64 // array positions +} + +type interimFreqNorm struct { + freq uint64 + norm float32 + numLocs int +} + +type interimLoc struct { + fieldID uint16 + pos uint64 + start uint64 + end uint64 + arrayposs []uint64 +} + +func (s *interim) convert() (uint64, uint64, uint64, []uint64, error) { + s.FieldsMap = map[string]uint16{} + + s.getOrDefineField("_id") // _id field is fieldID 0 + + for _, result := range s.results { + result.VisitComposite(func(field index.CompositeField) { + s.getOrDefineField(field.Name()) + }) + result.VisitFields(func(field index.Field) { + s.getOrDefineField(field.Name()) + }) + } + + sort.Strings(s.FieldsInv[1:]) // keep _id as first field + + for fieldID, fieldName := range s.FieldsInv { + s.FieldsMap[fieldName] = uint16(fieldID + 1) + } + + if cap(s.IncludeDocValues) >= len(s.FieldsInv) { + s.IncludeDocValues = s.IncludeDocValues[:len(s.FieldsInv)] + } else { + s.IncludeDocValues = make([]bool, len(s.FieldsInv)) + } + + s.prepareDicts() + + for _, dict := range s.DictKeys { + sort.Strings(dict) + } + + s.processDocuments() + + storedIndexOffset, err := s.writeStoredFields() + if err != nil { + return 0, 0, 0, nil, err + } + + var fdvIndexOffset uint64 + var dictOffsets []uint64 + + if len(s.results) > 0 { + fdvIndexOffset, dictOffsets, err = s.writeDicts() + if err != nil { + return 0, 0, 0, nil, err + } + } else { + dictOffsets = make([]uint64, len(s.FieldsInv)) + } + + fieldsIndexOffset, err := persistFields(s.FieldsInv, s.w, dictOffsets) + if err != nil { + return 0, 0, 0, nil, err + } + + return storedIndexOffset, fieldsIndexOffset, fdvIndexOffset, dictOffsets, nil +} + +func (s *interim) getOrDefineField(fieldName string) int { + fieldIDPlus1, exists := s.FieldsMap[fieldName] + if !exists { + fieldIDPlus1 = uint16(len(s.FieldsInv) + 1) + s.FieldsMap[fieldName] = fieldIDPlus1 + s.FieldsInv = append(s.FieldsInv, fieldName) + + s.Dicts = append(s.Dicts, make(map[string]uint64)) + + n := len(s.DictKeys) + if n < cap(s.DictKeys) { + s.DictKeys = s.DictKeys[:n+1] + s.DictKeys[n] = s.DictKeys[n][:0] + } else { + s.DictKeys = append(s.DictKeys, []string(nil)) + } + } + + return int(fieldIDPlus1 - 1) +} + +// fill Dicts and DictKeys from analysis results +func (s *interim) prepareDicts() { + var pidNext int + + var totTFs int + var totLocs int + + visitField := func(field index.Field) { + fieldID := uint16(s.getOrDefineField(field.Name())) + + dict := s.Dicts[fieldID] + dictKeys := s.DictKeys[fieldID] + + tfs := field.AnalyzedTokenFrequencies() + for term, tf := range tfs { + pidPlus1, exists := dict[term] + if !exists { + pidNext++ + pidPlus1 = uint64(pidNext) + + dict[term] = pidPlus1 + dictKeys = append(dictKeys, term) + + s.numTermsPerPostingsList = append(s.numTermsPerPostingsList, 0) + s.numLocsPerPostingsList = append(s.numLocsPerPostingsList, 0) + } + + pid := pidPlus1 - 1 + + s.numTermsPerPostingsList[pid] += 1 + s.numLocsPerPostingsList[pid] += len(tf.Locations) + + totLocs += len(tf.Locations) + } + + totTFs += len(tfs) + + s.DictKeys[fieldID] = dictKeys + } + + for _, result := range s.results { + // walk each composite field + result.VisitComposite(func(field index.CompositeField) { + visitField(field) + }) + + // walk each field + result.VisitFields(visitField) + } + + numPostingsLists := pidNext + + if cap(s.Postings) >= numPostingsLists { + s.Postings = s.Postings[:numPostingsLists] + } else { + postings := make([]*roaring.Bitmap, numPostingsLists) + copy(postings, s.Postings[:cap(s.Postings)]) + for i := 0; i < numPostingsLists; i++ { + if postings[i] == nil { + postings[i] = roaring.New() + } + } + s.Postings = postings + } + + if cap(s.FreqNorms) >= numPostingsLists { + s.FreqNorms = s.FreqNorms[:numPostingsLists] + } else { + s.FreqNorms = make([][]interimFreqNorm, numPostingsLists) + } + + if cap(s.freqNormsBacking) >= totTFs { + s.freqNormsBacking = s.freqNormsBacking[:totTFs] + } else { + s.freqNormsBacking = make([]interimFreqNorm, totTFs) + } + + freqNormsBacking := s.freqNormsBacking + for pid, numTerms := range s.numTermsPerPostingsList { + s.FreqNorms[pid] = freqNormsBacking[0:0] + freqNormsBacking = freqNormsBacking[numTerms:] + } + + if cap(s.Locs) >= numPostingsLists { + s.Locs = s.Locs[:numPostingsLists] + } else { + s.Locs = make([][]interimLoc, numPostingsLists) + } + + if cap(s.locsBacking) >= totLocs { + s.locsBacking = s.locsBacking[:totLocs] + } else { + s.locsBacking = make([]interimLoc, totLocs) + } + + locsBacking := s.locsBacking + for pid, numLocs := range s.numLocsPerPostingsList { + s.Locs[pid] = locsBacking[0:0] + locsBacking = locsBacking[numLocs:] + } +} + +func (s *interim) processDocuments() { + numFields := len(s.FieldsInv) + reuseFieldLens := make([]int, numFields) + reuseFieldTFs := make([]index.TokenFrequencies, numFields) + + for docNum, result := range s.results { + for i := 0; i < numFields; i++ { // clear these for reuse + reuseFieldLens[i] = 0 + reuseFieldTFs[i] = nil + } + + s.processDocument(uint64(docNum), result, + reuseFieldLens, reuseFieldTFs) + } +} + +func (s *interim) processDocument(docNum uint64, + result index.Document, + fieldLens []int, fieldTFs []index.TokenFrequencies) { + visitField := func(field index.Field) { + fieldID := uint16(s.getOrDefineField(field.Name())) + fieldLens[fieldID] += field.AnalyzedLength() + + existingFreqs := fieldTFs[fieldID] + if existingFreqs != nil { + existingFreqs.MergeAll(field.Name(), field.AnalyzedTokenFrequencies()) + } else { + fieldTFs[fieldID] = field.AnalyzedTokenFrequencies() + } + } + + // walk each composite field + result.VisitComposite(func(field index.CompositeField) { + visitField(field) + }) + + // walk each field + result.VisitFields(visitField) + + // now that it's been rolled up into fieldTFs, walk that + for fieldID, tfs := range fieldTFs { + dict := s.Dicts[fieldID] + norm := float32(1.0 / math.Sqrt(float64(fieldLens[fieldID]))) + + for term, tf := range tfs { + pid := dict[term] - 1 + bs := s.Postings[pid] + bs.Add(uint32(docNum)) + + s.FreqNorms[pid] = append(s.FreqNorms[pid], + interimFreqNorm{ + freq: uint64(tf.Frequency()), + norm: norm, + numLocs: len(tf.Locations), + }) + + if len(tf.Locations) > 0 { + locs := s.Locs[pid] + + for _, loc := range tf.Locations { + var locf = uint16(fieldID) + if loc.Field != "" { + locf = uint16(s.getOrDefineField(loc.Field)) + } + var arrayposs []uint64 + if len(loc.ArrayPositions) > 0 { + arrayposs = loc.ArrayPositions + } + locs = append(locs, interimLoc{ + fieldID: locf, + pos: uint64(loc.Position), + start: uint64(loc.Start), + end: uint64(loc.End), + arrayposs: arrayposs, + }) + } + + s.Locs[pid] = locs + } + } + } +} + +func (s *interim) writeStoredFields() ( + storedIndexOffset uint64, err error) { + varBuf := make([]byte, binary.MaxVarintLen64) + metaEncode := func(val uint64) (int, error) { + wb := binary.PutUvarint(varBuf, val) + return s.metaBuf.Write(varBuf[:wb]) + } + + data, compressed := s.tmp0[:0], s.tmp1[:0] + defer func() { s.tmp0, s.tmp1 = data, compressed }() + + // keyed by docNum + docStoredOffsets := make([]uint64, len(s.results)) + + // keyed by fieldID, for the current doc in the loop + docStoredFields := map[uint16]interimStoredField{} + + for docNum, result := range s.results { + for fieldID := range docStoredFields { // reset for next doc + delete(docStoredFields, fieldID) + } + + var validationErr error + result.VisitFields(func(field index.Field) { + fieldID := uint16(s.getOrDefineField(field.Name())) + + if field.Options().IsStored() { + isf := docStoredFields[fieldID] + isf.vals = append(isf.vals, field.Value()) + isf.typs = append(isf.typs, field.EncodedFieldType()) + isf.arrayposs = append(isf.arrayposs, field.ArrayPositions()) + docStoredFields[fieldID] = isf + } + + if field.Options().IncludeDocValues() { + s.IncludeDocValues[fieldID] = true + } + + err := ValidateDocFields(field) + if err != nil && validationErr == nil { + validationErr = err + } + }) + if validationErr != nil { + return 0, validationErr + } + + var curr int + + s.metaBuf.Reset() + data = data[:0] + + // _id field special case optimizes ExternalID() lookups + idFieldVal := docStoredFields[uint16(0)].vals[0] + _, err = metaEncode(uint64(len(idFieldVal))) + if err != nil { + return 0, err + } + + // handle non-"_id" fields + for fieldID := 1; fieldID < len(s.FieldsInv); fieldID++ { + isf, exists := docStoredFields[uint16(fieldID)] + if exists { + curr, data, err = persistStoredFieldValues( + fieldID, isf.vals, isf.typs, isf.arrayposs, + curr, metaEncode, data) + if err != nil { + return 0, err + } + } + } + + metaBytes := s.metaBuf.Bytes() + + compressed = snappy.Encode(compressed[:cap(compressed)], data) + + docStoredOffsets[docNum] = uint64(s.w.Count()) + + _, err := writeUvarints(s.w, + uint64(len(metaBytes)), + uint64(len(idFieldVal)+len(compressed))) + if err != nil { + return 0, err + } + + _, err = s.w.Write(metaBytes) + if err != nil { + return 0, err + } + + _, err = s.w.Write(idFieldVal) + if err != nil { + return 0, err + } + + _, err = s.w.Write(compressed) + if err != nil { + return 0, err + } + } + + storedIndexOffset = uint64(s.w.Count()) + + for _, docStoredOffset := range docStoredOffsets { + err = binary.Write(s.w, binary.BigEndian, docStoredOffset) + if err != nil { + return 0, err + } + } + + return storedIndexOffset, nil +} + +func (s *interim) writeDicts() (fdvIndexOffset uint64, dictOffsets []uint64, err error) { + dictOffsets = make([]uint64, len(s.FieldsInv)) + + fdvOffsetsStart := make([]uint64, len(s.FieldsInv)) + fdvOffsetsEnd := make([]uint64, len(s.FieldsInv)) + + buf := s.grabBuf(binary.MaxVarintLen64) + + // these int coders are initialized with chunk size 1024 + // however this will be reset to the correct chunk size + // while processing each individual field-term section + tfEncoder := newChunkedIntCoder(1024, uint64(len(s.results)-1)) + locEncoder := newChunkedIntCoder(1024, uint64(len(s.results)-1)) + + var docTermMap [][]byte + + if s.builder == nil { + s.builder, err = vellum.New(&s.builderBuf, nil) + if err != nil { + return 0, nil, err + } + } + + for fieldID, terms := range s.DictKeys { + if cap(docTermMap) < len(s.results) { + docTermMap = make([][]byte, len(s.results)) + } else { + docTermMap = docTermMap[0:len(s.results)] + for docNum := range docTermMap { // reset the docTermMap + docTermMap[docNum] = docTermMap[docNum][:0] + } + } + + dict := s.Dicts[fieldID] + + for _, term := range terms { // terms are already sorted + pid := dict[term] - 1 + + postingsBS := s.Postings[pid] + + freqNorms := s.FreqNorms[pid] + freqNormOffset := 0 + + locs := s.Locs[pid] + locOffset := 0 + + chunkSize, err := getChunkSize(s.chunkMode, postingsBS.GetCardinality(), uint64(len(s.results))) + if err != nil { + return 0, nil, err + } + tfEncoder.SetChunkSize(chunkSize, uint64(len(s.results)-1)) + locEncoder.SetChunkSize(chunkSize, uint64(len(s.results)-1)) + + postingsItr := postingsBS.Iterator() + for postingsItr.HasNext() { + docNum := uint64(postingsItr.Next()) + + freqNorm := freqNorms[freqNormOffset] + + err = tfEncoder.Add(docNum, + encodeFreqHasLocs(freqNorm.freq, freqNorm.numLocs > 0), + uint64(math.Float32bits(freqNorm.norm))) + if err != nil { + return 0, nil, err + } + + if freqNorm.numLocs > 0 { + numBytesLocs := 0 + for _, loc := range locs[locOffset : locOffset+freqNorm.numLocs] { + numBytesLocs += totalUvarintBytes( + uint64(loc.fieldID), loc.pos, loc.start, loc.end, + uint64(len(loc.arrayposs)), loc.arrayposs) + } + + err = locEncoder.Add(docNum, uint64(numBytesLocs)) + if err != nil { + return 0, nil, err + } + + for _, loc := range locs[locOffset : locOffset+freqNorm.numLocs] { + err = locEncoder.Add(docNum, + uint64(loc.fieldID), loc.pos, loc.start, loc.end, + uint64(len(loc.arrayposs))) + if err != nil { + return 0, nil, err + } + + err = locEncoder.Add(docNum, loc.arrayposs...) + if err != nil { + return 0, nil, err + } + } + + locOffset += freqNorm.numLocs + } + + freqNormOffset++ + + docTermMap[docNum] = append( + append(docTermMap[docNum], term...), + termSeparator) + } + + tfEncoder.Close() + locEncoder.Close() + + postingsOffset, err := + writePostings(postingsBS, tfEncoder, locEncoder, nil, s.w, buf) + if err != nil { + return 0, nil, err + } + + if postingsOffset > uint64(0) { + err = s.builder.Insert([]byte(term), postingsOffset) + if err != nil { + return 0, nil, err + } + } + + tfEncoder.Reset() + locEncoder.Reset() + } + + err = s.builder.Close() + if err != nil { + return 0, nil, err + } + + // record where this dictionary starts + dictOffsets[fieldID] = uint64(s.w.Count()) + + vellumData := s.builderBuf.Bytes() + + // write out the length of the vellum data + n := binary.PutUvarint(buf, uint64(len(vellumData))) + _, err = s.w.Write(buf[:n]) + if err != nil { + return 0, nil, err + } + + // write this vellum to disk + _, err = s.w.Write(vellumData) + if err != nil { + return 0, nil, err + } + + // reset vellum for reuse + s.builderBuf.Reset() + + err = s.builder.Reset(&s.builderBuf) + if err != nil { + return 0, nil, err + } + + // write the field doc values + // NOTE: doc values continue to use legacy chunk mode + chunkSize, err := getChunkSize(LegacyChunkMode, 0, 0) + if err != nil { + return 0, nil, err + } + fdvEncoder := newChunkedContentCoder(chunkSize, uint64(len(s.results)-1), s.w, false) + if s.IncludeDocValues[fieldID] { + for docNum, docTerms := range docTermMap { + if len(docTerms) > 0 { + err = fdvEncoder.Add(uint64(docNum), docTerms) + if err != nil { + return 0, nil, err + } + } + } + err = fdvEncoder.Close() + if err != nil { + return 0, nil, err + } + + fdvOffsetsStart[fieldID] = uint64(s.w.Count()) + + _, err = fdvEncoder.Write() + if err != nil { + return 0, nil, err + } + + fdvOffsetsEnd[fieldID] = uint64(s.w.Count()) + + fdvEncoder.Reset() + } else { + fdvOffsetsStart[fieldID] = fieldNotUninverted + fdvOffsetsEnd[fieldID] = fieldNotUninverted + } + } + + fdvIndexOffset = uint64(s.w.Count()) + + for i := 0; i < len(fdvOffsetsStart); i++ { + n := binary.PutUvarint(buf, fdvOffsetsStart[i]) + _, err := s.w.Write(buf[:n]) + if err != nil { + return 0, nil, err + } + n = binary.PutUvarint(buf, fdvOffsetsEnd[i]) + _, err = s.w.Write(buf[:n]) + if err != nil { + return 0, nil, err + } + } + + return fdvIndexOffset, dictOffsets, nil +} + +// returns the total # of bytes needed to encode the given uint64's +// into binary.PutUVarint() encoding +func totalUvarintBytes(a, b, c, d, e uint64, more []uint64) (n int) { + n = numUvarintBytes(a) + n += numUvarintBytes(b) + n += numUvarintBytes(c) + n += numUvarintBytes(d) + n += numUvarintBytes(e) + for _, v := range more { + n += numUvarintBytes(v) + } + return n +} + +// returns # of bytes needed to encode x in binary.PutUvarint() encoding +func numUvarintBytes(x uint64) (n int) { + for x >= 0x80 { + x >>= 7 + n++ + } + return n + 1 +} diff --git a/vendor/github.com/blevesearch/zapx/v13/plugin.go b/vendor/github.com/blevesearch/zapx/v13/plugin.go new file mode 100644 index 00000000..f67297ec --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v13/plugin.go @@ -0,0 +1,27 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +// ZapPlugin implements the Plugin interface of +// the blevesearch/scorch_segment_api pkg +type ZapPlugin struct{} + +func (*ZapPlugin) Type() string { + return Type +} + +func (*ZapPlugin) Version() uint32 { + return Version +} diff --git a/vendor/github.com/blevesearch/zapx/v13/posting.go b/vendor/github.com/blevesearch/zapx/v13/posting.go new file mode 100644 index 00000000..1ba133d6 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v13/posting.go @@ -0,0 +1,837 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "encoding/binary" + "fmt" + "math" + "reflect" + + "github.com/RoaringBitmap/roaring" + segment "github.com/blevesearch/scorch_segment_api/v2" +) + +var reflectStaticSizePostingsList int +var reflectStaticSizePostingsIterator int +var reflectStaticSizePosting int +var reflectStaticSizeLocation int + +func init() { + var pl PostingsList + reflectStaticSizePostingsList = int(reflect.TypeOf(pl).Size()) + var pi PostingsIterator + reflectStaticSizePostingsIterator = int(reflect.TypeOf(pi).Size()) + var p Posting + reflectStaticSizePosting = int(reflect.TypeOf(p).Size()) + var l Location + reflectStaticSizeLocation = int(reflect.TypeOf(l).Size()) +} + +// FST or vellum value (uint64) encoding is determined by the top two +// highest-order or most significant bits... +// +// encoding : MSB +// name : 63 62 61...to...bit #0 (LSB) +// ----------+---+---+--------------------------------------------------- +// general : 0 | 0 | 62-bits of postingsOffset. +// ~ : 0 | 1 | reserved for future. +// 1-hit : 1 | 0 | 31-bits of positive float31 norm | 31-bits docNum. +// ~ : 1 | 1 | reserved for future. +// +// Encoding "general" is able to handle all cases, where the +// postingsOffset points to more information about the postings for +// the term. +// +// Encoding "1-hit" is used to optimize a commonly seen case when a +// term has only a single hit. For example, a term in the _id field +// will have only 1 hit. The "1-hit" encoding is used for a term +// in a field when... +// +// - term vector info is disabled for that field; +// - and, the term appears in only a single doc for that field; +// - and, the term's freq is exactly 1 in that single doc for that field; +// - and, the docNum must fit into 31-bits; +// +// Otherwise, the "general" encoding is used instead. +// +// In the "1-hit" encoding, the field in that single doc may have +// other terms, which is supported in the "1-hit" encoding by the +// positive float31 norm. + +const FSTValEncodingMask = uint64(0xc000000000000000) +const FSTValEncodingGeneral = uint64(0x0000000000000000) +const FSTValEncoding1Hit = uint64(0x8000000000000000) + +func FSTValEncode1Hit(docNum uint64, normBits uint64) uint64 { + return FSTValEncoding1Hit | ((mask31Bits & normBits) << 31) | (mask31Bits & docNum) +} + +func FSTValDecode1Hit(v uint64) (docNum uint64, normBits uint64) { + return (mask31Bits & v), (mask31Bits & (v >> 31)) +} + +const mask31Bits = uint64(0x000000007fffffff) + +func under32Bits(x uint64) bool { + return x <= mask31Bits +} + +const DocNum1HitFinished = math.MaxUint64 + +var NormBits1Hit = uint64(math.Float32bits(float32(1))) + +// PostingsList is an in-memory representation of a postings list +type PostingsList struct { + sb *SegmentBase + postingsOffset uint64 + freqOffset uint64 + locOffset uint64 + postings *roaring.Bitmap + except *roaring.Bitmap + + // when normBits1Hit != 0, then this postings list came from a + // 1-hit encoding, and only the docNum1Hit & normBits1Hit apply + docNum1Hit uint64 + normBits1Hit uint64 +} + +// represents an immutable, empty postings list +var emptyPostingsList = &PostingsList{} + +// Following methods are dummy implementations for the interface +// DiskStatsReporter (for backward compatibility). +// The working implementations are supported only in zap v15.x +// and not in the earlier versions of zap. +func (i *PostingsList) ResetBytesRead(uint64) {} + +func (i *PostingsList) BytesRead() uint64 { + return 0 +} + +func (i *PostingsList) incrementBytesRead(uint64) {} + +func (i *PostingsList) BytesWritten() uint64 { + return 0 +} + +func (p *PostingsList) Size() int { + sizeInBytes := reflectStaticSizePostingsList + SizeOfPtr + + if p.except != nil { + sizeInBytes += int(p.except.GetSizeInBytes()) + } + + return sizeInBytes +} + +func (p *PostingsList) OrInto(receiver *roaring.Bitmap) { + if p.normBits1Hit != 0 { + receiver.Add(uint32(p.docNum1Hit)) + return + } + + if p.postings != nil { + receiver.Or(p.postings) + } +} + +// Iterator returns an iterator for this postings list +func (p *PostingsList) Iterator(includeFreq, includeNorm, includeLocs bool, + prealloc segment.PostingsIterator) segment.PostingsIterator { + if p.normBits1Hit == 0 && p.postings == nil { + return emptyPostingsIterator + } + + var preallocPI *PostingsIterator + pi, ok := prealloc.(*PostingsIterator) + if ok && pi != nil { + preallocPI = pi + } + if preallocPI == emptyPostingsIterator { + preallocPI = nil + } + + return p.iterator(includeFreq, includeNorm, includeLocs, preallocPI) +} + +func (p *PostingsList) iterator(includeFreq, includeNorm, includeLocs bool, + rv *PostingsIterator) *PostingsIterator { + if rv == nil { + rv = &PostingsIterator{} + } else { + freqNormReader := rv.freqNormReader + if freqNormReader != nil { + freqNormReader.reset() + } + + locReader := rv.locReader + if locReader != nil { + locReader.reset() + } + + nextLocs := rv.nextLocs[:0] + nextSegmentLocs := rv.nextSegmentLocs[:0] + + buf := rv.buf + + *rv = PostingsIterator{} // clear the struct + + rv.freqNormReader = freqNormReader + rv.locReader = locReader + + rv.nextLocs = nextLocs + rv.nextSegmentLocs = nextSegmentLocs + + rv.buf = buf + } + + rv.postings = p + rv.includeFreqNorm = includeFreq || includeNorm || includeLocs + rv.includeLocs = includeLocs + + if p.normBits1Hit != 0 { + // "1-hit" encoding + rv.docNum1Hit = p.docNum1Hit + rv.normBits1Hit = p.normBits1Hit + + if p.except != nil && p.except.Contains(uint32(rv.docNum1Hit)) { + rv.docNum1Hit = DocNum1HitFinished + } + + return rv + } + + // "general" encoding, check if empty + if p.postings == nil { + return rv + } + + // initialize freq chunk reader + if rv.includeFreqNorm { + rv.freqNormReader = newChunkedIntDecoder(p.sb.mem, p.freqOffset) + } + + // initialize the loc chunk reader + if rv.includeLocs { + rv.locReader = newChunkedIntDecoder(p.sb.mem, p.locOffset) + } + + rv.all = p.postings.Iterator() + if p.except != nil { + rv.ActualBM = roaring.AndNot(p.postings, p.except) + rv.Actual = rv.ActualBM.Iterator() + } else { + rv.ActualBM = p.postings + rv.Actual = rv.all // Optimize to use same iterator for all & Actual. + } + + return rv +} + +// Count returns the number of items on this postings list +func (p *PostingsList) Count() uint64 { + var n, e uint64 + if p.normBits1Hit != 0 { + n = 1 + if p.except != nil && p.except.Contains(uint32(p.docNum1Hit)) { + e = 1 + } + } else if p.postings != nil { + n = p.postings.GetCardinality() + if p.except != nil { + e = p.postings.AndCardinality(p.except) + } + } + return n - e +} + +func (rv *PostingsList) read(postingsOffset uint64, d *Dictionary) error { + rv.postingsOffset = postingsOffset + + // handle "1-hit" encoding special case + if rv.postingsOffset&FSTValEncodingMask == FSTValEncoding1Hit { + return rv.init1Hit(postingsOffset) + } + + // read the location of the freq/norm details + var n uint64 + var read int + + rv.freqOffset, read = binary.Uvarint(d.sb.mem[postingsOffset+n : postingsOffset+binary.MaxVarintLen64]) + n += uint64(read) + + rv.locOffset, read = binary.Uvarint(d.sb.mem[postingsOffset+n : postingsOffset+n+binary.MaxVarintLen64]) + n += uint64(read) + + var postingsLen uint64 + postingsLen, read = binary.Uvarint(d.sb.mem[postingsOffset+n : postingsOffset+n+binary.MaxVarintLen64]) + n += uint64(read) + + roaringBytes := d.sb.mem[postingsOffset+n : postingsOffset+n+postingsLen] + + if rv.postings == nil { + rv.postings = roaring.NewBitmap() + } + _, err := rv.postings.FromBuffer(roaringBytes) + if err != nil { + return fmt.Errorf("error loading roaring bitmap: %v", err) + } + + return nil +} + +func (rv *PostingsList) init1Hit(fstVal uint64) error { + docNum, normBits := FSTValDecode1Hit(fstVal) + + rv.docNum1Hit = docNum + rv.normBits1Hit = normBits + + return nil +} + +// PostingsIterator provides a way to iterate through the postings list +type PostingsIterator struct { + postings *PostingsList + all roaring.IntPeekable + Actual roaring.IntPeekable + ActualBM *roaring.Bitmap + + currChunk uint32 + freqNormReader *chunkedIntDecoder + locReader *chunkedIntDecoder + + next Posting // reused across Next() calls + nextLocs []Location // reused across Next() calls + nextSegmentLocs []segment.Location // reused across Next() calls + + docNum1Hit uint64 + normBits1Hit uint64 + + buf []byte + + includeFreqNorm bool + includeLocs bool +} + +var emptyPostingsIterator = &PostingsIterator{} + +// Following methods are dummy implementations for the interface +// DiskStatsReporter (for backward compatibility). +// The working implementations are supported only in zap v15.x +// and not in the earlier versions of zap. +func (i *PostingsIterator) ResetBytesRead(uint64) {} + +func (i *PostingsIterator) BytesRead() uint64 { + return 0 +} + +func (i *PostingsIterator) incrementBytesRead(uint64) {} + +func (i *PostingsIterator) BytesWritten() uint64 { + return 0 +} + +func (i *PostingsIterator) Size() int { + sizeInBytes := reflectStaticSizePostingsIterator + SizeOfPtr + + i.next.Size() + // account for freqNormReader, locReader if we start using this. + for _, entry := range i.nextLocs { + sizeInBytes += entry.Size() + } + + return sizeInBytes +} + +func (i *PostingsIterator) loadChunk(chunk int) error { + if i.includeFreqNorm { + err := i.freqNormReader.loadChunk(chunk) + if err != nil { + return err + } + } + + if i.includeLocs { + err := i.locReader.loadChunk(chunk) + if err != nil { + return err + } + } + + i.currChunk = uint32(chunk) + return nil +} + +func (i *PostingsIterator) readFreqNormHasLocs() (uint64, uint64, bool, error) { + if i.normBits1Hit != 0 { + return 1, i.normBits1Hit, false, nil + } + + freqHasLocs, err := i.freqNormReader.readUvarint() + if err != nil { + return 0, 0, false, fmt.Errorf("error reading frequency: %v", err) + } + + freq, hasLocs := decodeFreqHasLocs(freqHasLocs) + + normBits, err := i.freqNormReader.readUvarint() + if err != nil { + return 0, 0, false, fmt.Errorf("error reading norm: %v", err) + } + + return freq, normBits, hasLocs, nil +} + +func (i *PostingsIterator) skipFreqNormReadHasLocs() (bool, error) { + if i.normBits1Hit != 0 { + return false, nil + } + + freqHasLocs, err := i.freqNormReader.readUvarint() + if err != nil { + return false, fmt.Errorf("error reading freqHasLocs: %v", err) + } + + i.freqNormReader.SkipUvarint() // Skip normBits. + + return freqHasLocs&0x01 != 0, nil // See decodeFreqHasLocs() / hasLocs. +} + +func encodeFreqHasLocs(freq uint64, hasLocs bool) uint64 { + rv := freq << 1 + if hasLocs { + rv = rv | 0x01 // 0'th LSB encodes whether there are locations + } + return rv +} + +func decodeFreqHasLocs(freqHasLocs uint64) (uint64, bool) { + freq := freqHasLocs >> 1 + hasLocs := freqHasLocs&0x01 != 0 + return freq, hasLocs +} + +// readLocation processes all the integers on the stream representing a single +// location. +func (i *PostingsIterator) readLocation(l *Location) error { + // read off field + fieldID, err := i.locReader.readUvarint() + if err != nil { + return fmt.Errorf("error reading location field: %v", err) + } + // read off pos + pos, err := i.locReader.readUvarint() + if err != nil { + return fmt.Errorf("error reading location pos: %v", err) + } + // read off start + start, err := i.locReader.readUvarint() + if err != nil { + return fmt.Errorf("error reading location start: %v", err) + } + // read off end + end, err := i.locReader.readUvarint() + if err != nil { + return fmt.Errorf("error reading location end: %v", err) + } + // read off num array pos + numArrayPos, err := i.locReader.readUvarint() + if err != nil { + return fmt.Errorf("error reading location num array pos: %v", err) + } + + l.field = i.postings.sb.fieldsInv[fieldID] + l.pos = pos + l.start = start + l.end = end + + if cap(l.ap) < int(numArrayPos) { + l.ap = make([]uint64, int(numArrayPos)) + } else { + l.ap = l.ap[:int(numArrayPos)] + } + + // read off array positions + for k := 0; k < int(numArrayPos); k++ { + ap, err := i.locReader.readUvarint() + if err != nil { + return fmt.Errorf("error reading array position: %v", err) + } + + l.ap[k] = ap + } + + return nil +} + +// Next returns the next posting on the postings list, or nil at the end +func (i *PostingsIterator) Next() (segment.Posting, error) { + return i.nextAtOrAfter(0) +} + +// Advance returns the posting at the specified docNum or it is not present +// the next posting, or if the end is reached, nil +func (i *PostingsIterator) Advance(docNum uint64) (segment.Posting, error) { + return i.nextAtOrAfter(docNum) +} + +// Next returns the next posting on the postings list, or nil at the end +func (i *PostingsIterator) nextAtOrAfter(atOrAfter uint64) (segment.Posting, error) { + docNum, exists, err := i.nextDocNumAtOrAfter(atOrAfter) + if err != nil || !exists { + return nil, err + } + + i.next = Posting{} // clear the struct + rv := &i.next + rv.docNum = docNum + + if !i.includeFreqNorm { + return rv, nil + } + + var normBits uint64 + var hasLocs bool + + rv.freq, normBits, hasLocs, err = i.readFreqNormHasLocs() + if err != nil { + return nil, err + } + + rv.norm = math.Float32frombits(uint32(normBits)) + + if i.includeLocs && hasLocs { + // prepare locations into reused slices, where we assume + // rv.freq >= "number of locs", since in a composite field, + // some component fields might have their IncludeTermVector + // flags disabled while other component fields are enabled + if cap(i.nextLocs) >= int(rv.freq) { + i.nextLocs = i.nextLocs[0:rv.freq] + } else { + i.nextLocs = make([]Location, rv.freq, rv.freq*2) + } + if cap(i.nextSegmentLocs) < int(rv.freq) { + i.nextSegmentLocs = make([]segment.Location, rv.freq, rv.freq*2) + } + rv.locs = i.nextSegmentLocs[:0] + + numLocsBytes, err := i.locReader.readUvarint() + if err != nil { + return nil, fmt.Errorf("error reading location numLocsBytes: %v", err) + } + + j := 0 + startBytesRemaining := i.locReader.Len() // # bytes remaining in the locReader + for startBytesRemaining-i.locReader.Len() < int(numLocsBytes) { + err := i.readLocation(&i.nextLocs[j]) + if err != nil { + return nil, err + } + rv.locs = append(rv.locs, &i.nextLocs[j]) + j++ + } + } + + return rv, nil +} + +// nextDocNum returns the next docNum on the postings list, and also +// sets up the currChunk / loc related fields of the iterator. +func (i *PostingsIterator) nextDocNumAtOrAfter(atOrAfter uint64) (uint64, bool, error) { + if i.normBits1Hit != 0 { + if i.docNum1Hit == DocNum1HitFinished { + return 0, false, nil + } + if i.docNum1Hit < atOrAfter { + // advanced past our 1-hit + i.docNum1Hit = DocNum1HitFinished // consume our 1-hit docNum + return 0, false, nil + } + docNum := i.docNum1Hit + i.docNum1Hit = DocNum1HitFinished // consume our 1-hit docNum + return docNum, true, nil + } + + if i.Actual == nil || !i.Actual.HasNext() { + return 0, false, nil + } + + if i.postings == nil || i.postings == emptyPostingsList { + // couldn't find anything + return 0, false, nil + } + + if i.postings.postings == i.ActualBM { + return i.nextDocNumAtOrAfterClean(atOrAfter) + } + + i.Actual.AdvanceIfNeeded(uint32(atOrAfter)) + + if !i.Actual.HasNext() || !i.all.HasNext() { + // couldn't find anything + return 0, false, nil + } + + n := i.Actual.Next() + allN := i.all.Next() + + chunkSize, err := getChunkSize(i.postings.sb.chunkMode, i.postings.postings.GetCardinality(), i.postings.sb.numDocs) + if err != nil { + return 0, false, err + } + nChunk := n / uint32(chunkSize) + + // when allN becomes >= to here, then allN is in the same chunk as nChunk. + allNReachesNChunk := nChunk * uint32(chunkSize) + + // n is the next actual hit (excluding some postings), and + // allN is the next hit in the full postings, and + // if they don't match, move 'all' forwards until they do + for allN != n { + // we've reached same chunk, so move the freq/norm/loc decoders forward + if i.includeFreqNorm && allN >= allNReachesNChunk { + err := i.currChunkNext(nChunk) + if err != nil { + return 0, false, err + } + } + + if !i.all.HasNext() { + return 0, false, nil + } + + allN = i.all.Next() + } + + if i.includeFreqNorm && (i.currChunk != nChunk || i.freqNormReader.isNil()) { + err := i.loadChunk(int(nChunk)) + if err != nil { + return 0, false, fmt.Errorf("error loading chunk: %v", err) + } + } + + return uint64(n), true, nil +} + +// optimization when the postings list is "clean" (e.g., no updates & +// no deletions) where the all bitmap is the same as the actual bitmap +func (i *PostingsIterator) nextDocNumAtOrAfterClean( + atOrAfter uint64) (uint64, bool, error) { + if !i.includeFreqNorm { + i.Actual.AdvanceIfNeeded(uint32(atOrAfter)) + + if !i.Actual.HasNext() { + return 0, false, nil // couldn't find anything + } + + return uint64(i.Actual.Next()), true, nil + } + + chunkSize, err := getChunkSize(i.postings.sb.chunkMode, i.postings.postings.GetCardinality(), i.postings.sb.numDocs) + if err != nil { + return 0, false, err + } + + // freq-norm's needed, so maintain freq-norm chunk reader + sameChunkNexts := 0 // # of times we called Next() in the same chunk + n := i.Actual.Next() + nChunk := n / uint32(chunkSize) + + for uint64(n) < atOrAfter && i.Actual.HasNext() { + n = i.Actual.Next() + + nChunkPrev := nChunk + nChunk = n / uint32(chunkSize) + + if nChunk != nChunkPrev { + sameChunkNexts = 0 + } else { + sameChunkNexts += 1 + } + } + + if uint64(n) < atOrAfter { + // couldn't find anything + return 0, false, nil + } + + for j := 0; j < sameChunkNexts; j++ { + err := i.currChunkNext(nChunk) + if err != nil { + return 0, false, fmt.Errorf("error optimized currChunkNext: %v", err) + } + } + + if i.currChunk != nChunk || i.freqNormReader.isNil() { + err := i.loadChunk(int(nChunk)) + if err != nil { + return 0, false, fmt.Errorf("error loading chunk: %v", err) + } + } + + return uint64(n), true, nil +} + +func (i *PostingsIterator) currChunkNext(nChunk uint32) error { + if i.currChunk != nChunk || i.freqNormReader.isNil() { + err := i.loadChunk(int(nChunk)) + if err != nil { + return fmt.Errorf("error loading chunk: %v", err) + } + } + + // read off freq/offsets even though we don't care about them + hasLocs, err := i.skipFreqNormReadHasLocs() + if err != nil { + return err + } + + if i.includeLocs && hasLocs { + numLocsBytes, err := i.locReader.readUvarint() + if err != nil { + return fmt.Errorf("error reading location numLocsBytes: %v", err) + } + + // skip over all the location bytes + i.locReader.SkipBytes(int(numLocsBytes)) + } + + return nil +} + +// DocNum1Hit returns the docNum and true if this is "1-hit" optimized +// and the docNum is available. +func (p *PostingsIterator) DocNum1Hit() (uint64, bool) { + if p.normBits1Hit != 0 && p.docNum1Hit != DocNum1HitFinished { + return p.docNum1Hit, true + } + return 0, false +} + +// ActualBitmap returns the underlying actual bitmap +// which can be used up the stack for optimizations +func (p *PostingsIterator) ActualBitmap() *roaring.Bitmap { + return p.ActualBM +} + +// ReplaceActual replaces the ActualBM with the provided +// bitmap +func (p *PostingsIterator) ReplaceActual(abm *roaring.Bitmap) { + p.ActualBM = abm + p.Actual = abm.Iterator() +} + +// PostingsIteratorFromBitmap constructs a PostingsIterator given an +// "actual" bitmap. +func PostingsIteratorFromBitmap(bm *roaring.Bitmap, + includeFreqNorm, includeLocs bool) (segment.PostingsIterator, error) { + return &PostingsIterator{ + ActualBM: bm, + Actual: bm.Iterator(), + includeFreqNorm: includeFreqNorm, + includeLocs: includeLocs, + }, nil +} + +// PostingsIteratorFrom1Hit constructs a PostingsIterator given a +// 1-hit docNum. +func PostingsIteratorFrom1Hit(docNum1Hit uint64, + includeFreqNorm, includeLocs bool) (segment.PostingsIterator, error) { + return &PostingsIterator{ + docNum1Hit: docNum1Hit, + normBits1Hit: NormBits1Hit, + includeFreqNorm: includeFreqNorm, + includeLocs: includeLocs, + }, nil +} + +// Posting is a single entry in a postings list +type Posting struct { + docNum uint64 + freq uint64 + norm float32 + locs []segment.Location +} + +func (p *Posting) Size() int { + sizeInBytes := reflectStaticSizePosting + + for _, entry := range p.locs { + sizeInBytes += entry.Size() + } + + return sizeInBytes +} + +// Number returns the document number of this posting in this segment +func (p *Posting) Number() uint64 { + return p.docNum +} + +// Frequency returns the frequencies of occurrence of this term in this doc/field +func (p *Posting) Frequency() uint64 { + return p.freq +} + +// Norm returns the normalization factor for this posting +func (p *Posting) Norm() float64 { + return float64(p.norm) +} + +// Locations returns the location information for each occurrence +func (p *Posting) Locations() []segment.Location { + return p.locs +} + +// Location represents the location of a single occurrence +type Location struct { + field string + pos uint64 + start uint64 + end uint64 + ap []uint64 +} + +func (l *Location) Size() int { + return reflectStaticSizeLocation + + len(l.field) + + len(l.ap)*SizeOfUint64 +} + +// Field returns the name of the field (useful in composite fields to know +// which original field the value came from) +func (l *Location) Field() string { + return l.field +} + +// Start returns the start byte offset of this occurrence +func (l *Location) Start() uint64 { + return l.start +} + +// End returns the end byte offset of this occurrence +func (l *Location) End() uint64 { + return l.end +} + +// Pos returns the 1-based phrase position of this occurrence +func (l *Location) Pos() uint64 { + return l.pos +} + +// ArrayPositions returns the array position vector associated with this occurrence +func (l *Location) ArrayPositions() []uint64 { + return l.ap +} diff --git a/vendor/github.com/blevesearch/zap/v13/read.go b/vendor/github.com/blevesearch/zapx/v13/read.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v13/read.go rename to vendor/github.com/blevesearch/zapx/v13/read.go diff --git a/vendor/github.com/blevesearch/zapx/v13/segment.go b/vendor/github.com/blevesearch/zapx/v13/segment.go new file mode 100644 index 00000000..1fbf7848 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v13/segment.go @@ -0,0 +1,600 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "os" + "sync" + "unsafe" + + "github.com/RoaringBitmap/roaring" + mmap "github.com/blevesearch/mmap-go" + segment "github.com/blevesearch/scorch_segment_api/v2" + "github.com/blevesearch/vellum" + "github.com/golang/snappy" +) + +var reflectStaticSizeSegmentBase int + +func init() { + var sb SegmentBase + reflectStaticSizeSegmentBase = int(unsafe.Sizeof(sb)) +} + +// Open returns a zap impl of a segment +func (*ZapPlugin) Open(path string) (segment.Segment, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + mm, err := mmap.Map(f, mmap.RDONLY, 0) + if err != nil { + // mmap failed, try to close the file + _ = f.Close() + return nil, err + } + + rv := &Segment{ + SegmentBase: SegmentBase{ + mem: mm[0 : len(mm)-FooterSize], + fieldsMap: make(map[string]uint16), + fieldDvReaders: make(map[uint16]*docValueReader), + fieldFSTs: make(map[uint16]*vellum.FST), + }, + f: f, + mm: mm, + path: path, + refs: 1, + } + rv.SegmentBase.updateSize() + + err = rv.loadConfig() + if err != nil { + _ = rv.Close() + return nil, err + } + + err = rv.loadFields() + if err != nil { + _ = rv.Close() + return nil, err + } + + err = rv.loadDvReaders() + if err != nil { + _ = rv.Close() + return nil, err + } + + return rv, nil +} + +// SegmentBase is a memory only, read-only implementation of the +// segment.Segment interface, using zap's data representation. +type SegmentBase struct { + mem []byte + memCRC uint32 + chunkMode uint32 + fieldsMap map[string]uint16 // fieldName -> fieldID+1 + fieldsInv []string // fieldID -> fieldName + numDocs uint64 + storedIndexOffset uint64 + fieldsIndexOffset uint64 + docValueOffset uint64 + dictLocs []uint64 + fieldDvReaders map[uint16]*docValueReader // naive chunk cache per field + fieldDvNames []string // field names cached in fieldDvReaders + size uint64 + + m sync.Mutex + fieldFSTs map[uint16]*vellum.FST +} + +func (sb *SegmentBase) Size() int { + return int(sb.size) +} + +func (sb *SegmentBase) updateSize() { + sizeInBytes := reflectStaticSizeSegmentBase + + cap(sb.mem) + + // fieldsMap + for k := range sb.fieldsMap { + sizeInBytes += (len(k) + SizeOfString) + SizeOfUint16 + } + + // fieldsInv, dictLocs + for _, entry := range sb.fieldsInv { + sizeInBytes += len(entry) + SizeOfString + } + sizeInBytes += len(sb.dictLocs) * SizeOfUint64 + + // fieldDvReaders + for _, v := range sb.fieldDvReaders { + sizeInBytes += SizeOfUint16 + SizeOfPtr + if v != nil { + sizeInBytes += v.size() + } + } + + sb.size = uint64(sizeInBytes) +} + +func (sb *SegmentBase) AddRef() {} +func (sb *SegmentBase) DecRef() (err error) { return nil } +func (sb *SegmentBase) Close() (err error) { return nil } + +// Segment implements a persisted segment.Segment interface, by +// embedding an mmap()'ed SegmentBase. +type Segment struct { + SegmentBase + + f *os.File + mm mmap.MMap + path string + version uint32 + crc uint32 + + m sync.Mutex // Protects the fields that follow. + refs int64 +} + +func (s *Segment) Size() int { + // 8 /* size of file pointer */ + // 4 /* size of version -> uint32 */ + // 4 /* size of crc -> uint32 */ + sizeOfUints := 16 + + sizeInBytes := (len(s.path) + SizeOfString) + sizeOfUints + + // mutex, refs -> int64 + sizeInBytes += 16 + + // do not include the mmap'ed part + return sizeInBytes + s.SegmentBase.Size() - cap(s.mem) +} + +func (s *Segment) AddRef() { + s.m.Lock() + s.refs++ + s.m.Unlock() +} + +func (s *Segment) DecRef() (err error) { + s.m.Lock() + s.refs-- + if s.refs == 0 { + err = s.closeActual() + } + s.m.Unlock() + return err +} + +func (s *Segment) loadConfig() error { + crcOffset := len(s.mm) - 4 + s.crc = binary.BigEndian.Uint32(s.mm[crcOffset : crcOffset+4]) + + verOffset := crcOffset - 4 + s.version = binary.BigEndian.Uint32(s.mm[verOffset : verOffset+4]) + if s.version != Version { + return fmt.Errorf("unsupported version %d", s.version) + } + + chunkOffset := verOffset - 4 + s.chunkMode = binary.BigEndian.Uint32(s.mm[chunkOffset : chunkOffset+4]) + + docValueOffset := chunkOffset - 8 + s.docValueOffset = binary.BigEndian.Uint64(s.mm[docValueOffset : docValueOffset+8]) + + fieldsIndexOffset := docValueOffset - 8 + s.fieldsIndexOffset = binary.BigEndian.Uint64(s.mm[fieldsIndexOffset : fieldsIndexOffset+8]) + + storedIndexOffset := fieldsIndexOffset - 8 + s.storedIndexOffset = binary.BigEndian.Uint64(s.mm[storedIndexOffset : storedIndexOffset+8]) + + numDocsOffset := storedIndexOffset - 8 + s.numDocs = binary.BigEndian.Uint64(s.mm[numDocsOffset : numDocsOffset+8]) + return nil +} + +// Following methods are dummy implementations for the interface +// DiskStatsReporter (for backward compatibility). +// The working implementations are supported only in zap v15.x +// and not in the earlier versions of zap. +func (s *Segment) ResetBytesRead(uint64) {} + +func (s *Segment) BytesRead() uint64 { + return 0 +} + +func (s *Segment) BytesWritten() uint64 { + return 0 +} + +func (s *Segment) incrementBytesRead(uint64) {} + +func (s *SegmentBase) BytesWritten() uint64 { + return 0 +} + +func (s *SegmentBase) setBytesWritten(uint64) {} + +func (s *SegmentBase) BytesRead() uint64 { + return 0 +} + +func (s *SegmentBase) ResetBytesRead(uint64) {} + +func (s *SegmentBase) incrementBytesRead(uint64) {} + +func (s *SegmentBase) loadFields() error { + // NOTE for now we assume the fields index immediately precedes + // the footer, and if this changes, need to adjust accordingly (or + // store explicit length), where s.mem was sliced from s.mm in Open(). + fieldsIndexEnd := uint64(len(s.mem)) + + // iterate through fields index + var fieldID uint64 + for s.fieldsIndexOffset+(8*fieldID) < fieldsIndexEnd { + addr := binary.BigEndian.Uint64(s.mem[s.fieldsIndexOffset+(8*fieldID) : s.fieldsIndexOffset+(8*fieldID)+8]) + + dictLoc, read := binary.Uvarint(s.mem[addr:fieldsIndexEnd]) + n := uint64(read) + s.dictLocs = append(s.dictLocs, dictLoc) + + var nameLen uint64 + nameLen, read = binary.Uvarint(s.mem[addr+n : fieldsIndexEnd]) + n += uint64(read) + + name := string(s.mem[addr+n : addr+n+nameLen]) + s.fieldsInv = append(s.fieldsInv, name) + s.fieldsMap[name] = uint16(fieldID + 1) + + fieldID++ + } + return nil +} + +// Dictionary returns the term dictionary for the specified field +func (s *SegmentBase) Dictionary(field string) (segment.TermDictionary, error) { + dict, err := s.dictionary(field) + if err == nil && dict == nil { + return emptyDictionary, nil + } + return dict, err +} + +func (sb *SegmentBase) dictionary(field string) (rv *Dictionary, err error) { + fieldIDPlus1 := sb.fieldsMap[field] + if fieldIDPlus1 > 0 { + rv = &Dictionary{ + sb: sb, + field: field, + fieldID: fieldIDPlus1 - 1, + } + + dictStart := sb.dictLocs[rv.fieldID] + if dictStart > 0 { + var ok bool + sb.m.Lock() + if rv.fst, ok = sb.fieldFSTs[rv.fieldID]; !ok { + // read the length of the vellum data + vellumLen, read := binary.Uvarint(sb.mem[dictStart : dictStart+binary.MaxVarintLen64]) + fstBytes := sb.mem[dictStart+uint64(read) : dictStart+uint64(read)+vellumLen] + rv.fst, err = vellum.Load(fstBytes) + if err != nil { + sb.m.Unlock() + return nil, fmt.Errorf("dictionary field %s vellum err: %v", field, err) + } + + sb.fieldFSTs[rv.fieldID] = rv.fst + } + + sb.m.Unlock() + rv.fstReader, err = rv.fst.Reader() + if err != nil { + return nil, fmt.Errorf("dictionary field %s vellum reader err: %v", field, err) + } + } + } + + return rv, nil +} + +// visitDocumentCtx holds data structures that are reusable across +// multiple VisitDocument() calls to avoid memory allocations +type visitDocumentCtx struct { + buf []byte + reader bytes.Reader + arrayPos []uint64 +} + +var visitDocumentCtxPool = sync.Pool{ + New: func() interface{} { + reuse := &visitDocumentCtx{} + return reuse + }, +} + +// VisitStoredFields invokes the StoredFieldValueVisitor for each stored field +// for the specified doc number +func (s *SegmentBase) VisitStoredFields(num uint64, visitor segment.StoredFieldValueVisitor) error { + vdc := visitDocumentCtxPool.Get().(*visitDocumentCtx) + defer visitDocumentCtxPool.Put(vdc) + return s.visitStoredFields(vdc, num, visitor) +} + +func (s *SegmentBase) visitStoredFields(vdc *visitDocumentCtx, num uint64, + visitor segment.StoredFieldValueVisitor) error { + // first make sure this is a valid number in this segment + if num < s.numDocs { + meta, compressed := s.getDocStoredMetaAndCompressed(num) + + vdc.reader.Reset(meta) + + // handle _id field special case + idFieldValLen, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return err + } + idFieldVal := compressed[:idFieldValLen] + + keepGoing := visitor("_id", byte('t'), idFieldVal, nil) + if !keepGoing { + visitDocumentCtxPool.Put(vdc) + return nil + } + + // handle non-"_id" fields + compressed = compressed[idFieldValLen:] + + uncompressed, err := snappy.Decode(vdc.buf[:cap(vdc.buf)], compressed) + if err != nil { + return err + } + + for keepGoing { + field, err := binary.ReadUvarint(&vdc.reader) + if err == io.EOF { + break + } + if err != nil { + return err + } + typ, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return err + } + offset, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return err + } + l, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return err + } + numap, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return err + } + var arrayPos []uint64 + if numap > 0 { + if cap(vdc.arrayPos) < int(numap) { + vdc.arrayPos = make([]uint64, numap) + } + arrayPos = vdc.arrayPos[:numap] + for i := 0; i < int(numap); i++ { + ap, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return err + } + arrayPos[i] = ap + } + } + + value := uncompressed[offset : offset+l] + keepGoing = visitor(s.fieldsInv[field], byte(typ), value, arrayPos) + } + + vdc.buf = uncompressed + } + return nil +} + +// DocID returns the value of the _id field for the given docNum +func (s *SegmentBase) DocID(num uint64) ([]byte, error) { + if num >= s.numDocs { + return nil, nil + } + + vdc := visitDocumentCtxPool.Get().(*visitDocumentCtx) + + meta, compressed := s.getDocStoredMetaAndCompressed(num) + + vdc.reader.Reset(meta) + + // handle _id field special case + idFieldValLen, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return nil, err + } + idFieldVal := compressed[:idFieldValLen] + + visitDocumentCtxPool.Put(vdc) + + return idFieldVal, nil +} + +// Count returns the number of documents in this segment. +func (s *SegmentBase) Count() uint64 { + return s.numDocs +} + +// DocNumbers returns a bitset corresponding to the doc numbers of all the +// provided _id strings +func (s *SegmentBase) DocNumbers(ids []string) (*roaring.Bitmap, error) { + rv := roaring.New() + + if len(s.fieldsMap) > 0 { + idDict, err := s.dictionary("_id") + if err != nil { + return nil, err + } + + postingsList := emptyPostingsList + + sMax, err := idDict.fst.GetMaxKey() + if err != nil { + return nil, err + } + sMaxStr := string(sMax) + filteredIds := make([]string, 0, len(ids)) + for _, id := range ids { + if id <= sMaxStr { + filteredIds = append(filteredIds, id) + } + } + + for _, id := range filteredIds { + postingsList, err = idDict.postingsList([]byte(id), nil, postingsList) + if err != nil { + return nil, err + } + postingsList.OrInto(rv) + } + } + + return rv, nil +} + +// Fields returns the field names used in this segment +func (s *SegmentBase) Fields() []string { + return s.fieldsInv +} + +// Path returns the path of this segment on disk +func (s *Segment) Path() string { + return s.path +} + +// Close releases all resources associated with this segment +func (s *Segment) Close() (err error) { + return s.DecRef() +} + +func (s *Segment) closeActual() (err error) { + if s.mm != nil { + err = s.mm.Unmap() + } + // try to close file even if unmap failed + if s.f != nil { + err2 := s.f.Close() + if err == nil { + // try to return first error + err = err2 + } + } + return +} + +// some helpers i started adding for the command-line utility + +// Data returns the underlying mmaped data slice +func (s *Segment) Data() []byte { + return s.mm +} + +// CRC returns the CRC value stored in the file footer +func (s *Segment) CRC() uint32 { + return s.crc +} + +// Version returns the file version in the file footer +func (s *Segment) Version() uint32 { + return s.version +} + +// ChunkFactor returns the chunk factor in the file footer +func (s *Segment) ChunkMode() uint32 { + return s.chunkMode +} + +// FieldsIndexOffset returns the fields index offset in the file footer +func (s *Segment) FieldsIndexOffset() uint64 { + return s.fieldsIndexOffset +} + +// StoredIndexOffset returns the stored value index offset in the file footer +func (s *Segment) StoredIndexOffset() uint64 { + return s.storedIndexOffset +} + +// DocValueOffset returns the docValue offset in the file footer +func (s *Segment) DocValueOffset() uint64 { + return s.docValueOffset +} + +// NumDocs returns the number of documents in the file footer +func (s *Segment) NumDocs() uint64 { + return s.numDocs +} + +// DictAddr is a helper function to compute the file offset where the +// dictionary is stored for the specified field. +func (s *Segment) DictAddr(field string) (uint64, error) { + fieldIDPlus1, ok := s.fieldsMap[field] + if !ok { + return 0, fmt.Errorf("no such field '%s'", field) + } + + return s.dictLocs[fieldIDPlus1-1], nil +} + +func (s *SegmentBase) loadDvReaders() error { + if s.docValueOffset == fieldNotUninverted || s.numDocs == 0 { + return nil + } + + var read uint64 + for fieldID, field := range s.fieldsInv { + var fieldLocStart, fieldLocEnd uint64 + var n int + fieldLocStart, n = binary.Uvarint(s.mem[s.docValueOffset+read : s.docValueOffset+read+binary.MaxVarintLen64]) + if n <= 0 { + return fmt.Errorf("loadDvReaders: failed to read the docvalue offset start for field %d", fieldID) + } + read += uint64(n) + fieldLocEnd, n = binary.Uvarint(s.mem[s.docValueOffset+read : s.docValueOffset+read+binary.MaxVarintLen64]) + if n <= 0 { + return fmt.Errorf("loadDvReaders: failed to read the docvalue offset end for field %d", fieldID) + } + read += uint64(n) + + fieldDvReader, err := s.loadFieldDocValueReader(field, fieldLocStart, fieldLocEnd) + if err != nil { + return err + } + if fieldDvReader != nil { + s.fieldDvReaders[uint16(fieldID)] = fieldDvReader + s.fieldDvNames = append(s.fieldDvNames, field) + } + } + + return nil +} diff --git a/vendor/github.com/blevesearch/zapx/v13/sizes.go b/vendor/github.com/blevesearch/zapx/v13/sizes.go new file mode 100644 index 00000000..34166ea3 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v13/sizes.go @@ -0,0 +1,59 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "reflect" +) + +func init() { + var b bool + SizeOfBool = int(reflect.TypeOf(b).Size()) + var f32 float32 + SizeOfFloat32 = int(reflect.TypeOf(f32).Size()) + var f64 float64 + SizeOfFloat64 = int(reflect.TypeOf(f64).Size()) + var i int + SizeOfInt = int(reflect.TypeOf(i).Size()) + var m map[int]int + SizeOfMap = int(reflect.TypeOf(m).Size()) + var ptr *int + SizeOfPtr = int(reflect.TypeOf(ptr).Size()) + var slice []int + SizeOfSlice = int(reflect.TypeOf(slice).Size()) + var str string + SizeOfString = int(reflect.TypeOf(str).Size()) + var u8 uint8 + SizeOfUint8 = int(reflect.TypeOf(u8).Size()) + var u16 uint16 + SizeOfUint16 = int(reflect.TypeOf(u16).Size()) + var u32 uint32 + SizeOfUint32 = int(reflect.TypeOf(u32).Size()) + var u64 uint64 + SizeOfUint64 = int(reflect.TypeOf(u64).Size()) +} + +var SizeOfBool int +var SizeOfFloat32 int +var SizeOfFloat64 int +var SizeOfInt int +var SizeOfMap int +var SizeOfPtr int +var SizeOfSlice int +var SizeOfString int +var SizeOfUint8 int +var SizeOfUint16 int +var SizeOfUint32 int +var SizeOfUint64 int diff --git a/vendor/github.com/blevesearch/zap/v13/write.go b/vendor/github.com/blevesearch/zapx/v13/write.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v13/write.go rename to vendor/github.com/blevesearch/zapx/v13/write.go diff --git a/vendor/github.com/blevesearch/zap/v13/zap.md b/vendor/github.com/blevesearch/zapx/v13/zap.md similarity index 100% rename from vendor/github.com/blevesearch/zap/v13/zap.md rename to vendor/github.com/blevesearch/zapx/v13/zap.md diff --git a/vendor/github.com/blevesearch/zap/v14/.gitignore b/vendor/github.com/blevesearch/zapx/v14/.gitignore similarity index 100% rename from vendor/github.com/blevesearch/zap/v14/.gitignore rename to vendor/github.com/blevesearch/zapx/v14/.gitignore diff --git a/vendor/github.com/blevesearch/zapx/v14/.golangci.yml b/vendor/github.com/blevesearch/zapx/v14/.golangci.yml new file mode 100644 index 00000000..1d55bfc0 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v14/.golangci.yml @@ -0,0 +1,28 @@ +linters: + # please, do not use `enable-all`: it's deprecated and will be removed soon. + # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint + disable-all: true + enable: + - bodyclose + - deadcode + - depguard + - dupl + - errcheck + - gofmt + - goimports + - goprintffuncname + - gosec + - gosimple + - govet + - ineffassign + - misspell + - nakedret + - nolintlint + - rowserrcheck + - staticcheck + - structcheck + - typecheck + - unused + - varcheck + - whitespace + diff --git a/vendor/github.com/blevesearch/zapx/v14/LICENSE b/vendor/github.com/blevesearch/zapx/v14/LICENSE new file mode 100644 index 00000000..7a4a3ea2 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v14/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/vendor/github.com/blevesearch/zapx/v14/README.md b/vendor/github.com/blevesearch/zapx/v14/README.md new file mode 100644 index 00000000..4cbf1a14 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v14/README.md @@ -0,0 +1,163 @@ +# zapx file format + +The zapx module is fork of [zap](https://github.com/blevesearch/zap) module which maintains file format compatibility, but removes dependency on bleve, and instead depends only on the indepenent interface modules: + +- [bleve_index_api](https://github.com/blevesearch/scorch_segment_api) +- [scorch_segment_api](https://github.com/blevesearch/scorch_segment_api) + +Advanced ZAP File Format Documentation is [here](zap.md). + +The file is written in the reverse order that we typically access data. This helps us write in one pass since later sections of the file require file offsets of things we've already written. + +Current usage: + +- mmap the entire file +- crc-32 bytes and version are in fixed position at end of the file +- reading remainder of footer could be version specific +- remainder of footer gives us: + - 3 important offsets (docValue , fields index and stored data index) + - 2 important values (number of docs and chunk factor) +- field data is processed once and memoized onto the heap so that we never have to go back to disk for it +- access to stored data by doc number means first navigating to the stored data index, then accessing a fixed position offset into that slice, which gives us the actual address of the data. the first bytes of that section tell us the size of data so that we know where it ends. +- access to all other indexed data follows the following pattern: + - first know the field name -> convert to id + - next navigate to term dictionary for that field + - some operations stop here and do dictionary ops + - next use dictionary to navigate to posting list for a specific term + - walk posting list + - if necessary, walk posting details as we go + - if location info is desired, consult location bitmap to see if it is there + +## stored fields section + +- for each document + - preparation phase: + - produce a slice of metadata bytes and data bytes + - produce these slices in field id order + - field value is appended to the data slice + - metadata slice is varint encoded with the following values for each field value + - field id (uint16) + - field type (byte) + - field value start offset in uncompressed data slice (uint64) + - field value length (uint64) + - field number of array positions (uint64) + - one additional value for each array position (uint64) + - compress the data slice using snappy + - file writing phase: + - remember the start offset for this document + - write out meta data length (varint uint64) + - write out compressed data length (varint uint64) + - write out the metadata bytes + - write out the compressed data bytes + +## stored fields idx + +- for each document + - write start offset (remembered from previous section) of stored data (big endian uint64) + +With this index and a known document number, we have direct access to all the stored field data. + +## posting details (freq/norm) section + +- for each posting list + - produce a slice containing multiple consecutive chunks (each chunk is varint stream) + - produce a slice remembering offsets of where each chunk starts + - preparation phase: + - for each hit in the posting list + - if this hit is in next chunk close out encoding of last chunk and record offset start of next + - encode term frequency (uint64) + - encode norm factor (float32) + - file writing phase: + - remember start position for this posting list details + - write out number of chunks that follow (varint uint64) + - write out length of each chunk (each a varint uint64) + - write out the byte slice containing all the chunk data + +If you know the doc number you're interested in, this format lets you jump to the correct chunk (docNum/chunkFactor) directly and then seek within that chunk until you find it. + +## posting details (location) section + +- for each posting list + - produce a slice containing multiple consecutive chunks (each chunk is varint stream) + - produce a slice remembering offsets of where each chunk starts + - preparation phase: + - for each hit in the posting list + - if this hit is in next chunk close out encoding of last chunk and record offset start of next + - encode field (uint16) + - encode field pos (uint64) + - encode field start (uint64) + - encode field end (uint64) + - encode number of array positions to follow (uint64) + - encode each array position (each uint64) + - file writing phase: + - remember start position for this posting list details + - write out number of chunks that follow (varint uint64) + - write out length of each chunk (each a varint uint64) + - write out the byte slice containing all the chunk data + +If you know the doc number you're interested in, this format lets you jump to the correct chunk (docNum/chunkFactor) directly and then seek within that chunk until you find it. + +## postings list section + +- for each posting list + - preparation phase: + - encode roaring bitmap posting list to bytes (so we know the length) + - file writing phase: + - remember the start position for this posting list + - write freq/norm details offset (remembered from previous, as varint uint64) + - write location details offset (remembered from previous, as varint uint64) + - write length of encoded roaring bitmap + - write the serialized roaring bitmap data + +## dictionary + +- for each field + - preparation phase: + - encode vellum FST with dictionary data pointing to file offset of posting list (remembered from previous) + - file writing phase: + - remember the start position of this persistDictionary + - write length of vellum data (varint uint64) + - write out vellum data + +## fields section + +- for each field + - file writing phase: + - remember start offset for each field + - write dictionary address (remembered from previous) (varint uint64) + - write length of field name (varint uint64) + - write field name bytes + +## fields idx + +- for each field + - file writing phase: + - write big endian uint64 of start offset for each field + +NOTE: currently we don't know or record the length of this fields index. Instead we rely on the fact that we know it immediately precedes a footer of known size. + +## fields DocValue + +- for each field + - preparation phase: + - produce a slice containing multiple consecutive chunks, where each chunk is composed of a meta section followed by compressed columnar field data + - produce a slice remembering the length of each chunk + - file writing phase: + - remember the start position of this first field DocValue offset in the footer + - write out number of chunks that follow (varint uint64) + - write out length of each chunk (each a varint uint64) + - write out the byte slice containing all the chunk data + +NOTE: currently the meta header inside each chunk gives clue to the location offsets and size of the data pertaining to a given docID and any +read operation leverage that meta information to extract the document specific data from the file. + +## footer + +- file writing phase + - write number of docs (big endian uint64) + - write stored field index location (big endian uint64) + - write field index location (big endian uint64) + - write field docValue location (big endian uint64) + - write out chunk factor (big endian uint32) + - write out version (big endian uint32) + - write out file CRC of everything preceding this (big endian uint32) diff --git a/vendor/github.com/blevesearch/zapx/v14/build.go b/vendor/github.com/blevesearch/zapx/v14/build.go new file mode 100644 index 00000000..b36878ab --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v14/build.go @@ -0,0 +1,186 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "bufio" + "fmt" + "io" + "math" + "os" + + "github.com/blevesearch/vellum" +) + +const Version uint32 = 14 + +const Type string = "zap" + +const fieldNotUninverted = math.MaxUint64 + +func (sb *SegmentBase) Persist(path string) error { + return PersistSegmentBase(sb, path) +} + +// WriteTo is an implementation of io.WriterTo interface. +func (sb *SegmentBase) WriteTo(w io.Writer) (int64, error) { + if w == nil { + return 0, fmt.Errorf("invalid writer found") + } + + n, err := persistSegmentBaseToWriter(sb, w) + return int64(n), err +} + +// PersistSegmentBase persists SegmentBase in the zap file format. +func PersistSegmentBase(sb *SegmentBase, path string) error { + flag := os.O_RDWR | os.O_CREATE + + f, err := os.OpenFile(path, flag, 0600) + if err != nil { + return err + } + + cleanup := func() { + _ = f.Close() + _ = os.Remove(path) + } + + _, err = persistSegmentBaseToWriter(sb, f) + if err != nil { + cleanup() + return err + } + + err = f.Sync() + if err != nil { + cleanup() + return err + } + + err = f.Close() + if err != nil { + cleanup() + return err + } + + return err +} + +type bufWriter struct { + w *bufio.Writer + n int +} + +func (br *bufWriter) Write(in []byte) (int, error) { + n, err := br.w.Write(in) + br.n += n + return n, err +} + +func persistSegmentBaseToWriter(sb *SegmentBase, w io.Writer) (int, error) { + br := &bufWriter{w: bufio.NewWriter(w)} + + _, err := br.Write(sb.mem) + if err != nil { + return 0, err + } + + err = persistFooter(sb.numDocs, sb.storedIndexOffset, sb.fieldsIndexOffset, + sb.docValueOffset, sb.chunkMode, sb.memCRC, br) + if err != nil { + return 0, err + } + + err = br.w.Flush() + if err != nil { + return 0, err + } + + return br.n, nil +} + +func persistStoredFieldValues(fieldID int, + storedFieldValues [][]byte, stf []byte, spf [][]uint64, + curr int, metaEncode varintEncoder, data []byte) ( + int, []byte, error) { + for i := 0; i < len(storedFieldValues); i++ { + // encode field + _, err := metaEncode(uint64(fieldID)) + if err != nil { + return 0, nil, err + } + // encode type + _, err = metaEncode(uint64(stf[i])) + if err != nil { + return 0, nil, err + } + // encode start offset + _, err = metaEncode(uint64(curr)) + if err != nil { + return 0, nil, err + } + // end len + _, err = metaEncode(uint64(len(storedFieldValues[i]))) + if err != nil { + return 0, nil, err + } + // encode number of array pos + _, err = metaEncode(uint64(len(spf[i]))) + if err != nil { + return 0, nil, err + } + // encode all array positions + for _, pos := range spf[i] { + _, err = metaEncode(pos) + if err != nil { + return 0, nil, err + } + } + + data = append(data, storedFieldValues[i]...) + curr += len(storedFieldValues[i]) + } + + return curr, data, nil +} + +func InitSegmentBase(mem []byte, memCRC uint32, chunkMode uint32, + fieldsMap map[string]uint16, fieldsInv []string, numDocs uint64, + storedIndexOffset uint64, fieldsIndexOffset uint64, docValueOffset uint64, + dictLocs []uint64) (*SegmentBase, error) { + sb := &SegmentBase{ + mem: mem, + memCRC: memCRC, + chunkMode: chunkMode, + fieldsMap: fieldsMap, + fieldsInv: fieldsInv, + numDocs: numDocs, + storedIndexOffset: storedIndexOffset, + fieldsIndexOffset: fieldsIndexOffset, + docValueOffset: docValueOffset, + dictLocs: dictLocs, + fieldDvReaders: make(map[uint16]*docValueReader), + fieldFSTs: make(map[uint16]*vellum.FST), + } + sb.updateSize() + + err := sb.loadDvReaders() + if err != nil { + return nil, err + } + + return sb, nil +} diff --git a/vendor/github.com/blevesearch/zap/v14/chunk.go b/vendor/github.com/blevesearch/zapx/v14/chunk.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v14/chunk.go rename to vendor/github.com/blevesearch/zapx/v14/chunk.go diff --git a/vendor/github.com/blevesearch/zap/v14/contentcoder.go b/vendor/github.com/blevesearch/zapx/v14/contentcoder.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v14/contentcoder.go rename to vendor/github.com/blevesearch/zapx/v14/contentcoder.go diff --git a/vendor/github.com/blevesearch/zapx/v14/count.go b/vendor/github.com/blevesearch/zapx/v14/count.go new file mode 100644 index 00000000..b6135359 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v14/count.go @@ -0,0 +1,61 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "hash/crc32" + "io" + + segment "github.com/blevesearch/scorch_segment_api/v2" +) + +// CountHashWriter is a wrapper around a Writer which counts the number of +// bytes which have been written and computes a crc32 hash +type CountHashWriter struct { + w io.Writer + crc uint32 + n int + s segment.StatsReporter +} + +// NewCountHashWriter returns a CountHashWriter which wraps the provided Writer +func NewCountHashWriter(w io.Writer) *CountHashWriter { + return &CountHashWriter{w: w} +} + +func NewCountHashWriterWithStatsReporter(w io.Writer, s segment.StatsReporter) *CountHashWriter { + return &CountHashWriter{w: w, s: s} +} + +// Write writes the provided bytes to the wrapped writer and counts the bytes +func (c *CountHashWriter) Write(b []byte) (int, error) { + n, err := c.w.Write(b) + c.crc = crc32.Update(c.crc, crc32.IEEETable, b[:n]) + c.n += n + if c.s != nil { + c.s.ReportBytesWritten(uint64(n)) + } + return n, err +} + +// Count returns the number of bytes written +func (c *CountHashWriter) Count() int { + return c.n +} + +// Sum32 returns the CRC-32 hash of the content written to this writer +func (c *CountHashWriter) Sum32() uint32 { + return c.crc +} diff --git a/vendor/github.com/blevesearch/zapx/v14/dict.go b/vendor/github.com/blevesearch/zapx/v14/dict.go new file mode 100644 index 00000000..e30bf242 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v14/dict.go @@ -0,0 +1,158 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "fmt" + + "github.com/RoaringBitmap/roaring" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" + "github.com/blevesearch/vellum" +) + +// Dictionary is the zap representation of the term dictionary +type Dictionary struct { + sb *SegmentBase + field string + fieldID uint16 + fst *vellum.FST + fstReader *vellum.Reader +} + +// represents an immutable, empty dictionary +var emptyDictionary = &Dictionary{} + +// PostingsList returns the postings list for the specified term +func (d *Dictionary) PostingsList(term []byte, except *roaring.Bitmap, + prealloc segment.PostingsList) (segment.PostingsList, error) { + var preallocPL *PostingsList + pl, ok := prealloc.(*PostingsList) + if ok && pl != nil { + preallocPL = pl + } + return d.postingsList(term, except, preallocPL) +} + +func (d *Dictionary) postingsList(term []byte, except *roaring.Bitmap, rv *PostingsList) (*PostingsList, error) { + if d.fstReader == nil { + if rv == nil || rv == emptyPostingsList { + return emptyPostingsList, nil + } + return d.postingsListInit(rv, except), nil + } + + postingsOffset, exists, err := d.fstReader.Get(term) + if err != nil { + return nil, fmt.Errorf("vellum err: %v", err) + } + if !exists { + if rv == nil || rv == emptyPostingsList { + return emptyPostingsList, nil + } + return d.postingsListInit(rv, except), nil + } + + return d.postingsListFromOffset(postingsOffset, except, rv) +} + +func (d *Dictionary) postingsListFromOffset(postingsOffset uint64, except *roaring.Bitmap, rv *PostingsList) (*PostingsList, error) { + rv = d.postingsListInit(rv, except) + + err := rv.read(postingsOffset, d) + if err != nil { + return nil, err + } + + return rv, nil +} + +func (d *Dictionary) postingsListInit(rv *PostingsList, except *roaring.Bitmap) *PostingsList { + if rv == nil || rv == emptyPostingsList { + rv = &PostingsList{} + } else { + postings := rv.postings + if postings != nil { + postings.Clear() + } + + *rv = PostingsList{} // clear the struct + + rv.postings = postings + } + rv.sb = d.sb + rv.except = except + return rv +} + +func (d *Dictionary) Contains(key []byte) (bool, error) { + if d.fst != nil { + return d.fst.Contains(key) + } + return false, nil +} + +// AutomatonIterator returns an iterator which only visits terms +// having the the vellum automaton and start/end key range +func (d *Dictionary) AutomatonIterator(a segment.Automaton, + startKeyInclusive, endKeyExclusive []byte) segment.DictionaryIterator { + if d.fst != nil { + rv := &DictionaryIterator{ + d: d, + } + + itr, err := d.fst.Search(a, startKeyInclusive, endKeyExclusive) + if err == nil { + rv.itr = itr + } else if err != vellum.ErrIteratorDone { + rv.err = err + } + + return rv + } + return emptyDictionaryIterator +} + +// DictionaryIterator is an iterator for term dictionary +type DictionaryIterator struct { + d *Dictionary + itr vellum.Iterator + err error + tmp PostingsList + entry index.DictEntry + omitCount bool +} + +var emptyDictionaryIterator = &DictionaryIterator{} + +// Next returns the next entry in the dictionary +func (i *DictionaryIterator) Next() (*index.DictEntry, error) { + if i.err != nil && i.err != vellum.ErrIteratorDone { + return nil, i.err + } else if i.itr == nil || i.err == vellum.ErrIteratorDone { + return nil, nil + } + term, postingsOffset := i.itr.Current() + i.entry.Term = string(term) + if !i.omitCount { + i.err = i.tmp.read(postingsOffset, i.d) + if i.err != nil { + return nil, i.err + } + i.entry.Count = i.tmp.Count() + } + i.err = i.itr.Next() + return &i.entry, nil +} diff --git a/vendor/github.com/blevesearch/zapx/v14/docvalues.go b/vendor/github.com/blevesearch/zapx/v14/docvalues.go new file mode 100644 index 00000000..a36485c8 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v14/docvalues.go @@ -0,0 +1,323 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "bytes" + "encoding/binary" + "fmt" + "math" + "reflect" + "sort" + + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" + "github.com/golang/snappy" +) + +var reflectStaticSizedocValueReader int + +func init() { + var dvi docValueReader + reflectStaticSizedocValueReader = int(reflect.TypeOf(dvi).Size()) +} + +type docNumTermsVisitor func(docNum uint64, terms []byte) error + +type docVisitState struct { + dvrs map[uint16]*docValueReader + segment *SegmentBase +} + +// No-op implementations for DiskStatsReporter interface. +// Supported only in v15 +func (d *docVisitState) BytesRead() uint64 { + return 0 +} + +func (d *docVisitState) BytesWritten() uint64 { + return 0 +} + +func (d *docVisitState) ResetBytesRead(val uint64) {} + +type docValueReader struct { + field string + curChunkNum uint64 + chunkOffsets []uint64 + dvDataLoc uint64 + curChunkHeader []MetaData + curChunkData []byte // compressed data cache + uncompressed []byte // temp buf for snappy decompression +} + +func (di *docValueReader) size() int { + return reflectStaticSizedocValueReader + SizeOfPtr + + len(di.field) + + len(di.chunkOffsets)*SizeOfUint64 + + len(di.curChunkHeader)*reflectStaticSizeMetaData + + len(di.curChunkData) +} + +func (di *docValueReader) cloneInto(rv *docValueReader) *docValueReader { + if rv == nil { + rv = &docValueReader{} + } + + rv.field = di.field + rv.curChunkNum = math.MaxUint64 + rv.chunkOffsets = di.chunkOffsets // immutable, so it's sharable + rv.dvDataLoc = di.dvDataLoc + rv.curChunkHeader = rv.curChunkHeader[:0] + rv.curChunkData = nil + rv.uncompressed = rv.uncompressed[:0] + + return rv +} + +func (di *docValueReader) curChunkNumber() uint64 { + return di.curChunkNum +} + +func (s *SegmentBase) loadFieldDocValueReader(field string, + fieldDvLocStart, fieldDvLocEnd uint64) (*docValueReader, error) { + // get the docValue offset for the given fields + if fieldDvLocStart == fieldNotUninverted { + // no docValues found, nothing to do + return nil, nil + } + + // read the number of chunks, and chunk offsets position + var numChunks, chunkOffsetsPosition uint64 + + if fieldDvLocEnd-fieldDvLocStart > 16 { + numChunks = binary.BigEndian.Uint64(s.mem[fieldDvLocEnd-8 : fieldDvLocEnd]) + // read the length of chunk offsets + chunkOffsetsLen := binary.BigEndian.Uint64(s.mem[fieldDvLocEnd-16 : fieldDvLocEnd-8]) + // acquire position of chunk offsets + chunkOffsetsPosition = (fieldDvLocEnd - 16) - chunkOffsetsLen + } else { + return nil, fmt.Errorf("loadFieldDocValueReader: fieldDvLoc too small: %d-%d", fieldDvLocEnd, fieldDvLocStart) + } + + fdvIter := &docValueReader{ + curChunkNum: math.MaxUint64, + field: field, + chunkOffsets: make([]uint64, int(numChunks)), + } + + // read the chunk offsets + var offset uint64 + for i := 0; i < int(numChunks); i++ { + loc, read := binary.Uvarint(s.mem[chunkOffsetsPosition+offset : chunkOffsetsPosition+offset+binary.MaxVarintLen64]) + if read <= 0 { + return nil, fmt.Errorf("corrupted chunk offset during segment load") + } + fdvIter.chunkOffsets[i] = loc + offset += uint64(read) + } + + // set the data offset + fdvIter.dvDataLoc = fieldDvLocStart + + return fdvIter, nil +} + +func (di *docValueReader) loadDvChunk(chunkNumber uint64, s *SegmentBase) error { + // advance to the chunk where the docValues + // reside for the given docNum + destChunkDataLoc, curChunkEnd := di.dvDataLoc, di.dvDataLoc + start, end := readChunkBoundary(int(chunkNumber), di.chunkOffsets) + if start >= end { + di.curChunkHeader = di.curChunkHeader[:0] + di.curChunkData = nil + di.curChunkNum = chunkNumber + di.uncompressed = di.uncompressed[:0] + return nil + } + + destChunkDataLoc += start + curChunkEnd += end + + // read the number of docs reside in the chunk + numDocs, read := binary.Uvarint(s.mem[destChunkDataLoc : destChunkDataLoc+binary.MaxVarintLen64]) + if read <= 0 { + return fmt.Errorf("failed to read the chunk") + } + chunkMetaLoc := destChunkDataLoc + uint64(read) + + offset := uint64(0) + if cap(di.curChunkHeader) < int(numDocs) { + di.curChunkHeader = make([]MetaData, int(numDocs)) + } else { + di.curChunkHeader = di.curChunkHeader[:int(numDocs)] + } + for i := 0; i < int(numDocs); i++ { + di.curChunkHeader[i].DocNum, read = binary.Uvarint(s.mem[chunkMetaLoc+offset : chunkMetaLoc+offset+binary.MaxVarintLen64]) + offset += uint64(read) + di.curChunkHeader[i].DocDvOffset, read = binary.Uvarint(s.mem[chunkMetaLoc+offset : chunkMetaLoc+offset+binary.MaxVarintLen64]) + offset += uint64(read) + } + + compressedDataLoc := chunkMetaLoc + offset + dataLength := curChunkEnd - compressedDataLoc + di.curChunkData = s.mem[compressedDataLoc : compressedDataLoc+dataLength] + di.curChunkNum = chunkNumber + di.uncompressed = di.uncompressed[:0] + return nil +} + +func (di *docValueReader) iterateAllDocValues(s *SegmentBase, visitor docNumTermsVisitor) error { + for i := 0; i < len(di.chunkOffsets); i++ { + err := di.loadDvChunk(uint64(i), s) + if err != nil { + return err + } + if di.curChunkData == nil || len(di.curChunkHeader) == 0 { + continue + } + + // uncompress the already loaded data + uncompressed, err := snappy.Decode(di.uncompressed[:cap(di.uncompressed)], di.curChunkData) + if err != nil { + return err + } + di.uncompressed = uncompressed + + start := uint64(0) + for _, entry := range di.curChunkHeader { + err = visitor(entry.DocNum, uncompressed[start:entry.DocDvOffset]) + if err != nil { + return err + } + + start = entry.DocDvOffset + } + } + + return nil +} + +func (di *docValueReader) visitDocValues(docNum uint64, + visitor index.DocValueVisitor) error { + // binary search the term locations for the docNum + start, end := di.getDocValueLocs(docNum) + if start == math.MaxUint64 || end == math.MaxUint64 || start == end { + return nil + } + + var uncompressed []byte + var err error + // use the uncompressed copy if available + if len(di.uncompressed) > 0 { + uncompressed = di.uncompressed + } else { + // uncompress the already loaded data + uncompressed, err = snappy.Decode(di.uncompressed[:cap(di.uncompressed)], di.curChunkData) + if err != nil { + return err + } + di.uncompressed = uncompressed + } + + // pick the terms for the given docNum + uncompressed = uncompressed[start:end] + for { + i := bytes.Index(uncompressed, termSeparatorSplitSlice) + if i < 0 { + break + } + + visitor(di.field, uncompressed[0:i]) + uncompressed = uncompressed[i+1:] + } + + return nil +} + +func (di *docValueReader) getDocValueLocs(docNum uint64) (uint64, uint64) { + i := sort.Search(len(di.curChunkHeader), func(i int) bool { + return di.curChunkHeader[i].DocNum >= docNum + }) + if i < len(di.curChunkHeader) && di.curChunkHeader[i].DocNum == docNum { + return ReadDocValueBoundary(i, di.curChunkHeader) + } + return math.MaxUint64, math.MaxUint64 +} + +// VisitDocValues is an implementation of the +// DocValueVisitable interface +func (s *SegmentBase) VisitDocValues(localDocNum uint64, fields []string, + visitor index.DocValueVisitor, dvsIn segment.DocVisitState) ( + segment.DocVisitState, error) { + dvs, ok := dvsIn.(*docVisitState) + if !ok || dvs == nil { + dvs = &docVisitState{} + } else { + if dvs.segment != s { + dvs.segment = s + dvs.dvrs = nil + } + } + + var fieldIDPlus1 uint16 + if dvs.dvrs == nil { + dvs.dvrs = make(map[uint16]*docValueReader, len(fields)) + for _, field := range fields { + if fieldIDPlus1, ok = s.fieldsMap[field]; !ok { + continue + } + fieldID := fieldIDPlus1 - 1 + if dvIter, exists := s.fieldDvReaders[fieldID]; exists && + dvIter != nil { + dvs.dvrs[fieldID] = dvIter.cloneInto(dvs.dvrs[fieldID]) + } + } + } + + // find the chunkNumber where the docValues are stored + // NOTE: doc values continue to use legacy chunk mode + chunkFactor, err := getChunkSize(LegacyChunkMode, 0, 0) + if err != nil { + return nil, err + } + docInChunk := localDocNum / chunkFactor + var dvr *docValueReader + for _, field := range fields { + if fieldIDPlus1, ok = s.fieldsMap[field]; !ok { + continue + } + fieldID := fieldIDPlus1 - 1 + if dvr, ok = dvs.dvrs[fieldID]; ok && dvr != nil { + // check if the chunk is already loaded + if docInChunk != dvr.curChunkNumber() { + err := dvr.loadDvChunk(docInChunk, s) + if err != nil { + return dvs, err + } + } + + _ = dvr.visitDocValues(localDocNum, visitor) + } + } + return dvs, nil +} + +// VisitableDocValueFields returns the list of fields with +// persisted doc value terms ready to be visitable using the +// VisitDocumentFieldTerms method. +func (s *SegmentBase) VisitableDocValueFields() ([]string, error) { + return s.fieldDvNames, nil +} diff --git a/vendor/github.com/blevesearch/zapx/v14/enumerator.go b/vendor/github.com/blevesearch/zapx/v14/enumerator.go new file mode 100644 index 00000000..972a2241 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v14/enumerator.go @@ -0,0 +1,138 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "bytes" + + "github.com/blevesearch/vellum" +) + +// enumerator provides an ordered traversal of multiple vellum +// iterators. Like JOIN of iterators, the enumerator produces a +// sequence of (key, iteratorIndex, value) tuples, sorted by key ASC, +// then iteratorIndex ASC, where the same key might be seen or +// repeated across multiple child iterators. +type enumerator struct { + itrs []vellum.Iterator + currKs [][]byte + currVs []uint64 + + lowK []byte + lowIdxs []int + lowCurr int +} + +// newEnumerator returns a new enumerator over the vellum Iterators +func newEnumerator(itrs []vellum.Iterator) (*enumerator, error) { + rv := &enumerator{ + itrs: itrs, + currKs: make([][]byte, len(itrs)), + currVs: make([]uint64, len(itrs)), + lowIdxs: make([]int, 0, len(itrs)), + } + for i, itr := range rv.itrs { + rv.currKs[i], rv.currVs[i] = itr.Current() + } + rv.updateMatches(false) + if rv.lowK == nil && len(rv.lowIdxs) == 0 { + return rv, vellum.ErrIteratorDone + } + return rv, nil +} + +// updateMatches maintains the low key matches based on the currKs +func (m *enumerator) updateMatches(skipEmptyKey bool) { + m.lowK = nil + m.lowIdxs = m.lowIdxs[:0] + m.lowCurr = 0 + + for i, key := range m.currKs { + if (key == nil && m.currVs[i] == 0) || // in case of empty iterator + (len(key) == 0 && skipEmptyKey) { // skip empty keys + continue + } + + cmp := bytes.Compare(key, m.lowK) + if cmp < 0 || len(m.lowIdxs) == 0 { + // reached a new low + m.lowK = key + m.lowIdxs = m.lowIdxs[:0] + m.lowIdxs = append(m.lowIdxs, i) + } else if cmp == 0 { + m.lowIdxs = append(m.lowIdxs, i) + } + } +} + +// Current returns the enumerator's current key, iterator-index, and +// value. If the enumerator is not pointing at a valid value (because +// Next returned an error previously), Current will return nil,0,0. +func (m *enumerator) Current() ([]byte, int, uint64) { + var i int + var v uint64 + if m.lowCurr < len(m.lowIdxs) { + i = m.lowIdxs[m.lowCurr] + v = m.currVs[i] + } + return m.lowK, i, v +} + +// GetLowIdxsAndValues will return all of the iterator indices +// which point to the current key, and their corresponding +// values. This can be used by advanced caller which may need +// to peek into these other sets of data before processing. +func (m *enumerator) GetLowIdxsAndValues() ([]int, []uint64) { + values := make([]uint64, 0, len(m.lowIdxs)) + for _, idx := range m.lowIdxs { + values = append(values, m.currVs[idx]) + } + return m.lowIdxs, values +} + +// Next advances the enumerator to the next key/iterator/value result, +// else vellum.ErrIteratorDone is returned. +func (m *enumerator) Next() error { + m.lowCurr += 1 + if m.lowCurr >= len(m.lowIdxs) { + // move all the current low iterators forwards + for _, vi := range m.lowIdxs { + err := m.itrs[vi].Next() + if err != nil && err != vellum.ErrIteratorDone { + return err + } + m.currKs[vi], m.currVs[vi] = m.itrs[vi].Current() + } + // can skip any empty keys encountered at this point + m.updateMatches(true) + } + if m.lowK == nil && len(m.lowIdxs) == 0 { + return vellum.ErrIteratorDone + } + return nil +} + +// Close all the underlying Iterators. The first error, if any, will +// be returned. +func (m *enumerator) Close() error { + var rv error + for _, itr := range m.itrs { + err := itr.Close() + if rv == nil { + rv = err + } + } + return rv +} diff --git a/vendor/github.com/blevesearch/zapx/v14/intDecoder.go b/vendor/github.com/blevesearch/zapx/v14/intDecoder.go new file mode 100644 index 00000000..1b839e1c --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v14/intDecoder.go @@ -0,0 +1,116 @@ +// Copyright (c) 2019 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "encoding/binary" + "fmt" +) + +type chunkedIntDecoder struct { + startOffset uint64 + dataStartOffset uint64 + chunkOffsets []uint64 + curChunkBytes []byte + data []byte + r *memUvarintReader +} + +// newChunkedIntDecoder expects an optional or reset chunkedIntDecoder for better reuse. +func newChunkedIntDecoder(buf []byte, offset uint64, rv *chunkedIntDecoder) *chunkedIntDecoder { + if rv == nil { + rv = &chunkedIntDecoder{startOffset: offset, data: buf} + } else { + rv.startOffset = offset + rv.data = buf + } + + var n, numChunks uint64 + var read int + if offset == termNotEncoded { + numChunks = 0 + } else { + numChunks, read = binary.Uvarint(buf[offset+n : offset+n+binary.MaxVarintLen64]) + } + + n += uint64(read) + if cap(rv.chunkOffsets) >= int(numChunks) { + rv.chunkOffsets = rv.chunkOffsets[:int(numChunks)] + } else { + rv.chunkOffsets = make([]uint64, int(numChunks)) + } + for i := 0; i < int(numChunks); i++ { + rv.chunkOffsets[i], read = binary.Uvarint(buf[offset+n : offset+n+binary.MaxVarintLen64]) + n += uint64(read) + } + rv.dataStartOffset = offset + n + return rv +} + +func (d *chunkedIntDecoder) loadChunk(chunk int) error { + if d.startOffset == termNotEncoded { + d.r = newMemUvarintReader([]byte(nil)) + return nil + } + + if chunk >= len(d.chunkOffsets) { + return fmt.Errorf("tried to load freq chunk that doesn't exist %d/(%d)", + chunk, len(d.chunkOffsets)) + } + + end, start := d.dataStartOffset, d.dataStartOffset + s, e := readChunkBoundary(chunk, d.chunkOffsets) + start += s + end += e + d.curChunkBytes = d.data[start:end] + if d.r == nil { + d.r = newMemUvarintReader(d.curChunkBytes) + } else { + d.r.Reset(d.curChunkBytes) + } + + return nil +} + +func (d *chunkedIntDecoder) reset() { + d.startOffset = 0 + d.dataStartOffset = 0 + d.chunkOffsets = d.chunkOffsets[:0] + d.curChunkBytes = d.curChunkBytes[:0] + d.data = d.data[:0] + if d.r != nil { + d.r.Reset([]byte(nil)) + } +} + +func (d *chunkedIntDecoder) isNil() bool { + return d.curChunkBytes == nil || len(d.curChunkBytes) == 0 +} + +func (d *chunkedIntDecoder) readUvarint() (uint64, error) { + return d.r.ReadUvarint() +} + +func (d *chunkedIntDecoder) SkipUvarint() { + d.r.SkipUvarint() +} + +func (d *chunkedIntDecoder) SkipBytes(count int) { + d.r.SkipBytes(count) +} + +func (d *chunkedIntDecoder) Len() int { + return d.r.Len() +} diff --git a/vendor/github.com/blevesearch/zap/v14/intcoder.go b/vendor/github.com/blevesearch/zapx/v14/intcoder.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v14/intcoder.go rename to vendor/github.com/blevesearch/zapx/v14/intcoder.go diff --git a/vendor/github.com/blevesearch/zapx/v14/memuvarint.go b/vendor/github.com/blevesearch/zapx/v14/memuvarint.go new file mode 100644 index 00000000..48a57f9c --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v14/memuvarint.go @@ -0,0 +1,103 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "fmt" +) + +type memUvarintReader struct { + C int // index of next byte to read from S + S []byte +} + +func newMemUvarintReader(s []byte) *memUvarintReader { + return &memUvarintReader{S: s} +} + +// Len returns the number of unread bytes. +func (r *memUvarintReader) Len() int { + n := len(r.S) - r.C + if n < 0 { + return 0 + } + return n +} + +// ReadUvarint reads an encoded uint64. The original code this was +// based on is at encoding/binary/ReadUvarint(). +func (r *memUvarintReader) ReadUvarint() (uint64, error) { + if r.C >= len(r.S) { + // nothing else to read + return 0, nil + } + + var x uint64 + var s uint + var C = r.C + var S = r.S + + for { + b := S[C] + C++ + + if b < 0x80 { + r.C = C + + // why 63? The original code had an 'i += 1' loop var and + // checked for i > 9 || i == 9 ...; but, we no longer + // check for the i var, but instead check here for s, + // which is incremented by 7. So, 7*9 == 63. + // + // why the "extra" >= check? The normal case is that s < + // 63, so we check this single >= guard first so that we + // hit the normal, nil-error return pathway sooner. + if s >= 63 && (s > 63 || b > 1) { + return 0, fmt.Errorf("memUvarintReader overflow") + } + + return x | uint64(b)<= len(r.S) { + return + } + + b := r.S[r.C] + r.C++ + + if b < 0x80 { + return + } + } +} + +// SkipBytes skips a count number of bytes. +func (r *memUvarintReader) SkipBytes(count int) { + r.C = r.C + count +} + +func (r *memUvarintReader) Reset(s []byte) { + r.C = 0 + r.S = s +} diff --git a/vendor/github.com/blevesearch/zapx/v14/merge.go b/vendor/github.com/blevesearch/zapx/v14/merge.go new file mode 100644 index 00000000..6a853a16 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v14/merge.go @@ -0,0 +1,843 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "bufio" + "bytes" + "encoding/binary" + "fmt" + "math" + "os" + "sort" + + "github.com/RoaringBitmap/roaring" + seg "github.com/blevesearch/scorch_segment_api/v2" + "github.com/blevesearch/vellum" + "github.com/golang/snappy" +) + +var DefaultFileMergerBufferSize = 1024 * 1024 + +const docDropped = math.MaxUint64 // sentinel docNum to represent a deleted doc + +// Merge takes a slice of segments and bit masks describing which +// documents may be dropped, and creates a new segment containing the +// remaining data. This new segment is built at the specified path. +func (*ZapPlugin) Merge(segments []seg.Segment, drops []*roaring.Bitmap, path string, + closeCh chan struct{}, s seg.StatsReporter) ( + [][]uint64, uint64, error) { + segmentBases := make([]*SegmentBase, len(segments)) + for segmenti, segment := range segments { + switch segmentx := segment.(type) { + case *Segment: + segmentBases[segmenti] = &segmentx.SegmentBase + case *SegmentBase: + segmentBases[segmenti] = segmentx + default: + panic(fmt.Sprintf("oops, unexpected segment type: %T", segment)) + } + } + return mergeSegmentBases(segmentBases, drops, path, DefaultChunkMode, closeCh, s) +} + +func mergeSegmentBases(segmentBases []*SegmentBase, drops []*roaring.Bitmap, path string, + chunkMode uint32, closeCh chan struct{}, s seg.StatsReporter) ( + [][]uint64, uint64, error) { + flag := os.O_RDWR | os.O_CREATE + + f, err := os.OpenFile(path, flag, 0600) + if err != nil { + return nil, 0, err + } + + cleanup := func() { + _ = f.Close() + _ = os.Remove(path) + } + + // buffer the output + br := bufio.NewWriterSize(f, DefaultFileMergerBufferSize) + + // wrap it for counting (tracking offsets) + cr := NewCountHashWriterWithStatsReporter(br, s) + + newDocNums, numDocs, storedIndexOffset, fieldsIndexOffset, docValueOffset, _, _, _, err := + MergeToWriter(segmentBases, drops, chunkMode, cr, closeCh) + if err != nil { + cleanup() + return nil, 0, err + } + + err = persistFooter(numDocs, storedIndexOffset, fieldsIndexOffset, + docValueOffset, chunkMode, cr.Sum32(), cr) + if err != nil { + cleanup() + return nil, 0, err + } + + err = br.Flush() + if err != nil { + cleanup() + return nil, 0, err + } + + err = f.Sync() + if err != nil { + cleanup() + return nil, 0, err + } + + err = f.Close() + if err != nil { + cleanup() + return nil, 0, err + } + + return newDocNums, uint64(cr.Count()), nil +} + +func MergeToWriter(segments []*SegmentBase, drops []*roaring.Bitmap, + chunkMode uint32, cr *CountHashWriter, closeCh chan struct{}) ( + newDocNums [][]uint64, + numDocs, storedIndexOffset, fieldsIndexOffset, docValueOffset uint64, + dictLocs []uint64, fieldsInv []string, fieldsMap map[string]uint16, + err error) { + docValueOffset = uint64(fieldNotUninverted) + + var fieldsSame bool + fieldsSame, fieldsInv = mergeFields(segments) + fieldsMap = mapFields(fieldsInv) + + numDocs = computeNewDocCount(segments, drops) + + if isClosed(closeCh) { + return nil, 0, 0, 0, 0, nil, nil, nil, seg.ErrClosed + } + + if numDocs > 0 { + storedIndexOffset, newDocNums, err = mergeStoredAndRemap(segments, drops, + fieldsMap, fieldsInv, fieldsSame, numDocs, cr, closeCh) + if err != nil { + return nil, 0, 0, 0, 0, nil, nil, nil, err + } + + dictLocs, docValueOffset, err = persistMergedRest(segments, drops, + fieldsInv, fieldsMap, fieldsSame, + newDocNums, numDocs, chunkMode, cr, closeCh) + if err != nil { + return nil, 0, 0, 0, 0, nil, nil, nil, err + } + } else { + dictLocs = make([]uint64, len(fieldsInv)) + } + + fieldsIndexOffset, err = persistFields(fieldsInv, cr, dictLocs) + if err != nil { + return nil, 0, 0, 0, 0, nil, nil, nil, err + } + + return newDocNums, numDocs, storedIndexOffset, fieldsIndexOffset, docValueOffset, dictLocs, fieldsInv, fieldsMap, nil +} + +// mapFields takes the fieldsInv list and returns a map of fieldName +// to fieldID+1 +func mapFields(fields []string) map[string]uint16 { + rv := make(map[string]uint16, len(fields)) + for i, fieldName := range fields { + rv[fieldName] = uint16(i) + 1 + } + return rv +} + +// computeNewDocCount determines how many documents will be in the newly +// merged segment when obsoleted docs are dropped +func computeNewDocCount(segments []*SegmentBase, drops []*roaring.Bitmap) uint64 { + var newDocCount uint64 + for segI, segment := range segments { + newDocCount += segment.numDocs + if drops[segI] != nil { + newDocCount -= drops[segI].GetCardinality() + } + } + return newDocCount +} + +func persistMergedRest(segments []*SegmentBase, dropsIn []*roaring.Bitmap, + fieldsInv []string, fieldsMap map[string]uint16, fieldsSame bool, + newDocNumsIn [][]uint64, newSegDocCount uint64, chunkMode uint32, + w *CountHashWriter, closeCh chan struct{}) ([]uint64, uint64, error) { + var bufMaxVarintLen64 []byte = make([]byte, binary.MaxVarintLen64) + var bufLoc []uint64 + + var postings *PostingsList + var postItr *PostingsIterator + + rv := make([]uint64, len(fieldsInv)) + fieldDvLocsStart := make([]uint64, len(fieldsInv)) + fieldDvLocsEnd := make([]uint64, len(fieldsInv)) + + // these int coders are initialized with chunk size 1024 + // however this will be reset to the correct chunk size + // while processing each individual field-term section + tfEncoder := newChunkedIntCoder(1024, newSegDocCount-1) + locEncoder := newChunkedIntCoder(1024, newSegDocCount-1) + + var vellumBuf bytes.Buffer + newVellum, err := vellum.New(&vellumBuf, nil) + if err != nil { + return nil, 0, err + } + + newRoaring := roaring.NewBitmap() + + // for each field + for fieldID, fieldName := range fieldsInv { + // collect FST iterators from all active segments for this field + var newDocNums [][]uint64 + var drops []*roaring.Bitmap + var dicts []*Dictionary + var itrs []vellum.Iterator + + var segmentsInFocus []*SegmentBase + + for segmentI, segment := range segments { + // check for the closure in meantime + if isClosed(closeCh) { + return nil, 0, seg.ErrClosed + } + + dict, err2 := segment.dictionary(fieldName) + if err2 != nil { + return nil, 0, err2 + } + if dict != nil && dict.fst != nil { + itr, err2 := dict.fst.Iterator(nil, nil) + if err2 != nil && err2 != vellum.ErrIteratorDone { + return nil, 0, err2 + } + if itr != nil { + newDocNums = append(newDocNums, newDocNumsIn[segmentI]) + if dropsIn[segmentI] != nil && !dropsIn[segmentI].IsEmpty() { + drops = append(drops, dropsIn[segmentI]) + } else { + drops = append(drops, nil) + } + dicts = append(dicts, dict) + itrs = append(itrs, itr) + segmentsInFocus = append(segmentsInFocus, segment) + } + } + } + + var prevTerm []byte + + newRoaring.Clear() + + var lastDocNum, lastFreq, lastNorm uint64 + + // determines whether to use "1-hit" encoding optimization + // when a term appears in only 1 doc, with no loc info, + // has freq of 1, and the docNum fits into 31-bits + use1HitEncoding := func(termCardinality uint64) (bool, uint64, uint64) { + if termCardinality == uint64(1) && locEncoder.FinalSize() <= 0 { + docNum := uint64(newRoaring.Minimum()) + if under32Bits(docNum) && docNum == lastDocNum && lastFreq == 1 { + return true, docNum, lastNorm + } + } + return false, 0, 0 + } + + finishTerm := func(term []byte) error { + tfEncoder.Close() + locEncoder.Close() + + postingsOffset, err := writePostings(newRoaring, + tfEncoder, locEncoder, use1HitEncoding, w, bufMaxVarintLen64) + if err != nil { + return err + } + + if postingsOffset > 0 { + err = newVellum.Insert(term, postingsOffset) + if err != nil { + return err + } + } + + newRoaring.Clear() + + tfEncoder.Reset() + locEncoder.Reset() + + lastDocNum = 0 + lastFreq = 0 + lastNorm = 0 + + return nil + } + + enumerator, err := newEnumerator(itrs) + + for err == nil { + term, itrI, postingsOffset := enumerator.Current() + + if !bytes.Equal(prevTerm, term) { + // check for the closure in meantime + if isClosed(closeCh) { + return nil, 0, seg.ErrClosed + } + + // if the term changed, write out the info collected + // for the previous term + err = finishTerm(prevTerm) + if err != nil { + return nil, 0, err + } + } + if !bytes.Equal(prevTerm, term) || prevTerm == nil { + // compute cardinality of field-term in new seg + var newCard uint64 + lowItrIdxs, lowItrVals := enumerator.GetLowIdxsAndValues() + for i, idx := range lowItrIdxs { + pl, err := dicts[idx].postingsListFromOffset(lowItrVals[i], drops[idx], nil) + if err != nil { + return nil, 0, err + } + newCard += pl.Count() + } + // compute correct chunk size with this + chunkSize, err := getChunkSize(chunkMode, newCard, newSegDocCount) + if err != nil { + return nil, 0, err + } + // update encoders chunk + tfEncoder.SetChunkSize(chunkSize, newSegDocCount-1) + locEncoder.SetChunkSize(chunkSize, newSegDocCount-1) + } + + postings, err = dicts[itrI].postingsListFromOffset( + postingsOffset, drops[itrI], postings) + if err != nil { + return nil, 0, err + } + + postItr = postings.iterator(true, true, true, postItr) + + // can no longer optimize by copying, since chunk factor could have changed + lastDocNum, lastFreq, lastNorm, bufLoc, err = mergeTermFreqNormLocs( + fieldsMap, term, postItr, newDocNums[itrI], newRoaring, + tfEncoder, locEncoder, bufLoc) + + if err != nil { + return nil, 0, err + } + + prevTerm = prevTerm[:0] // copy to prevTerm in case Next() reuses term mem + prevTerm = append(prevTerm, term...) + + err = enumerator.Next() + } + if err != vellum.ErrIteratorDone { + return nil, 0, err + } + + err = finishTerm(prevTerm) + if err != nil { + return nil, 0, err + } + + dictOffset := uint64(w.Count()) + + err = newVellum.Close() + if err != nil { + return nil, 0, err + } + vellumData := vellumBuf.Bytes() + + // write out the length of the vellum data + n := binary.PutUvarint(bufMaxVarintLen64, uint64(len(vellumData))) + _, err = w.Write(bufMaxVarintLen64[:n]) + if err != nil { + return nil, 0, err + } + + // write this vellum to disk + _, err = w.Write(vellumData) + if err != nil { + return nil, 0, err + } + + rv[fieldID] = dictOffset + + // get the field doc value offset (start) + fieldDvLocsStart[fieldID] = uint64(w.Count()) + + // update the field doc values + // NOTE: doc values continue to use legacy chunk mode + chunkSize, err := getChunkSize(LegacyChunkMode, 0, 0) + if err != nil { + return nil, 0, err + } + fdvEncoder := newChunkedContentCoder(chunkSize, newSegDocCount-1, w, true) + + fdvReadersAvailable := false + var dvIterClone *docValueReader + for segmentI, segment := range segmentsInFocus { + // check for the closure in meantime + if isClosed(closeCh) { + return nil, 0, seg.ErrClosed + } + + fieldIDPlus1 := uint16(segment.fieldsMap[fieldName]) + if dvIter, exists := segment.fieldDvReaders[fieldIDPlus1-1]; exists && + dvIter != nil { + fdvReadersAvailable = true + dvIterClone = dvIter.cloneInto(dvIterClone) + err = dvIterClone.iterateAllDocValues(segment, func(docNum uint64, terms []byte) error { + if newDocNums[segmentI][docNum] == docDropped { + return nil + } + err := fdvEncoder.Add(newDocNums[segmentI][docNum], terms) + if err != nil { + return err + } + return nil + }) + if err != nil { + return nil, 0, err + } + } + } + + if fdvReadersAvailable { + err = fdvEncoder.Close() + if err != nil { + return nil, 0, err + } + + // persist the doc value details for this field + _, err = fdvEncoder.Write() + if err != nil { + return nil, 0, err + } + + // get the field doc value offset (end) + fieldDvLocsEnd[fieldID] = uint64(w.Count()) + } else { + fieldDvLocsStart[fieldID] = fieldNotUninverted + fieldDvLocsEnd[fieldID] = fieldNotUninverted + } + + // reset vellum buffer and vellum builder + vellumBuf.Reset() + err = newVellum.Reset(&vellumBuf) + if err != nil { + return nil, 0, err + } + } + + fieldDvLocsOffset := uint64(w.Count()) + + buf := bufMaxVarintLen64 + for i := 0; i < len(fieldDvLocsStart); i++ { + n := binary.PutUvarint(buf, fieldDvLocsStart[i]) + _, err := w.Write(buf[:n]) + if err != nil { + return nil, 0, err + } + n = binary.PutUvarint(buf, fieldDvLocsEnd[i]) + _, err = w.Write(buf[:n]) + if err != nil { + return nil, 0, err + } + } + + return rv, fieldDvLocsOffset, nil +} + +func mergeTermFreqNormLocs(fieldsMap map[string]uint16, term []byte, postItr *PostingsIterator, + newDocNums []uint64, newRoaring *roaring.Bitmap, + tfEncoder *chunkedIntCoder, locEncoder *chunkedIntCoder, bufLoc []uint64) ( + lastDocNum uint64, lastFreq uint64, lastNorm uint64, bufLocOut []uint64, err error) { + next, err := postItr.Next() + for next != nil && err == nil { + hitNewDocNum := newDocNums[next.Number()] + if hitNewDocNum == docDropped { + return 0, 0, 0, nil, fmt.Errorf("see hit with dropped docNum") + } + + newRoaring.Add(uint32(hitNewDocNum)) + + nextFreq := next.Frequency() + nextNorm := uint64(math.Float32bits(float32(next.Norm()))) + + locs := next.Locations() + + err = tfEncoder.Add(hitNewDocNum, + encodeFreqHasLocs(nextFreq, len(locs) > 0), nextNorm) + if err != nil { + return 0, 0, 0, nil, err + } + + if len(locs) > 0 { + numBytesLocs := 0 + for _, loc := range locs { + ap := loc.ArrayPositions() + numBytesLocs += totalUvarintBytes(uint64(fieldsMap[loc.Field()]-1), + loc.Pos(), loc.Start(), loc.End(), uint64(len(ap)), ap) + } + + err = locEncoder.Add(hitNewDocNum, uint64(numBytesLocs)) + if err != nil { + return 0, 0, 0, nil, err + } + + for _, loc := range locs { + ap := loc.ArrayPositions() + if cap(bufLoc) < 5+len(ap) { + bufLoc = make([]uint64, 0, 5+len(ap)) + } + args := bufLoc[0:5] + args[0] = uint64(fieldsMap[loc.Field()] - 1) + args[1] = loc.Pos() + args[2] = loc.Start() + args[3] = loc.End() + args[4] = uint64(len(ap)) + args = append(args, ap...) + err = locEncoder.Add(hitNewDocNum, args...) + if err != nil { + return 0, 0, 0, nil, err + } + } + } + + lastDocNum = hitNewDocNum + lastFreq = nextFreq + lastNorm = nextNorm + + next, err = postItr.Next() + } + + return lastDocNum, lastFreq, lastNorm, bufLoc, err +} + +func writePostings(postings *roaring.Bitmap, tfEncoder, locEncoder *chunkedIntCoder, + use1HitEncoding func(uint64) (bool, uint64, uint64), + w *CountHashWriter, bufMaxVarintLen64 []byte) ( + offset uint64, err error) { + termCardinality := postings.GetCardinality() + if termCardinality <= 0 { + return 0, nil + } + + if use1HitEncoding != nil { + encodeAs1Hit, docNum1Hit, normBits1Hit := use1HitEncoding(termCardinality) + if encodeAs1Hit { + return FSTValEncode1Hit(docNum1Hit, normBits1Hit), nil + } + } + + var tfOffset uint64 + tfOffset, _, err = tfEncoder.writeAt(w) + if err != nil { + return 0, err + } + + var locOffset uint64 + locOffset, _, err = locEncoder.writeAt(w) + if err != nil { + return 0, err + } + + postingsOffset := uint64(w.Count()) + + n := binary.PutUvarint(bufMaxVarintLen64, tfOffset) + _, err = w.Write(bufMaxVarintLen64[:n]) + if err != nil { + return 0, err + } + + n = binary.PutUvarint(bufMaxVarintLen64, locOffset) + _, err = w.Write(bufMaxVarintLen64[:n]) + if err != nil { + return 0, err + } + + _, err = writeRoaringWithLen(postings, w, bufMaxVarintLen64) + if err != nil { + return 0, err + } + + return postingsOffset, nil +} + +type varintEncoder func(uint64) (int, error) + +func mergeStoredAndRemap(segments []*SegmentBase, drops []*roaring.Bitmap, + fieldsMap map[string]uint16, fieldsInv []string, fieldsSame bool, newSegDocCount uint64, + w *CountHashWriter, closeCh chan struct{}) (uint64, [][]uint64, error) { + var rv [][]uint64 // The remapped or newDocNums for each segment. + + var newDocNum uint64 + + var curr int + var data, compressed []byte + var metaBuf bytes.Buffer + varBuf := make([]byte, binary.MaxVarintLen64) + metaEncode := func(val uint64) (int, error) { + wb := binary.PutUvarint(varBuf, val) + return metaBuf.Write(varBuf[:wb]) + } + + vals := make([][][]byte, len(fieldsInv)) + typs := make([][]byte, len(fieldsInv)) + poss := make([][][]uint64, len(fieldsInv)) + + var posBuf []uint64 + + docNumOffsets := make([]uint64, newSegDocCount) + + vdc := visitDocumentCtxPool.Get().(*visitDocumentCtx) + defer visitDocumentCtxPool.Put(vdc) + + // for each segment + for segI, segment := range segments { + // check for the closure in meantime + if isClosed(closeCh) { + return 0, nil, seg.ErrClosed + } + + segNewDocNums := make([]uint64, segment.numDocs) + + dropsI := drops[segI] + + // optimize when the field mapping is the same across all + // segments and there are no deletions, via byte-copying + // of stored docs bytes directly to the writer + if fieldsSame && (dropsI == nil || dropsI.GetCardinality() == 0) { + err := segment.copyStoredDocs(newDocNum, docNumOffsets, w) + if err != nil { + return 0, nil, err + } + + for i := uint64(0); i < segment.numDocs; i++ { + segNewDocNums[i] = newDocNum + newDocNum++ + } + rv = append(rv, segNewDocNums) + + continue + } + + // for each doc num + for docNum := uint64(0); docNum < segment.numDocs; docNum++ { + // TODO: roaring's API limits docNums to 32-bits? + if dropsI != nil && dropsI.Contains(uint32(docNum)) { + segNewDocNums[docNum] = docDropped + continue + } + + segNewDocNums[docNum] = newDocNum + + curr = 0 + metaBuf.Reset() + data = data[:0] + + posTemp := posBuf + + // collect all the data + for i := 0; i < len(fieldsInv); i++ { + vals[i] = vals[i][:0] + typs[i] = typs[i][:0] + poss[i] = poss[i][:0] + } + err := segment.visitStoredFields(vdc, docNum, func(field string, typ byte, value []byte, pos []uint64) bool { + fieldID := int(fieldsMap[field]) - 1 + vals[fieldID] = append(vals[fieldID], value) + typs[fieldID] = append(typs[fieldID], typ) + + // copy array positions to preserve them beyond the scope of this callback + var curPos []uint64 + if len(pos) > 0 { + if cap(posTemp) < len(pos) { + posBuf = make([]uint64, len(pos)*len(fieldsInv)) + posTemp = posBuf + } + curPos = posTemp[0:len(pos)] + copy(curPos, pos) + posTemp = posTemp[len(pos):] + } + poss[fieldID] = append(poss[fieldID], curPos) + + return true + }) + if err != nil { + return 0, nil, err + } + + // _id field special case optimizes ExternalID() lookups + idFieldVal := vals[uint16(0)][0] + _, err = metaEncode(uint64(len(idFieldVal))) + if err != nil { + return 0, nil, err + } + + // now walk the non-"_id" fields in order + for fieldID := 1; fieldID < len(fieldsInv); fieldID++ { + storedFieldValues := vals[fieldID] + + stf := typs[fieldID] + spf := poss[fieldID] + + var err2 error + curr, data, err2 = persistStoredFieldValues(fieldID, + storedFieldValues, stf, spf, curr, metaEncode, data) + if err2 != nil { + return 0, nil, err2 + } + } + + metaBytes := metaBuf.Bytes() + + compressed = snappy.Encode(compressed[:cap(compressed)], data) + + // record where we're about to start writing + docNumOffsets[newDocNum] = uint64(w.Count()) + + // write out the meta len and compressed data len + _, err = writeUvarints(w, + uint64(len(metaBytes)), + uint64(len(idFieldVal)+len(compressed))) + if err != nil { + return 0, nil, err + } + // now write the meta + _, err = w.Write(metaBytes) + if err != nil { + return 0, nil, err + } + // now write the _id field val (counted as part of the 'compressed' data) + _, err = w.Write(idFieldVal) + if err != nil { + return 0, nil, err + } + // now write the compressed data + _, err = w.Write(compressed) + if err != nil { + return 0, nil, err + } + + newDocNum++ + } + + rv = append(rv, segNewDocNums) + } + + // return value is the start of the stored index + storedIndexOffset := uint64(w.Count()) + + // now write out the stored doc index + for _, docNumOffset := range docNumOffsets { + err := binary.Write(w, binary.BigEndian, docNumOffset) + if err != nil { + return 0, nil, err + } + } + + return storedIndexOffset, rv, nil +} + +// copyStoredDocs writes out a segment's stored doc info, optimized by +// using a single Write() call for the entire set of bytes. The +// newDocNumOffsets is filled with the new offsets for each doc. +func (s *SegmentBase) copyStoredDocs(newDocNum uint64, newDocNumOffsets []uint64, + w *CountHashWriter) error { + if s.numDocs <= 0 { + return nil + } + + indexOffset0, storedOffset0, _, _, _ := + s.getDocStoredOffsets(0) // the segment's first doc + + indexOffsetN, storedOffsetN, readN, metaLenN, dataLenN := + s.getDocStoredOffsets(s.numDocs - 1) // the segment's last doc + + storedOffset0New := uint64(w.Count()) + + storedBytes := s.mem[storedOffset0 : storedOffsetN+readN+metaLenN+dataLenN] + _, err := w.Write(storedBytes) + if err != nil { + return err + } + + // remap the storedOffset's for the docs into new offsets relative + // to storedOffset0New, filling the given docNumOffsetsOut array + for indexOffset := indexOffset0; indexOffset <= indexOffsetN; indexOffset += 8 { + storedOffset := binary.BigEndian.Uint64(s.mem[indexOffset : indexOffset+8]) + storedOffsetNew := storedOffset - storedOffset0 + storedOffset0New + newDocNumOffsets[newDocNum] = storedOffsetNew + newDocNum += 1 + } + + return nil +} + +// mergeFields builds a unified list of fields used across all the +// input segments, and computes whether the fields are the same across +// segments (which depends on fields to be sorted in the same way +// across segments) +func mergeFields(segments []*SegmentBase) (bool, []string) { + fieldsSame := true + + var segment0Fields []string + if len(segments) > 0 { + segment0Fields = segments[0].Fields() + } + + fieldsExist := map[string]struct{}{} + for _, segment := range segments { + fields := segment.Fields() + for fieldi, field := range fields { + fieldsExist[field] = struct{}{} + if len(segment0Fields) != len(fields) || segment0Fields[fieldi] != field { + fieldsSame = false + } + } + } + + rv := make([]string, 0, len(fieldsExist)) + // ensure _id stays first + rv = append(rv, "_id") + for k := range fieldsExist { + if k != "_id" { + rv = append(rv, k) + } + } + + sort.Strings(rv[1:]) // leave _id as first + + return fieldsSame, rv +} + +func isClosed(closeCh chan struct{}) bool { + select { + case <-closeCh: + return true + default: + return false + } +} diff --git a/vendor/github.com/blevesearch/zapx/v14/new.go b/vendor/github.com/blevesearch/zapx/v14/new.go new file mode 100644 index 00000000..b4e0d034 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v14/new.go @@ -0,0 +1,830 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "bytes" + "encoding/binary" + "math" + "sort" + "sync" + + "github.com/RoaringBitmap/roaring" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" + "github.com/blevesearch/vellum" + "github.com/golang/snappy" +) + +var NewSegmentBufferNumResultsBump int = 100 +var NewSegmentBufferNumResultsFactor float64 = 1.0 +var NewSegmentBufferAvgBytesPerDocFactor float64 = 1.0 + +// ValidateDocFields can be set by applications to perform additional checks +// on fields in a document being added to a new segment, by default it does +// nothing. +// This API is experimental and may be removed at any time. +var ValidateDocFields = func(field index.Field) error { + return nil +} + +// New creates an in-memory zap-encoded SegmentBase from a set of Documents +func (z *ZapPlugin) New(results []index.Document) ( + segment.Segment, uint64, error) { + return z.newWithChunkMode(results, DefaultChunkMode) +} + +func (*ZapPlugin) newWithChunkMode(results []index.Document, + chunkMode uint32) (segment.Segment, uint64, error) { + s := interimPool.Get().(*interim) + + var br bytes.Buffer + if s.lastNumDocs > 0 { + // use previous results to initialize the buf with an estimate + // size, but note that the interim instance comes from a + // global interimPool, so multiple scorch instances indexing + // different docs can lead to low quality estimates + estimateAvgBytesPerDoc := int(float64(s.lastOutSize/s.lastNumDocs) * + NewSegmentBufferNumResultsFactor) + estimateNumResults := int(float64(len(results)+NewSegmentBufferNumResultsBump) * + NewSegmentBufferAvgBytesPerDocFactor) + br.Grow(estimateAvgBytesPerDoc * estimateNumResults) + } + + s.results = results + s.chunkMode = chunkMode + s.w = NewCountHashWriter(&br) + + storedIndexOffset, fieldsIndexOffset, fdvIndexOffset, dictOffsets, + err := s.convert() + if err != nil { + return nil, uint64(0), err + } + + sb, err := InitSegmentBase(br.Bytes(), s.w.Sum32(), chunkMode, + s.FieldsMap, s.FieldsInv, uint64(len(results)), + storedIndexOffset, fieldsIndexOffset, fdvIndexOffset, dictOffsets) + + if err == nil && s.reset() == nil { + s.lastNumDocs = len(results) + s.lastOutSize = len(br.Bytes()) + interimPool.Put(s) + } + + return sb, uint64(len(br.Bytes())), err +} + +var interimPool = sync.Pool{New: func() interface{} { return &interim{} }} + +// interim holds temporary working data used while converting from +// analysis results to a zap-encoded segment +type interim struct { + results []index.Document + + chunkMode uint32 + + w *CountHashWriter + + // FieldsMap adds 1 to field id to avoid zero value issues + // name -> field id + 1 + FieldsMap map[string]uint16 + + // FieldsInv is the inverse of FieldsMap + // field id -> name + FieldsInv []string + + // Term dictionaries for each field + // field id -> term -> postings list id + 1 + Dicts []map[string]uint64 + + // Terms for each field, where terms are sorted ascending + // field id -> []term + DictKeys [][]string + + // Fields whose IncludeDocValues is true + // field id -> bool + IncludeDocValues []bool + + // postings id -> bitmap of docNums + Postings []*roaring.Bitmap + + // postings id -> freq/norm's, one for each docNum in postings + FreqNorms [][]interimFreqNorm + freqNormsBacking []interimFreqNorm + + // postings id -> locs, one for each freq + Locs [][]interimLoc + locsBacking []interimLoc + + numTermsPerPostingsList []int // key is postings list id + numLocsPerPostingsList []int // key is postings list id + + builder *vellum.Builder + builderBuf bytes.Buffer + + metaBuf bytes.Buffer + + tmp0 []byte + tmp1 []byte + + lastNumDocs int + lastOutSize int +} + +func (s *interim) reset() (err error) { + s.results = nil + s.chunkMode = 0 + s.w = nil + s.FieldsMap = nil + s.FieldsInv = nil + for i := range s.Dicts { + s.Dicts[i] = nil + } + s.Dicts = s.Dicts[:0] + for i := range s.DictKeys { + s.DictKeys[i] = s.DictKeys[i][:0] + } + s.DictKeys = s.DictKeys[:0] + for i := range s.IncludeDocValues { + s.IncludeDocValues[i] = false + } + s.IncludeDocValues = s.IncludeDocValues[:0] + for _, idn := range s.Postings { + idn.Clear() + } + s.Postings = s.Postings[:0] + s.FreqNorms = s.FreqNorms[:0] + for i := range s.freqNormsBacking { + s.freqNormsBacking[i] = interimFreqNorm{} + } + s.freqNormsBacking = s.freqNormsBacking[:0] + s.Locs = s.Locs[:0] + for i := range s.locsBacking { + s.locsBacking[i] = interimLoc{} + } + s.locsBacking = s.locsBacking[:0] + s.numTermsPerPostingsList = s.numTermsPerPostingsList[:0] + s.numLocsPerPostingsList = s.numLocsPerPostingsList[:0] + s.builderBuf.Reset() + if s.builder != nil { + err = s.builder.Reset(&s.builderBuf) + } + s.metaBuf.Reset() + s.tmp0 = s.tmp0[:0] + s.tmp1 = s.tmp1[:0] + s.lastNumDocs = 0 + s.lastOutSize = 0 + + return err +} + +func (s *interim) grabBuf(size int) []byte { + buf := s.tmp0 + if cap(buf) < size { + buf = make([]byte, size) + s.tmp0 = buf + } + return buf[0:size] +} + +type interimStoredField struct { + vals [][]byte + typs []byte + arrayposs [][]uint64 // array positions +} + +type interimFreqNorm struct { + freq uint64 + norm float32 + numLocs int +} + +type interimLoc struct { + fieldID uint16 + pos uint64 + start uint64 + end uint64 + arrayposs []uint64 +} + +func (s *interim) convert() (uint64, uint64, uint64, []uint64, error) { + s.FieldsMap = map[string]uint16{} + + s.getOrDefineField("_id") // _id field is fieldID 0 + + for _, result := range s.results { + result.VisitComposite(func(field index.CompositeField) { + s.getOrDefineField(field.Name()) + }) + result.VisitFields(func(field index.Field) { + s.getOrDefineField(field.Name()) + }) + } + + sort.Strings(s.FieldsInv[1:]) // keep _id as first field + + for fieldID, fieldName := range s.FieldsInv { + s.FieldsMap[fieldName] = uint16(fieldID + 1) + } + + if cap(s.IncludeDocValues) >= len(s.FieldsInv) { + s.IncludeDocValues = s.IncludeDocValues[:len(s.FieldsInv)] + } else { + s.IncludeDocValues = make([]bool, len(s.FieldsInv)) + } + + s.prepareDicts() + + for _, dict := range s.DictKeys { + sort.Strings(dict) + } + + s.processDocuments() + + storedIndexOffset, err := s.writeStoredFields() + if err != nil { + return 0, 0, 0, nil, err + } + + var fdvIndexOffset uint64 + var dictOffsets []uint64 + + if len(s.results) > 0 { + fdvIndexOffset, dictOffsets, err = s.writeDicts() + if err != nil { + return 0, 0, 0, nil, err + } + } else { + dictOffsets = make([]uint64, len(s.FieldsInv)) + } + + fieldsIndexOffset, err := persistFields(s.FieldsInv, s.w, dictOffsets) + if err != nil { + return 0, 0, 0, nil, err + } + + return storedIndexOffset, fieldsIndexOffset, fdvIndexOffset, dictOffsets, nil +} + +func (s *interim) getOrDefineField(fieldName string) int { + fieldIDPlus1, exists := s.FieldsMap[fieldName] + if !exists { + fieldIDPlus1 = uint16(len(s.FieldsInv) + 1) + s.FieldsMap[fieldName] = fieldIDPlus1 + s.FieldsInv = append(s.FieldsInv, fieldName) + + s.Dicts = append(s.Dicts, make(map[string]uint64)) + + n := len(s.DictKeys) + if n < cap(s.DictKeys) { + s.DictKeys = s.DictKeys[:n+1] + s.DictKeys[n] = s.DictKeys[n][:0] + } else { + s.DictKeys = append(s.DictKeys, []string(nil)) + } + } + + return int(fieldIDPlus1 - 1) +} + +// fill Dicts and DictKeys from analysis results +func (s *interim) prepareDicts() { + var pidNext int + + var totTFs int + var totLocs int + + visitField := func(field index.Field) { + fieldID := uint16(s.getOrDefineField(field.Name())) + + dict := s.Dicts[fieldID] + dictKeys := s.DictKeys[fieldID] + + tfs := field.AnalyzedTokenFrequencies() + for term, tf := range tfs { + pidPlus1, exists := dict[term] + if !exists { + pidNext++ + pidPlus1 = uint64(pidNext) + + dict[term] = pidPlus1 + dictKeys = append(dictKeys, term) + + s.numTermsPerPostingsList = append(s.numTermsPerPostingsList, 0) + s.numLocsPerPostingsList = append(s.numLocsPerPostingsList, 0) + } + + pid := pidPlus1 - 1 + + s.numTermsPerPostingsList[pid] += 1 + s.numLocsPerPostingsList[pid] += len(tf.Locations) + + totLocs += len(tf.Locations) + } + + totTFs += len(tfs) + + s.DictKeys[fieldID] = dictKeys + } + + for _, result := range s.results { + // walk each composite field + result.VisitComposite(func(field index.CompositeField) { + visitField(field) + }) + + // walk each field + result.VisitFields(visitField) + } + + numPostingsLists := pidNext + + if cap(s.Postings) >= numPostingsLists { + s.Postings = s.Postings[:numPostingsLists] + } else { + postings := make([]*roaring.Bitmap, numPostingsLists) + copy(postings, s.Postings[:cap(s.Postings)]) + for i := 0; i < numPostingsLists; i++ { + if postings[i] == nil { + postings[i] = roaring.New() + } + } + s.Postings = postings + } + + if cap(s.FreqNorms) >= numPostingsLists { + s.FreqNorms = s.FreqNorms[:numPostingsLists] + } else { + s.FreqNorms = make([][]interimFreqNorm, numPostingsLists) + } + + if cap(s.freqNormsBacking) >= totTFs { + s.freqNormsBacking = s.freqNormsBacking[:totTFs] + } else { + s.freqNormsBacking = make([]interimFreqNorm, totTFs) + } + + freqNormsBacking := s.freqNormsBacking + for pid, numTerms := range s.numTermsPerPostingsList { + s.FreqNorms[pid] = freqNormsBacking[0:0] + freqNormsBacking = freqNormsBacking[numTerms:] + } + + if cap(s.Locs) >= numPostingsLists { + s.Locs = s.Locs[:numPostingsLists] + } else { + s.Locs = make([][]interimLoc, numPostingsLists) + } + + if cap(s.locsBacking) >= totLocs { + s.locsBacking = s.locsBacking[:totLocs] + } else { + s.locsBacking = make([]interimLoc, totLocs) + } + + locsBacking := s.locsBacking + for pid, numLocs := range s.numLocsPerPostingsList { + s.Locs[pid] = locsBacking[0:0] + locsBacking = locsBacking[numLocs:] + } +} + +func (s *interim) processDocuments() { + numFields := len(s.FieldsInv) + reuseFieldLens := make([]int, numFields) + reuseFieldTFs := make([]index.TokenFrequencies, numFields) + + for docNum, result := range s.results { + for i := 0; i < numFields; i++ { // clear these for reuse + reuseFieldLens[i] = 0 + reuseFieldTFs[i] = nil + } + + s.processDocument(uint64(docNum), result, + reuseFieldLens, reuseFieldTFs) + } +} + +func (s *interim) processDocument(docNum uint64, + result index.Document, + fieldLens []int, fieldTFs []index.TokenFrequencies) { + visitField := func(field index.Field) { + fieldID := uint16(s.getOrDefineField(field.Name())) + fieldLens[fieldID] += field.AnalyzedLength() + + existingFreqs := fieldTFs[fieldID] + if existingFreqs != nil { + existingFreqs.MergeAll(field.Name(), field.AnalyzedTokenFrequencies()) + } else { + fieldTFs[fieldID] = field.AnalyzedTokenFrequencies() + } + } + + // walk each composite field + result.VisitComposite(func(field index.CompositeField) { + visitField(field) + }) + + // walk each field + result.VisitFields(visitField) + + // now that it's been rolled up into fieldTFs, walk that + for fieldID, tfs := range fieldTFs { + dict := s.Dicts[fieldID] + norm := float32(1.0 / math.Sqrt(float64(fieldLens[fieldID]))) + + for term, tf := range tfs { + pid := dict[term] - 1 + bs := s.Postings[pid] + bs.Add(uint32(docNum)) + + s.FreqNorms[pid] = append(s.FreqNorms[pid], + interimFreqNorm{ + freq: uint64(tf.Frequency()), + norm: norm, + numLocs: len(tf.Locations), + }) + + if len(tf.Locations) > 0 { + locs := s.Locs[pid] + + for _, loc := range tf.Locations { + var locf = uint16(fieldID) + if loc.Field != "" { + locf = uint16(s.getOrDefineField(loc.Field)) + } + var arrayposs []uint64 + if len(loc.ArrayPositions) > 0 { + arrayposs = loc.ArrayPositions + } + locs = append(locs, interimLoc{ + fieldID: locf, + pos: uint64(loc.Position), + start: uint64(loc.Start), + end: uint64(loc.End), + arrayposs: arrayposs, + }) + } + + s.Locs[pid] = locs + } + } + } +} + +func (s *interim) writeStoredFields() ( + storedIndexOffset uint64, err error) { + varBuf := make([]byte, binary.MaxVarintLen64) + metaEncode := func(val uint64) (int, error) { + wb := binary.PutUvarint(varBuf, val) + return s.metaBuf.Write(varBuf[:wb]) + } + + data, compressed := s.tmp0[:0], s.tmp1[:0] + defer func() { s.tmp0, s.tmp1 = data, compressed }() + + // keyed by docNum + docStoredOffsets := make([]uint64, len(s.results)) + + // keyed by fieldID, for the current doc in the loop + docStoredFields := map[uint16]interimStoredField{} + + for docNum, result := range s.results { + for fieldID := range docStoredFields { // reset for next doc + delete(docStoredFields, fieldID) + } + + var validationErr error + result.VisitFields(func(field index.Field) { + fieldID := uint16(s.getOrDefineField(field.Name())) + + if field.Options().IsStored() { + isf := docStoredFields[fieldID] + isf.vals = append(isf.vals, field.Value()) + isf.typs = append(isf.typs, field.EncodedFieldType()) + isf.arrayposs = append(isf.arrayposs, field.ArrayPositions()) + docStoredFields[fieldID] = isf + } + + if field.Options().IncludeDocValues() { + s.IncludeDocValues[fieldID] = true + } + + err := ValidateDocFields(field) + if err != nil && validationErr == nil { + validationErr = err + } + }) + if validationErr != nil { + return 0, validationErr + } + + var curr int + + s.metaBuf.Reset() + data = data[:0] + + // _id field special case optimizes ExternalID() lookups + idFieldVal := docStoredFields[uint16(0)].vals[0] + _, err = metaEncode(uint64(len(idFieldVal))) + if err != nil { + return 0, err + } + + // handle non-"_id" fields + for fieldID := 1; fieldID < len(s.FieldsInv); fieldID++ { + isf, exists := docStoredFields[uint16(fieldID)] + if exists { + curr, data, err = persistStoredFieldValues( + fieldID, isf.vals, isf.typs, isf.arrayposs, + curr, metaEncode, data) + if err != nil { + return 0, err + } + } + } + + metaBytes := s.metaBuf.Bytes() + + compressed = snappy.Encode(compressed[:cap(compressed)], data) + + docStoredOffsets[docNum] = uint64(s.w.Count()) + + _, err := writeUvarints(s.w, + uint64(len(metaBytes)), + uint64(len(idFieldVal)+len(compressed))) + if err != nil { + return 0, err + } + + _, err = s.w.Write(metaBytes) + if err != nil { + return 0, err + } + + _, err = s.w.Write(idFieldVal) + if err != nil { + return 0, err + } + + _, err = s.w.Write(compressed) + if err != nil { + return 0, err + } + } + + storedIndexOffset = uint64(s.w.Count()) + + for _, docStoredOffset := range docStoredOffsets { + err = binary.Write(s.w, binary.BigEndian, docStoredOffset) + if err != nil { + return 0, err + } + } + + return storedIndexOffset, nil +} + +func (s *interim) writeDicts() (fdvIndexOffset uint64, dictOffsets []uint64, err error) { + dictOffsets = make([]uint64, len(s.FieldsInv)) + + fdvOffsetsStart := make([]uint64, len(s.FieldsInv)) + fdvOffsetsEnd := make([]uint64, len(s.FieldsInv)) + + buf := s.grabBuf(binary.MaxVarintLen64) + + // these int coders are initialized with chunk size 1024 + // however this will be reset to the correct chunk size + // while processing each individual field-term section + tfEncoder := newChunkedIntCoder(1024, uint64(len(s.results)-1)) + locEncoder := newChunkedIntCoder(1024, uint64(len(s.results)-1)) + + var docTermMap [][]byte + + if s.builder == nil { + s.builder, err = vellum.New(&s.builderBuf, nil) + if err != nil { + return 0, nil, err + } + } + + for fieldID, terms := range s.DictKeys { + if cap(docTermMap) < len(s.results) { + docTermMap = make([][]byte, len(s.results)) + } else { + docTermMap = docTermMap[0:len(s.results)] + for docNum := range docTermMap { // reset the docTermMap + docTermMap[docNum] = docTermMap[docNum][:0] + } + } + + dict := s.Dicts[fieldID] + + for _, term := range terms { // terms are already sorted + pid := dict[term] - 1 + + postingsBS := s.Postings[pid] + + freqNorms := s.FreqNorms[pid] + freqNormOffset := 0 + + locs := s.Locs[pid] + locOffset := 0 + + chunkSize, err := getChunkSize(s.chunkMode, postingsBS.GetCardinality(), uint64(len(s.results))) + if err != nil { + return 0, nil, err + } + tfEncoder.SetChunkSize(chunkSize, uint64(len(s.results)-1)) + locEncoder.SetChunkSize(chunkSize, uint64(len(s.results)-1)) + + postingsItr := postingsBS.Iterator() + for postingsItr.HasNext() { + docNum := uint64(postingsItr.Next()) + + freqNorm := freqNorms[freqNormOffset] + + err = tfEncoder.Add(docNum, + encodeFreqHasLocs(freqNorm.freq, freqNorm.numLocs > 0), + uint64(math.Float32bits(freqNorm.norm))) + if err != nil { + return 0, nil, err + } + + if freqNorm.numLocs > 0 { + numBytesLocs := 0 + for _, loc := range locs[locOffset : locOffset+freqNorm.numLocs] { + numBytesLocs += totalUvarintBytes( + uint64(loc.fieldID), loc.pos, loc.start, loc.end, + uint64(len(loc.arrayposs)), loc.arrayposs) + } + + err = locEncoder.Add(docNum, uint64(numBytesLocs)) + if err != nil { + return 0, nil, err + } + + for _, loc := range locs[locOffset : locOffset+freqNorm.numLocs] { + err = locEncoder.Add(docNum, + uint64(loc.fieldID), loc.pos, loc.start, loc.end, + uint64(len(loc.arrayposs))) + if err != nil { + return 0, nil, err + } + + err = locEncoder.Add(docNum, loc.arrayposs...) + if err != nil { + return 0, nil, err + } + } + + locOffset += freqNorm.numLocs + } + + freqNormOffset++ + + docTermMap[docNum] = append( + append(docTermMap[docNum], term...), + termSeparator) + } + + tfEncoder.Close() + locEncoder.Close() + + postingsOffset, err := + writePostings(postingsBS, tfEncoder, locEncoder, nil, s.w, buf) + if err != nil { + return 0, nil, err + } + + if postingsOffset > uint64(0) { + err = s.builder.Insert([]byte(term), postingsOffset) + if err != nil { + return 0, nil, err + } + } + + tfEncoder.Reset() + locEncoder.Reset() + } + + err = s.builder.Close() + if err != nil { + return 0, nil, err + } + + // record where this dictionary starts + dictOffsets[fieldID] = uint64(s.w.Count()) + + vellumData := s.builderBuf.Bytes() + + // write out the length of the vellum data + n := binary.PutUvarint(buf, uint64(len(vellumData))) + _, err = s.w.Write(buf[:n]) + if err != nil { + return 0, nil, err + } + + // write this vellum to disk + _, err = s.w.Write(vellumData) + if err != nil { + return 0, nil, err + } + + // reset vellum for reuse + s.builderBuf.Reset() + + err = s.builder.Reset(&s.builderBuf) + if err != nil { + return 0, nil, err + } + + // write the field doc values + // NOTE: doc values continue to use legacy chunk mode + chunkSize, err := getChunkSize(LegacyChunkMode, 0, 0) + if err != nil { + return 0, nil, err + } + fdvEncoder := newChunkedContentCoder(chunkSize, uint64(len(s.results)-1), s.w, false) + if s.IncludeDocValues[fieldID] { + for docNum, docTerms := range docTermMap { + if len(docTerms) > 0 { + err = fdvEncoder.Add(uint64(docNum), docTerms) + if err != nil { + return 0, nil, err + } + } + } + err = fdvEncoder.Close() + if err != nil { + return 0, nil, err + } + + fdvOffsetsStart[fieldID] = uint64(s.w.Count()) + + _, err = fdvEncoder.Write() + if err != nil { + return 0, nil, err + } + + fdvOffsetsEnd[fieldID] = uint64(s.w.Count()) + + fdvEncoder.Reset() + } else { + fdvOffsetsStart[fieldID] = fieldNotUninverted + fdvOffsetsEnd[fieldID] = fieldNotUninverted + } + } + + fdvIndexOffset = uint64(s.w.Count()) + + for i := 0; i < len(fdvOffsetsStart); i++ { + n := binary.PutUvarint(buf, fdvOffsetsStart[i]) + _, err := s.w.Write(buf[:n]) + if err != nil { + return 0, nil, err + } + n = binary.PutUvarint(buf, fdvOffsetsEnd[i]) + _, err = s.w.Write(buf[:n]) + if err != nil { + return 0, nil, err + } + } + + return fdvIndexOffset, dictOffsets, nil +} + +// returns the total # of bytes needed to encode the given uint64's +// into binary.PutUVarint() encoding +func totalUvarintBytes(a, b, c, d, e uint64, more []uint64) (n int) { + n = numUvarintBytes(a) + n += numUvarintBytes(b) + n += numUvarintBytes(c) + n += numUvarintBytes(d) + n += numUvarintBytes(e) + for _, v := range more { + n += numUvarintBytes(v) + } + return n +} + +// returns # of bytes needed to encode x in binary.PutUvarint() encoding +func numUvarintBytes(x uint64) (n int) { + for x >= 0x80 { + x >>= 7 + n++ + } + return n + 1 +} diff --git a/vendor/github.com/blevesearch/zapx/v14/plugin.go b/vendor/github.com/blevesearch/zapx/v14/plugin.go new file mode 100644 index 00000000..f67297ec --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v14/plugin.go @@ -0,0 +1,27 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +// ZapPlugin implements the Plugin interface of +// the blevesearch/scorch_segment_api pkg +type ZapPlugin struct{} + +func (*ZapPlugin) Type() string { + return Type +} + +func (*ZapPlugin) Version() uint32 { + return Version +} diff --git a/vendor/github.com/blevesearch/zapx/v14/posting.go b/vendor/github.com/blevesearch/zapx/v14/posting.go new file mode 100644 index 00000000..8d138509 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v14/posting.go @@ -0,0 +1,835 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "encoding/binary" + "fmt" + "math" + "reflect" + + "github.com/RoaringBitmap/roaring" + segment "github.com/blevesearch/scorch_segment_api/v2" +) + +var reflectStaticSizePostingsList int +var reflectStaticSizePostingsIterator int +var reflectStaticSizePosting int +var reflectStaticSizeLocation int + +func init() { + var pl PostingsList + reflectStaticSizePostingsList = int(reflect.TypeOf(pl).Size()) + var pi PostingsIterator + reflectStaticSizePostingsIterator = int(reflect.TypeOf(pi).Size()) + var p Posting + reflectStaticSizePosting = int(reflect.TypeOf(p).Size()) + var l Location + reflectStaticSizeLocation = int(reflect.TypeOf(l).Size()) +} + +// FST or vellum value (uint64) encoding is determined by the top two +// highest-order or most significant bits... +// +// encoding : MSB +// name : 63 62 61...to...bit #0 (LSB) +// ----------+---+---+--------------------------------------------------- +// general : 0 | 0 | 62-bits of postingsOffset. +// ~ : 0 | 1 | reserved for future. +// 1-hit : 1 | 0 | 31-bits of positive float31 norm | 31-bits docNum. +// ~ : 1 | 1 | reserved for future. +// +// Encoding "general" is able to handle all cases, where the +// postingsOffset points to more information about the postings for +// the term. +// +// Encoding "1-hit" is used to optimize a commonly seen case when a +// term has only a single hit. For example, a term in the _id field +// will have only 1 hit. The "1-hit" encoding is used for a term +// in a field when... +// +// - term vector info is disabled for that field; +// - and, the term appears in only a single doc for that field; +// - and, the term's freq is exactly 1 in that single doc for that field; +// - and, the docNum must fit into 31-bits; +// +// Otherwise, the "general" encoding is used instead. +// +// In the "1-hit" encoding, the field in that single doc may have +// other terms, which is supported in the "1-hit" encoding by the +// positive float31 norm. + +const FSTValEncodingMask = uint64(0xc000000000000000) +const FSTValEncodingGeneral = uint64(0x0000000000000000) +const FSTValEncoding1Hit = uint64(0x8000000000000000) + +func FSTValEncode1Hit(docNum uint64, normBits uint64) uint64 { + return FSTValEncoding1Hit | ((mask31Bits & normBits) << 31) | (mask31Bits & docNum) +} + +func FSTValDecode1Hit(v uint64) (docNum uint64, normBits uint64) { + return (mask31Bits & v), (mask31Bits & (v >> 31)) +} + +const mask31Bits = uint64(0x000000007fffffff) + +func under32Bits(x uint64) bool { + return x <= mask31Bits +} + +const DocNum1HitFinished = math.MaxUint64 + +var NormBits1Hit = uint64(math.Float32bits(float32(1))) + +// PostingsList is an in-memory representation of a postings list +type PostingsList struct { + sb *SegmentBase + postingsOffset uint64 + freqOffset uint64 + locOffset uint64 + postings *roaring.Bitmap + except *roaring.Bitmap + + // when normBits1Hit != 0, then this postings list came from a + // 1-hit encoding, and only the docNum1Hit & normBits1Hit apply + docNum1Hit uint64 + normBits1Hit uint64 + + chunkSize uint64 +} + +// represents an immutable, empty postings list +var emptyPostingsList = &PostingsList{} + +// Following methods are dummy implementations for the interface +// DiskStatsReporter (for backward compatibility). +// The working implementations are supported only in zap v15.x +// and not in the earlier versions of zap. +func (i *PostingsList) ResetBytesRead(uint64) {} + +func (i *PostingsList) BytesRead() uint64 { + return 0 +} + +func (i *PostingsList) incrementBytesRead(uint64) {} + +func (i *PostingsList) BytesWritten() uint64 { + return 0 +} + +func (p *PostingsList) Size() int { + sizeInBytes := reflectStaticSizePostingsList + SizeOfPtr + + if p.except != nil { + sizeInBytes += int(p.except.GetSizeInBytes()) + } + + return sizeInBytes +} + +func (p *PostingsList) OrInto(receiver *roaring.Bitmap) { + if p.normBits1Hit != 0 { + receiver.Add(uint32(p.docNum1Hit)) + return + } + + if p.postings != nil { + receiver.Or(p.postings) + } +} + +// Iterator returns an iterator for this postings list +func (p *PostingsList) Iterator(includeFreq, includeNorm, includeLocs bool, + prealloc segment.PostingsIterator) segment.PostingsIterator { + if p.normBits1Hit == 0 && p.postings == nil { + return emptyPostingsIterator + } + + var preallocPI *PostingsIterator + pi, ok := prealloc.(*PostingsIterator) + if ok && pi != nil { + preallocPI = pi + } + if preallocPI == emptyPostingsIterator { + preallocPI = nil + } + + return p.iterator(includeFreq, includeNorm, includeLocs, preallocPI) +} + +func (p *PostingsList) iterator(includeFreq, includeNorm, includeLocs bool, + rv *PostingsIterator) *PostingsIterator { + if rv == nil { + rv = &PostingsIterator{} + } else { + freqNormReader := rv.freqNormReader + if freqNormReader != nil { + freqNormReader.reset() + } + + locReader := rv.locReader + if locReader != nil { + locReader.reset() + } + + nextLocs := rv.nextLocs[:0] + nextSegmentLocs := rv.nextSegmentLocs[:0] + + buf := rv.buf + + *rv = PostingsIterator{} // clear the struct + + rv.freqNormReader = freqNormReader + rv.locReader = locReader + + rv.nextLocs = nextLocs + rv.nextSegmentLocs = nextSegmentLocs + + rv.buf = buf + } + + rv.postings = p + rv.includeFreqNorm = includeFreq || includeNorm || includeLocs + rv.includeLocs = includeLocs + + if p.normBits1Hit != 0 { + // "1-hit" encoding + rv.docNum1Hit = p.docNum1Hit + rv.normBits1Hit = p.normBits1Hit + + if p.except != nil && p.except.Contains(uint32(rv.docNum1Hit)) { + rv.docNum1Hit = DocNum1HitFinished + } + + return rv + } + + // "general" encoding, check if empty + if p.postings == nil { + return rv + } + + // initialize freq chunk reader + if rv.includeFreqNorm { + rv.freqNormReader = newChunkedIntDecoder(p.sb.mem, p.freqOffset, rv.freqNormReader) + } + + // initialize the loc chunk reader + if rv.includeLocs { + rv.locReader = newChunkedIntDecoder(p.sb.mem, p.locOffset, rv.locReader) + } + + rv.all = p.postings.Iterator() + if p.except != nil { + rv.ActualBM = roaring.AndNot(p.postings, p.except) + rv.Actual = rv.ActualBM.Iterator() + } else { + rv.ActualBM = p.postings + rv.Actual = rv.all // Optimize to use same iterator for all & Actual. + } + + return rv +} + +// Count returns the number of items on this postings list +func (p *PostingsList) Count() uint64 { + var n, e uint64 + if p.normBits1Hit != 0 { + n = 1 + if p.except != nil && p.except.Contains(uint32(p.docNum1Hit)) { + e = 1 + } + } else if p.postings != nil { + n = p.postings.GetCardinality() + if p.except != nil { + e = p.postings.AndCardinality(p.except) + } + } + return n - e +} + +func (rv *PostingsList) read(postingsOffset uint64, d *Dictionary) error { + rv.postingsOffset = postingsOffset + + // handle "1-hit" encoding special case + if rv.postingsOffset&FSTValEncodingMask == FSTValEncoding1Hit { + return rv.init1Hit(postingsOffset) + } + + // read the location of the freq/norm details + var n uint64 + var read int + + rv.freqOffset, read = binary.Uvarint(d.sb.mem[postingsOffset+n : postingsOffset+binary.MaxVarintLen64]) + n += uint64(read) + + rv.locOffset, read = binary.Uvarint(d.sb.mem[postingsOffset+n : postingsOffset+n+binary.MaxVarintLen64]) + n += uint64(read) + + var postingsLen uint64 + postingsLen, read = binary.Uvarint(d.sb.mem[postingsOffset+n : postingsOffset+n+binary.MaxVarintLen64]) + n += uint64(read) + + roaringBytes := d.sb.mem[postingsOffset+n : postingsOffset+n+postingsLen] + + if rv.postings == nil { + rv.postings = roaring.NewBitmap() + } + _, err := rv.postings.FromBuffer(roaringBytes) + if err != nil { + return fmt.Errorf("error loading roaring bitmap: %v", err) + } + + rv.chunkSize, err = getChunkSize(d.sb.chunkMode, + rv.postings.GetCardinality(), d.sb.numDocs) + if err != nil { + return err + } + + return nil +} + +func (rv *PostingsList) init1Hit(fstVal uint64) error { + docNum, normBits := FSTValDecode1Hit(fstVal) + + rv.docNum1Hit = docNum + rv.normBits1Hit = normBits + + return nil +} + +// PostingsIterator provides a way to iterate through the postings list +type PostingsIterator struct { + postings *PostingsList + all roaring.IntPeekable + Actual roaring.IntPeekable + ActualBM *roaring.Bitmap + + currChunk uint32 + freqNormReader *chunkedIntDecoder + locReader *chunkedIntDecoder + + next Posting // reused across Next() calls + nextLocs []Location // reused across Next() calls + nextSegmentLocs []segment.Location // reused across Next() calls + + docNum1Hit uint64 + normBits1Hit uint64 + + buf []byte + + includeFreqNorm bool + includeLocs bool +} + +var emptyPostingsIterator = &PostingsIterator{} + +// Following methods are dummy implementations for the interface +// DiskStatsReporter (for backward compatibility). +// The working implementations are supported only in zap v15.x +// and not in the earlier versions of zap. +func (i *PostingsIterator) ResetBytesRead(uint64) {} + +func (i *PostingsIterator) BytesRead() uint64 { + return 0 +} + +func (i *PostingsIterator) incrementBytesRead(uint64) {} + +func (i *PostingsIterator) BytesWritten() uint64 { + return 0 +} + +func (i *PostingsIterator) Size() int { + sizeInBytes := reflectStaticSizePostingsIterator + SizeOfPtr + + i.next.Size() + // account for freqNormReader, locReader if we start using this. + for _, entry := range i.nextLocs { + sizeInBytes += entry.Size() + } + + return sizeInBytes +} + +func (i *PostingsIterator) loadChunk(chunk int) error { + if i.includeFreqNorm { + err := i.freqNormReader.loadChunk(chunk) + if err != nil { + return err + } + } + + if i.includeLocs { + err := i.locReader.loadChunk(chunk) + if err != nil { + return err + } + } + + i.currChunk = uint32(chunk) + return nil +} + +func (i *PostingsIterator) readFreqNormHasLocs() (uint64, uint64, bool, error) { + if i.normBits1Hit != 0 { + return 1, i.normBits1Hit, false, nil + } + + freqHasLocs, err := i.freqNormReader.readUvarint() + if err != nil { + return 0, 0, false, fmt.Errorf("error reading frequency: %v", err) + } + + freq, hasLocs := decodeFreqHasLocs(freqHasLocs) + + normBits, err := i.freqNormReader.readUvarint() + if err != nil { + return 0, 0, false, fmt.Errorf("error reading norm: %v", err) + } + + return freq, normBits, hasLocs, nil +} + +func (i *PostingsIterator) skipFreqNormReadHasLocs() (bool, error) { + if i.normBits1Hit != 0 { + return false, nil + } + + freqHasLocs, err := i.freqNormReader.readUvarint() + if err != nil { + return false, fmt.Errorf("error reading freqHasLocs: %v", err) + } + + i.freqNormReader.SkipUvarint() // Skip normBits. + + return freqHasLocs&0x01 != 0, nil // See decodeFreqHasLocs() / hasLocs. +} + +func encodeFreqHasLocs(freq uint64, hasLocs bool) uint64 { + rv := freq << 1 + if hasLocs { + rv = rv | 0x01 // 0'th LSB encodes whether there are locations + } + return rv +} + +func decodeFreqHasLocs(freqHasLocs uint64) (uint64, bool) { + freq := freqHasLocs >> 1 + hasLocs := freqHasLocs&0x01 != 0 + return freq, hasLocs +} + +// readLocation processes all the integers on the stream representing a single +// location. +func (i *PostingsIterator) readLocation(l *Location) error { + // read off field + fieldID, err := i.locReader.readUvarint() + if err != nil { + return fmt.Errorf("error reading location field: %v", err) + } + // read off pos + pos, err := i.locReader.readUvarint() + if err != nil { + return fmt.Errorf("error reading location pos: %v", err) + } + // read off start + start, err := i.locReader.readUvarint() + if err != nil { + return fmt.Errorf("error reading location start: %v", err) + } + // read off end + end, err := i.locReader.readUvarint() + if err != nil { + return fmt.Errorf("error reading location end: %v", err) + } + // read off num array pos + numArrayPos, err := i.locReader.readUvarint() + if err != nil { + return fmt.Errorf("error reading location num array pos: %v", err) + } + + l.field = i.postings.sb.fieldsInv[fieldID] + l.pos = pos + l.start = start + l.end = end + + if cap(l.ap) < int(numArrayPos) { + l.ap = make([]uint64, int(numArrayPos)) + } else { + l.ap = l.ap[:int(numArrayPos)] + } + + // read off array positions + for k := 0; k < int(numArrayPos); k++ { + ap, err := i.locReader.readUvarint() + if err != nil { + return fmt.Errorf("error reading array position: %v", err) + } + + l.ap[k] = ap + } + + return nil +} + +// Next returns the next posting on the postings list, or nil at the end +func (i *PostingsIterator) Next() (segment.Posting, error) { + return i.nextAtOrAfter(0) +} + +// Advance returns the posting at the specified docNum or it is not present +// the next posting, or if the end is reached, nil +func (i *PostingsIterator) Advance(docNum uint64) (segment.Posting, error) { + return i.nextAtOrAfter(docNum) +} + +// Next returns the next posting on the postings list, or nil at the end +func (i *PostingsIterator) nextAtOrAfter(atOrAfter uint64) (segment.Posting, error) { + docNum, exists, err := i.nextDocNumAtOrAfter(atOrAfter) + if err != nil || !exists { + return nil, err + } + + i.next = Posting{} // clear the struct + rv := &i.next + rv.docNum = docNum + + if !i.includeFreqNorm { + return rv, nil + } + + var normBits uint64 + var hasLocs bool + + rv.freq, normBits, hasLocs, err = i.readFreqNormHasLocs() + if err != nil { + return nil, err + } + + rv.norm = math.Float32frombits(uint32(normBits)) + + if i.includeLocs && hasLocs { + // prepare locations into reused slices, where we assume + // rv.freq >= "number of locs", since in a composite field, + // some component fields might have their IncludeTermVector + // flags disabled while other component fields are enabled + if cap(i.nextLocs) >= int(rv.freq) { + i.nextLocs = i.nextLocs[0:rv.freq] + } else { + i.nextLocs = make([]Location, rv.freq, rv.freq*2) + } + if cap(i.nextSegmentLocs) < int(rv.freq) { + i.nextSegmentLocs = make([]segment.Location, rv.freq, rv.freq*2) + } + rv.locs = i.nextSegmentLocs[:0] + + numLocsBytes, err := i.locReader.readUvarint() + if err != nil { + return nil, fmt.Errorf("error reading location numLocsBytes: %v", err) + } + + j := 0 + startBytesRemaining := i.locReader.Len() // # bytes remaining in the locReader + for startBytesRemaining-i.locReader.Len() < int(numLocsBytes) { + err := i.readLocation(&i.nextLocs[j]) + if err != nil { + return nil, err + } + rv.locs = append(rv.locs, &i.nextLocs[j]) + j++ + } + } + + return rv, nil +} + +// nextDocNum returns the next docNum on the postings list, and also +// sets up the currChunk / loc related fields of the iterator. +func (i *PostingsIterator) nextDocNumAtOrAfter(atOrAfter uint64) (uint64, bool, error) { + if i.normBits1Hit != 0 { + if i.docNum1Hit == DocNum1HitFinished { + return 0, false, nil + } + if i.docNum1Hit < atOrAfter { + // advanced past our 1-hit + i.docNum1Hit = DocNum1HitFinished // consume our 1-hit docNum + return 0, false, nil + } + docNum := i.docNum1Hit + i.docNum1Hit = DocNum1HitFinished // consume our 1-hit docNum + return docNum, true, nil + } + + if i.Actual == nil || !i.Actual.HasNext() { + return 0, false, nil + } + + if i.postings == nil || i.postings == emptyPostingsList { + // couldn't find anything + return 0, false, nil + } + + if i.postings.postings == i.ActualBM { + return i.nextDocNumAtOrAfterClean(atOrAfter) + } + + i.Actual.AdvanceIfNeeded(uint32(atOrAfter)) + + if !i.Actual.HasNext() || !i.all.HasNext() { + // couldn't find anything + return 0, false, nil + } + + n := i.Actual.Next() + allN := i.all.Next() + nChunk := n / uint32(i.postings.chunkSize) + + // when allN becomes >= to here, then allN is in the same chunk as nChunk. + allNReachesNChunk := nChunk * uint32(i.postings.chunkSize) + + // n is the next actual hit (excluding some postings), and + // allN is the next hit in the full postings, and + // if they don't match, move 'all' forwards until they do + for allN != n { + // we've reached same chunk, so move the freq/norm/loc decoders forward + if i.includeFreqNorm && allN >= allNReachesNChunk { + err := i.currChunkNext(nChunk) + if err != nil { + return 0, false, err + } + } + + if !i.all.HasNext() { + return 0, false, nil + } + + allN = i.all.Next() + } + + if i.includeFreqNorm && (i.currChunk != nChunk || i.freqNormReader.isNil()) { + err := i.loadChunk(int(nChunk)) + if err != nil { + return 0, false, fmt.Errorf("error loading chunk: %v", err) + } + } + + return uint64(n), true, nil +} + +// optimization when the postings list is "clean" (e.g., no updates & +// no deletions) where the all bitmap is the same as the actual bitmap +func (i *PostingsIterator) nextDocNumAtOrAfterClean( + atOrAfter uint64) (uint64, bool, error) { + if !i.includeFreqNorm { + i.Actual.AdvanceIfNeeded(uint32(atOrAfter)) + + if !i.Actual.HasNext() { + return 0, false, nil // couldn't find anything + } + + return uint64(i.Actual.Next()), true, nil + } + + // freq-norm's needed, so maintain freq-norm chunk reader + sameChunkNexts := 0 // # of times we called Next() in the same chunk + n := i.Actual.Next() + nChunk := n / uint32(i.postings.chunkSize) + + for uint64(n) < atOrAfter && i.Actual.HasNext() { + n = i.Actual.Next() + + nChunkPrev := nChunk + nChunk = n / uint32(i.postings.chunkSize) + + if nChunk != nChunkPrev { + sameChunkNexts = 0 + } else { + sameChunkNexts += 1 + } + } + + if uint64(n) < atOrAfter { + // couldn't find anything + return 0, false, nil + } + + for j := 0; j < sameChunkNexts; j++ { + err := i.currChunkNext(nChunk) + if err != nil { + return 0, false, fmt.Errorf("error optimized currChunkNext: %v", err) + } + } + + if i.currChunk != nChunk || i.freqNormReader.isNil() { + err := i.loadChunk(int(nChunk)) + if err != nil { + return 0, false, fmt.Errorf("error loading chunk: %v", err) + } + } + + return uint64(n), true, nil +} + +func (i *PostingsIterator) currChunkNext(nChunk uint32) error { + if i.currChunk != nChunk || i.freqNormReader.isNil() { + err := i.loadChunk(int(nChunk)) + if err != nil { + return fmt.Errorf("error loading chunk: %v", err) + } + } + + // read off freq/offsets even though we don't care about them + hasLocs, err := i.skipFreqNormReadHasLocs() + if err != nil { + return err + } + + if i.includeLocs && hasLocs { + numLocsBytes, err := i.locReader.readUvarint() + if err != nil { + return fmt.Errorf("error reading location numLocsBytes: %v", err) + } + + // skip over all the location bytes + i.locReader.SkipBytes(int(numLocsBytes)) + } + + return nil +} + +// DocNum1Hit returns the docNum and true if this is "1-hit" optimized +// and the docNum is available. +func (p *PostingsIterator) DocNum1Hit() (uint64, bool) { + if p.normBits1Hit != 0 && p.docNum1Hit != DocNum1HitFinished { + return p.docNum1Hit, true + } + return 0, false +} + +// ActualBitmap returns the underlying actual bitmap +// which can be used up the stack for optimizations +func (p *PostingsIterator) ActualBitmap() *roaring.Bitmap { + return p.ActualBM +} + +// ReplaceActual replaces the ActualBM with the provided +// bitmap +func (p *PostingsIterator) ReplaceActual(abm *roaring.Bitmap) { + p.ActualBM = abm + p.Actual = abm.Iterator() +} + +// PostingsIteratorFromBitmap constructs a PostingsIterator given an +// "actual" bitmap. +func PostingsIteratorFromBitmap(bm *roaring.Bitmap, + includeFreqNorm, includeLocs bool) (segment.PostingsIterator, error) { + return &PostingsIterator{ + ActualBM: bm, + Actual: bm.Iterator(), + includeFreqNorm: includeFreqNorm, + includeLocs: includeLocs, + }, nil +} + +// PostingsIteratorFrom1Hit constructs a PostingsIterator given a +// 1-hit docNum. +func PostingsIteratorFrom1Hit(docNum1Hit uint64, + includeFreqNorm, includeLocs bool) (segment.PostingsIterator, error) { + return &PostingsIterator{ + docNum1Hit: docNum1Hit, + normBits1Hit: NormBits1Hit, + includeFreqNorm: includeFreqNorm, + includeLocs: includeLocs, + }, nil +} + +// Posting is a single entry in a postings list +type Posting struct { + docNum uint64 + freq uint64 + norm float32 + locs []segment.Location +} + +func (p *Posting) Size() int { + sizeInBytes := reflectStaticSizePosting + + for _, entry := range p.locs { + sizeInBytes += entry.Size() + } + + return sizeInBytes +} + +// Number returns the document number of this posting in this segment +func (p *Posting) Number() uint64 { + return p.docNum +} + +// Frequency returns the frequencies of occurrence of this term in this doc/field +func (p *Posting) Frequency() uint64 { + return p.freq +} + +// Norm returns the normalization factor for this posting +func (p *Posting) Norm() float64 { + return float64(p.norm) +} + +// Locations returns the location information for each occurrence +func (p *Posting) Locations() []segment.Location { + return p.locs +} + +// Location represents the location of a single occurrence +type Location struct { + field string + pos uint64 + start uint64 + end uint64 + ap []uint64 +} + +func (l *Location) Size() int { + return reflectStaticSizeLocation + + len(l.field) + + len(l.ap)*SizeOfUint64 +} + +// Field returns the name of the field (useful in composite fields to know +// which original field the value came from) +func (l *Location) Field() string { + return l.field +} + +// Start returns the start byte offset of this occurrence +func (l *Location) Start() uint64 { + return l.start +} + +// End returns the end byte offset of this occurrence +func (l *Location) End() uint64 { + return l.end +} + +// Pos returns the 1-based phrase position of this occurrence +func (l *Location) Pos() uint64 { + return l.pos +} + +// ArrayPositions returns the array position vector associated with this occurrence +func (l *Location) ArrayPositions() []uint64 { + return l.ap +} diff --git a/vendor/github.com/blevesearch/zap/v14/read.go b/vendor/github.com/blevesearch/zapx/v14/read.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v14/read.go rename to vendor/github.com/blevesearch/zapx/v14/read.go diff --git a/vendor/github.com/blevesearch/zapx/v14/segment.go b/vendor/github.com/blevesearch/zapx/v14/segment.go new file mode 100644 index 00000000..1fbf7848 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v14/segment.go @@ -0,0 +1,600 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "os" + "sync" + "unsafe" + + "github.com/RoaringBitmap/roaring" + mmap "github.com/blevesearch/mmap-go" + segment "github.com/blevesearch/scorch_segment_api/v2" + "github.com/blevesearch/vellum" + "github.com/golang/snappy" +) + +var reflectStaticSizeSegmentBase int + +func init() { + var sb SegmentBase + reflectStaticSizeSegmentBase = int(unsafe.Sizeof(sb)) +} + +// Open returns a zap impl of a segment +func (*ZapPlugin) Open(path string) (segment.Segment, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + mm, err := mmap.Map(f, mmap.RDONLY, 0) + if err != nil { + // mmap failed, try to close the file + _ = f.Close() + return nil, err + } + + rv := &Segment{ + SegmentBase: SegmentBase{ + mem: mm[0 : len(mm)-FooterSize], + fieldsMap: make(map[string]uint16), + fieldDvReaders: make(map[uint16]*docValueReader), + fieldFSTs: make(map[uint16]*vellum.FST), + }, + f: f, + mm: mm, + path: path, + refs: 1, + } + rv.SegmentBase.updateSize() + + err = rv.loadConfig() + if err != nil { + _ = rv.Close() + return nil, err + } + + err = rv.loadFields() + if err != nil { + _ = rv.Close() + return nil, err + } + + err = rv.loadDvReaders() + if err != nil { + _ = rv.Close() + return nil, err + } + + return rv, nil +} + +// SegmentBase is a memory only, read-only implementation of the +// segment.Segment interface, using zap's data representation. +type SegmentBase struct { + mem []byte + memCRC uint32 + chunkMode uint32 + fieldsMap map[string]uint16 // fieldName -> fieldID+1 + fieldsInv []string // fieldID -> fieldName + numDocs uint64 + storedIndexOffset uint64 + fieldsIndexOffset uint64 + docValueOffset uint64 + dictLocs []uint64 + fieldDvReaders map[uint16]*docValueReader // naive chunk cache per field + fieldDvNames []string // field names cached in fieldDvReaders + size uint64 + + m sync.Mutex + fieldFSTs map[uint16]*vellum.FST +} + +func (sb *SegmentBase) Size() int { + return int(sb.size) +} + +func (sb *SegmentBase) updateSize() { + sizeInBytes := reflectStaticSizeSegmentBase + + cap(sb.mem) + + // fieldsMap + for k := range sb.fieldsMap { + sizeInBytes += (len(k) + SizeOfString) + SizeOfUint16 + } + + // fieldsInv, dictLocs + for _, entry := range sb.fieldsInv { + sizeInBytes += len(entry) + SizeOfString + } + sizeInBytes += len(sb.dictLocs) * SizeOfUint64 + + // fieldDvReaders + for _, v := range sb.fieldDvReaders { + sizeInBytes += SizeOfUint16 + SizeOfPtr + if v != nil { + sizeInBytes += v.size() + } + } + + sb.size = uint64(sizeInBytes) +} + +func (sb *SegmentBase) AddRef() {} +func (sb *SegmentBase) DecRef() (err error) { return nil } +func (sb *SegmentBase) Close() (err error) { return nil } + +// Segment implements a persisted segment.Segment interface, by +// embedding an mmap()'ed SegmentBase. +type Segment struct { + SegmentBase + + f *os.File + mm mmap.MMap + path string + version uint32 + crc uint32 + + m sync.Mutex // Protects the fields that follow. + refs int64 +} + +func (s *Segment) Size() int { + // 8 /* size of file pointer */ + // 4 /* size of version -> uint32 */ + // 4 /* size of crc -> uint32 */ + sizeOfUints := 16 + + sizeInBytes := (len(s.path) + SizeOfString) + sizeOfUints + + // mutex, refs -> int64 + sizeInBytes += 16 + + // do not include the mmap'ed part + return sizeInBytes + s.SegmentBase.Size() - cap(s.mem) +} + +func (s *Segment) AddRef() { + s.m.Lock() + s.refs++ + s.m.Unlock() +} + +func (s *Segment) DecRef() (err error) { + s.m.Lock() + s.refs-- + if s.refs == 0 { + err = s.closeActual() + } + s.m.Unlock() + return err +} + +func (s *Segment) loadConfig() error { + crcOffset := len(s.mm) - 4 + s.crc = binary.BigEndian.Uint32(s.mm[crcOffset : crcOffset+4]) + + verOffset := crcOffset - 4 + s.version = binary.BigEndian.Uint32(s.mm[verOffset : verOffset+4]) + if s.version != Version { + return fmt.Errorf("unsupported version %d", s.version) + } + + chunkOffset := verOffset - 4 + s.chunkMode = binary.BigEndian.Uint32(s.mm[chunkOffset : chunkOffset+4]) + + docValueOffset := chunkOffset - 8 + s.docValueOffset = binary.BigEndian.Uint64(s.mm[docValueOffset : docValueOffset+8]) + + fieldsIndexOffset := docValueOffset - 8 + s.fieldsIndexOffset = binary.BigEndian.Uint64(s.mm[fieldsIndexOffset : fieldsIndexOffset+8]) + + storedIndexOffset := fieldsIndexOffset - 8 + s.storedIndexOffset = binary.BigEndian.Uint64(s.mm[storedIndexOffset : storedIndexOffset+8]) + + numDocsOffset := storedIndexOffset - 8 + s.numDocs = binary.BigEndian.Uint64(s.mm[numDocsOffset : numDocsOffset+8]) + return nil +} + +// Following methods are dummy implementations for the interface +// DiskStatsReporter (for backward compatibility). +// The working implementations are supported only in zap v15.x +// and not in the earlier versions of zap. +func (s *Segment) ResetBytesRead(uint64) {} + +func (s *Segment) BytesRead() uint64 { + return 0 +} + +func (s *Segment) BytesWritten() uint64 { + return 0 +} + +func (s *Segment) incrementBytesRead(uint64) {} + +func (s *SegmentBase) BytesWritten() uint64 { + return 0 +} + +func (s *SegmentBase) setBytesWritten(uint64) {} + +func (s *SegmentBase) BytesRead() uint64 { + return 0 +} + +func (s *SegmentBase) ResetBytesRead(uint64) {} + +func (s *SegmentBase) incrementBytesRead(uint64) {} + +func (s *SegmentBase) loadFields() error { + // NOTE for now we assume the fields index immediately precedes + // the footer, and if this changes, need to adjust accordingly (or + // store explicit length), where s.mem was sliced from s.mm in Open(). + fieldsIndexEnd := uint64(len(s.mem)) + + // iterate through fields index + var fieldID uint64 + for s.fieldsIndexOffset+(8*fieldID) < fieldsIndexEnd { + addr := binary.BigEndian.Uint64(s.mem[s.fieldsIndexOffset+(8*fieldID) : s.fieldsIndexOffset+(8*fieldID)+8]) + + dictLoc, read := binary.Uvarint(s.mem[addr:fieldsIndexEnd]) + n := uint64(read) + s.dictLocs = append(s.dictLocs, dictLoc) + + var nameLen uint64 + nameLen, read = binary.Uvarint(s.mem[addr+n : fieldsIndexEnd]) + n += uint64(read) + + name := string(s.mem[addr+n : addr+n+nameLen]) + s.fieldsInv = append(s.fieldsInv, name) + s.fieldsMap[name] = uint16(fieldID + 1) + + fieldID++ + } + return nil +} + +// Dictionary returns the term dictionary for the specified field +func (s *SegmentBase) Dictionary(field string) (segment.TermDictionary, error) { + dict, err := s.dictionary(field) + if err == nil && dict == nil { + return emptyDictionary, nil + } + return dict, err +} + +func (sb *SegmentBase) dictionary(field string) (rv *Dictionary, err error) { + fieldIDPlus1 := sb.fieldsMap[field] + if fieldIDPlus1 > 0 { + rv = &Dictionary{ + sb: sb, + field: field, + fieldID: fieldIDPlus1 - 1, + } + + dictStart := sb.dictLocs[rv.fieldID] + if dictStart > 0 { + var ok bool + sb.m.Lock() + if rv.fst, ok = sb.fieldFSTs[rv.fieldID]; !ok { + // read the length of the vellum data + vellumLen, read := binary.Uvarint(sb.mem[dictStart : dictStart+binary.MaxVarintLen64]) + fstBytes := sb.mem[dictStart+uint64(read) : dictStart+uint64(read)+vellumLen] + rv.fst, err = vellum.Load(fstBytes) + if err != nil { + sb.m.Unlock() + return nil, fmt.Errorf("dictionary field %s vellum err: %v", field, err) + } + + sb.fieldFSTs[rv.fieldID] = rv.fst + } + + sb.m.Unlock() + rv.fstReader, err = rv.fst.Reader() + if err != nil { + return nil, fmt.Errorf("dictionary field %s vellum reader err: %v", field, err) + } + } + } + + return rv, nil +} + +// visitDocumentCtx holds data structures that are reusable across +// multiple VisitDocument() calls to avoid memory allocations +type visitDocumentCtx struct { + buf []byte + reader bytes.Reader + arrayPos []uint64 +} + +var visitDocumentCtxPool = sync.Pool{ + New: func() interface{} { + reuse := &visitDocumentCtx{} + return reuse + }, +} + +// VisitStoredFields invokes the StoredFieldValueVisitor for each stored field +// for the specified doc number +func (s *SegmentBase) VisitStoredFields(num uint64, visitor segment.StoredFieldValueVisitor) error { + vdc := visitDocumentCtxPool.Get().(*visitDocumentCtx) + defer visitDocumentCtxPool.Put(vdc) + return s.visitStoredFields(vdc, num, visitor) +} + +func (s *SegmentBase) visitStoredFields(vdc *visitDocumentCtx, num uint64, + visitor segment.StoredFieldValueVisitor) error { + // first make sure this is a valid number in this segment + if num < s.numDocs { + meta, compressed := s.getDocStoredMetaAndCompressed(num) + + vdc.reader.Reset(meta) + + // handle _id field special case + idFieldValLen, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return err + } + idFieldVal := compressed[:idFieldValLen] + + keepGoing := visitor("_id", byte('t'), idFieldVal, nil) + if !keepGoing { + visitDocumentCtxPool.Put(vdc) + return nil + } + + // handle non-"_id" fields + compressed = compressed[idFieldValLen:] + + uncompressed, err := snappy.Decode(vdc.buf[:cap(vdc.buf)], compressed) + if err != nil { + return err + } + + for keepGoing { + field, err := binary.ReadUvarint(&vdc.reader) + if err == io.EOF { + break + } + if err != nil { + return err + } + typ, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return err + } + offset, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return err + } + l, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return err + } + numap, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return err + } + var arrayPos []uint64 + if numap > 0 { + if cap(vdc.arrayPos) < int(numap) { + vdc.arrayPos = make([]uint64, numap) + } + arrayPos = vdc.arrayPos[:numap] + for i := 0; i < int(numap); i++ { + ap, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return err + } + arrayPos[i] = ap + } + } + + value := uncompressed[offset : offset+l] + keepGoing = visitor(s.fieldsInv[field], byte(typ), value, arrayPos) + } + + vdc.buf = uncompressed + } + return nil +} + +// DocID returns the value of the _id field for the given docNum +func (s *SegmentBase) DocID(num uint64) ([]byte, error) { + if num >= s.numDocs { + return nil, nil + } + + vdc := visitDocumentCtxPool.Get().(*visitDocumentCtx) + + meta, compressed := s.getDocStoredMetaAndCompressed(num) + + vdc.reader.Reset(meta) + + // handle _id field special case + idFieldValLen, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return nil, err + } + idFieldVal := compressed[:idFieldValLen] + + visitDocumentCtxPool.Put(vdc) + + return idFieldVal, nil +} + +// Count returns the number of documents in this segment. +func (s *SegmentBase) Count() uint64 { + return s.numDocs +} + +// DocNumbers returns a bitset corresponding to the doc numbers of all the +// provided _id strings +func (s *SegmentBase) DocNumbers(ids []string) (*roaring.Bitmap, error) { + rv := roaring.New() + + if len(s.fieldsMap) > 0 { + idDict, err := s.dictionary("_id") + if err != nil { + return nil, err + } + + postingsList := emptyPostingsList + + sMax, err := idDict.fst.GetMaxKey() + if err != nil { + return nil, err + } + sMaxStr := string(sMax) + filteredIds := make([]string, 0, len(ids)) + for _, id := range ids { + if id <= sMaxStr { + filteredIds = append(filteredIds, id) + } + } + + for _, id := range filteredIds { + postingsList, err = idDict.postingsList([]byte(id), nil, postingsList) + if err != nil { + return nil, err + } + postingsList.OrInto(rv) + } + } + + return rv, nil +} + +// Fields returns the field names used in this segment +func (s *SegmentBase) Fields() []string { + return s.fieldsInv +} + +// Path returns the path of this segment on disk +func (s *Segment) Path() string { + return s.path +} + +// Close releases all resources associated with this segment +func (s *Segment) Close() (err error) { + return s.DecRef() +} + +func (s *Segment) closeActual() (err error) { + if s.mm != nil { + err = s.mm.Unmap() + } + // try to close file even if unmap failed + if s.f != nil { + err2 := s.f.Close() + if err == nil { + // try to return first error + err = err2 + } + } + return +} + +// some helpers i started adding for the command-line utility + +// Data returns the underlying mmaped data slice +func (s *Segment) Data() []byte { + return s.mm +} + +// CRC returns the CRC value stored in the file footer +func (s *Segment) CRC() uint32 { + return s.crc +} + +// Version returns the file version in the file footer +func (s *Segment) Version() uint32 { + return s.version +} + +// ChunkFactor returns the chunk factor in the file footer +func (s *Segment) ChunkMode() uint32 { + return s.chunkMode +} + +// FieldsIndexOffset returns the fields index offset in the file footer +func (s *Segment) FieldsIndexOffset() uint64 { + return s.fieldsIndexOffset +} + +// StoredIndexOffset returns the stored value index offset in the file footer +func (s *Segment) StoredIndexOffset() uint64 { + return s.storedIndexOffset +} + +// DocValueOffset returns the docValue offset in the file footer +func (s *Segment) DocValueOffset() uint64 { + return s.docValueOffset +} + +// NumDocs returns the number of documents in the file footer +func (s *Segment) NumDocs() uint64 { + return s.numDocs +} + +// DictAddr is a helper function to compute the file offset where the +// dictionary is stored for the specified field. +func (s *Segment) DictAddr(field string) (uint64, error) { + fieldIDPlus1, ok := s.fieldsMap[field] + if !ok { + return 0, fmt.Errorf("no such field '%s'", field) + } + + return s.dictLocs[fieldIDPlus1-1], nil +} + +func (s *SegmentBase) loadDvReaders() error { + if s.docValueOffset == fieldNotUninverted || s.numDocs == 0 { + return nil + } + + var read uint64 + for fieldID, field := range s.fieldsInv { + var fieldLocStart, fieldLocEnd uint64 + var n int + fieldLocStart, n = binary.Uvarint(s.mem[s.docValueOffset+read : s.docValueOffset+read+binary.MaxVarintLen64]) + if n <= 0 { + return fmt.Errorf("loadDvReaders: failed to read the docvalue offset start for field %d", fieldID) + } + read += uint64(n) + fieldLocEnd, n = binary.Uvarint(s.mem[s.docValueOffset+read : s.docValueOffset+read+binary.MaxVarintLen64]) + if n <= 0 { + return fmt.Errorf("loadDvReaders: failed to read the docvalue offset end for field %d", fieldID) + } + read += uint64(n) + + fieldDvReader, err := s.loadFieldDocValueReader(field, fieldLocStart, fieldLocEnd) + if err != nil { + return err + } + if fieldDvReader != nil { + s.fieldDvReaders[uint16(fieldID)] = fieldDvReader + s.fieldDvNames = append(s.fieldDvNames, field) + } + } + + return nil +} diff --git a/vendor/github.com/blevesearch/zapx/v14/sizes.go b/vendor/github.com/blevesearch/zapx/v14/sizes.go new file mode 100644 index 00000000..34166ea3 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v14/sizes.go @@ -0,0 +1,59 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "reflect" +) + +func init() { + var b bool + SizeOfBool = int(reflect.TypeOf(b).Size()) + var f32 float32 + SizeOfFloat32 = int(reflect.TypeOf(f32).Size()) + var f64 float64 + SizeOfFloat64 = int(reflect.TypeOf(f64).Size()) + var i int + SizeOfInt = int(reflect.TypeOf(i).Size()) + var m map[int]int + SizeOfMap = int(reflect.TypeOf(m).Size()) + var ptr *int + SizeOfPtr = int(reflect.TypeOf(ptr).Size()) + var slice []int + SizeOfSlice = int(reflect.TypeOf(slice).Size()) + var str string + SizeOfString = int(reflect.TypeOf(str).Size()) + var u8 uint8 + SizeOfUint8 = int(reflect.TypeOf(u8).Size()) + var u16 uint16 + SizeOfUint16 = int(reflect.TypeOf(u16).Size()) + var u32 uint32 + SizeOfUint32 = int(reflect.TypeOf(u32).Size()) + var u64 uint64 + SizeOfUint64 = int(reflect.TypeOf(u64).Size()) +} + +var SizeOfBool int +var SizeOfFloat32 int +var SizeOfFloat64 int +var SizeOfInt int +var SizeOfMap int +var SizeOfPtr int +var SizeOfSlice int +var SizeOfString int +var SizeOfUint8 int +var SizeOfUint16 int +var SizeOfUint32 int +var SizeOfUint64 int diff --git a/vendor/github.com/blevesearch/zap/v14/write.go b/vendor/github.com/blevesearch/zapx/v14/write.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v14/write.go rename to vendor/github.com/blevesearch/zapx/v14/write.go diff --git a/vendor/github.com/blevesearch/zap/v14/zap.md b/vendor/github.com/blevesearch/zapx/v14/zap.md similarity index 100% rename from vendor/github.com/blevesearch/zap/v14/zap.md rename to vendor/github.com/blevesearch/zapx/v14/zap.md diff --git a/vendor/github.com/blevesearch/zap/v15/.gitignore b/vendor/github.com/blevesearch/zapx/v15/.gitignore similarity index 100% rename from vendor/github.com/blevesearch/zap/v15/.gitignore rename to vendor/github.com/blevesearch/zapx/v15/.gitignore diff --git a/vendor/github.com/blevesearch/zapx/v15/.golangci.yml b/vendor/github.com/blevesearch/zapx/v15/.golangci.yml new file mode 100644 index 00000000..1d55bfc0 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v15/.golangci.yml @@ -0,0 +1,28 @@ +linters: + # please, do not use `enable-all`: it's deprecated and will be removed soon. + # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint + disable-all: true + enable: + - bodyclose + - deadcode + - depguard + - dupl + - errcheck + - gofmt + - goimports + - goprintffuncname + - gosec + - gosimple + - govet + - ineffassign + - misspell + - nakedret + - nolintlint + - rowserrcheck + - staticcheck + - structcheck + - typecheck + - unused + - varcheck + - whitespace + diff --git a/vendor/github.com/blevesearch/zapx/v15/LICENSE b/vendor/github.com/blevesearch/zapx/v15/LICENSE new file mode 100644 index 00000000..7a4a3ea2 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v15/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/vendor/github.com/blevesearch/zapx/v15/README.md b/vendor/github.com/blevesearch/zapx/v15/README.md new file mode 100644 index 00000000..4cbf1a14 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v15/README.md @@ -0,0 +1,163 @@ +# zapx file format + +The zapx module is fork of [zap](https://github.com/blevesearch/zap) module which maintains file format compatibility, but removes dependency on bleve, and instead depends only on the indepenent interface modules: + +- [bleve_index_api](https://github.com/blevesearch/scorch_segment_api) +- [scorch_segment_api](https://github.com/blevesearch/scorch_segment_api) + +Advanced ZAP File Format Documentation is [here](zap.md). + +The file is written in the reverse order that we typically access data. This helps us write in one pass since later sections of the file require file offsets of things we've already written. + +Current usage: + +- mmap the entire file +- crc-32 bytes and version are in fixed position at end of the file +- reading remainder of footer could be version specific +- remainder of footer gives us: + - 3 important offsets (docValue , fields index and stored data index) + - 2 important values (number of docs and chunk factor) +- field data is processed once and memoized onto the heap so that we never have to go back to disk for it +- access to stored data by doc number means first navigating to the stored data index, then accessing a fixed position offset into that slice, which gives us the actual address of the data. the first bytes of that section tell us the size of data so that we know where it ends. +- access to all other indexed data follows the following pattern: + - first know the field name -> convert to id + - next navigate to term dictionary for that field + - some operations stop here and do dictionary ops + - next use dictionary to navigate to posting list for a specific term + - walk posting list + - if necessary, walk posting details as we go + - if location info is desired, consult location bitmap to see if it is there + +## stored fields section + +- for each document + - preparation phase: + - produce a slice of metadata bytes and data bytes + - produce these slices in field id order + - field value is appended to the data slice + - metadata slice is varint encoded with the following values for each field value + - field id (uint16) + - field type (byte) + - field value start offset in uncompressed data slice (uint64) + - field value length (uint64) + - field number of array positions (uint64) + - one additional value for each array position (uint64) + - compress the data slice using snappy + - file writing phase: + - remember the start offset for this document + - write out meta data length (varint uint64) + - write out compressed data length (varint uint64) + - write out the metadata bytes + - write out the compressed data bytes + +## stored fields idx + +- for each document + - write start offset (remembered from previous section) of stored data (big endian uint64) + +With this index and a known document number, we have direct access to all the stored field data. + +## posting details (freq/norm) section + +- for each posting list + - produce a slice containing multiple consecutive chunks (each chunk is varint stream) + - produce a slice remembering offsets of where each chunk starts + - preparation phase: + - for each hit in the posting list + - if this hit is in next chunk close out encoding of last chunk and record offset start of next + - encode term frequency (uint64) + - encode norm factor (float32) + - file writing phase: + - remember start position for this posting list details + - write out number of chunks that follow (varint uint64) + - write out length of each chunk (each a varint uint64) + - write out the byte slice containing all the chunk data + +If you know the doc number you're interested in, this format lets you jump to the correct chunk (docNum/chunkFactor) directly and then seek within that chunk until you find it. + +## posting details (location) section + +- for each posting list + - produce a slice containing multiple consecutive chunks (each chunk is varint stream) + - produce a slice remembering offsets of where each chunk starts + - preparation phase: + - for each hit in the posting list + - if this hit is in next chunk close out encoding of last chunk and record offset start of next + - encode field (uint16) + - encode field pos (uint64) + - encode field start (uint64) + - encode field end (uint64) + - encode number of array positions to follow (uint64) + - encode each array position (each uint64) + - file writing phase: + - remember start position for this posting list details + - write out number of chunks that follow (varint uint64) + - write out length of each chunk (each a varint uint64) + - write out the byte slice containing all the chunk data + +If you know the doc number you're interested in, this format lets you jump to the correct chunk (docNum/chunkFactor) directly and then seek within that chunk until you find it. + +## postings list section + +- for each posting list + - preparation phase: + - encode roaring bitmap posting list to bytes (so we know the length) + - file writing phase: + - remember the start position for this posting list + - write freq/norm details offset (remembered from previous, as varint uint64) + - write location details offset (remembered from previous, as varint uint64) + - write length of encoded roaring bitmap + - write the serialized roaring bitmap data + +## dictionary + +- for each field + - preparation phase: + - encode vellum FST with dictionary data pointing to file offset of posting list (remembered from previous) + - file writing phase: + - remember the start position of this persistDictionary + - write length of vellum data (varint uint64) + - write out vellum data + +## fields section + +- for each field + - file writing phase: + - remember start offset for each field + - write dictionary address (remembered from previous) (varint uint64) + - write length of field name (varint uint64) + - write field name bytes + +## fields idx + +- for each field + - file writing phase: + - write big endian uint64 of start offset for each field + +NOTE: currently we don't know or record the length of this fields index. Instead we rely on the fact that we know it immediately precedes a footer of known size. + +## fields DocValue + +- for each field + - preparation phase: + - produce a slice containing multiple consecutive chunks, where each chunk is composed of a meta section followed by compressed columnar field data + - produce a slice remembering the length of each chunk + - file writing phase: + - remember the start position of this first field DocValue offset in the footer + - write out number of chunks that follow (varint uint64) + - write out length of each chunk (each a varint uint64) + - write out the byte slice containing all the chunk data + +NOTE: currently the meta header inside each chunk gives clue to the location offsets and size of the data pertaining to a given docID and any +read operation leverage that meta information to extract the document specific data from the file. + +## footer + +- file writing phase + - write number of docs (big endian uint64) + - write stored field index location (big endian uint64) + - write field index location (big endian uint64) + - write field docValue location (big endian uint64) + - write out chunk factor (big endian uint32) + - write out version (big endian uint32) + - write out file CRC of everything preceding this (big endian uint32) diff --git a/vendor/github.com/blevesearch/zapx/v15/build.go b/vendor/github.com/blevesearch/zapx/v15/build.go new file mode 100644 index 00000000..5db1d9ee --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v15/build.go @@ -0,0 +1,186 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "bufio" + "fmt" + "io" + "math" + "os" + + "github.com/blevesearch/vellum" +) + +const Version uint32 = 15 + +const Type string = "zap" + +const fieldNotUninverted = math.MaxUint64 + +func (sb *SegmentBase) Persist(path string) error { + return PersistSegmentBase(sb, path) +} + +// WriteTo is an implementation of io.WriterTo interface. +func (sb *SegmentBase) WriteTo(w io.Writer) (int64, error) { + if w == nil { + return 0, fmt.Errorf("invalid writer found") + } + + n, err := persistSegmentBaseToWriter(sb, w) + return int64(n), err +} + +// PersistSegmentBase persists SegmentBase in the zap file format. +func PersistSegmentBase(sb *SegmentBase, path string) error { + flag := os.O_RDWR | os.O_CREATE + + f, err := os.OpenFile(path, flag, 0600) + if err != nil { + return err + } + + cleanup := func() { + _ = f.Close() + _ = os.Remove(path) + } + + _, err = persistSegmentBaseToWriter(sb, f) + if err != nil { + cleanup() + return err + } + + err = f.Sync() + if err != nil { + cleanup() + return err + } + + err = f.Close() + if err != nil { + cleanup() + return err + } + + return err +} + +type bufWriter struct { + w *bufio.Writer + n int +} + +func (br *bufWriter) Write(in []byte) (int, error) { + n, err := br.w.Write(in) + br.n += n + return n, err +} + +func persistSegmentBaseToWriter(sb *SegmentBase, w io.Writer) (int, error) { + br := &bufWriter{w: bufio.NewWriter(w)} + + _, err := br.Write(sb.mem) + if err != nil { + return 0, err + } + + err = persistFooter(sb.numDocs, sb.storedIndexOffset, sb.fieldsIndexOffset, + sb.docValueOffset, sb.chunkMode, sb.memCRC, br) + if err != nil { + return 0, err + } + + err = br.w.Flush() + if err != nil { + return 0, err + } + + return br.n, nil +} + +func persistStoredFieldValues(fieldID int, + storedFieldValues [][]byte, stf []byte, spf [][]uint64, + curr int, metaEncode varintEncoder, data []byte) ( + int, []byte, error) { + for i := 0; i < len(storedFieldValues); i++ { + // encode field + _, err := metaEncode(uint64(fieldID)) + if err != nil { + return 0, nil, err + } + // encode type + _, err = metaEncode(uint64(stf[i])) + if err != nil { + return 0, nil, err + } + // encode start offset + _, err = metaEncode(uint64(curr)) + if err != nil { + return 0, nil, err + } + // end len + _, err = metaEncode(uint64(len(storedFieldValues[i]))) + if err != nil { + return 0, nil, err + } + // encode number of array pos + _, err = metaEncode(uint64(len(spf[i]))) + if err != nil { + return 0, nil, err + } + // encode all array positions + for _, pos := range spf[i] { + _, err = metaEncode(pos) + if err != nil { + return 0, nil, err + } + } + + data = append(data, storedFieldValues[i]...) + curr += len(storedFieldValues[i]) + } + + return curr, data, nil +} + +func InitSegmentBase(mem []byte, memCRC uint32, chunkMode uint32, + fieldsMap map[string]uint16, fieldsInv []string, numDocs uint64, + storedIndexOffset uint64, fieldsIndexOffset uint64, docValueOffset uint64, + dictLocs []uint64) (*SegmentBase, error) { + sb := &SegmentBase{ + mem: mem, + memCRC: memCRC, + chunkMode: chunkMode, + fieldsMap: fieldsMap, + fieldsInv: fieldsInv, + numDocs: numDocs, + storedIndexOffset: storedIndexOffset, + fieldsIndexOffset: fieldsIndexOffset, + docValueOffset: docValueOffset, + dictLocs: dictLocs, + fieldDvReaders: make(map[uint16]*docValueReader), + fieldFSTs: make(map[uint16]*vellum.FST), + } + sb.updateSize() + + err := sb.loadDvReaders() + if err != nil { + return nil, err + } + + return sb, nil +} diff --git a/vendor/github.com/blevesearch/zap/v15/chunk.go b/vendor/github.com/blevesearch/zapx/v15/chunk.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v15/chunk.go rename to vendor/github.com/blevesearch/zapx/v15/chunk.go diff --git a/vendor/github.com/blevesearch/zap/v15/contentcoder.go b/vendor/github.com/blevesearch/zapx/v15/contentcoder.go similarity index 90% rename from vendor/github.com/blevesearch/zap/v15/contentcoder.go rename to vendor/github.com/blevesearch/zapx/v15/contentcoder.go index c145b5a1..cd8b3fc8 100644 --- a/vendor/github.com/blevesearch/zap/v15/contentcoder.go +++ b/vendor/github.com/blevesearch/zapx/v15/contentcoder.go @@ -19,6 +19,7 @@ import ( "encoding/binary" "io" "reflect" + "sync/atomic" "github.com/golang/snappy" ) @@ -30,24 +31,27 @@ func init() { reflectStaticSizeMetaData = int(reflect.TypeOf(md).Size()) } -var termSeparator byte = 0xff -var termSeparatorSplitSlice = []byte{termSeparator} +var ( + termSeparator byte = 0xff + termSeparatorSplitSlice = []byte{termSeparator} +) type chunkedContentCoder struct { + bytesWritten uint64 // atomic access to this variable, moved to top to correct alignment issues on ARM, 386 and 32-bit MIPS. + final []byte chunkSize uint64 currChunk uint64 chunkLens []uint64 + compressed []byte // temp buf for snappy compression + w io.Writer progressiveWrite bool + chunkMeta []MetaData chunkMetaBuf bytes.Buffer chunkBuf bytes.Buffer - - chunkMeta []MetaData - - compressed []byte // temp buf for snappy compression } // MetaData represents the data information inside a @@ -60,7 +64,8 @@ type MetaData struct { // newChunkedContentCoder returns a new chunk content coder which // packs data into chunks based on the provided chunkSize func newChunkedContentCoder(chunkSize uint64, maxDocNum uint64, - w io.Writer, progressiveWrite bool) *chunkedContentCoder { + w io.Writer, progressiveWrite bool, +) *chunkedContentCoder { total := maxDocNum/chunkSize + 1 rv := &chunkedContentCoder{ chunkSize: chunkSize, @@ -77,6 +82,7 @@ func newChunkedContentCoder(chunkSize uint64, maxDocNum uint64, // and re used. You cannot change the chunk size. func (c *chunkedContentCoder) Reset() { c.currChunk = 0 + c.bytesWritten = 0 c.final = c.final[:0] c.chunkBuf.Reset() c.chunkMetaBuf.Reset() @@ -105,6 +111,14 @@ func (c *chunkedContentCoder) Close() error { return c.flushContents() } +func (c *chunkedContentCoder) incrementBytesWritten(val uint64) { + atomic.AddUint64(&c.bytesWritten, val) +} + +func (c *chunkedContentCoder) getBytesWritten() uint64 { + return atomic.LoadUint64(&c.bytesWritten) +} + func (c *chunkedContentCoder) flushContents() error { // flush the contents, with meta information at first buf := make([]byte, binary.MaxVarintLen64) @@ -127,6 +141,7 @@ func (c *chunkedContentCoder) flushContents() error { c.final = append(c.final, c.chunkMetaBuf.Bytes()...) // write the compressed data to the final data c.compressed = snappy.Encode(c.compressed[:cap(c.compressed)], c.chunkBuf.Bytes()) + c.incrementBytesWritten(uint64(len(c.compressed))) c.final = append(c.final, c.compressed...) c.chunkLens[c.currChunk] = uint64(len(c.compressed) + len(metaData)) @@ -177,7 +192,6 @@ func (c *chunkedContentCoder) Add(docNum uint64, vals []byte) error { // // | ..... data ..... | chunk offsets (varints) // | position of chunk offsets (uint64) | number of offsets (uint64) | -// func (c *chunkedContentCoder) Write() (int, error) { var tw int diff --git a/vendor/github.com/blevesearch/zapx/v15/count.go b/vendor/github.com/blevesearch/zapx/v15/count.go new file mode 100644 index 00000000..b6135359 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v15/count.go @@ -0,0 +1,61 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "hash/crc32" + "io" + + segment "github.com/blevesearch/scorch_segment_api/v2" +) + +// CountHashWriter is a wrapper around a Writer which counts the number of +// bytes which have been written and computes a crc32 hash +type CountHashWriter struct { + w io.Writer + crc uint32 + n int + s segment.StatsReporter +} + +// NewCountHashWriter returns a CountHashWriter which wraps the provided Writer +func NewCountHashWriter(w io.Writer) *CountHashWriter { + return &CountHashWriter{w: w} +} + +func NewCountHashWriterWithStatsReporter(w io.Writer, s segment.StatsReporter) *CountHashWriter { + return &CountHashWriter{w: w, s: s} +} + +// Write writes the provided bytes to the wrapped writer and counts the bytes +func (c *CountHashWriter) Write(b []byte) (int, error) { + n, err := c.w.Write(b) + c.crc = crc32.Update(c.crc, crc32.IEEETable, b[:n]) + c.n += n + if c.s != nil { + c.s.ReportBytesWritten(uint64(n)) + } + return n, err +} + +// Count returns the number of bytes written +func (c *CountHashWriter) Count() int { + return c.n +} + +// Sum32 returns the CRC-32 hash of the content written to this writer +func (c *CountHashWriter) Sum32() uint32 { + return c.crc +} diff --git a/vendor/github.com/blevesearch/zapx/v15/dict.go b/vendor/github.com/blevesearch/zapx/v15/dict.go new file mode 100644 index 00000000..6b8acf52 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v15/dict.go @@ -0,0 +1,176 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "fmt" + + "github.com/RoaringBitmap/roaring" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" + "github.com/blevesearch/vellum" +) + +// Dictionary is the zap representation of the term dictionary +type Dictionary struct { + sb *SegmentBase + field string + fieldID uint16 + fst *vellum.FST + fstReader *vellum.Reader + + bytesRead uint64 +} + +// represents an immutable, empty dictionary +var emptyDictionary = &Dictionary{} + +// PostingsList returns the postings list for the specified term +func (d *Dictionary) PostingsList(term []byte, except *roaring.Bitmap, + prealloc segment.PostingsList) (segment.PostingsList, error) { + var preallocPL *PostingsList + pl, ok := prealloc.(*PostingsList) + if ok && pl != nil { + preallocPL = pl + } + return d.postingsList(term, except, preallocPL) +} + +func (d *Dictionary) postingsList(term []byte, except *roaring.Bitmap, rv *PostingsList) (*PostingsList, error) { + if d.fstReader == nil { + if rv == nil || rv == emptyPostingsList { + return emptyPostingsList, nil + } + return d.postingsListInit(rv, except), nil + } + + postingsOffset, exists, err := d.fstReader.Get(term) + if err != nil { + return nil, fmt.Errorf("vellum err: %v", err) + } + if !exists { + if rv == nil || rv == emptyPostingsList { + return emptyPostingsList, nil + } + return d.postingsListInit(rv, except), nil + } + + return d.postingsListFromOffset(postingsOffset, except, rv) +} + +func (d *Dictionary) postingsListFromOffset(postingsOffset uint64, except *roaring.Bitmap, rv *PostingsList) (*PostingsList, error) { + rv = d.postingsListInit(rv, except) + + err := rv.read(postingsOffset, d) + if err != nil { + return nil, err + } + + return rv, nil +} + +func (d *Dictionary) postingsListInit(rv *PostingsList, except *roaring.Bitmap) *PostingsList { + if rv == nil || rv == emptyPostingsList { + rv = &PostingsList{} + } else { + postings := rv.postings + if postings != nil { + postings.Clear() + } + + *rv = PostingsList{} // clear the struct + + rv.postings = postings + } + rv.sb = d.sb + rv.except = except + return rv +} + +func (d *Dictionary) Contains(key []byte) (bool, error) { + if d.fst != nil { + return d.fst.Contains(key) + } + return false, nil +} + +// AutomatonIterator returns an iterator which only visits terms +// having the the vellum automaton and start/end key range +func (d *Dictionary) AutomatonIterator(a segment.Automaton, + startKeyInclusive, endKeyExclusive []byte) segment.DictionaryIterator { + if d.fst != nil { + rv := &DictionaryIterator{ + d: d, + } + + itr, err := d.fst.Search(a, startKeyInclusive, endKeyExclusive) + if err == nil { + rv.itr = itr + } else if err != vellum.ErrIteratorDone { + rv.err = err + } + + return rv + } + return emptyDictionaryIterator +} + +func (d *Dictionary) incrementBytesRead(val uint64) { + d.bytesRead += val +} + +func (d *Dictionary) BytesRead() uint64 { + return d.bytesRead +} + +func (d *Dictionary) ResetBytesRead(val uint64) { + d.bytesRead = val +} + +func (d *Dictionary) BytesWritten() uint64 { + return 0 +} + +// DictionaryIterator is an iterator for term dictionary +type DictionaryIterator struct { + d *Dictionary + itr vellum.Iterator + err error + tmp PostingsList + entry index.DictEntry + omitCount bool +} + +var emptyDictionaryIterator = &DictionaryIterator{} + +// Next returns the next entry in the dictionary +func (i *DictionaryIterator) Next() (*index.DictEntry, error) { + if i.err != nil && i.err != vellum.ErrIteratorDone { + return nil, i.err + } else if i.itr == nil || i.err == vellum.ErrIteratorDone { + return nil, nil + } + term, postingsOffset := i.itr.Current() + i.entry.Term = string(term) + if !i.omitCount { + i.err = i.tmp.read(postingsOffset, i.d) + if i.err != nil { + return nil, i.err + } + i.entry.Count = i.tmp.Count() + } + i.err = i.itr.Next() + return &i.entry, nil +} diff --git a/vendor/github.com/blevesearch/zapx/v15/docvalues.go b/vendor/github.com/blevesearch/zapx/v15/docvalues.go new file mode 100644 index 00000000..046244d1 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v15/docvalues.go @@ -0,0 +1,356 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "bytes" + "encoding/binary" + "fmt" + "math" + "reflect" + "sort" + + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" + "github.com/golang/snappy" +) + +var reflectStaticSizedocValueReader int + +func init() { + var dvi docValueReader + reflectStaticSizedocValueReader = int(reflect.TypeOf(dvi).Size()) +} + +type docNumTermsVisitor func(docNum uint64, terms []byte) error + +type docVisitState struct { + dvrs map[uint16]*docValueReader + segment *SegmentBase + + bytesRead uint64 +} + +// Implements the segment.DiskStatsReporter interface +// The purpose of this implementation is to get +// the bytes read from the disk (pertaining to the +// docvalues) while querying. +// the loadDvChunk retrieves the next chunk of docvalues +// and the bytes retrieved off the disk pertaining to that +// is accounted as well. +func (d *docVisitState) incrementBytesRead(val uint64) { + d.bytesRead += val +} + +func (d *docVisitState) BytesRead() uint64 { + return d.bytesRead +} + +func (d *docVisitState) BytesWritten() uint64 { + return 0 +} + +func (d *docVisitState) ResetBytesRead(val uint64) { + d.bytesRead = val +} + +type docValueReader struct { + field string + curChunkNum uint64 + chunkOffsets []uint64 + dvDataLoc uint64 + curChunkHeader []MetaData + curChunkData []byte // compressed data cache + uncompressed []byte // temp buf for snappy decompression + + // atomic access to this variable + bytesRead uint64 +} + +func (di *docValueReader) size() int { + return reflectStaticSizedocValueReader + SizeOfPtr + + len(di.field) + + len(di.chunkOffsets)*SizeOfUint64 + + len(di.curChunkHeader)*reflectStaticSizeMetaData + + len(di.curChunkData) +} + +func (di *docValueReader) cloneInto(rv *docValueReader) *docValueReader { + if rv == nil { + rv = &docValueReader{} + } + + rv.field = di.field + rv.curChunkNum = math.MaxUint64 + rv.chunkOffsets = di.chunkOffsets // immutable, so it's sharable + rv.dvDataLoc = di.dvDataLoc + rv.curChunkHeader = rv.curChunkHeader[:0] + rv.curChunkData = nil + rv.uncompressed = rv.uncompressed[:0] + + return rv +} + +func (di *docValueReader) curChunkNumber() uint64 { + return di.curChunkNum +} + +func (s *SegmentBase) loadFieldDocValueReader(field string, + fieldDvLocStart, fieldDvLocEnd uint64) (*docValueReader, error) { + // get the docValue offset for the given fields + if fieldDvLocStart == fieldNotUninverted { + // no docValues found, nothing to do + return nil, nil + } + + // read the number of chunks, and chunk offsets position + var numChunks, chunkOffsetsPosition uint64 + + if fieldDvLocEnd-fieldDvLocStart > 16 { + numChunks = binary.BigEndian.Uint64(s.mem[fieldDvLocEnd-8 : fieldDvLocEnd]) + // read the length of chunk offsets + chunkOffsetsLen := binary.BigEndian.Uint64(s.mem[fieldDvLocEnd-16 : fieldDvLocEnd-8]) + // acquire position of chunk offsets + chunkOffsetsPosition = (fieldDvLocEnd - 16) - chunkOffsetsLen + + // 16 bytes since it corresponds to the length + // of chunk offsets and the position of the offsets + s.incrementBytesRead(16) + } else { + return nil, fmt.Errorf("loadFieldDocValueReader: fieldDvLoc too small: %d-%d", fieldDvLocEnd, fieldDvLocStart) + } + + fdvIter := &docValueReader{ + curChunkNum: math.MaxUint64, + field: field, + chunkOffsets: make([]uint64, int(numChunks)), + } + + // read the chunk offsets + var offset uint64 + for i := 0; i < int(numChunks); i++ { + loc, read := binary.Uvarint(s.mem[chunkOffsetsPosition+offset : chunkOffsetsPosition+offset+binary.MaxVarintLen64]) + if read <= 0 { + return nil, fmt.Errorf("corrupted chunk offset during segment load") + } + fdvIter.chunkOffsets[i] = loc + offset += uint64(read) + } + s.incrementBytesRead(offset) + // set the data offset + fdvIter.dvDataLoc = fieldDvLocStart + + return fdvIter, nil +} + +func (d *docValueReader) getBytesRead() uint64 { + return d.bytesRead +} + +func (d *docValueReader) incrementBytesRead(val uint64) { + d.bytesRead += val +} + +func (di *docValueReader) loadDvChunk(chunkNumber uint64, s *SegmentBase) error { + // advance to the chunk where the docValues + // reside for the given docNum + destChunkDataLoc, curChunkEnd := di.dvDataLoc, di.dvDataLoc + start, end := readChunkBoundary(int(chunkNumber), di.chunkOffsets) + if start >= end { + di.curChunkHeader = di.curChunkHeader[:0] + di.curChunkData = nil + di.curChunkNum = chunkNumber + di.uncompressed = di.uncompressed[:0] + return nil + } + + destChunkDataLoc += start + curChunkEnd += end + + // read the number of docs reside in the chunk + numDocs, read := binary.Uvarint(s.mem[destChunkDataLoc : destChunkDataLoc+binary.MaxVarintLen64]) + if read <= 0 { + return fmt.Errorf("failed to read the chunk") + } + chunkMetaLoc := destChunkDataLoc + uint64(read) + di.incrementBytesRead(uint64(read)) + offset := uint64(0) + if cap(di.curChunkHeader) < int(numDocs) { + di.curChunkHeader = make([]MetaData, int(numDocs)) + } else { + di.curChunkHeader = di.curChunkHeader[:int(numDocs)] + } + for i := 0; i < int(numDocs); i++ { + di.curChunkHeader[i].DocNum, read = binary.Uvarint(s.mem[chunkMetaLoc+offset : chunkMetaLoc+offset+binary.MaxVarintLen64]) + offset += uint64(read) + di.curChunkHeader[i].DocDvOffset, read = binary.Uvarint(s.mem[chunkMetaLoc+offset : chunkMetaLoc+offset+binary.MaxVarintLen64]) + offset += uint64(read) + } + + compressedDataLoc := chunkMetaLoc + offset + dataLength := curChunkEnd - compressedDataLoc + di.incrementBytesRead(uint64(dataLength + offset)) + di.curChunkData = s.mem[compressedDataLoc : compressedDataLoc+dataLength] + di.curChunkNum = chunkNumber + di.uncompressed = di.uncompressed[:0] + return nil +} + +func (di *docValueReader) iterateAllDocValues(s *SegmentBase, visitor docNumTermsVisitor) error { + for i := 0; i < len(di.chunkOffsets); i++ { + err := di.loadDvChunk(uint64(i), s) + if err != nil { + return err + } + if di.curChunkData == nil || len(di.curChunkHeader) == 0 { + continue + } + + // uncompress the already loaded data + uncompressed, err := snappy.Decode(di.uncompressed[:cap(di.uncompressed)], di.curChunkData) + if err != nil { + return err + } + di.uncompressed = uncompressed + + start := uint64(0) + for _, entry := range di.curChunkHeader { + err = visitor(entry.DocNum, uncompressed[start:entry.DocDvOffset]) + if err != nil { + return err + } + + start = entry.DocDvOffset + } + } + + return nil +} + +func (di *docValueReader) visitDocValues(docNum uint64, + visitor index.DocValueVisitor) error { + // binary search the term locations for the docNum + start, end := di.getDocValueLocs(docNum) + if start == math.MaxUint64 || end == math.MaxUint64 || start == end { + return nil + } + + var uncompressed []byte + var err error + // use the uncompressed copy if available + if len(di.uncompressed) > 0 { + uncompressed = di.uncompressed + } else { + // uncompress the already loaded data + uncompressed, err = snappy.Decode(di.uncompressed[:cap(di.uncompressed)], di.curChunkData) + if err != nil { + return err + } + di.uncompressed = uncompressed + } + + // pick the terms for the given docNum + uncompressed = uncompressed[start:end] + for { + i := bytes.Index(uncompressed, termSeparatorSplitSlice) + if i < 0 { + break + } + + visitor(di.field, uncompressed[0:i]) + uncompressed = uncompressed[i+1:] + } + + return nil +} + +func (di *docValueReader) getDocValueLocs(docNum uint64) (uint64, uint64) { + i := sort.Search(len(di.curChunkHeader), func(i int) bool { + return di.curChunkHeader[i].DocNum >= docNum + }) + if i < len(di.curChunkHeader) && di.curChunkHeader[i].DocNum == docNum { + return ReadDocValueBoundary(i, di.curChunkHeader) + } + return math.MaxUint64, math.MaxUint64 +} + +// VisitDocValues is an implementation of the +// DocValueVisitable interface +func (s *SegmentBase) VisitDocValues(localDocNum uint64, fields []string, + visitor index.DocValueVisitor, dvsIn segment.DocVisitState) ( + segment.DocVisitState, error) { + dvs, ok := dvsIn.(*docVisitState) + if !ok || dvs == nil { + dvs = &docVisitState{} + } else { + if dvs.segment != s { + dvs.segment = s + dvs.dvrs = nil + dvs.bytesRead = 0 + } + } + + var fieldIDPlus1 uint16 + if dvs.dvrs == nil { + dvs.dvrs = make(map[uint16]*docValueReader, len(fields)) + for _, field := range fields { + if fieldIDPlus1, ok = s.fieldsMap[field]; !ok { + continue + } + fieldID := fieldIDPlus1 - 1 + if dvIter, exists := s.fieldDvReaders[fieldID]; exists && + dvIter != nil { + dvs.dvrs[fieldID] = dvIter.cloneInto(dvs.dvrs[fieldID]) + } + } + } + + // find the chunkNumber where the docValues are stored + // NOTE: doc values continue to use legacy chunk mode + chunkFactor, err := getChunkSize(LegacyChunkMode, 0, 0) + if err != nil { + return nil, err + } + docInChunk := localDocNum / chunkFactor + var dvr *docValueReader + for _, field := range fields { + if fieldIDPlus1, ok = s.fieldsMap[field]; !ok { + continue + } + fieldID := fieldIDPlus1 - 1 + if dvr, ok = dvs.dvrs[fieldID]; ok && dvr != nil { + // check if the chunk is already loaded + if docInChunk != dvr.curChunkNumber() { + err := dvr.loadDvChunk(docInChunk, s) + if err != nil { + return dvs, err + } + dvs.ResetBytesRead(dvr.getBytesRead()) + } else { + dvs.ResetBytesRead(0) + } + + _ = dvr.visitDocValues(localDocNum, visitor) + } + } + return dvs, nil +} + +// VisitableDocValueFields returns the list of fields with +// persisted doc value terms ready to be visitable using the +// VisitDocumentFieldTerms method. +func (s *SegmentBase) VisitableDocValueFields() ([]string, error) { + return s.fieldDvNames, nil +} diff --git a/vendor/github.com/blevesearch/zapx/v15/enumerator.go b/vendor/github.com/blevesearch/zapx/v15/enumerator.go new file mode 100644 index 00000000..972a2241 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v15/enumerator.go @@ -0,0 +1,138 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "bytes" + + "github.com/blevesearch/vellum" +) + +// enumerator provides an ordered traversal of multiple vellum +// iterators. Like JOIN of iterators, the enumerator produces a +// sequence of (key, iteratorIndex, value) tuples, sorted by key ASC, +// then iteratorIndex ASC, where the same key might be seen or +// repeated across multiple child iterators. +type enumerator struct { + itrs []vellum.Iterator + currKs [][]byte + currVs []uint64 + + lowK []byte + lowIdxs []int + lowCurr int +} + +// newEnumerator returns a new enumerator over the vellum Iterators +func newEnumerator(itrs []vellum.Iterator) (*enumerator, error) { + rv := &enumerator{ + itrs: itrs, + currKs: make([][]byte, len(itrs)), + currVs: make([]uint64, len(itrs)), + lowIdxs: make([]int, 0, len(itrs)), + } + for i, itr := range rv.itrs { + rv.currKs[i], rv.currVs[i] = itr.Current() + } + rv.updateMatches(false) + if rv.lowK == nil && len(rv.lowIdxs) == 0 { + return rv, vellum.ErrIteratorDone + } + return rv, nil +} + +// updateMatches maintains the low key matches based on the currKs +func (m *enumerator) updateMatches(skipEmptyKey bool) { + m.lowK = nil + m.lowIdxs = m.lowIdxs[:0] + m.lowCurr = 0 + + for i, key := range m.currKs { + if (key == nil && m.currVs[i] == 0) || // in case of empty iterator + (len(key) == 0 && skipEmptyKey) { // skip empty keys + continue + } + + cmp := bytes.Compare(key, m.lowK) + if cmp < 0 || len(m.lowIdxs) == 0 { + // reached a new low + m.lowK = key + m.lowIdxs = m.lowIdxs[:0] + m.lowIdxs = append(m.lowIdxs, i) + } else if cmp == 0 { + m.lowIdxs = append(m.lowIdxs, i) + } + } +} + +// Current returns the enumerator's current key, iterator-index, and +// value. If the enumerator is not pointing at a valid value (because +// Next returned an error previously), Current will return nil,0,0. +func (m *enumerator) Current() ([]byte, int, uint64) { + var i int + var v uint64 + if m.lowCurr < len(m.lowIdxs) { + i = m.lowIdxs[m.lowCurr] + v = m.currVs[i] + } + return m.lowK, i, v +} + +// GetLowIdxsAndValues will return all of the iterator indices +// which point to the current key, and their corresponding +// values. This can be used by advanced caller which may need +// to peek into these other sets of data before processing. +func (m *enumerator) GetLowIdxsAndValues() ([]int, []uint64) { + values := make([]uint64, 0, len(m.lowIdxs)) + for _, idx := range m.lowIdxs { + values = append(values, m.currVs[idx]) + } + return m.lowIdxs, values +} + +// Next advances the enumerator to the next key/iterator/value result, +// else vellum.ErrIteratorDone is returned. +func (m *enumerator) Next() error { + m.lowCurr += 1 + if m.lowCurr >= len(m.lowIdxs) { + // move all the current low iterators forwards + for _, vi := range m.lowIdxs { + err := m.itrs[vi].Next() + if err != nil && err != vellum.ErrIteratorDone { + return err + } + m.currKs[vi], m.currVs[vi] = m.itrs[vi].Current() + } + // can skip any empty keys encountered at this point + m.updateMatches(true) + } + if m.lowK == nil && len(m.lowIdxs) == 0 { + return vellum.ErrIteratorDone + } + return nil +} + +// Close all the underlying Iterators. The first error, if any, will +// be returned. +func (m *enumerator) Close() error { + var rv error + for _, itr := range m.itrs { + err := itr.Close() + if rv == nil { + rv = err + } + } + return rv +} diff --git a/vendor/github.com/blevesearch/zapx/v15/intDecoder.go b/vendor/github.com/blevesearch/zapx/v15/intDecoder.go new file mode 100644 index 00000000..e50c4717 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v15/intDecoder.go @@ -0,0 +1,139 @@ +// Copyright (c) 2019 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "encoding/binary" + "fmt" +) + +type chunkedIntDecoder struct { + startOffset uint64 + dataStartOffset uint64 + chunkOffsets []uint64 + curChunkBytes []byte + data []byte + r *memUvarintReader + + // atomic access to this variable + bytesRead uint64 +} + +// newChunkedIntDecoder expects an optional or reset chunkedIntDecoder for better reuse. +func newChunkedIntDecoder(buf []byte, offset uint64, rv *chunkedIntDecoder) *chunkedIntDecoder { + if rv == nil { + rv = &chunkedIntDecoder{startOffset: offset, data: buf} + } else { + rv.startOffset = offset + rv.data = buf + } + + var n, numChunks uint64 + var read int + if offset == termNotEncoded { + numChunks = 0 + } else { + numChunks, read = binary.Uvarint(buf[offset+n : offset+n+binary.MaxVarintLen64]) + } + + n += uint64(read) + if cap(rv.chunkOffsets) >= int(numChunks) { + rv.chunkOffsets = rv.chunkOffsets[:int(numChunks)] + } else { + rv.chunkOffsets = make([]uint64, int(numChunks)) + } + for i := 0; i < int(numChunks); i++ { + rv.chunkOffsets[i], read = binary.Uvarint(buf[offset+n : offset+n+binary.MaxVarintLen64]) + n += uint64(read) + } + rv.bytesRead += n + rv.dataStartOffset = offset + n + return rv +} + +// A util function which fetches the query time +// specific bytes encoded by intcoder (for eg the +// freqNorm and location details of a term in document) +// the loadChunk retrieves the next chunk and the +// number of bytes retrieve in that operation is accounted +func (d *chunkedIntDecoder) getBytesRead() uint64 { + return d.bytesRead +} + +func (d *chunkedIntDecoder) loadChunk(chunk int) error { + if d.startOffset == termNotEncoded { + d.r = newMemUvarintReader([]byte(nil)) + return nil + } + + if chunk >= len(d.chunkOffsets) { + return fmt.Errorf("tried to load freq chunk that doesn't exist %d/(%d)", + chunk, len(d.chunkOffsets)) + } + + end, start := d.dataStartOffset, d.dataStartOffset + s, e := readChunkBoundary(chunk, d.chunkOffsets) + start += s + end += e + d.curChunkBytes = d.data[start:end] + d.bytesRead += uint64(len(d.curChunkBytes)) + if d.r == nil { + d.r = newMemUvarintReader(d.curChunkBytes) + } else { + d.r.Reset(d.curChunkBytes) + } + + return nil +} + +func (d *chunkedIntDecoder) reset() { + d.startOffset = 0 + d.dataStartOffset = 0 + d.chunkOffsets = d.chunkOffsets[:0] + d.curChunkBytes = d.curChunkBytes[:0] + d.bytesRead = 0 + d.data = d.data[:0] + if d.r != nil { + d.r.Reset([]byte(nil)) + } +} + +func (d *chunkedIntDecoder) isNil() bool { + return d.curChunkBytes == nil || len(d.curChunkBytes) == 0 +} + +func (d *chunkedIntDecoder) readUvarint() (uint64, error) { + return d.r.ReadUvarint() +} + +func (d *chunkedIntDecoder) readBytes(start, end int) []byte { + return d.curChunkBytes[start:end] +} + +func (d *chunkedIntDecoder) SkipUvarint() { + d.r.SkipUvarint() +} + +func (d *chunkedIntDecoder) SkipBytes(count int) { + d.r.SkipBytes(count) +} + +func (d *chunkedIntDecoder) Len() int { + return d.r.Len() +} + +func (d *chunkedIntDecoder) remainingLen() int { + return len(d.curChunkBytes) - d.r.Len() +} diff --git a/vendor/github.com/blevesearch/zap/v15/intcoder.go b/vendor/github.com/blevesearch/zapx/v15/intcoder.go similarity index 94% rename from vendor/github.com/blevesearch/zap/v15/intcoder.go rename to vendor/github.com/blevesearch/zapx/v15/intcoder.go index c3c488fb..2957fbd0 100644 --- a/vendor/github.com/blevesearch/zap/v15/intcoder.go +++ b/vendor/github.com/blevesearch/zapx/v15/intcoder.go @@ -18,6 +18,7 @@ import ( "bytes" "encoding/binary" "io" + "sync/atomic" ) // We can safely use 0 to represent termNotEncoded since 0 @@ -34,6 +35,9 @@ type chunkedIntCoder struct { currChunk uint64 buf []byte + + // atomic access to this variable + bytesWritten uint64 } // newChunkedIntCoder returns a new chunk int coder which packs data into @@ -54,6 +58,7 @@ func newChunkedIntCoder(chunkSize uint64, maxDocNum uint64) *chunkedIntCoder { // from previous use. you cannot change the chunk size or max doc num. func (c *chunkedIntCoder) Reset() { c.final = c.final[:0] + c.bytesWritten = 0 c.chunkBuf.Reset() c.currChunk = 0 for i := range c.chunkLens { @@ -73,6 +78,14 @@ func (c *chunkedIntCoder) SetChunkSize(chunkSize uint64, maxDocNum uint64) { } } +func (c *chunkedIntCoder) incrementBytesWritten(val uint64) { + atomic.AddUint64(&c.bytesWritten, val) +} + +func (c *chunkedIntCoder) getBytesWritten() uint64 { + return atomic.LoadUint64(&c.bytesWritten) +} + // Add encodes the provided integers into the correct chunk for the provided // doc num. You MUST call Add() with increasing docNums. func (c *chunkedIntCoder) Add(docNum uint64, vals ...uint64) error { @@ -116,6 +129,7 @@ func (c *chunkedIntCoder) AddBytes(docNum uint64, buf []byte) error { // to be encoded. func (c *chunkedIntCoder) Close() { encodingBytes := c.chunkBuf.Bytes() + c.incrementBytesWritten(uint64(len(encodingBytes))) c.chunkLens[c.currChunk] = uint64(len(encodingBytes)) c.final = append(c.final, encodingBytes...) c.currChunk = uint64(cap(c.chunkLens)) // sentinel to detect double close diff --git a/vendor/github.com/blevesearch/zapx/v15/memuvarint.go b/vendor/github.com/blevesearch/zapx/v15/memuvarint.go new file mode 100644 index 00000000..48a57f9c --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v15/memuvarint.go @@ -0,0 +1,103 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "fmt" +) + +type memUvarintReader struct { + C int // index of next byte to read from S + S []byte +} + +func newMemUvarintReader(s []byte) *memUvarintReader { + return &memUvarintReader{S: s} +} + +// Len returns the number of unread bytes. +func (r *memUvarintReader) Len() int { + n := len(r.S) - r.C + if n < 0 { + return 0 + } + return n +} + +// ReadUvarint reads an encoded uint64. The original code this was +// based on is at encoding/binary/ReadUvarint(). +func (r *memUvarintReader) ReadUvarint() (uint64, error) { + if r.C >= len(r.S) { + // nothing else to read + return 0, nil + } + + var x uint64 + var s uint + var C = r.C + var S = r.S + + for { + b := S[C] + C++ + + if b < 0x80 { + r.C = C + + // why 63? The original code had an 'i += 1' loop var and + // checked for i > 9 || i == 9 ...; but, we no longer + // check for the i var, but instead check here for s, + // which is incremented by 7. So, 7*9 == 63. + // + // why the "extra" >= check? The normal case is that s < + // 63, so we check this single >= guard first so that we + // hit the normal, nil-error return pathway sooner. + if s >= 63 && (s > 63 || b > 1) { + return 0, fmt.Errorf("memUvarintReader overflow") + } + + return x | uint64(b)<= len(r.S) { + return + } + + b := r.S[r.C] + r.C++ + + if b < 0x80 { + return + } + } +} + +// SkipBytes skips a count number of bytes. +func (r *memUvarintReader) SkipBytes(count int) { + r.C = r.C + count +} + +func (r *memUvarintReader) Reset(s []byte) { + r.C = 0 + r.S = s +} diff --git a/vendor/github.com/blevesearch/zapx/v15/merge.go b/vendor/github.com/blevesearch/zapx/v15/merge.go new file mode 100644 index 00000000..fa406e6d --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v15/merge.go @@ -0,0 +1,899 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "bufio" + "bytes" + "encoding/binary" + "fmt" + "math" + "os" + "sort" + + "github.com/RoaringBitmap/roaring" + seg "github.com/blevesearch/scorch_segment_api/v2" + "github.com/blevesearch/vellum" + "github.com/golang/snappy" +) + +var DefaultFileMergerBufferSize = 1024 * 1024 + +const docDropped = math.MaxUint64 // sentinel docNum to represent a deleted doc + +// Merge takes a slice of segments and bit masks describing which +// documents may be dropped, and creates a new segment containing the +// remaining data. This new segment is built at the specified path. +func (*ZapPlugin) Merge(segments []seg.Segment, drops []*roaring.Bitmap, path string, + closeCh chan struct{}, s seg.StatsReporter) ( + [][]uint64, uint64, error) { + segmentBases := make([]*SegmentBase, len(segments)) + for segmenti, segment := range segments { + switch segmentx := segment.(type) { + case *Segment: + segmentBases[segmenti] = &segmentx.SegmentBase + case *SegmentBase: + segmentBases[segmenti] = segmentx + default: + panic(fmt.Sprintf("oops, unexpected segment type: %T", segment)) + } + } + return mergeSegmentBases(segmentBases, drops, path, DefaultChunkMode, closeCh, s) +} + +func mergeSegmentBases(segmentBases []*SegmentBase, drops []*roaring.Bitmap, path string, + chunkMode uint32, closeCh chan struct{}, s seg.StatsReporter) ( + [][]uint64, uint64, error) { + flag := os.O_RDWR | os.O_CREATE + + f, err := os.OpenFile(path, flag, 0600) + if err != nil { + return nil, 0, err + } + + cleanup := func() { + _ = f.Close() + _ = os.Remove(path) + } + + // buffer the output + br := bufio.NewWriterSize(f, DefaultFileMergerBufferSize) + + // wrap it for counting (tracking offsets) + cr := NewCountHashWriterWithStatsReporter(br, s) + + newDocNums, numDocs, storedIndexOffset, fieldsIndexOffset, docValueOffset, _, _, _, err := + MergeToWriter(segmentBases, drops, chunkMode, cr, closeCh) + if err != nil { + cleanup() + return nil, 0, err + } + + err = persistFooter(numDocs, storedIndexOffset, fieldsIndexOffset, + docValueOffset, chunkMode, cr.Sum32(), cr) + if err != nil { + cleanup() + return nil, 0, err + } + + err = br.Flush() + if err != nil { + cleanup() + return nil, 0, err + } + + err = f.Sync() + if err != nil { + cleanup() + return nil, 0, err + } + + err = f.Close() + if err != nil { + cleanup() + return nil, 0, err + } + + return newDocNums, uint64(cr.Count()), nil +} + +func MergeToWriter(segments []*SegmentBase, drops []*roaring.Bitmap, + chunkMode uint32, cr *CountHashWriter, closeCh chan struct{}) ( + newDocNums [][]uint64, + numDocs, storedIndexOffset, fieldsIndexOffset, docValueOffset uint64, + dictLocs []uint64, fieldsInv []string, fieldsMap map[string]uint16, + err error) { + docValueOffset = uint64(fieldNotUninverted) + + var fieldsSame bool + fieldsSame, fieldsInv = mergeFields(segments) + fieldsMap = mapFields(fieldsInv) + + numDocs = computeNewDocCount(segments, drops) + + if isClosed(closeCh) { + return nil, 0, 0, 0, 0, nil, nil, nil, seg.ErrClosed + } + + if numDocs > 0 { + storedIndexOffset, newDocNums, err = mergeStoredAndRemap(segments, drops, + fieldsMap, fieldsInv, fieldsSame, numDocs, cr, closeCh) + if err != nil { + return nil, 0, 0, 0, 0, nil, nil, nil, err + } + + dictLocs, docValueOffset, err = persistMergedRest(segments, drops, + fieldsInv, fieldsMap, fieldsSame, + newDocNums, numDocs, chunkMode, cr, closeCh) + if err != nil { + return nil, 0, 0, 0, 0, nil, nil, nil, err + } + } else { + dictLocs = make([]uint64, len(fieldsInv)) + } + + fieldsIndexOffset, err = persistFields(fieldsInv, cr, dictLocs) + if err != nil { + return nil, 0, 0, 0, 0, nil, nil, nil, err + } + + return newDocNums, numDocs, storedIndexOffset, fieldsIndexOffset, docValueOffset, dictLocs, fieldsInv, fieldsMap, nil +} + +// mapFields takes the fieldsInv list and returns a map of fieldName +// to fieldID+1 +func mapFields(fields []string) map[string]uint16 { + rv := make(map[string]uint16, len(fields)) + for i, fieldName := range fields { + rv[fieldName] = uint16(i) + 1 + } + return rv +} + +// computeNewDocCount determines how many documents will be in the newly +// merged segment when obsoleted docs are dropped +func computeNewDocCount(segments []*SegmentBase, drops []*roaring.Bitmap) uint64 { + var newDocCount uint64 + for segI, segment := range segments { + newDocCount += segment.numDocs + if drops[segI] != nil { + newDocCount -= drops[segI].GetCardinality() + } + } + return newDocCount +} + +func persistMergedRest(segments []*SegmentBase, dropsIn []*roaring.Bitmap, + fieldsInv []string, fieldsMap map[string]uint16, fieldsSame bool, + newDocNumsIn [][]uint64, newSegDocCount uint64, chunkMode uint32, + w *CountHashWriter, closeCh chan struct{}) ([]uint64, uint64, error) { + var bufMaxVarintLen64 []byte = make([]byte, binary.MaxVarintLen64) + var bufLoc []uint64 + + var postings *PostingsList + var postItr *PostingsIterator + + rv := make([]uint64, len(fieldsInv)) + fieldDvLocsStart := make([]uint64, len(fieldsInv)) + fieldDvLocsEnd := make([]uint64, len(fieldsInv)) + + // these int coders are initialized with chunk size 1024 + // however this will be reset to the correct chunk size + // while processing each individual field-term section + tfEncoder := newChunkedIntCoder(1024, newSegDocCount-1) + locEncoder := newChunkedIntCoder(1024, newSegDocCount-1) + + var vellumBuf bytes.Buffer + newVellum, err := vellum.New(&vellumBuf, nil) + if err != nil { + return nil, 0, err + } + + newRoaring := roaring.NewBitmap() + + // for each field + for fieldID, fieldName := range fieldsInv { + // collect FST iterators from all active segments for this field + var newDocNums [][]uint64 + var drops []*roaring.Bitmap + var dicts []*Dictionary + var itrs []vellum.Iterator + + var segmentsInFocus []*SegmentBase + + for segmentI, segment := range segments { + // check for the closure in meantime + if isClosed(closeCh) { + return nil, 0, seg.ErrClosed + } + + dict, err2 := segment.dictionary(fieldName) + if err2 != nil { + return nil, 0, err2 + } + if dict != nil && dict.fst != nil { + itr, err2 := dict.fst.Iterator(nil, nil) + if err2 != nil && err2 != vellum.ErrIteratorDone { + return nil, 0, err2 + } + if itr != nil { + newDocNums = append(newDocNums, newDocNumsIn[segmentI]) + if dropsIn[segmentI] != nil && !dropsIn[segmentI].IsEmpty() { + drops = append(drops, dropsIn[segmentI]) + } else { + drops = append(drops, nil) + } + dicts = append(dicts, dict) + itrs = append(itrs, itr) + segmentsInFocus = append(segmentsInFocus, segment) + } + } + } + + var prevTerm []byte + + newRoaring.Clear() + + var lastDocNum, lastFreq, lastNorm uint64 + + // determines whether to use "1-hit" encoding optimization + // when a term appears in only 1 doc, with no loc info, + // has freq of 1, and the docNum fits into 31-bits + use1HitEncoding := func(termCardinality uint64) (bool, uint64, uint64) { + if termCardinality == uint64(1) && locEncoder.FinalSize() <= 0 { + docNum := uint64(newRoaring.Minimum()) + if under32Bits(docNum) && docNum == lastDocNum && lastFreq == 1 { + return true, docNum, lastNorm + } + } + return false, 0, 0 + } + + finishTerm := func(term []byte) error { + tfEncoder.Close() + locEncoder.Close() + + postingsOffset, err := writePostings(newRoaring, + tfEncoder, locEncoder, use1HitEncoding, w, bufMaxVarintLen64) + if err != nil { + return err + } + + if postingsOffset > 0 { + err = newVellum.Insert(term, postingsOffset) + if err != nil { + return err + } + } + + newRoaring.Clear() + + tfEncoder.Reset() + locEncoder.Reset() + + lastDocNum = 0 + lastFreq = 0 + lastNorm = 0 + + return nil + } + + enumerator, err := newEnumerator(itrs) + + for err == nil { + term, itrI, postingsOffset := enumerator.Current() + + if !bytes.Equal(prevTerm, term) { + // check for the closure in meantime + if isClosed(closeCh) { + return nil, 0, seg.ErrClosed + } + + // if the term changed, write out the info collected + // for the previous term + err = finishTerm(prevTerm) + if err != nil { + return nil, 0, err + } + } + if !bytes.Equal(prevTerm, term) || prevTerm == nil { + // compute cardinality of field-term in new seg + var newCard uint64 + lowItrIdxs, lowItrVals := enumerator.GetLowIdxsAndValues() + for i, idx := range lowItrIdxs { + pl, err := dicts[idx].postingsListFromOffset(lowItrVals[i], drops[idx], nil) + if err != nil { + return nil, 0, err + } + newCard += pl.Count() + } + // compute correct chunk size with this + chunkSize, err := getChunkSize(chunkMode, newCard, newSegDocCount) + if err != nil { + return nil, 0, err + } + // update encoders chunk + tfEncoder.SetChunkSize(chunkSize, newSegDocCount-1) + locEncoder.SetChunkSize(chunkSize, newSegDocCount-1) + } + + postings, err = dicts[itrI].postingsListFromOffset( + postingsOffset, drops[itrI], postings) + if err != nil { + return nil, 0, err + } + + postItr = postings.iterator(true, true, true, postItr) + + if fieldsSame { + // can optimize by copying freq/norm/loc bytes directly + lastDocNum, lastFreq, lastNorm, err = mergeTermFreqNormLocsByCopying( + term, postItr, newDocNums[itrI], newRoaring, + tfEncoder, locEncoder) + } else { + lastDocNum, lastFreq, lastNorm, bufLoc, err = mergeTermFreqNormLocs( + fieldsMap, term, postItr, newDocNums[itrI], newRoaring, + tfEncoder, locEncoder, bufLoc) + } + if err != nil { + return nil, 0, err + } + + prevTerm = prevTerm[:0] // copy to prevTerm in case Next() reuses term mem + prevTerm = append(prevTerm, term...) + + err = enumerator.Next() + } + if err != vellum.ErrIteratorDone { + return nil, 0, err + } + + err = finishTerm(prevTerm) + if err != nil { + return nil, 0, err + } + + dictOffset := uint64(w.Count()) + + err = newVellum.Close() + if err != nil { + return nil, 0, err + } + vellumData := vellumBuf.Bytes() + + // write out the length of the vellum data + n := binary.PutUvarint(bufMaxVarintLen64, uint64(len(vellumData))) + _, err = w.Write(bufMaxVarintLen64[:n]) + if err != nil { + return nil, 0, err + } + + // write this vellum to disk + _, err = w.Write(vellumData) + if err != nil { + return nil, 0, err + } + + rv[fieldID] = dictOffset + + // get the field doc value offset (start) + fieldDvLocsStart[fieldID] = uint64(w.Count()) + + // update the field doc values + // NOTE: doc values continue to use legacy chunk mode + chunkSize, err := getChunkSize(LegacyChunkMode, 0, 0) + if err != nil { + return nil, 0, err + } + fdvEncoder := newChunkedContentCoder(chunkSize, newSegDocCount-1, w, true) + + fdvReadersAvailable := false + var dvIterClone *docValueReader + for segmentI, segment := range segmentsInFocus { + // check for the closure in meantime + if isClosed(closeCh) { + return nil, 0, seg.ErrClosed + } + + fieldIDPlus1 := uint16(segment.fieldsMap[fieldName]) + if dvIter, exists := segment.fieldDvReaders[fieldIDPlus1-1]; exists && + dvIter != nil { + fdvReadersAvailable = true + dvIterClone = dvIter.cloneInto(dvIterClone) + err = dvIterClone.iterateAllDocValues(segment, func(docNum uint64, terms []byte) error { + if newDocNums[segmentI][docNum] == docDropped { + return nil + } + err := fdvEncoder.Add(newDocNums[segmentI][docNum], terms) + if err != nil { + return err + } + return nil + }) + if err != nil { + return nil, 0, err + } + } + } + + if fdvReadersAvailable { + err = fdvEncoder.Close() + if err != nil { + return nil, 0, err + } + + // persist the doc value details for this field + _, err = fdvEncoder.Write() + if err != nil { + return nil, 0, err + } + + // get the field doc value offset (end) + fieldDvLocsEnd[fieldID] = uint64(w.Count()) + } else { + fieldDvLocsStart[fieldID] = fieldNotUninverted + fieldDvLocsEnd[fieldID] = fieldNotUninverted + } + + // reset vellum buffer and vellum builder + vellumBuf.Reset() + err = newVellum.Reset(&vellumBuf) + if err != nil { + return nil, 0, err + } + } + + fieldDvLocsOffset := uint64(w.Count()) + + buf := bufMaxVarintLen64 + for i := 0; i < len(fieldDvLocsStart); i++ { + n := binary.PutUvarint(buf, fieldDvLocsStart[i]) + _, err := w.Write(buf[:n]) + if err != nil { + return nil, 0, err + } + n = binary.PutUvarint(buf, fieldDvLocsEnd[i]) + _, err = w.Write(buf[:n]) + if err != nil { + return nil, 0, err + } + } + + return rv, fieldDvLocsOffset, nil +} + +func mergeTermFreqNormLocsByCopying(term []byte, postItr *PostingsIterator, + newDocNums []uint64, newRoaring *roaring.Bitmap, + tfEncoder *chunkedIntCoder, locEncoder *chunkedIntCoder) ( + lastDocNum uint64, lastFreq uint64, lastNorm uint64, err error) { + nextDocNum, nextFreq, nextNorm, nextFreqNormBytes, nextLocBytes, err := + postItr.nextBytes() + for err == nil && len(nextFreqNormBytes) > 0 { + hitNewDocNum := newDocNums[nextDocNum] + if hitNewDocNum == docDropped { + return 0, 0, 0, fmt.Errorf("see hit with dropped doc num") + } + + newRoaring.Add(uint32(hitNewDocNum)) + + err = tfEncoder.AddBytes(hitNewDocNum, nextFreqNormBytes) + if err != nil { + return 0, 0, 0, err + } + + if len(nextLocBytes) > 0 { + err = locEncoder.AddBytes(hitNewDocNum, nextLocBytes) + if err != nil { + return 0, 0, 0, err + } + } + + lastDocNum = hitNewDocNum + lastFreq = nextFreq + lastNorm = nextNorm + + nextDocNum, nextFreq, nextNorm, nextFreqNormBytes, nextLocBytes, err = + postItr.nextBytes() + } + + return lastDocNum, lastFreq, lastNorm, err +} + +func mergeTermFreqNormLocs(fieldsMap map[string]uint16, term []byte, postItr *PostingsIterator, + newDocNums []uint64, newRoaring *roaring.Bitmap, + tfEncoder *chunkedIntCoder, locEncoder *chunkedIntCoder, bufLoc []uint64) ( + lastDocNum uint64, lastFreq uint64, lastNorm uint64, bufLocOut []uint64, err error) { + next, err := postItr.Next() + for next != nil && err == nil { + hitNewDocNum := newDocNums[next.Number()] + if hitNewDocNum == docDropped { + return 0, 0, 0, nil, fmt.Errorf("see hit with dropped docNum") + } + + newRoaring.Add(uint32(hitNewDocNum)) + + nextFreq := next.Frequency() + var nextNorm uint64 + if pi, ok := next.(*Posting); ok { + nextNorm = pi.NormUint64() + } else { + return 0, 0, 0, nil, fmt.Errorf("unexpected posting type %T", next) + } + + locs := next.Locations() + + if nextFreq > 0 { + err = tfEncoder.Add(hitNewDocNum, + encodeFreqHasLocs(nextFreq, len(locs) > 0), nextNorm) + } else { + err = tfEncoder.Add(hitNewDocNum, + encodeFreqHasLocs(nextFreq, len(locs) > 0)) + } + if err != nil { + return 0, 0, 0, nil, err + } + + if len(locs) > 0 { + numBytesLocs := 0 + for _, loc := range locs { + ap := loc.ArrayPositions() + numBytesLocs += totalUvarintBytes(uint64(fieldsMap[loc.Field()]-1), + loc.Pos(), loc.Start(), loc.End(), uint64(len(ap)), ap) + } + + err = locEncoder.Add(hitNewDocNum, uint64(numBytesLocs)) + if err != nil { + return 0, 0, 0, nil, err + } + + for _, loc := range locs { + ap := loc.ArrayPositions() + if cap(bufLoc) < 5+len(ap) { + bufLoc = make([]uint64, 0, 5+len(ap)) + } + args := bufLoc[0:5] + args[0] = uint64(fieldsMap[loc.Field()] - 1) + args[1] = loc.Pos() + args[2] = loc.Start() + args[3] = loc.End() + args[4] = uint64(len(ap)) + args = append(args, ap...) + err = locEncoder.Add(hitNewDocNum, args...) + if err != nil { + return 0, 0, 0, nil, err + } + } + } + + lastDocNum = hitNewDocNum + lastFreq = nextFreq + lastNorm = nextNorm + + next, err = postItr.Next() + } + + return lastDocNum, lastFreq, lastNorm, bufLoc, err +} + +func writePostings(postings *roaring.Bitmap, tfEncoder, locEncoder *chunkedIntCoder, + use1HitEncoding func(uint64) (bool, uint64, uint64), + w *CountHashWriter, bufMaxVarintLen64 []byte) ( + offset uint64, err error) { + termCardinality := postings.GetCardinality() + if termCardinality <= 0 { + return 0, nil + } + + if use1HitEncoding != nil { + encodeAs1Hit, docNum1Hit, normBits1Hit := use1HitEncoding(termCardinality) + if encodeAs1Hit { + return FSTValEncode1Hit(docNum1Hit, normBits1Hit), nil + } + } + + var tfOffset uint64 + tfOffset, _, err = tfEncoder.writeAt(w) + if err != nil { + return 0, err + } + + var locOffset uint64 + locOffset, _, err = locEncoder.writeAt(w) + if err != nil { + return 0, err + } + + postingsOffset := uint64(w.Count()) + + n := binary.PutUvarint(bufMaxVarintLen64, tfOffset) + _, err = w.Write(bufMaxVarintLen64[:n]) + if err != nil { + return 0, err + } + + n = binary.PutUvarint(bufMaxVarintLen64, locOffset) + _, err = w.Write(bufMaxVarintLen64[:n]) + if err != nil { + return 0, err + } + + _, err = writeRoaringWithLen(postings, w, bufMaxVarintLen64) + if err != nil { + return 0, err + } + + return postingsOffset, nil +} + +type varintEncoder func(uint64) (int, error) + +func mergeStoredAndRemap(segments []*SegmentBase, drops []*roaring.Bitmap, + fieldsMap map[string]uint16, fieldsInv []string, fieldsSame bool, newSegDocCount uint64, + w *CountHashWriter, closeCh chan struct{}) (uint64, [][]uint64, error) { + var rv [][]uint64 // The remapped or newDocNums for each segment. + + var newDocNum uint64 + + var curr int + var data, compressed []byte + var metaBuf bytes.Buffer + varBuf := make([]byte, binary.MaxVarintLen64) + metaEncode := func(val uint64) (int, error) { + wb := binary.PutUvarint(varBuf, val) + return metaBuf.Write(varBuf[:wb]) + } + + vals := make([][][]byte, len(fieldsInv)) + typs := make([][]byte, len(fieldsInv)) + poss := make([][][]uint64, len(fieldsInv)) + + var posBuf []uint64 + + docNumOffsets := make([]uint64, newSegDocCount) + + vdc := visitDocumentCtxPool.Get().(*visitDocumentCtx) + defer visitDocumentCtxPool.Put(vdc) + + // for each segment + for segI, segment := range segments { + // check for the closure in meantime + if isClosed(closeCh) { + return 0, nil, seg.ErrClosed + } + + segNewDocNums := make([]uint64, segment.numDocs) + + dropsI := drops[segI] + + // optimize when the field mapping is the same across all + // segments and there are no deletions, via byte-copying + // of stored docs bytes directly to the writer + if fieldsSame && (dropsI == nil || dropsI.GetCardinality() == 0) { + err := segment.copyStoredDocs(newDocNum, docNumOffsets, w) + if err != nil { + return 0, nil, err + } + + for i := uint64(0); i < segment.numDocs; i++ { + segNewDocNums[i] = newDocNum + newDocNum++ + } + rv = append(rv, segNewDocNums) + + continue + } + + // for each doc num + for docNum := uint64(0); docNum < segment.numDocs; docNum++ { + // TODO: roaring's API limits docNums to 32-bits? + if dropsI != nil && dropsI.Contains(uint32(docNum)) { + segNewDocNums[docNum] = docDropped + continue + } + + segNewDocNums[docNum] = newDocNum + + curr = 0 + metaBuf.Reset() + data = data[:0] + + posTemp := posBuf + + // collect all the data + for i := 0; i < len(fieldsInv); i++ { + vals[i] = vals[i][:0] + typs[i] = typs[i][:0] + poss[i] = poss[i][:0] + } + err := segment.visitStoredFields(vdc, docNum, func(field string, typ byte, value []byte, pos []uint64) bool { + fieldID := int(fieldsMap[field]) - 1 + if fieldID < 0 { + // no entry for field in fieldsMap + return false + } + vals[fieldID] = append(vals[fieldID], value) + typs[fieldID] = append(typs[fieldID], typ) + + // copy array positions to preserve them beyond the scope of this callback + var curPos []uint64 + if len(pos) > 0 { + if cap(posTemp) < len(pos) { + posBuf = make([]uint64, len(pos)*len(fieldsInv)) + posTemp = posBuf + } + curPos = posTemp[0:len(pos)] + copy(curPos, pos) + posTemp = posTemp[len(pos):] + } + poss[fieldID] = append(poss[fieldID], curPos) + + return true + }) + if err != nil { + return 0, nil, err + } + + // _id field special case optimizes ExternalID() lookups + idFieldVal := vals[uint16(0)][0] + _, err = metaEncode(uint64(len(idFieldVal))) + if err != nil { + return 0, nil, err + } + + // now walk the non-"_id" fields in order + for fieldID := 1; fieldID < len(fieldsInv); fieldID++ { + storedFieldValues := vals[fieldID] + + stf := typs[fieldID] + spf := poss[fieldID] + + var err2 error + curr, data, err2 = persistStoredFieldValues(fieldID, + storedFieldValues, stf, spf, curr, metaEncode, data) + if err2 != nil { + return 0, nil, err2 + } + } + + metaBytes := metaBuf.Bytes() + + compressed = snappy.Encode(compressed[:cap(compressed)], data) + + // record where we're about to start writing + docNumOffsets[newDocNum] = uint64(w.Count()) + + // write out the meta len and compressed data len + _, err = writeUvarints(w, + uint64(len(metaBytes)), + uint64(len(idFieldVal)+len(compressed))) + if err != nil { + return 0, nil, err + } + // now write the meta + _, err = w.Write(metaBytes) + if err != nil { + return 0, nil, err + } + // now write the _id field val (counted as part of the 'compressed' data) + _, err = w.Write(idFieldVal) + if err != nil { + return 0, nil, err + } + // now write the compressed data + _, err = w.Write(compressed) + if err != nil { + return 0, nil, err + } + + newDocNum++ + } + + rv = append(rv, segNewDocNums) + } + + // return value is the start of the stored index + storedIndexOffset := uint64(w.Count()) + + // now write out the stored doc index + for _, docNumOffset := range docNumOffsets { + err := binary.Write(w, binary.BigEndian, docNumOffset) + if err != nil { + return 0, nil, err + } + } + + return storedIndexOffset, rv, nil +} + +// copyStoredDocs writes out a segment's stored doc info, optimized by +// using a single Write() call for the entire set of bytes. The +// newDocNumOffsets is filled with the new offsets for each doc. +func (s *SegmentBase) copyStoredDocs(newDocNum uint64, newDocNumOffsets []uint64, + w *CountHashWriter) error { + if s.numDocs <= 0 { + return nil + } + + indexOffset0, storedOffset0, _, _, _ := + s.getDocStoredOffsets(0) // the segment's first doc + + indexOffsetN, storedOffsetN, readN, metaLenN, dataLenN := + s.getDocStoredOffsets(s.numDocs - 1) // the segment's last doc + + storedOffset0New := uint64(w.Count()) + + storedBytes := s.mem[storedOffset0 : storedOffsetN+readN+metaLenN+dataLenN] + _, err := w.Write(storedBytes) + if err != nil { + return err + } + + // remap the storedOffset's for the docs into new offsets relative + // to storedOffset0New, filling the given docNumOffsetsOut array + for indexOffset := indexOffset0; indexOffset <= indexOffsetN; indexOffset += 8 { + storedOffset := binary.BigEndian.Uint64(s.mem[indexOffset : indexOffset+8]) + storedOffsetNew := storedOffset - storedOffset0 + storedOffset0New + newDocNumOffsets[newDocNum] = storedOffsetNew + newDocNum += 1 + } + + return nil +} + +// mergeFields builds a unified list of fields used across all the +// input segments, and computes whether the fields are the same across +// segments (which depends on fields to be sorted in the same way +// across segments) +func mergeFields(segments []*SegmentBase) (bool, []string) { + fieldsSame := true + + var segment0Fields []string + if len(segments) > 0 { + segment0Fields = segments[0].Fields() + } + + fieldsExist := map[string]struct{}{} + for _, segment := range segments { + fields := segment.Fields() + for fieldi, field := range fields { + fieldsExist[field] = struct{}{} + if len(segment0Fields) != len(fields) || segment0Fields[fieldi] != field { + fieldsSame = false + } + } + } + + rv := make([]string, 0, len(fieldsExist)) + // ensure _id stays first + rv = append(rv, "_id") + for k := range fieldsExist { + if k != "_id" { + rv = append(rv, k) + } + } + + sort.Strings(rv[1:]) // leave _id as first + + return fieldsSame, rv +} + +func isClosed(closeCh chan struct{}) bool { + select { + case <-closeCh: + return true + default: + return false + } +} diff --git a/vendor/github.com/blevesearch/zapx/v15/new.go b/vendor/github.com/blevesearch/zapx/v15/new.go new file mode 100644 index 00000000..f659a5d0 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v15/new.go @@ -0,0 +1,865 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "bytes" + "encoding/binary" + "math" + "sort" + "sync" + "sync/atomic" + + "github.com/RoaringBitmap/roaring" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" + "github.com/blevesearch/vellum" + "github.com/golang/snappy" +) + +var NewSegmentBufferNumResultsBump int = 100 +var NewSegmentBufferNumResultsFactor float64 = 1.0 +var NewSegmentBufferAvgBytesPerDocFactor float64 = 1.0 + +// ValidateDocFields can be set by applications to perform additional checks +// on fields in a document being added to a new segment, by default it does +// nothing. +// This API is experimental and may be removed at any time. +var ValidateDocFields = func(field index.Field) error { + return nil +} + +// New creates an in-memory zap-encoded SegmentBase from a set of Documents +func (z *ZapPlugin) New(results []index.Document) ( + segment.Segment, uint64, error) { + return z.newWithChunkMode(results, DefaultChunkMode) +} + +func (*ZapPlugin) newWithChunkMode(results []index.Document, + chunkMode uint32) (segment.Segment, uint64, error) { + s := interimPool.Get().(*interim) + + var br bytes.Buffer + if s.lastNumDocs > 0 { + // use previous results to initialize the buf with an estimate + // size, but note that the interim instance comes from a + // global interimPool, so multiple scorch instances indexing + // different docs can lead to low quality estimates + estimateAvgBytesPerDoc := int(float64(s.lastOutSize/s.lastNumDocs) * + NewSegmentBufferNumResultsFactor) + estimateNumResults := int(float64(len(results)+NewSegmentBufferNumResultsBump) * + NewSegmentBufferAvgBytesPerDocFactor) + br.Grow(estimateAvgBytesPerDoc * estimateNumResults) + } + + s.results = results + s.chunkMode = chunkMode + s.w = NewCountHashWriter(&br) + + storedIndexOffset, fieldsIndexOffset, fdvIndexOffset, dictOffsets, + err := s.convert() + if err != nil { + return nil, uint64(0), err + } + + sb, err := InitSegmentBase(br.Bytes(), s.w.Sum32(), chunkMode, + s.FieldsMap, s.FieldsInv, uint64(len(results)), + storedIndexOffset, fieldsIndexOffset, fdvIndexOffset, dictOffsets) + + // get the bytes written before the interim's reset() call + // write it to the newly formed segment base. + totalBytesWritten := s.getBytesWritten() + if err == nil && s.reset() == nil { + s.lastNumDocs = len(results) + s.lastOutSize = len(br.Bytes()) + sb.setBytesWritten(totalBytesWritten) + interimPool.Put(s) + } + + return sb, uint64(len(br.Bytes())), err +} + +var interimPool = sync.Pool{New: func() interface{} { return &interim{} }} + +// interim holds temporary working data used while converting from +// analysis results to a zap-encoded segment +type interim struct { + results []index.Document + + chunkMode uint32 + + w *CountHashWriter + + // FieldsMap adds 1 to field id to avoid zero value issues + // name -> field id + 1 + FieldsMap map[string]uint16 + + // FieldsInv is the inverse of FieldsMap + // field id -> name + FieldsInv []string + + // Term dictionaries for each field + // field id -> term -> postings list id + 1 + Dicts []map[string]uint64 + + // Terms for each field, where terms are sorted ascending + // field id -> []term + DictKeys [][]string + + // Fields whose IncludeDocValues is true + // field id -> bool + IncludeDocValues []bool + + // postings id -> bitmap of docNums + Postings []*roaring.Bitmap + + // postings id -> freq/norm's, one for each docNum in postings + FreqNorms [][]interimFreqNorm + freqNormsBacking []interimFreqNorm + + // postings id -> locs, one for each freq + Locs [][]interimLoc + locsBacking []interimLoc + + numTermsPerPostingsList []int // key is postings list id + numLocsPerPostingsList []int // key is postings list id + + builder *vellum.Builder + builderBuf bytes.Buffer + + metaBuf bytes.Buffer + + tmp0 []byte + tmp1 []byte + + lastNumDocs int + lastOutSize int + + // atomic access to this variable + bytesWritten uint64 +} + +func (s *interim) reset() (err error) { + s.results = nil + s.chunkMode = 0 + s.w = nil + s.FieldsMap = nil + s.FieldsInv = nil + for i := range s.Dicts { + s.Dicts[i] = nil + } + s.Dicts = s.Dicts[:0] + for i := range s.DictKeys { + s.DictKeys[i] = s.DictKeys[i][:0] + } + s.DictKeys = s.DictKeys[:0] + for i := range s.IncludeDocValues { + s.IncludeDocValues[i] = false + } + s.IncludeDocValues = s.IncludeDocValues[:0] + for _, idn := range s.Postings { + idn.Clear() + } + s.Postings = s.Postings[:0] + s.FreqNorms = s.FreqNorms[:0] + for i := range s.freqNormsBacking { + s.freqNormsBacking[i] = interimFreqNorm{} + } + s.freqNormsBacking = s.freqNormsBacking[:0] + s.Locs = s.Locs[:0] + for i := range s.locsBacking { + s.locsBacking[i] = interimLoc{} + } + s.locsBacking = s.locsBacking[:0] + s.numTermsPerPostingsList = s.numTermsPerPostingsList[:0] + s.numLocsPerPostingsList = s.numLocsPerPostingsList[:0] + s.builderBuf.Reset() + if s.builder != nil { + err = s.builder.Reset(&s.builderBuf) + } + s.metaBuf.Reset() + s.tmp0 = s.tmp0[:0] + s.tmp1 = s.tmp1[:0] + s.lastNumDocs = 0 + s.lastOutSize = 0 + + // reset the bytes written stat count + // to avoid leaking of bytesWritten across reuse cycles. + s.setBytesWritten(0) + + return err +} + +func (s *interim) grabBuf(size int) []byte { + buf := s.tmp0 + if cap(buf) < size { + buf = make([]byte, size) + s.tmp0 = buf + } + return buf[0:size] +} + +type interimStoredField struct { + vals [][]byte + typs []byte + arrayposs [][]uint64 // array positions +} + +type interimFreqNorm struct { + freq uint64 + norm float32 + numLocs int +} + +type interimLoc struct { + fieldID uint16 + pos uint64 + start uint64 + end uint64 + arrayposs []uint64 +} + +func (s *interim) convert() (uint64, uint64, uint64, []uint64, error) { + s.FieldsMap = map[string]uint16{} + + s.getOrDefineField("_id") // _id field is fieldID 0 + + for _, result := range s.results { + result.VisitComposite(func(field index.CompositeField) { + s.getOrDefineField(field.Name()) + }) + result.VisitFields(func(field index.Field) { + s.getOrDefineField(field.Name()) + }) + } + + sort.Strings(s.FieldsInv[1:]) // keep _id as first field + + for fieldID, fieldName := range s.FieldsInv { + s.FieldsMap[fieldName] = uint16(fieldID + 1) + } + + if cap(s.IncludeDocValues) >= len(s.FieldsInv) { + s.IncludeDocValues = s.IncludeDocValues[:len(s.FieldsInv)] + } else { + s.IncludeDocValues = make([]bool, len(s.FieldsInv)) + } + + s.prepareDicts() + + for _, dict := range s.DictKeys { + sort.Strings(dict) + } + + s.processDocuments() + + storedIndexOffset, err := s.writeStoredFields() + if err != nil { + return 0, 0, 0, nil, err + } + + var fdvIndexOffset uint64 + var dictOffsets []uint64 + + if len(s.results) > 0 { + fdvIndexOffset, dictOffsets, err = s.writeDicts() + if err != nil { + return 0, 0, 0, nil, err + } + } else { + dictOffsets = make([]uint64, len(s.FieldsInv)) + } + + fieldsIndexOffset, err := persistFields(s.FieldsInv, s.w, dictOffsets) + if err != nil { + return 0, 0, 0, nil, err + } + + return storedIndexOffset, fieldsIndexOffset, fdvIndexOffset, dictOffsets, nil +} + +func (s *interim) getOrDefineField(fieldName string) int { + fieldIDPlus1, exists := s.FieldsMap[fieldName] + if !exists { + fieldIDPlus1 = uint16(len(s.FieldsInv) + 1) + s.FieldsMap[fieldName] = fieldIDPlus1 + s.FieldsInv = append(s.FieldsInv, fieldName) + + s.Dicts = append(s.Dicts, make(map[string]uint64)) + + n := len(s.DictKeys) + if n < cap(s.DictKeys) { + s.DictKeys = s.DictKeys[:n+1] + s.DictKeys[n] = s.DictKeys[n][:0] + } else { + s.DictKeys = append(s.DictKeys, []string(nil)) + } + } + + return int(fieldIDPlus1 - 1) +} + +// fill Dicts and DictKeys from analysis results +func (s *interim) prepareDicts() { + var pidNext int + + var totTFs int + var totLocs int + + visitField := func(field index.Field) { + fieldID := uint16(s.getOrDefineField(field.Name())) + + dict := s.Dicts[fieldID] + dictKeys := s.DictKeys[fieldID] + + tfs := field.AnalyzedTokenFrequencies() + for term, tf := range tfs { + pidPlus1, exists := dict[term] + if !exists { + pidNext++ + pidPlus1 = uint64(pidNext) + + dict[term] = pidPlus1 + dictKeys = append(dictKeys, term) + + s.numTermsPerPostingsList = append(s.numTermsPerPostingsList, 0) + s.numLocsPerPostingsList = append(s.numLocsPerPostingsList, 0) + } + + pid := pidPlus1 - 1 + + s.numTermsPerPostingsList[pid] += 1 + s.numLocsPerPostingsList[pid] += len(tf.Locations) + + totLocs += len(tf.Locations) + } + + totTFs += len(tfs) + + s.DictKeys[fieldID] = dictKeys + } + + for _, result := range s.results { + // walk each composite field + result.VisitComposite(func(field index.CompositeField) { + visitField(field) + }) + + // walk each field + result.VisitFields(visitField) + } + + numPostingsLists := pidNext + + if cap(s.Postings) >= numPostingsLists { + s.Postings = s.Postings[:numPostingsLists] + } else { + postings := make([]*roaring.Bitmap, numPostingsLists) + copy(postings, s.Postings[:cap(s.Postings)]) + for i := 0; i < numPostingsLists; i++ { + if postings[i] == nil { + postings[i] = roaring.New() + } + } + s.Postings = postings + } + + if cap(s.FreqNorms) >= numPostingsLists { + s.FreqNorms = s.FreqNorms[:numPostingsLists] + } else { + s.FreqNorms = make([][]interimFreqNorm, numPostingsLists) + } + + if cap(s.freqNormsBacking) >= totTFs { + s.freqNormsBacking = s.freqNormsBacking[:totTFs] + } else { + s.freqNormsBacking = make([]interimFreqNorm, totTFs) + } + + freqNormsBacking := s.freqNormsBacking + for pid, numTerms := range s.numTermsPerPostingsList { + s.FreqNorms[pid] = freqNormsBacking[0:0] + freqNormsBacking = freqNormsBacking[numTerms:] + } + + if cap(s.Locs) >= numPostingsLists { + s.Locs = s.Locs[:numPostingsLists] + } else { + s.Locs = make([][]interimLoc, numPostingsLists) + } + + if cap(s.locsBacking) >= totLocs { + s.locsBacking = s.locsBacking[:totLocs] + } else { + s.locsBacking = make([]interimLoc, totLocs) + } + + locsBacking := s.locsBacking + for pid, numLocs := range s.numLocsPerPostingsList { + s.Locs[pid] = locsBacking[0:0] + locsBacking = locsBacking[numLocs:] + } +} + +func (s *interim) processDocuments() { + numFields := len(s.FieldsInv) + reuseFieldLens := make([]int, numFields) + reuseFieldTFs := make([]index.TokenFrequencies, numFields) + + for docNum, result := range s.results { + for i := 0; i < numFields; i++ { // clear these for reuse + reuseFieldLens[i] = 0 + reuseFieldTFs[i] = nil + } + + s.processDocument(uint64(docNum), result, + reuseFieldLens, reuseFieldTFs) + } +} + +func (s *interim) processDocument(docNum uint64, + result index.Document, + fieldLens []int, fieldTFs []index.TokenFrequencies) { + visitField := func(field index.Field) { + fieldID := uint16(s.getOrDefineField(field.Name())) + fieldLens[fieldID] += field.AnalyzedLength() + + existingFreqs := fieldTFs[fieldID] + if existingFreqs != nil { + existingFreqs.MergeAll(field.Name(), field.AnalyzedTokenFrequencies()) + } else { + fieldTFs[fieldID] = field.AnalyzedTokenFrequencies() + } + } + + // walk each composite field + result.VisitComposite(func(field index.CompositeField) { + visitField(field) + }) + + // walk each field + result.VisitFields(visitField) + + // now that it's been rolled up into fieldTFs, walk that + for fieldID, tfs := range fieldTFs { + dict := s.Dicts[fieldID] + norm := math.Float32frombits(uint32(fieldLens[fieldID])) + + for term, tf := range tfs { + pid := dict[term] - 1 + bs := s.Postings[pid] + bs.Add(uint32(docNum)) + + s.FreqNorms[pid] = append(s.FreqNorms[pid], + interimFreqNorm{ + freq: uint64(tf.Frequency()), + norm: norm, + numLocs: len(tf.Locations), + }) + + if len(tf.Locations) > 0 { + locs := s.Locs[pid] + + for _, loc := range tf.Locations { + var locf = uint16(fieldID) + if loc.Field != "" { + locf = uint16(s.getOrDefineField(loc.Field)) + } + var arrayposs []uint64 + if len(loc.ArrayPositions) > 0 { + arrayposs = loc.ArrayPositions + } + locs = append(locs, interimLoc{ + fieldID: locf, + pos: uint64(loc.Position), + start: uint64(loc.Start), + end: uint64(loc.End), + arrayposs: arrayposs, + }) + } + + s.Locs[pid] = locs + } + } + } +} + +func (s *interim) getBytesWritten() uint64 { + return atomic.LoadUint64(&s.bytesWritten) +} + +func (s *interim) incrementBytesWritten(val uint64) { + atomic.AddUint64(&s.bytesWritten, val) +} + +func (s *interim) writeStoredFields() ( + storedIndexOffset uint64, err error) { + varBuf := make([]byte, binary.MaxVarintLen64) + metaEncode := func(val uint64) (int, error) { + wb := binary.PutUvarint(varBuf, val) + return s.metaBuf.Write(varBuf[:wb]) + } + + data, compressed := s.tmp0[:0], s.tmp1[:0] + defer func() { s.tmp0, s.tmp1 = data, compressed }() + + // keyed by docNum + docStoredOffsets := make([]uint64, len(s.results)) + + // keyed by fieldID, for the current doc in the loop + docStoredFields := map[uint16]interimStoredField{} + + for docNum, result := range s.results { + for fieldID := range docStoredFields { // reset for next doc + delete(docStoredFields, fieldID) + } + + var validationErr error + result.VisitFields(func(field index.Field) { + fieldID := uint16(s.getOrDefineField(field.Name())) + + if field.Options().IsStored() { + isf := docStoredFields[fieldID] + isf.vals = append(isf.vals, field.Value()) + isf.typs = append(isf.typs, field.EncodedFieldType()) + isf.arrayposs = append(isf.arrayposs, field.ArrayPositions()) + docStoredFields[fieldID] = isf + } + + if field.Options().IncludeDocValues() { + s.IncludeDocValues[fieldID] = true + } + + err := ValidateDocFields(field) + if err != nil && validationErr == nil { + validationErr = err + } + }) + if validationErr != nil { + return 0, validationErr + } + + var curr int + + s.metaBuf.Reset() + data = data[:0] + + // _id field special case optimizes ExternalID() lookups + idFieldVal := docStoredFields[uint16(0)].vals[0] + _, err = metaEncode(uint64(len(idFieldVal))) + if err != nil { + return 0, err + } + + // handle non-"_id" fields + for fieldID := 1; fieldID < len(s.FieldsInv); fieldID++ { + isf, exists := docStoredFields[uint16(fieldID)] + if exists { + curr, data, err = persistStoredFieldValues( + fieldID, isf.vals, isf.typs, isf.arrayposs, + curr, metaEncode, data) + if err != nil { + return 0, err + } + } + } + + metaBytes := s.metaBuf.Bytes() + + compressed = snappy.Encode(compressed[:cap(compressed)], data) + s.incrementBytesWritten(uint64(len(compressed))) + docStoredOffsets[docNum] = uint64(s.w.Count()) + + _, err := writeUvarints(s.w, + uint64(len(metaBytes)), + uint64(len(idFieldVal)+len(compressed))) + if err != nil { + return 0, err + } + + _, err = s.w.Write(metaBytes) + if err != nil { + return 0, err + } + + _, err = s.w.Write(idFieldVal) + if err != nil { + return 0, err + } + + _, err = s.w.Write(compressed) + if err != nil { + return 0, err + } + } + + storedIndexOffset = uint64(s.w.Count()) + + for _, docStoredOffset := range docStoredOffsets { + err = binary.Write(s.w, binary.BigEndian, docStoredOffset) + if err != nil { + return 0, err + } + } + + return storedIndexOffset, nil +} + +func (s *interim) setBytesWritten(val uint64) { + atomic.StoreUint64(&s.bytesWritten, val) +} + +func (s *interim) writeDicts() (fdvIndexOffset uint64, dictOffsets []uint64, err error) { + dictOffsets = make([]uint64, len(s.FieldsInv)) + + fdvOffsetsStart := make([]uint64, len(s.FieldsInv)) + fdvOffsetsEnd := make([]uint64, len(s.FieldsInv)) + + buf := s.grabBuf(binary.MaxVarintLen64) + + // these int coders are initialized with chunk size 1024 + // however this will be reset to the correct chunk size + // while processing each individual field-term section + tfEncoder := newChunkedIntCoder(1024, uint64(len(s.results)-1)) + locEncoder := newChunkedIntCoder(1024, uint64(len(s.results)-1)) + + var docTermMap [][]byte + + if s.builder == nil { + s.builder, err = vellum.New(&s.builderBuf, nil) + if err != nil { + return 0, nil, err + } + } + + for fieldID, terms := range s.DictKeys { + if cap(docTermMap) < len(s.results) { + docTermMap = make([][]byte, len(s.results)) + } else { + docTermMap = docTermMap[0:len(s.results)] + for docNum := range docTermMap { // reset the docTermMap + docTermMap[docNum] = docTermMap[docNum][:0] + } + } + + dict := s.Dicts[fieldID] + + for _, term := range terms { // terms are already sorted + pid := dict[term] - 1 + + postingsBS := s.Postings[pid] + + freqNorms := s.FreqNorms[pid] + freqNormOffset := 0 + + locs := s.Locs[pid] + locOffset := 0 + + chunkSize, err := getChunkSize(s.chunkMode, postingsBS.GetCardinality(), uint64(len(s.results))) + if err != nil { + return 0, nil, err + } + tfEncoder.SetChunkSize(chunkSize, uint64(len(s.results)-1)) + locEncoder.SetChunkSize(chunkSize, uint64(len(s.results)-1)) + + postingsItr := postingsBS.Iterator() + for postingsItr.HasNext() { + docNum := uint64(postingsItr.Next()) + + freqNorm := freqNorms[freqNormOffset] + + // check if freq/norm is enabled + if freqNorm.freq > 0 { + err = tfEncoder.Add(docNum, + encodeFreqHasLocs(freqNorm.freq, freqNorm.numLocs > 0), + uint64(math.Float32bits(freqNorm.norm))) + } else { + // if disabled, then skip the norm part + err = tfEncoder.Add(docNum, + encodeFreqHasLocs(freqNorm.freq, freqNorm.numLocs > 0)) + } + if err != nil { + return 0, nil, err + } + + if freqNorm.numLocs > 0 { + numBytesLocs := 0 + for _, loc := range locs[locOffset : locOffset+freqNorm.numLocs] { + numBytesLocs += totalUvarintBytes( + uint64(loc.fieldID), loc.pos, loc.start, loc.end, + uint64(len(loc.arrayposs)), loc.arrayposs) + } + + err = locEncoder.Add(docNum, uint64(numBytesLocs)) + if err != nil { + return 0, nil, err + } + for _, loc := range locs[locOffset : locOffset+freqNorm.numLocs] { + err = locEncoder.Add(docNum, + uint64(loc.fieldID), loc.pos, loc.start, loc.end, + uint64(len(loc.arrayposs))) + if err != nil { + return 0, nil, err + } + + err = locEncoder.Add(docNum, loc.arrayposs...) + if err != nil { + return 0, nil, err + } + } + locOffset += freqNorm.numLocs + } + + freqNormOffset++ + + docTermMap[docNum] = append( + append(docTermMap[docNum], term...), + termSeparator) + } + + tfEncoder.Close() + locEncoder.Close() + s.incrementBytesWritten(locEncoder.getBytesWritten()) + + postingsOffset, err := + writePostings(postingsBS, tfEncoder, locEncoder, nil, s.w, buf) + if err != nil { + return 0, nil, err + } + + if postingsOffset > uint64(0) { + err = s.builder.Insert([]byte(term), postingsOffset) + if err != nil { + return 0, nil, err + } + } + + tfEncoder.Reset() + locEncoder.Reset() + } + + err = s.builder.Close() + if err != nil { + return 0, nil, err + } + + // record where this dictionary starts + dictOffsets[fieldID] = uint64(s.w.Count()) + + vellumData := s.builderBuf.Bytes() + + // write out the length of the vellum data + n := binary.PutUvarint(buf, uint64(len(vellumData))) + _, err = s.w.Write(buf[:n]) + if err != nil { + return 0, nil, err + } + + // write this vellum to disk + _, err = s.w.Write(vellumData) + if err != nil { + return 0, nil, err + } + + s.incrementBytesWritten(uint64(len(vellumData))) + + // reset vellum for reuse + s.builderBuf.Reset() + + err = s.builder.Reset(&s.builderBuf) + if err != nil { + return 0, nil, err + } + + // write the field doc values + // NOTE: doc values continue to use legacy chunk mode + chunkSize, err := getChunkSize(LegacyChunkMode, 0, 0) + if err != nil { + return 0, nil, err + } + + fdvEncoder := newChunkedContentCoder(chunkSize, uint64(len(s.results)-1), s.w, false) + if s.IncludeDocValues[fieldID] { + for docNum, docTerms := range docTermMap { + if len(docTerms) > 0 { + err = fdvEncoder.Add(uint64(docNum), docTerms) + if err != nil { + return 0, nil, err + } + } + } + err = fdvEncoder.Close() + if err != nil { + return 0, nil, err + } + + s.incrementBytesWritten(fdvEncoder.getBytesWritten()) + + fdvOffsetsStart[fieldID] = uint64(s.w.Count()) + + _, err = fdvEncoder.Write() + if err != nil { + return 0, nil, err + } + + fdvOffsetsEnd[fieldID] = uint64(s.w.Count()) + + fdvEncoder.Reset() + } else { + fdvOffsetsStart[fieldID] = fieldNotUninverted + fdvOffsetsEnd[fieldID] = fieldNotUninverted + } + } + + fdvIndexOffset = uint64(s.w.Count()) + + for i := 0; i < len(fdvOffsetsStart); i++ { + n := binary.PutUvarint(buf, fdvOffsetsStart[i]) + _, err := s.w.Write(buf[:n]) + if err != nil { + return 0, nil, err + } + n = binary.PutUvarint(buf, fdvOffsetsEnd[i]) + _, err = s.w.Write(buf[:n]) + if err != nil { + return 0, nil, err + } + } + + return fdvIndexOffset, dictOffsets, nil +} + +// returns the total # of bytes needed to encode the given uint64's +// into binary.PutUVarint() encoding +func totalUvarintBytes(a, b, c, d, e uint64, more []uint64) (n int) { + n = numUvarintBytes(a) + n += numUvarintBytes(b) + n += numUvarintBytes(c) + n += numUvarintBytes(d) + n += numUvarintBytes(e) + for _, v := range more { + n += numUvarintBytes(v) + } + return n +} + +// returns # of bytes needed to encode x in binary.PutUvarint() encoding +func numUvarintBytes(x uint64) (n int) { + for x >= 0x80 { + x >>= 7 + n++ + } + return n + 1 +} diff --git a/vendor/github.com/blevesearch/zapx/v15/plugin.go b/vendor/github.com/blevesearch/zapx/v15/plugin.go new file mode 100644 index 00000000..f67297ec --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v15/plugin.go @@ -0,0 +1,27 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +// ZapPlugin implements the Plugin interface of +// the blevesearch/scorch_segment_api pkg +type ZapPlugin struct{} + +func (*ZapPlugin) Type() string { + return Type +} + +func (*ZapPlugin) Version() uint32 { + return Version +} diff --git a/vendor/github.com/blevesearch/zapx/v15/posting.go b/vendor/github.com/blevesearch/zapx/v15/posting.go new file mode 100644 index 00000000..ad47df0d --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v15/posting.go @@ -0,0 +1,939 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "encoding/binary" + "fmt" + "math" + "reflect" + + "github.com/RoaringBitmap/roaring" + segment "github.com/blevesearch/scorch_segment_api/v2" +) + +var reflectStaticSizePostingsList int +var reflectStaticSizePostingsIterator int +var reflectStaticSizePosting int +var reflectStaticSizeLocation int + +func init() { + var pl PostingsList + reflectStaticSizePostingsList = int(reflect.TypeOf(pl).Size()) + var pi PostingsIterator + reflectStaticSizePostingsIterator = int(reflect.TypeOf(pi).Size()) + var p Posting + reflectStaticSizePosting = int(reflect.TypeOf(p).Size()) + var l Location + reflectStaticSizeLocation = int(reflect.TypeOf(l).Size()) +} + +// FST or vellum value (uint64) encoding is determined by the top two +// highest-order or most significant bits... +// +// encoding : MSB +// name : 63 62 61...to...bit #0 (LSB) +// ----------+---+---+--------------------------------------------------- +// general : 0 | 0 | 62-bits of postingsOffset. +// ~ : 0 | 1 | reserved for future. +// 1-hit : 1 | 0 | 31-bits of positive float31 norm | 31-bits docNum. +// ~ : 1 | 1 | reserved for future. +// +// Encoding "general" is able to handle all cases, where the +// postingsOffset points to more information about the postings for +// the term. +// +// Encoding "1-hit" is used to optimize a commonly seen case when a +// term has only a single hit. For example, a term in the _id field +// will have only 1 hit. The "1-hit" encoding is used for a term +// in a field when... +// +// - term vector info is disabled for that field; +// - and, the term appears in only a single doc for that field; +// - and, the term's freq is exactly 1 in that single doc for that field; +// - and, the docNum must fit into 31-bits; +// +// Otherwise, the "general" encoding is used instead. +// +// In the "1-hit" encoding, the field in that single doc may have +// other terms, which is supported in the "1-hit" encoding by the +// positive float31 norm. + +const FSTValEncodingMask = uint64(0xc000000000000000) +const FSTValEncodingGeneral = uint64(0x0000000000000000) +const FSTValEncoding1Hit = uint64(0x8000000000000000) + +func FSTValEncode1Hit(docNum uint64, normBits uint64) uint64 { + return FSTValEncoding1Hit | ((mask31Bits & normBits) << 31) | (mask31Bits & docNum) +} + +func FSTValDecode1Hit(v uint64) (docNum uint64, normBits uint64) { + return (mask31Bits & v), (mask31Bits & (v >> 31)) +} + +const mask31Bits = uint64(0x000000007fffffff) + +func under32Bits(x uint64) bool { + return x <= mask31Bits +} + +const DocNum1HitFinished = math.MaxUint64 + +var NormBits1Hit = uint64(1) + +// PostingsList is an in-memory representation of a postings list +type PostingsList struct { + sb *SegmentBase + postingsOffset uint64 + freqOffset uint64 + locOffset uint64 + postings *roaring.Bitmap + except *roaring.Bitmap + + // when normBits1Hit != 0, then this postings list came from a + // 1-hit encoding, and only the docNum1Hit & normBits1Hit apply + docNum1Hit uint64 + normBits1Hit uint64 + + chunkSize uint64 + + bytesRead uint64 +} + +// represents an immutable, empty postings list +var emptyPostingsList = &PostingsList{} + +func (p *PostingsList) Size() int { + sizeInBytes := reflectStaticSizePostingsList + SizeOfPtr + + if p.except != nil { + sizeInBytes += int(p.except.GetSizeInBytes()) + } + + return sizeInBytes +} + +func (p *PostingsList) OrInto(receiver *roaring.Bitmap) { + if p.normBits1Hit != 0 { + receiver.Add(uint32(p.docNum1Hit)) + return + } + + if p.postings != nil { + receiver.Or(p.postings) + } +} + +// Iterator returns an iterator for this postings list +func (p *PostingsList) Iterator(includeFreq, includeNorm, includeLocs bool, + prealloc segment.PostingsIterator) segment.PostingsIterator { + if p.normBits1Hit == 0 && p.postings == nil { + return emptyPostingsIterator + } + + var preallocPI *PostingsIterator + pi, ok := prealloc.(*PostingsIterator) + if ok && pi != nil { + preallocPI = pi + } + if preallocPI == emptyPostingsIterator { + preallocPI = nil + } + + return p.iterator(includeFreq, includeNorm, includeLocs, preallocPI) +} + +func (p *PostingsList) iterator(includeFreq, includeNorm, includeLocs bool, + rv *PostingsIterator) *PostingsIterator { + if rv == nil { + rv = &PostingsIterator{} + } else { + freqNormReader := rv.freqNormReader + if freqNormReader != nil { + freqNormReader.reset() + } + + locReader := rv.locReader + if locReader != nil { + locReader.reset() + } + + nextLocs := rv.nextLocs[:0] + nextSegmentLocs := rv.nextSegmentLocs[:0] + + buf := rv.buf + + *rv = PostingsIterator{} // clear the struct + + rv.freqNormReader = freqNormReader + rv.locReader = locReader + + rv.nextLocs = nextLocs + rv.nextSegmentLocs = nextSegmentLocs + + rv.buf = buf + } + + rv.postings = p + rv.includeFreqNorm = includeFreq || includeNorm || includeLocs + rv.includeLocs = includeLocs + + if p.normBits1Hit != 0 { + // "1-hit" encoding + rv.docNum1Hit = p.docNum1Hit + rv.normBits1Hit = p.normBits1Hit + + if p.except != nil && p.except.Contains(uint32(rv.docNum1Hit)) { + rv.docNum1Hit = DocNum1HitFinished + } + + return rv + } + + // "general" encoding, check if empty + if p.postings == nil { + return rv + } + + // initialize freq chunk reader + if rv.includeFreqNorm { + rv.freqNormReader = newChunkedIntDecoder(p.sb.mem, p.freqOffset, rv.freqNormReader) + rv.incrementBytesRead(rv.freqNormReader.getBytesRead()) + } + + // initialize the loc chunk reader + if rv.includeLocs { + rv.locReader = newChunkedIntDecoder(p.sb.mem, p.locOffset, rv.locReader) + rv.incrementBytesRead(rv.locReader.getBytesRead()) + } + + rv.all = p.postings.Iterator() + if p.except != nil { + rv.ActualBM = roaring.AndNot(p.postings, p.except) + rv.Actual = rv.ActualBM.Iterator() + } else { + rv.ActualBM = p.postings + rv.Actual = rv.all // Optimize to use same iterator for all & Actual. + } + + return rv +} + +// Count returns the number of items on this postings list +func (p *PostingsList) Count() uint64 { + var n, e uint64 + if p.normBits1Hit != 0 { + n = 1 + if p.except != nil && p.except.Contains(uint32(p.docNum1Hit)) { + e = 1 + } + } else if p.postings != nil { + n = p.postings.GetCardinality() + if p.except != nil { + e = p.postings.AndCardinality(p.except) + } + } + return n - e +} + +// Implements the segment.DiskStatsReporter interface +// The purpose of this implementation is to get +// the bytes read from the postings lists stored +// on disk, while querying +func (p *PostingsList) ResetBytesRead(val uint64) { + p.bytesRead = val +} + +func (p *PostingsList) BytesRead() uint64 { + return p.bytesRead +} + +func (p *PostingsList) incrementBytesRead(val uint64) { + p.bytesRead += val +} + +func (p *PostingsList) BytesWritten() uint64 { + return 0 +} + +func (rv *PostingsList) read(postingsOffset uint64, d *Dictionary) error { + rv.postingsOffset = postingsOffset + + // handle "1-hit" encoding special case + if rv.postingsOffset&FSTValEncodingMask == FSTValEncoding1Hit { + return rv.init1Hit(postingsOffset) + } + + // read the location of the freq/norm details + var n uint64 + var read int + + rv.freqOffset, read = binary.Uvarint(d.sb.mem[postingsOffset+n : postingsOffset+binary.MaxVarintLen64]) + n += uint64(read) + + rv.locOffset, read = binary.Uvarint(d.sb.mem[postingsOffset+n : postingsOffset+n+binary.MaxVarintLen64]) + n += uint64(read) + + var postingsLen uint64 + postingsLen, read = binary.Uvarint(d.sb.mem[postingsOffset+n : postingsOffset+n+binary.MaxVarintLen64]) + n += uint64(read) + + roaringBytes := d.sb.mem[postingsOffset+n : postingsOffset+n+postingsLen] + + rv.incrementBytesRead(n + postingsLen) + + if rv.postings == nil { + rv.postings = roaring.NewBitmap() + } + _, err := rv.postings.FromBuffer(roaringBytes) + if err != nil { + return fmt.Errorf("error loading roaring bitmap: %v", err) + } + + chunkSize, err := getChunkSize(d.sb.chunkMode, + rv.postings.GetCardinality(), d.sb.numDocs) + if err != nil { + return err + } else if chunkSize == 0 { + return fmt.Errorf("chunk size is zero, chunkMode: %v, numDocs: %v", + d.sb.chunkMode, d.sb.numDocs) + } + + rv.chunkSize = chunkSize + + return nil +} + +func (rv *PostingsList) init1Hit(fstVal uint64) error { + docNum, normBits := FSTValDecode1Hit(fstVal) + + rv.docNum1Hit = docNum + rv.normBits1Hit = normBits + + return nil +} + +// PostingsIterator provides a way to iterate through the postings list +type PostingsIterator struct { + postings *PostingsList + all roaring.IntPeekable + Actual roaring.IntPeekable + ActualBM *roaring.Bitmap + + currChunk uint32 + freqNormReader *chunkedIntDecoder + locReader *chunkedIntDecoder + + next Posting // reused across Next() calls + nextLocs []Location // reused across Next() calls + nextSegmentLocs []segment.Location // reused across Next() calls + + docNum1Hit uint64 + normBits1Hit uint64 + + buf []byte + + includeFreqNorm bool + includeLocs bool + + bytesRead uint64 +} + +var emptyPostingsIterator = &PostingsIterator{} + +func (i *PostingsIterator) Size() int { + sizeInBytes := reflectStaticSizePostingsIterator + SizeOfPtr + + i.next.Size() + // account for freqNormReader, locReader if we start using this. + for _, entry := range i.nextLocs { + sizeInBytes += entry.Size() + } + + return sizeInBytes +} + +// Implements the segment.DiskStatsReporter interface +// The purpose of this implementation is to get +// the bytes read from the disk which includes +// the freqNorm and location specific information +// of a hit +func (i *PostingsIterator) ResetBytesRead(val uint64) { + i.bytesRead = val +} + +func (i *PostingsIterator) BytesRead() uint64 { + return i.bytesRead +} + +func (i *PostingsIterator) incrementBytesRead(val uint64) { + i.bytesRead += val +} + +func (i *PostingsIterator) BytesWritten() uint64 { + return 0 +} + +func (i *PostingsIterator) loadChunk(chunk int) error { + if i.includeFreqNorm { + err := i.freqNormReader.loadChunk(chunk) + if err != nil { + return err + } + + // assign the bytes read at this point, since + // the postingsIterator is tracking only the chunk loaded + // and the cumulation is tracked correctly in the downstream + // intDecoder + i.ResetBytesRead(i.freqNormReader.getBytesRead()) + } + + if i.includeLocs { + err := i.locReader.loadChunk(chunk) + if err != nil { + return err + } + i.ResetBytesRead(i.locReader.getBytesRead()) + } + + i.currChunk = uint32(chunk) + return nil +} + +func (i *PostingsIterator) readFreqNormHasLocs() (uint64, uint64, bool, error) { + if i.normBits1Hit != 0 { + return 1, i.normBits1Hit, false, nil + } + + freqHasLocs, err := i.freqNormReader.readUvarint() + if err != nil { + return 0, 0, false, fmt.Errorf("error reading frequency: %v", err) + } + + freq, hasLocs := decodeFreqHasLocs(freqHasLocs) + if freq == 0 { + return freq, 0, hasLocs, nil + } + + normBits, err := i.freqNormReader.readUvarint() + if err != nil { + return 0, 0, false, fmt.Errorf("error reading norm: %v", err) + } + + return freq, normBits, hasLocs, nil +} + +func (i *PostingsIterator) skipFreqNormReadHasLocs() (bool, error) { + if i.normBits1Hit != 0 { + return false, nil + } + + freqHasLocs, err := i.freqNormReader.readUvarint() + if err != nil { + return false, fmt.Errorf("error reading freqHasLocs: %v", err) + } + + freq, hasLocs := decodeFreqHasLocs(freqHasLocs) + if freq == 0 { + return hasLocs, nil + } + + i.freqNormReader.SkipUvarint() // Skip normBits. + + return hasLocs, nil // See decodeFreqHasLocs() / hasLocs. +} + +func encodeFreqHasLocs(freq uint64, hasLocs bool) uint64 { + rv := freq << 1 + if hasLocs { + rv = rv | 0x01 // 0'th LSB encodes whether there are locations + } + return rv +} + +func decodeFreqHasLocs(freqHasLocs uint64) (uint64, bool) { + freq := freqHasLocs >> 1 + hasLocs := freqHasLocs&0x01 != 0 + return freq, hasLocs +} + +// readLocation processes all the integers on the stream representing a single +// location. +func (i *PostingsIterator) readLocation(l *Location) error { + // read off field + fieldID, err := i.locReader.readUvarint() + if err != nil { + return fmt.Errorf("error reading location field: %v", err) + } + // read off pos + pos, err := i.locReader.readUvarint() + if err != nil { + return fmt.Errorf("error reading location pos: %v", err) + } + // read off start + start, err := i.locReader.readUvarint() + if err != nil { + return fmt.Errorf("error reading location start: %v", err) + } + // read off end + end, err := i.locReader.readUvarint() + if err != nil { + return fmt.Errorf("error reading location end: %v", err) + } + // read off num array pos + numArrayPos, err := i.locReader.readUvarint() + if err != nil { + return fmt.Errorf("error reading location num array pos: %v", err) + } + + l.field = i.postings.sb.fieldsInv[fieldID] + l.pos = pos + l.start = start + l.end = end + + if cap(l.ap) < int(numArrayPos) { + l.ap = make([]uint64, int(numArrayPos)) + } else { + l.ap = l.ap[:int(numArrayPos)] + } + + // read off array positions + for k := 0; k < int(numArrayPos); k++ { + ap, err := i.locReader.readUvarint() + if err != nil { + return fmt.Errorf("error reading array position: %v", err) + } + + l.ap[k] = ap + } + + return nil +} + +// Next returns the next posting on the postings list, or nil at the end +func (i *PostingsIterator) Next() (segment.Posting, error) { + return i.nextAtOrAfter(0) +} + +// Advance returns the posting at the specified docNum or it is not present +// the next posting, or if the end is reached, nil +func (i *PostingsIterator) Advance(docNum uint64) (segment.Posting, error) { + return i.nextAtOrAfter(docNum) +} + +// Next returns the next posting on the postings list, or nil at the end +func (i *PostingsIterator) nextAtOrAfter(atOrAfter uint64) (segment.Posting, error) { + docNum, exists, err := i.nextDocNumAtOrAfter(atOrAfter) + if err != nil || !exists { + return nil, err + } + + i.next = Posting{} // clear the struct + rv := &i.next + rv.docNum = docNum + + if !i.includeFreqNorm { + return rv, nil + } + + var normBits uint64 + var hasLocs bool + + rv.freq, normBits, hasLocs, err = i.readFreqNormHasLocs() + if err != nil { + return nil, err + } + + rv.norm = math.Float32frombits(uint32(normBits)) + + if i.includeLocs && hasLocs { + // prepare locations into reused slices, where we assume + // rv.freq >= "number of locs", since in a composite field, + // some component fields might have their IncludeTermVector + // flags disabled while other component fields are enabled + if rv.freq > 0 { + if cap(i.nextLocs) >= int(rv.freq) { + i.nextLocs = i.nextLocs[0:rv.freq] + } else { + i.nextLocs = make([]Location, rv.freq, rv.freq*2) + } + if cap(i.nextSegmentLocs) < int(rv.freq) { + i.nextSegmentLocs = make([]segment.Location, rv.freq, rv.freq*2) + } + rv.locs = i.nextSegmentLocs[:0] + } + + numLocsBytes, err := i.locReader.readUvarint() + if err != nil { + return nil, fmt.Errorf("error reading location numLocsBytes: %v", err) + } + + j := 0 + var nextLoc *Location + startBytesRemaining := i.locReader.Len() // # bytes remaining in the locReader + for startBytesRemaining-i.locReader.Len() < int(numLocsBytes) { + if len(i.nextLocs) > j { + nextLoc = &i.nextLocs[j] + } else { + nextLoc = &Location{} + } + + err := i.readLocation(nextLoc) + if err != nil { + return nil, err + } + + rv.locs = append(rv.locs, nextLoc) + j++ + } + } + + return rv, nil +} + +// nextDocNum returns the next docNum on the postings list, and also +// sets up the currChunk / loc related fields of the iterator. +func (i *PostingsIterator) nextDocNumAtOrAfter(atOrAfter uint64) (uint64, bool, error) { + if i.normBits1Hit != 0 { + if i.docNum1Hit == DocNum1HitFinished { + return 0, false, nil + } + if i.docNum1Hit < atOrAfter { + // advanced past our 1-hit + i.docNum1Hit = DocNum1HitFinished // consume our 1-hit docNum + return 0, false, nil + } + docNum := i.docNum1Hit + i.docNum1Hit = DocNum1HitFinished // consume our 1-hit docNum + return docNum, true, nil + } + + if i.Actual == nil || !i.Actual.HasNext() { + return 0, false, nil + } + + if i.postings == nil || i.postings == emptyPostingsList { + // couldn't find anything + return 0, false, nil + } + + if i.postings.postings == i.ActualBM { + return i.nextDocNumAtOrAfterClean(atOrAfter) + } + + i.Actual.AdvanceIfNeeded(uint32(atOrAfter)) + + if !i.Actual.HasNext() || !i.all.HasNext() { + // couldn't find anything + return 0, false, nil + } + + n := i.Actual.Next() + allN := i.all.Next() + nChunk := n / uint32(i.postings.chunkSize) + + // when allN becomes >= to here, then allN is in the same chunk as nChunk. + allNReachesNChunk := nChunk * uint32(i.postings.chunkSize) + + // n is the next actual hit (excluding some postings), and + // allN is the next hit in the full postings, and + // if they don't match, move 'all' forwards until they do + for allN != n { + // we've reached same chunk, so move the freq/norm/loc decoders forward + if i.includeFreqNorm && allN >= allNReachesNChunk { + err := i.currChunkNext(nChunk) + if err != nil { + return 0, false, err + } + } + + if !i.all.HasNext() { + return 0, false, nil + } + + allN = i.all.Next() + } + + if i.includeFreqNorm && (i.currChunk != nChunk || i.freqNormReader.isNil()) { + err := i.loadChunk(int(nChunk)) + if err != nil { + return 0, false, fmt.Errorf("error loading chunk: %v", err) + } + } + + return uint64(n), true, nil +} + +var freqHasLocs1Hit = encodeFreqHasLocs(1, false) + +// nextBytes returns the docNum and the encoded freq & loc bytes for +// the next posting +func (i *PostingsIterator) nextBytes() ( + docNumOut uint64, freq uint64, normBits uint64, + bytesFreqNorm []byte, bytesLoc []byte, err error) { + docNum, exists, err := i.nextDocNumAtOrAfter(0) + if err != nil || !exists { + return 0, 0, 0, nil, nil, err + } + + if i.normBits1Hit != 0 { + if i.buf == nil { + i.buf = make([]byte, binary.MaxVarintLen64*2) + } + n := binary.PutUvarint(i.buf, freqHasLocs1Hit) + n += binary.PutUvarint(i.buf[n:], i.normBits1Hit) + return docNum, uint64(1), i.normBits1Hit, i.buf[:n], nil, nil + } + + startFreqNorm := i.freqNormReader.remainingLen() + + var hasLocs bool + + freq, normBits, hasLocs, err = i.readFreqNormHasLocs() + if err != nil { + return 0, 0, 0, nil, nil, err + } + + endFreqNorm := i.freqNormReader.remainingLen() + bytesFreqNorm = i.freqNormReader.readBytes(startFreqNorm, endFreqNorm) + + if hasLocs { + startLoc := i.locReader.remainingLen() + + numLocsBytes, err := i.locReader.readUvarint() + if err != nil { + return 0, 0, 0, nil, nil, + fmt.Errorf("error reading location nextBytes numLocs: %v", err) + } + + // skip over all the location bytes + i.locReader.SkipBytes(int(numLocsBytes)) + + endLoc := i.locReader.remainingLen() + bytesLoc = i.locReader.readBytes(startLoc, endLoc) + } + + return docNum, freq, normBits, bytesFreqNorm, bytesLoc, nil +} + +// optimization when the postings list is "clean" (e.g., no updates & +// no deletions) where the all bitmap is the same as the actual bitmap +func (i *PostingsIterator) nextDocNumAtOrAfterClean( + atOrAfter uint64) (uint64, bool, error) { + if !i.includeFreqNorm { + i.Actual.AdvanceIfNeeded(uint32(atOrAfter)) + + if !i.Actual.HasNext() { + return 0, false, nil // couldn't find anything + } + + return uint64(i.Actual.Next()), true, nil + } + + // freq-norm's needed, so maintain freq-norm chunk reader + sameChunkNexts := 0 // # of times we called Next() in the same chunk + n := i.Actual.Next() + nChunk := n / uint32(i.postings.chunkSize) + + for uint64(n) < atOrAfter && i.Actual.HasNext() { + n = i.Actual.Next() + + nChunkPrev := nChunk + nChunk = n / uint32(i.postings.chunkSize) + + if nChunk != nChunkPrev { + sameChunkNexts = 0 + } else { + sameChunkNexts += 1 + } + } + + if uint64(n) < atOrAfter { + // couldn't find anything + return 0, false, nil + } + + for j := 0; j < sameChunkNexts; j++ { + err := i.currChunkNext(nChunk) + if err != nil { + return 0, false, fmt.Errorf("error optimized currChunkNext: %v", err) + } + } + + if i.currChunk != nChunk || i.freqNormReader.isNil() { + err := i.loadChunk(int(nChunk)) + if err != nil { + return 0, false, fmt.Errorf("error loading chunk: %v", err) + } + } + + return uint64(n), true, nil +} + +func (i *PostingsIterator) currChunkNext(nChunk uint32) error { + if i.currChunk != nChunk || i.freqNormReader.isNil() { + err := i.loadChunk(int(nChunk)) + if err != nil { + return fmt.Errorf("error loading chunk: %v", err) + } + } + + // read off freq/offsets even though we don't care about them + hasLocs, err := i.skipFreqNormReadHasLocs() + if err != nil { + return err + } + + if i.includeLocs && hasLocs { + numLocsBytes, err := i.locReader.readUvarint() + if err != nil { + return fmt.Errorf("error reading location numLocsBytes: %v", err) + } + + // skip over all the location bytes + i.locReader.SkipBytes(int(numLocsBytes)) + } + + return nil +} + +// DocNum1Hit returns the docNum and true if this is "1-hit" optimized +// and the docNum is available. +func (p *PostingsIterator) DocNum1Hit() (uint64, bool) { + if p.normBits1Hit != 0 && p.docNum1Hit != DocNum1HitFinished { + return p.docNum1Hit, true + } + return 0, false +} + +// ActualBitmap returns the underlying actual bitmap +// which can be used up the stack for optimizations +func (p *PostingsIterator) ActualBitmap() *roaring.Bitmap { + return p.ActualBM +} + +// ReplaceActual replaces the ActualBM with the provided +// bitmap +func (p *PostingsIterator) ReplaceActual(abm *roaring.Bitmap) { + p.ActualBM = abm + p.Actual = abm.Iterator() +} + +// PostingsIteratorFromBitmap constructs a PostingsIterator given an +// "actual" bitmap. +func PostingsIteratorFromBitmap(bm *roaring.Bitmap, + includeFreqNorm, includeLocs bool) (segment.PostingsIterator, error) { + return &PostingsIterator{ + ActualBM: bm, + Actual: bm.Iterator(), + includeFreqNorm: includeFreqNorm, + includeLocs: includeLocs, + }, nil +} + +// PostingsIteratorFrom1Hit constructs a PostingsIterator given a +// 1-hit docNum. +func PostingsIteratorFrom1Hit(docNum1Hit uint64, + includeFreqNorm, includeLocs bool) (segment.PostingsIterator, error) { + return &PostingsIterator{ + docNum1Hit: docNum1Hit, + normBits1Hit: NormBits1Hit, + includeFreqNorm: includeFreqNorm, + includeLocs: includeLocs, + }, nil +} + +// Posting is a single entry in a postings list +type Posting struct { + docNum uint64 + freq uint64 + norm float32 + locs []segment.Location +} + +func (p *Posting) Size() int { + sizeInBytes := reflectStaticSizePosting + + for _, entry := range p.locs { + sizeInBytes += entry.Size() + } + + return sizeInBytes +} + +// Number returns the document number of this posting in this segment +func (p *Posting) Number() uint64 { + return p.docNum +} + +// Frequency returns the frequencies of occurrence of this term in this doc/field +func (p *Posting) Frequency() uint64 { + return p.freq +} + +// Norm returns the normalization factor for this posting +func (p *Posting) Norm() float64 { + return float64(float32(1.0 / math.Sqrt(float64(math.Float32bits(p.norm))))) +} + +// Locations returns the location information for each occurrence +func (p *Posting) Locations() []segment.Location { + return p.locs +} + +// NormUint64 returns the norm value as uint64 +func (p *Posting) NormUint64() uint64 { + return uint64(math.Float32bits(p.norm)) +} + +// Location represents the location of a single occurrence +type Location struct { + field string + pos uint64 + start uint64 + end uint64 + ap []uint64 +} + +func (l *Location) Size() int { + return reflectStaticSizeLocation + + len(l.field) + + len(l.ap)*SizeOfUint64 +} + +// Field returns the name of the field (useful in composite fields to know +// which original field the value came from) +func (l *Location) Field() string { + return l.field +} + +// Start returns the start byte offset of this occurrence +func (l *Location) Start() uint64 { + return l.start +} + +// End returns the end byte offset of this occurrence +func (l *Location) End() uint64 { + return l.end +} + +// Pos returns the 1-based phrase position of this occurrence +func (l *Location) Pos() uint64 { + return l.pos +} + +// ArrayPositions returns the array position vector associated with this occurrence +func (l *Location) ArrayPositions() []uint64 { + return l.ap +} diff --git a/vendor/github.com/blevesearch/zap/v15/read.go b/vendor/github.com/blevesearch/zapx/v15/read.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v15/read.go rename to vendor/github.com/blevesearch/zapx/v15/read.go diff --git a/vendor/github.com/blevesearch/zapx/v15/segment.go b/vendor/github.com/blevesearch/zapx/v15/segment.go new file mode 100644 index 00000000..15bc911a --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v15/segment.go @@ -0,0 +1,629 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "os" + "sync" + "sync/atomic" + "unsafe" + + "github.com/RoaringBitmap/roaring" + mmap "github.com/blevesearch/mmap-go" + segment "github.com/blevesearch/scorch_segment_api/v2" + "github.com/blevesearch/vellum" + "github.com/golang/snappy" +) + +var reflectStaticSizeSegmentBase int + +func init() { + var sb SegmentBase + reflectStaticSizeSegmentBase = int(unsafe.Sizeof(sb)) +} + +// Open returns a zap impl of a segment +func (*ZapPlugin) Open(path string) (segment.Segment, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + mm, err := mmap.Map(f, mmap.RDONLY, 0) + if err != nil { + // mmap failed, try to close the file + _ = f.Close() + return nil, err + } + + rv := &Segment{ + SegmentBase: SegmentBase{ + mem: mm[0 : len(mm)-FooterSize], + fieldsMap: make(map[string]uint16), + fieldDvReaders: make(map[uint16]*docValueReader), + fieldFSTs: make(map[uint16]*vellum.FST), + }, + f: f, + mm: mm, + path: path, + refs: 1, + } + rv.SegmentBase.updateSize() + + err = rv.loadConfig() + if err != nil { + _ = rv.Close() + return nil, err + } + + err = rv.loadFields() + if err != nil { + _ = rv.Close() + return nil, err + } + + err = rv.loadDvReaders() + if err != nil { + _ = rv.Close() + return nil, err + } + + return rv, nil +} + +// SegmentBase is a memory only, read-only implementation of the +// segment.Segment interface, using zap's data representation. +type SegmentBase struct { + mem []byte + memCRC uint32 + chunkMode uint32 + fieldsMap map[string]uint16 // fieldName -> fieldID+1 + fieldsInv []string // fieldID -> fieldName + numDocs uint64 + storedIndexOffset uint64 + fieldsIndexOffset uint64 + docValueOffset uint64 + dictLocs []uint64 + fieldDvReaders map[uint16]*docValueReader // naive chunk cache per field + fieldDvNames []string // field names cached in fieldDvReaders + size uint64 + + // atomic access to these variables + bytesRead uint64 + bytesWritten uint64 + + m sync.Mutex + fieldFSTs map[uint16]*vellum.FST +} + +func (sb *SegmentBase) Size() int { + return int(sb.size) +} + +func (sb *SegmentBase) updateSize() { + sizeInBytes := reflectStaticSizeSegmentBase + + cap(sb.mem) + + // fieldsMap + for k := range sb.fieldsMap { + sizeInBytes += (len(k) + SizeOfString) + SizeOfUint16 + } + + // fieldsInv, dictLocs + for _, entry := range sb.fieldsInv { + sizeInBytes += len(entry) + SizeOfString + } + sizeInBytes += len(sb.dictLocs) * SizeOfUint64 + + // fieldDvReaders + for _, v := range sb.fieldDvReaders { + sizeInBytes += SizeOfUint16 + SizeOfPtr + if v != nil { + sizeInBytes += v.size() + } + } + + sb.size = uint64(sizeInBytes) +} + +func (sb *SegmentBase) AddRef() {} +func (sb *SegmentBase) DecRef() (err error) { return nil } +func (sb *SegmentBase) Close() (err error) { return nil } + +// Segment implements a persisted segment.Segment interface, by +// embedding an mmap()'ed SegmentBase. +type Segment struct { + SegmentBase + + f *os.File + mm mmap.MMap + path string + version uint32 + crc uint32 + + m sync.Mutex // Protects the fields that follow. + refs int64 +} + +func (s *Segment) Size() int { + // 8 /* size of file pointer */ + // 4 /* size of version -> uint32 */ + // 4 /* size of crc -> uint32 */ + sizeOfUints := 16 + + sizeInBytes := (len(s.path) + SizeOfString) + sizeOfUints + + // mutex, refs -> int64 + sizeInBytes += 16 + + // do not include the mmap'ed part + return sizeInBytes + s.SegmentBase.Size() - cap(s.mem) +} + +func (s *Segment) AddRef() { + s.m.Lock() + s.refs++ + s.m.Unlock() +} + +func (s *Segment) DecRef() (err error) { + s.m.Lock() + s.refs-- + if s.refs == 0 { + err = s.closeActual() + } + s.m.Unlock() + return err +} + +func (s *Segment) loadConfig() error { + crcOffset := len(s.mm) - 4 + s.crc = binary.BigEndian.Uint32(s.mm[crcOffset : crcOffset+4]) + + verOffset := crcOffset - 4 + s.version = binary.BigEndian.Uint32(s.mm[verOffset : verOffset+4]) + if s.version != Version { + return fmt.Errorf("unsupported version %d != %d", s.version, Version) + } + + chunkOffset := verOffset - 4 + s.chunkMode = binary.BigEndian.Uint32(s.mm[chunkOffset : chunkOffset+4]) + + docValueOffset := chunkOffset - 8 + s.docValueOffset = binary.BigEndian.Uint64(s.mm[docValueOffset : docValueOffset+8]) + + fieldsIndexOffset := docValueOffset - 8 + s.fieldsIndexOffset = binary.BigEndian.Uint64(s.mm[fieldsIndexOffset : fieldsIndexOffset+8]) + + storedIndexOffset := fieldsIndexOffset - 8 + s.storedIndexOffset = binary.BigEndian.Uint64(s.mm[storedIndexOffset : storedIndexOffset+8]) + + numDocsOffset := storedIndexOffset - 8 + s.numDocs = binary.BigEndian.Uint64(s.mm[numDocsOffset : numDocsOffset+8]) + + // 8*4 + 4*3 = 44 bytes being accounted from all the offsets + // above being read from the file + s.incrementBytesRead(44) + return nil +} + +// Implements the segment.DiskStatsReporter interface +// Only the persistedSegment type implments the +// interface, as the intention is to retrieve the bytes +// read from the on-disk segment as part of the current +// query. +func (s *Segment) ResetBytesRead(val uint64) { + atomic.StoreUint64(&s.SegmentBase.bytesRead, val) +} + +func (s *Segment) BytesRead() uint64 { + return atomic.LoadUint64(&s.bytesRead) +} + +func (s *Segment) BytesWritten() uint64 { + return 0 +} + +func (s *Segment) incrementBytesRead(val uint64) { + atomic.AddUint64(&s.bytesRead, val) +} + +func (s *SegmentBase) BytesWritten() uint64 { + return atomic.LoadUint64(&s.bytesWritten) +} + +func (s *SegmentBase) setBytesWritten(val uint64) { + atomic.AddUint64(&s.bytesWritten, val) +} + +func (s *SegmentBase) BytesRead() uint64 { + return 0 +} + +func (s *SegmentBase) ResetBytesRead(val uint64) {} + +func (s *SegmentBase) incrementBytesRead(val uint64) { + atomic.AddUint64(&s.bytesRead, val) +} + +func (s *SegmentBase) loadFields() error { + // NOTE for now we assume the fields index immediately precedes + // the footer, and if this changes, need to adjust accordingly (or + // store explicit length), where s.mem was sliced from s.mm in Open(). + fieldsIndexEnd := uint64(len(s.mem)) + + // iterate through fields index + var fieldID uint64 + for s.fieldsIndexOffset+(8*fieldID) < fieldsIndexEnd { + addr := binary.BigEndian.Uint64(s.mem[s.fieldsIndexOffset+(8*fieldID) : s.fieldsIndexOffset+(8*fieldID)+8]) + + // accounting the address of the dictLoc being read from file + s.incrementBytesRead(8) + + dictLoc, read := binary.Uvarint(s.mem[addr:fieldsIndexEnd]) + n := uint64(read) + s.dictLocs = append(s.dictLocs, dictLoc) + + var nameLen uint64 + nameLen, read = binary.Uvarint(s.mem[addr+n : fieldsIndexEnd]) + n += uint64(read) + + name := string(s.mem[addr+n : addr+n+nameLen]) + + s.incrementBytesRead(n + nameLen) + s.fieldsInv = append(s.fieldsInv, name) + s.fieldsMap[name] = uint16(fieldID + 1) + + fieldID++ + } + return nil +} + +// Dictionary returns the term dictionary for the specified field +func (s *SegmentBase) Dictionary(field string) (segment.TermDictionary, error) { + dict, err := s.dictionary(field) + if err == nil && dict == nil { + return emptyDictionary, nil + } + return dict, err +} + +func (sb *SegmentBase) dictionary(field string) (rv *Dictionary, err error) { + fieldIDPlus1 := sb.fieldsMap[field] + if fieldIDPlus1 > 0 { + rv = &Dictionary{ + sb: sb, + field: field, + fieldID: fieldIDPlus1 - 1, + } + + dictStart := sb.dictLocs[rv.fieldID] + if dictStart > 0 { + var ok bool + sb.m.Lock() + if rv.fst, ok = sb.fieldFSTs[rv.fieldID]; !ok { + // read the length of the vellum data + vellumLen, read := binary.Uvarint(sb.mem[dictStart : dictStart+binary.MaxVarintLen64]) + if vellumLen == 0 { + sb.m.Unlock() + return nil, fmt.Errorf("empty dictionary for field: %v", field) + } + fstBytes := sb.mem[dictStart+uint64(read) : dictStart+uint64(read)+vellumLen] + rv.incrementBytesRead(uint64(read) + vellumLen) + rv.fst, err = vellum.Load(fstBytes) + if err != nil { + sb.m.Unlock() + return nil, fmt.Errorf("dictionary field %s vellum err: %v", field, err) + } + + sb.fieldFSTs[rv.fieldID] = rv.fst + } + + sb.m.Unlock() + rv.fstReader, err = rv.fst.Reader() + if err != nil { + return nil, fmt.Errorf("dictionary field %s vellum reader err: %v", field, err) + } + } + } + + return rv, nil +} + +// visitDocumentCtx holds data structures that are reusable across +// multiple VisitDocument() calls to avoid memory allocations +type visitDocumentCtx struct { + buf []byte + reader bytes.Reader + arrayPos []uint64 +} + +var visitDocumentCtxPool = sync.Pool{ + New: func() interface{} { + reuse := &visitDocumentCtx{} + return reuse + }, +} + +// VisitStoredFields invokes the StoredFieldValueVisitor for each stored field +// for the specified doc number +func (s *SegmentBase) VisitStoredFields(num uint64, visitor segment.StoredFieldValueVisitor) error { + vdc := visitDocumentCtxPool.Get().(*visitDocumentCtx) + defer visitDocumentCtxPool.Put(vdc) + return s.visitStoredFields(vdc, num, visitor) +} + +func (s *SegmentBase) visitStoredFields(vdc *visitDocumentCtx, num uint64, + visitor segment.StoredFieldValueVisitor) error { + // first make sure this is a valid number in this segment + if num < s.numDocs { + meta, compressed := s.getDocStoredMetaAndCompressed(num) + + vdc.reader.Reset(meta) + + // handle _id field special case + idFieldValLen, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return err + } + idFieldVal := compressed[:idFieldValLen] + + keepGoing := visitor("_id", byte('t'), idFieldVal, nil) + if !keepGoing { + visitDocumentCtxPool.Put(vdc) + return nil + } + + // handle non-"_id" fields + compressed = compressed[idFieldValLen:] + + uncompressed, err := snappy.Decode(vdc.buf[:cap(vdc.buf)], compressed) + if err != nil { + return err + } + + for keepGoing { + field, err := binary.ReadUvarint(&vdc.reader) + if err == io.EOF { + break + } + if err != nil { + return err + } + typ, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return err + } + offset, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return err + } + l, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return err + } + numap, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return err + } + var arrayPos []uint64 + if numap > 0 { + if cap(vdc.arrayPos) < int(numap) { + vdc.arrayPos = make([]uint64, numap) + } + arrayPos = vdc.arrayPos[:numap] + for i := 0; i < int(numap); i++ { + ap, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return err + } + arrayPos[i] = ap + } + } + + value := uncompressed[offset : offset+l] + keepGoing = visitor(s.fieldsInv[field], byte(typ), value, arrayPos) + } + + vdc.buf = uncompressed + } + return nil +} + +// DocID returns the value of the _id field for the given docNum +func (s *SegmentBase) DocID(num uint64) ([]byte, error) { + if num >= s.numDocs { + return nil, nil + } + + vdc := visitDocumentCtxPool.Get().(*visitDocumentCtx) + + meta, compressed := s.getDocStoredMetaAndCompressed(num) + + vdc.reader.Reset(meta) + + // handle _id field special case + idFieldValLen, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return nil, err + } + idFieldVal := compressed[:idFieldValLen] + + visitDocumentCtxPool.Put(vdc) + + return idFieldVal, nil +} + +// Count returns the number of documents in this segment. +func (s *SegmentBase) Count() uint64 { + return s.numDocs +} + +// DocNumbers returns a bitset corresponding to the doc numbers of all the +// provided _id strings +func (s *SegmentBase) DocNumbers(ids []string) (*roaring.Bitmap, error) { + rv := roaring.New() + + if len(s.fieldsMap) > 0 { + idDict, err := s.dictionary("_id") + if err != nil { + return nil, err + } + + postingsList := emptyPostingsList + + sMax, err := idDict.fst.GetMaxKey() + if err != nil { + return nil, err + } + sMaxStr := string(sMax) + filteredIds := make([]string, 0, len(ids)) + for _, id := range ids { + if id <= sMaxStr { + filteredIds = append(filteredIds, id) + } + } + + for _, id := range filteredIds { + postingsList, err = idDict.postingsList([]byte(id), nil, postingsList) + if err != nil { + return nil, err + } + postingsList.OrInto(rv) + } + } + + return rv, nil +} + +// Fields returns the field names used in this segment +func (s *SegmentBase) Fields() []string { + return s.fieldsInv +} + +// Path returns the path of this segment on disk +func (s *Segment) Path() string { + return s.path +} + +// Close releases all resources associated with this segment +func (s *Segment) Close() (err error) { + return s.DecRef() +} + +func (s *Segment) closeActual() (err error) { + if s.mm != nil { + err = s.mm.Unmap() + } + // try to close file even if unmap failed + if s.f != nil { + err2 := s.f.Close() + if err == nil { + // try to return first error + err = err2 + } + } + return +} + +// some helpers i started adding for the command-line utility + +// Data returns the underlying mmaped data slice +func (s *Segment) Data() []byte { + return s.mm +} + +// CRC returns the CRC value stored in the file footer +func (s *Segment) CRC() uint32 { + return s.crc +} + +// Version returns the file version in the file footer +func (s *Segment) Version() uint32 { + return s.version +} + +// ChunkFactor returns the chunk factor in the file footer +func (s *Segment) ChunkMode() uint32 { + return s.chunkMode +} + +// FieldsIndexOffset returns the fields index offset in the file footer +func (s *Segment) FieldsIndexOffset() uint64 { + return s.fieldsIndexOffset +} + +// StoredIndexOffset returns the stored value index offset in the file footer +func (s *Segment) StoredIndexOffset() uint64 { + return s.storedIndexOffset +} + +// DocValueOffset returns the docValue offset in the file footer +func (s *Segment) DocValueOffset() uint64 { + return s.docValueOffset +} + +// NumDocs returns the number of documents in the file footer +func (s *Segment) NumDocs() uint64 { + return s.numDocs +} + +// DictAddr is a helper function to compute the file offset where the +// dictionary is stored for the specified field. +func (s *Segment) DictAddr(field string) (uint64, error) { + fieldIDPlus1, ok := s.fieldsMap[field] + if !ok { + return 0, fmt.Errorf("no such field '%s'", field) + } + + return s.dictLocs[fieldIDPlus1-1], nil +} + +func (s *SegmentBase) loadDvReaders() error { + if s.docValueOffset == fieldNotUninverted || s.numDocs == 0 { + return nil + } + + var read uint64 + for fieldID, field := range s.fieldsInv { + var fieldLocStart, fieldLocEnd uint64 + var n int + fieldLocStart, n = binary.Uvarint(s.mem[s.docValueOffset+read : s.docValueOffset+read+binary.MaxVarintLen64]) + if n <= 0 { + return fmt.Errorf("loadDvReaders: failed to read the docvalue offset start for field %d", fieldID) + } + read += uint64(n) + fieldLocEnd, n = binary.Uvarint(s.mem[s.docValueOffset+read : s.docValueOffset+read+binary.MaxVarintLen64]) + if n <= 0 { + return fmt.Errorf("loadDvReaders: failed to read the docvalue offset end for field %d", fieldID) + } + read += uint64(n) + + s.incrementBytesRead(read) + fieldDvReader, err := s.loadFieldDocValueReader(field, fieldLocStart, fieldLocEnd) + if err != nil { + return err + } + if fieldDvReader != nil { + s.fieldDvReaders[uint16(fieldID)] = fieldDvReader + s.fieldDvNames = append(s.fieldDvNames, field) + } + } + + return nil +} diff --git a/vendor/github.com/blevesearch/zapx/v15/sizes.go b/vendor/github.com/blevesearch/zapx/v15/sizes.go new file mode 100644 index 00000000..34166ea3 --- /dev/null +++ b/vendor/github.com/blevesearch/zapx/v15/sizes.go @@ -0,0 +1,59 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "reflect" +) + +func init() { + var b bool + SizeOfBool = int(reflect.TypeOf(b).Size()) + var f32 float32 + SizeOfFloat32 = int(reflect.TypeOf(f32).Size()) + var f64 float64 + SizeOfFloat64 = int(reflect.TypeOf(f64).Size()) + var i int + SizeOfInt = int(reflect.TypeOf(i).Size()) + var m map[int]int + SizeOfMap = int(reflect.TypeOf(m).Size()) + var ptr *int + SizeOfPtr = int(reflect.TypeOf(ptr).Size()) + var slice []int + SizeOfSlice = int(reflect.TypeOf(slice).Size()) + var str string + SizeOfString = int(reflect.TypeOf(str).Size()) + var u8 uint8 + SizeOfUint8 = int(reflect.TypeOf(u8).Size()) + var u16 uint16 + SizeOfUint16 = int(reflect.TypeOf(u16).Size()) + var u32 uint32 + SizeOfUint32 = int(reflect.TypeOf(u32).Size()) + var u64 uint64 + SizeOfUint64 = int(reflect.TypeOf(u64).Size()) +} + +var SizeOfBool int +var SizeOfFloat32 int +var SizeOfFloat64 int +var SizeOfInt int +var SizeOfMap int +var SizeOfPtr int +var SizeOfSlice int +var SizeOfString int +var SizeOfUint8 int +var SizeOfUint16 int +var SizeOfUint32 int +var SizeOfUint64 int diff --git a/vendor/github.com/blevesearch/zap/v15/write.go b/vendor/github.com/blevesearch/zapx/v15/write.go similarity index 100% rename from vendor/github.com/blevesearch/zap/v15/write.go rename to vendor/github.com/blevesearch/zapx/v15/write.go diff --git a/vendor/github.com/blevesearch/zap/v15/zap.md b/vendor/github.com/blevesearch/zapx/v15/zap.md similarity index 100% rename from vendor/github.com/blevesearch/zap/v15/zap.md rename to vendor/github.com/blevesearch/zapx/v15/zap.md diff --git a/vendor/github.com/couchbase/vellum/.travis.yml b/vendor/github.com/couchbase/vellum/.travis.yml deleted file mode 100644 index 229edf28..00000000 --- a/vendor/github.com/couchbase/vellum/.travis.yml +++ /dev/null @@ -1,22 +0,0 @@ -sudo: false - -language: go - -go: - - "1.9.x" - - "1.10.x" - - "1.11.x" - -script: - - go get github.com/mattn/goveralls - - go get -u github.com/kisielk/errcheck - - go test -v $(go list ./... | grep -v vendor/) - - go test -race - - go vet - - errcheck - - go test -coverprofile=profile.out -covermode=count - - 'if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then goveralls -service=travis-ci -coverprofile=profile.out -repotoken $COVERALLS; fi' - -notifications: - email: - - marty.schoch@gmail.com diff --git a/vendor/github.com/couchbase/vellum/builder.go b/vendor/github.com/couchbase/vellum/builder.go deleted file mode 100644 index f7933295..00000000 --- a/vendor/github.com/couchbase/vellum/builder.go +++ /dev/null @@ -1,452 +0,0 @@ -// Copyright (c) 2017 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package vellum - -import ( - "bytes" - "io" -) - -var defaultBuilderOpts = &BuilderOpts{ - Encoder: 1, - RegistryTableSize: 10000, - RegistryMRUSize: 2, -} - -// A Builder is used to build a new FST. When possible data is -// streamed out to the underlying Writer as soon as possible. -type Builder struct { - unfinished *unfinishedNodes - registry *registry - last []byte - len int - - lastAddr int - - encoder encoder - opts *BuilderOpts - - builderNodePool *builderNodePool -} - -const noneAddr = 1 -const emptyAddr = 0 - -// NewBuilder returns a new Builder which will stream out the -// underlying representation to the provided Writer as the set is built. -func newBuilder(w io.Writer, opts *BuilderOpts) (*Builder, error) { - if opts == nil { - opts = defaultBuilderOpts - } - builderNodePool := &builderNodePool{} - rv := &Builder{ - unfinished: newUnfinishedNodes(builderNodePool), - registry: newRegistry(builderNodePool, opts.RegistryTableSize, opts.RegistryMRUSize), - builderNodePool: builderNodePool, - opts: opts, - lastAddr: noneAddr, - } - - var err error - rv.encoder, err = loadEncoder(opts.Encoder, w) - if err != nil { - return nil, err - } - err = rv.encoder.start() - if err != nil { - return nil, err - } - return rv, nil -} - -func (b *Builder) Reset(w io.Writer) error { - b.unfinished.Reset() - b.registry.Reset() - b.lastAddr = noneAddr - b.encoder.reset(w) - b.last = nil - b.len = 0 - - err := b.encoder.start() - if err != nil { - return err - } - return nil -} - -// Insert the provided value to the set being built. -// NOTE: values must be inserted in lexicographical order. -func (b *Builder) Insert(key []byte, val uint64) error { - // ensure items are added in lexicographic order - if bytes.Compare(key, b.last) < 0 { - return ErrOutOfOrder - } - if len(key) == 0 { - b.len = 1 - b.unfinished.setRootOutput(val) - return nil - } - - prefixLen, out := b.unfinished.findCommonPrefixAndSetOutput(key, val) - b.len++ - err := b.compileFrom(prefixLen) - if err != nil { - return err - } - b.copyLastKey(key) - b.unfinished.addSuffix(key[prefixLen:], out) - - return nil -} - -func (b *Builder) copyLastKey(key []byte) { - if b.last == nil { - b.last = make([]byte, 0, 64) - } else { - b.last = b.last[:0] - } - b.last = append(b.last, key...) -} - -// Close MUST be called after inserting all values. -func (b *Builder) Close() error { - err := b.compileFrom(0) - if err != nil { - return err - } - root := b.unfinished.popRoot() - rootAddr, err := b.compile(root) - if err != nil { - return err - } - return b.encoder.finish(b.len, rootAddr) -} - -func (b *Builder) compileFrom(iState int) error { - addr := noneAddr - for iState+1 < len(b.unfinished.stack) { - var node *builderNode - if addr == noneAddr { - node = b.unfinished.popEmpty() - } else { - node = b.unfinished.popFreeze(addr) - } - var err error - addr, err = b.compile(node) - if err != nil { - return nil - } - } - b.unfinished.topLastFreeze(addr) - return nil -} - -func (b *Builder) compile(node *builderNode) (int, error) { - if node.final && len(node.trans) == 0 && - node.finalOutput == 0 { - return 0, nil - } - found, addr, entry := b.registry.entry(node) - if found { - return addr, nil - } - addr, err := b.encoder.encodeState(node, b.lastAddr) - if err != nil { - return 0, err - } - - b.lastAddr = addr - entry.addr = addr - return addr, nil -} - -type unfinishedNodes struct { - stack []*builderNodeUnfinished - - // cache allocates a reasonable number of builderNodeUnfinished - // objects up front and tries to keep reusing them - // because the main data structure is a stack, we assume the - // same access pattern, and don't track items separately - // this means calls get() and pushXYZ() must be paired, - // as well as calls put() and popXYZ() - cache []builderNodeUnfinished - - builderNodePool *builderNodePool -} - -func (u *unfinishedNodes) Reset() { - u.stack = u.stack[:0] - for i := 0; i < len(u.cache); i++ { - u.cache[i] = builderNodeUnfinished{} - } - u.pushEmpty(false) -} - -func newUnfinishedNodes(p *builderNodePool) *unfinishedNodes { - rv := &unfinishedNodes{ - stack: make([]*builderNodeUnfinished, 0, 64), - cache: make([]builderNodeUnfinished, 64), - builderNodePool: p, - } - rv.pushEmpty(false) - return rv -} - -// get new builderNodeUnfinished, reusing cache if possible -func (u *unfinishedNodes) get() *builderNodeUnfinished { - if len(u.stack) < len(u.cache) { - return &u.cache[len(u.stack)] - } - // full now allocate a new one - return &builderNodeUnfinished{} -} - -// return builderNodeUnfinished, clearing it for reuse -func (u *unfinishedNodes) put() { - if len(u.stack) >= len(u.cache) { - return - // do nothing, not part of cache - } - u.cache[len(u.stack)] = builderNodeUnfinished{} -} - -func (u *unfinishedNodes) findCommonPrefixAndSetOutput(key []byte, - out uint64) (int, uint64) { - var i int - for i < len(key) { - if i >= len(u.stack) { - break - } - var addPrefix uint64 - if !u.stack[i].hasLastT { - break - } - if u.stack[i].lastIn == key[i] { - commonPre := outputPrefix(u.stack[i].lastOut, out) - addPrefix = outputSub(u.stack[i].lastOut, commonPre) - out = outputSub(out, commonPre) - u.stack[i].lastOut = commonPre - i++ - } else { - break - } - - if addPrefix != 0 { - u.stack[i].addOutputPrefix(addPrefix) - } - } - - return i, out -} - -func (u *unfinishedNodes) pushEmpty(final bool) { - next := u.get() - next.node = u.builderNodePool.Get() - next.node.final = final - u.stack = append(u.stack, next) -} - -func (u *unfinishedNodes) popRoot() *builderNode { - l := len(u.stack) - var unfinished *builderNodeUnfinished - u.stack, unfinished = u.stack[:l-1], u.stack[l-1] - rv := unfinished.node - u.put() - return rv -} - -func (u *unfinishedNodes) popFreeze(addr int) *builderNode { - l := len(u.stack) - var unfinished *builderNodeUnfinished - u.stack, unfinished = u.stack[:l-1], u.stack[l-1] - unfinished.lastCompiled(addr) - rv := unfinished.node - u.put() - return rv -} - -func (u *unfinishedNodes) popEmpty() *builderNode { - l := len(u.stack) - var unfinished *builderNodeUnfinished - u.stack, unfinished = u.stack[:l-1], u.stack[l-1] - rv := unfinished.node - u.put() - return rv -} - -func (u *unfinishedNodes) setRootOutput(out uint64) { - u.stack[0].node.final = true - u.stack[0].node.finalOutput = out -} - -func (u *unfinishedNodes) topLastFreeze(addr int) { - last := len(u.stack) - 1 - u.stack[last].lastCompiled(addr) -} - -func (u *unfinishedNodes) addSuffix(bs []byte, out uint64) { - if len(bs) == 0 { - return - } - last := len(u.stack) - 1 - u.stack[last].hasLastT = true - u.stack[last].lastIn = bs[0] - u.stack[last].lastOut = out - for _, b := range bs[1:] { - next := u.get() - next.node = u.builderNodePool.Get() - next.hasLastT = true - next.lastIn = b - next.lastOut = 0 - u.stack = append(u.stack, next) - } - u.pushEmpty(true) -} - -type builderNodeUnfinished struct { - node *builderNode - lastOut uint64 - lastIn byte - hasLastT bool -} - -func (b *builderNodeUnfinished) lastCompiled(addr int) { - if b.hasLastT { - transIn := b.lastIn - transOut := b.lastOut - b.hasLastT = false - b.lastOut = 0 - b.node.trans = append(b.node.trans, transition{ - in: transIn, - out: transOut, - addr: addr, - }) - } -} - -func (b *builderNodeUnfinished) addOutputPrefix(prefix uint64) { - if b.node.final { - b.node.finalOutput = outputCat(prefix, b.node.finalOutput) - } - for i := range b.node.trans { - b.node.trans[i].out = outputCat(prefix, b.node.trans[i].out) - } - if b.hasLastT { - b.lastOut = outputCat(prefix, b.lastOut) - } -} - -type builderNode struct { - finalOutput uint64 - trans []transition - final bool - - // intrusive linked list - next *builderNode -} - -// reset resets the receiver builderNode to a re-usable state. -func (n *builderNode) reset() { - n.final = false - n.finalOutput = 0 - for i := range n.trans { - n.trans[i] = emptyTransition - } - n.trans = n.trans[:0] - n.next = nil -} - -func (n *builderNode) equiv(o *builderNode) bool { - if n.final != o.final { - return false - } - if n.finalOutput != o.finalOutput { - return false - } - if len(n.trans) != len(o.trans) { - return false - } - for i, ntrans := range n.trans { - otrans := o.trans[i] - if ntrans.in != otrans.in { - return false - } - if ntrans.addr != otrans.addr { - return false - } - if ntrans.out != otrans.out { - return false - } - } - return true -} - -var emptyTransition = transition{} - -type transition struct { - out uint64 - addr int - in byte -} - -func outputPrefix(l, r uint64) uint64 { - if l < r { - return l - } - return r -} - -func outputSub(l, r uint64) uint64 { - return l - r -} - -func outputCat(l, r uint64) uint64 { - return l + r -} - -// builderNodePool pools builderNodes using a singly linked list. -// -// NB: builderNode lifecylce is described by the following interactions - -// +------------------------+ +----------------------+ -// | Unfinished Nodes | Transfer once | Registry | -// |(not frozen builderNode)|-----builderNode is ------->| (frozen builderNode) | -// +------------------------+ marked frozen +----------------------+ -// ^ | -// | | -// | Put() -// | Get() on +-------------------+ when -// +-new char--------| builderNode Pool |<-----------evicted -// +-------------------+ -type builderNodePool struct { - head *builderNode -} - -func (p *builderNodePool) Get() *builderNode { - if p.head == nil { - return &builderNode{} - } - head := p.head - p.head = p.head.next - return head -} - -func (p *builderNodePool) Put(v *builderNode) { - if v == nil { - return - } - v.reset() - v.next = p.head - p.head = v -} diff --git a/vendor/github.com/dgraph-io/badger/v2/dir_plan9.go b/vendor/github.com/dgraph-io/badger/v2/dir_plan9.go new file mode 100644 index 00000000..ad323d70 --- /dev/null +++ b/vendor/github.com/dgraph-io/badger/v2/dir_plan9.go @@ -0,0 +1,150 @@ +/* + * Copyright 2020 Dgraph Labs, Inc. and Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package badger + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/pkg/errors" +) + +// directoryLockGuard holds a lock on a directory and a pid file inside. The pid file isn't part +// of the locking mechanism, it's just advisory. +type directoryLockGuard struct { + // File handle on the directory, which we've locked. + f *os.File + // The absolute path to our pid file. + path string +} + +// acquireDirectoryLock gets a lock on the directory. +// It will also write our pid to dirPath/pidFileName for convenience. +// readOnly is not supported on Plan 9. +func acquireDirectoryLock(dirPath string, pidFileName string, readOnly bool) ( + *directoryLockGuard, error) { + if readOnly { + return nil, ErrPlan9NotSupported + } + + // Convert to absolute path so that Release still works even if we do an unbalanced + // chdir in the meantime. + absPidFilePath, err := filepath.Abs(filepath.Join(dirPath, pidFileName)) + if err != nil { + return nil, errors.Wrap(err, "cannot get absolute path for pid lock file") + } + + // If the file was unpacked or created by some other program, it might not + // have the ModeExclusive bit set. Set it before we call OpenFile, so that we + // can be confident that a successful OpenFile implies exclusive use. + // + // OpenFile fails if the file ModeExclusive bit set *and* the file is already open. + // So, if the file is closed when the DB crashed, we're fine. When the process + // that was managing the DB crashes, the OS will close the file for us. + // + // This bit of code is copied from Go's lockedfile internal package: + // https://github.com/golang/go/blob/go1.15rc1/src/cmd/go/internal/lockedfile/lockedfile_plan9.go#L58 + if fi, err := os.Stat(absPidFilePath); err == nil { + if fi.Mode()&os.ModeExclusive == 0 { + if err := os.Chmod(absPidFilePath, fi.Mode()|os.ModeExclusive); err != nil { + return nil, errors.Wrapf(err, "could not set exclusive mode bit") + } + } + } else if !os.IsNotExist(err) { + return nil, err + } + f, err := os.OpenFile(absPidFilePath, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0666|os.ModeExclusive) + if err != nil { + if isLocked(err) { + return nil, errors.Wrapf(err, + "Cannot open pid lock file %q. Another process is using this Badger database", + absPidFilePath) + } + return nil, errors.Wrapf(err, "Cannot open pid lock file %q", absPidFilePath) + } + + if _, err = fmt.Fprintf(f, "%d\n", os.Getpid()); err != nil { + f.Close() + return nil, errors.Wrapf(err, "could not write pid") + } + return &directoryLockGuard{f, absPidFilePath}, nil +} + +// Release deletes the pid file and releases our lock on the directory. +func (guard *directoryLockGuard) release() error { + // It's important that we remove the pid file first. + err := os.Remove(guard.path) + + if closeErr := guard.f.Close(); err == nil { + err = closeErr + } + guard.path = "" + guard.f = nil + + return err +} + +// openDir opens a directory for syncing. +func openDir(path string) (*os.File, error) { return os.Open(path) } + +// When you create or delete a file, you have to ensure the directory entry for the file is synced +// in order to guarantee the file is visible (if the system crashes). (See the man page for fsync, +// or see https://github.com/coreos/etcd/issues/6368 for an example.) +func syncDir(dir string) error { + f, err := openDir(dir) + if err != nil { + return errors.Wrapf(err, "While opening directory: %s.", dir) + } + + err = f.Sync() + closeErr := f.Close() + if err != nil { + return errors.Wrapf(err, "While syncing directory: %s.", dir) + } + return errors.Wrapf(closeErr, "While closing directory: %s.", dir) +} + +// Opening an exclusive-use file returns an error. +// The expected error strings are: +// +// - "open/create -- file is locked" (cwfs, kfs) +// - "exclusive lock" (fossil) +// - "exclusive use file already open" (ramfs) +// +// See https://github.com/golang/go/blob/go1.15rc1/src/cmd/go/internal/lockedfile/lockedfile_plan9.go#L16 +var lockedErrStrings = [...]string{ + "file is locked", + "exclusive lock", + "exclusive use file already open", +} + +// Even though plan9 doesn't support the Lock/RLock/Unlock functions to +// manipulate already-open files, IsLocked is still meaningful: os.OpenFile +// itself may return errors that indicate that a file with the ModeExclusive bit +// set is already open. +func isLocked(err error) bool { + s := err.Error() + + for _, frag := range lockedErrStrings { + if strings.Contains(s, frag) { + return true + } + } + return false +} diff --git a/vendor/github.com/dgraph-io/badger/v2/y/mmap_plan9.go b/vendor/github.com/dgraph-io/badger/v2/y/mmap_plan9.go new file mode 100644 index 00000000..21db76bf --- /dev/null +++ b/vendor/github.com/dgraph-io/badger/v2/y/mmap_plan9.go @@ -0,0 +1,40 @@ +/* + * Copyright 2020 Dgraph Labs, Inc. and Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package y + +import ( + "os" + "syscall" +) + +// Mmap uses the mmap system call to memory-map a file. If writable is true, +// memory protection of the pages is set so that they may be written to as well. +func mmap(fd *os.File, writable bool, size int64) ([]byte, error) { + return nil, syscall.EPLAN9 +} + +// Munmap unmaps a previously mapped slice. +func munmap(b []byte) error { + return syscall.EPLAN9 +} + +// Madvise uses the madvise system call to give advise about the use of memory +// when using a slice that is memory-mapped to a file. Set the readahead flag to +// false if page references are expected in random order. +func madvise(b []byte, readahead bool) error { + return syscall.EPLAN9 +} diff --git a/vendor/github.com/dgraph-io/badger/v2/y/zstd.go b/vendor/github.com/dgraph-io/badger/v2/y/zstd.go new file mode 100644 index 00000000..57018680 --- /dev/null +++ b/vendor/github.com/dgraph-io/badger/v2/y/zstd.go @@ -0,0 +1,64 @@ +/* + * Copyright 2019 Dgraph Labs, Inc. and Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package y + +import ( + "sync" + + "github.com/klauspost/compress/zstd" +) + +var ( + decoder *zstd.Decoder + encoder *zstd.Encoder + + encOnce, decOnce sync.Once +) + +// ZSTDDecompress decompresses a block using ZSTD algorithm. +func ZSTDDecompress(dst, src []byte) ([]byte, error) { + decOnce.Do(func() { + var err error + decoder, err = zstd.NewReader(nil) + Check(err) + }) + return decoder.DecodeAll(src, dst[:0]) +} + +// ZSTDCompress compresses a block using ZSTD algorithm. +func ZSTDCompress(dst, src []byte, compressionLevel int) ([]byte, error) { + encOnce.Do(func() { + var err error + level := zstd.EncoderLevelFromZstd(compressionLevel) + encoder, err = zstd.NewWriter(nil, zstd.WithEncoderLevel(level)) + Check(err) + }) + return encoder.EncodeAll(src, dst[:0]), nil +} + +// ZSTDCompressBound returns the worst case size needed for a destination buffer. +// Klauspost ZSTD library does not provide any API for Compression Bound. This +// calculation is based on the DataDog ZSTD library. +// See https://pkg.go.dev/github.com/DataDog/zstd#CompressBound +func ZSTDCompressBound(srcSize int) int { + lowLimit := 128 << 10 // 128 kB + var margin int + if srcSize < lowLimit { + margin = (lowLimit - srcSize) >> 11 + } + return srcSize + (srcSize >> 8) + margin +} diff --git a/vendor/github.com/fatih/color/color_windows.go b/vendor/github.com/fatih/color/color_windows.go new file mode 100644 index 00000000..be01c558 --- /dev/null +++ b/vendor/github.com/fatih/color/color_windows.go @@ -0,0 +1,19 @@ +package color + +import ( + "os" + + "golang.org/x/sys/windows" +) + +func init() { + // Opt-in for ansi color support for current process. + // https://learn.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences#output-sequences + var outMode uint32 + out := windows.Handle(os.Stdout.Fd()) + if err := windows.GetConsoleMode(out, &outMode); err != nil { + return + } + outMode |= windows.ENABLE_PROCESSED_OUTPUT | windows.ENABLE_VIRTUAL_TERMINAL_PROCESSING + _ = windows.SetConsoleMode(out, outMode) +} diff --git a/vendor/github.com/fsnotify/fsnotify/.cirrus.yml b/vendor/github.com/fsnotify/fsnotify/.cirrus.yml new file mode 100644 index 00000000..ffc7b992 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/.cirrus.yml @@ -0,0 +1,13 @@ +freebsd_task: + name: 'FreeBSD' + freebsd_instance: + image_family: freebsd-13-2 + install_script: + - pkg update -f + - pkg install -y go + test_script: + # run tests as user "cirrus" instead of root + - pw useradd cirrus -m + - chown -R cirrus:cirrus . + - FSNOTIFY_BUFFER=4096 sudo --preserve-env=FSNOTIFY_BUFFER -u cirrus go test -parallel 1 -race ./... + - sudo --preserve-env=FSNOTIFY_BUFFER -u cirrus go test -parallel 1 -race ./... diff --git a/vendor/github.com/fsnotify/fsnotify/.gitignore b/vendor/github.com/fsnotify/fsnotify/.gitignore index 4cd0cbaf..391cc076 100644 --- a/vendor/github.com/fsnotify/fsnotify/.gitignore +++ b/vendor/github.com/fsnotify/fsnotify/.gitignore @@ -1,6 +1,7 @@ -# Setup a Global .gitignore for OS and editor generated files: -# https://help.github.com/articles/ignoring-files -# git config --global core.excludesfile ~/.gitignore_global +# go test -c output +*.test +*.test.exe -.vagrant -*.sublime-project +# Output of go build ./cmd/fsnotify +/fsnotify +/fsnotify.exe diff --git a/vendor/github.com/fsnotify/fsnotify/.mailmap b/vendor/github.com/fsnotify/fsnotify/.mailmap new file mode 100644 index 00000000..a04f2907 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/.mailmap @@ -0,0 +1,2 @@ +Chris Howey +Nathan Youngman <4566+nathany@users.noreply.github.com> diff --git a/vendor/github.com/fsnotify/fsnotify/.travis.yml b/vendor/github.com/fsnotify/fsnotify/.travis.yml deleted file mode 100644 index a9c30165..00000000 --- a/vendor/github.com/fsnotify/fsnotify/.travis.yml +++ /dev/null @@ -1,36 +0,0 @@ -sudo: false -language: go - -go: - - "stable" - - "1.11.x" - - "1.10.x" - - "1.9.x" - -matrix: - include: - - go: "stable" - env: GOLINT=true - allow_failures: - - go: tip - fast_finish: true - - -before_install: - - if [ ! -z "${GOLINT}" ]; then go get -u golang.org/x/lint/golint; fi - -script: - - go test --race ./... - -after_script: - - test -z "$(gofmt -s -l -w . | tee /dev/stderr)" - - if [ ! -z "${GOLINT}" ]; then echo running golint; golint --set_exit_status ./...; else echo skipping golint; fi - - go vet ./... - -os: - - linux - - osx - - windows - -notifications: - email: false diff --git a/vendor/github.com/fsnotify/fsnotify/AUTHORS b/vendor/github.com/fsnotify/fsnotify/AUTHORS deleted file mode 100644 index 5ab5d41c..00000000 --- a/vendor/github.com/fsnotify/fsnotify/AUTHORS +++ /dev/null @@ -1,52 +0,0 @@ -# Names should be added to this file as -# Name or Organization -# The email address is not required for organizations. - -# You can update this list using the following command: -# -# $ git shortlog -se | awk '{print $2 " " $3 " " $4}' - -# Please keep the list sorted. - -Aaron L -Adrien Bustany -Amit Krishnan -Anmol Sethi -Bjørn Erik Pedersen -Bruno Bigras -Caleb Spare -Case Nelson -Chris Howey -Christoffer Buchholz -Daniel Wagner-Hall -Dave Cheney -Evan Phoenix -Francisco Souza -Hari haran -John C Barstow -Kelvin Fo -Ken-ichirou MATSUZAWA -Matt Layher -Nathan Youngman -Nickolai Zeldovich -Patrick -Paul Hammond -Pawel Knap -Pieter Droogendijk -Pursuit92 -Riku Voipio -Rob Figueiredo -Rodrigo Chiossi -Slawek Ligus -Soge Zhang -Tiffany Jernigan -Tilak Sharma -Tom Payne -Travis Cline -Tudor Golubenco -Vahe Khachikyan -Yukang -bronze1man -debrando -henrikedwards -铁哥 diff --git a/vendor/github.com/fsnotify/fsnotify/CHANGELOG.md b/vendor/github.com/fsnotify/fsnotify/CHANGELOG.md index be4d7ea2..e0e57575 100644 --- a/vendor/github.com/fsnotify/fsnotify/CHANGELOG.md +++ b/vendor/github.com/fsnotify/fsnotify/CHANGELOG.md @@ -1,6 +1,230 @@ # Changelog -## v1.4.7 / 2018-01-09 +Unreleased +---------- +Nothing yet. + +1.7.0 - 2023-10-22 +------------------ +This version of fsnotify needs Go 1.17. + +### Additions + +- illumos: add FEN backend to support illumos and Solaris. ([#371]) + +- all: add `NewBufferedWatcher()` to use a buffered channel, which can be useful + in cases where you can't control the kernel buffer and receive a large number + of events in bursts. ([#550], [#572]) + +- all: add `AddWith()`, which is identical to `Add()` but allows passing + options. ([#521]) + +- windows: allow setting the ReadDirectoryChangesW() buffer size with + `fsnotify.WithBufferSize()`; the default of 64K is the highest value that + works on all platforms and is enough for most purposes, but in some cases a + highest buffer is needed. ([#521]) + +### Changes and fixes + +- inotify: remove watcher if a watched path is renamed ([#518]) + + After a rename the reported name wasn't updated, or even an empty string. + Inotify doesn't provide any good facilities to update it, so just remove the + watcher. This is already how it worked on kqueue and FEN. + + On Windows this does work, and remains working. + +- windows: don't listen for file attribute changes ([#520]) + + File attribute changes are sent as `FILE_ACTION_MODIFIED` by the Windows API, + with no way to see if they're a file write or attribute change, so would show + up as a fsnotify.Write event. This is never useful, and could result in many + spurious Write events. + +- windows: return `ErrEventOverflow` if the buffer is full ([#525]) + + Before it would merely return "short read", making it hard to detect this + error. + +- kqueue: make sure events for all files are delivered properly when removing a + watched directory ([#526]) + + Previously they would get sent with `""` (empty string) or `"."` as the path + name. + +- kqueue: don't emit spurious Create events for symbolic links ([#524]) + + The link would get resolved but kqueue would "forget" it already saw the link + itself, resulting on a Create for every Write event for the directory. + +- all: return `ErrClosed` on `Add()` when the watcher is closed ([#516]) + +- other: add `Watcher.Errors` and `Watcher.Events` to the no-op `Watcher` in + `backend_other.go`, making it easier to use on unsupported platforms such as + WASM, AIX, etc. ([#528]) + +- other: use the `backend_other.go` no-op if the `appengine` build tag is set; + Google AppEngine forbids usage of the unsafe package so the inotify backend + won't compile there. + +[#371]: https://github.com/fsnotify/fsnotify/pull/371 +[#516]: https://github.com/fsnotify/fsnotify/pull/516 +[#518]: https://github.com/fsnotify/fsnotify/pull/518 +[#520]: https://github.com/fsnotify/fsnotify/pull/520 +[#521]: https://github.com/fsnotify/fsnotify/pull/521 +[#524]: https://github.com/fsnotify/fsnotify/pull/524 +[#525]: https://github.com/fsnotify/fsnotify/pull/525 +[#526]: https://github.com/fsnotify/fsnotify/pull/526 +[#528]: https://github.com/fsnotify/fsnotify/pull/528 +[#537]: https://github.com/fsnotify/fsnotify/pull/537 +[#550]: https://github.com/fsnotify/fsnotify/pull/550 +[#572]: https://github.com/fsnotify/fsnotify/pull/572 + +1.6.0 - 2022-10-13 +------------------ +This version of fsnotify needs Go 1.16 (this was already the case since 1.5.1, +but not documented). It also increases the minimum Linux version to 2.6.32. + +### Additions + +- all: add `Event.Has()` and `Op.Has()` ([#477]) + + This makes checking events a lot easier; for example: + + if event.Op&Write == Write && !(event.Op&Remove == Remove) { + } + + Becomes: + + if event.Has(Write) && !event.Has(Remove) { + } + +- all: add cmd/fsnotify ([#463]) + + A command-line utility for testing and some examples. + +### Changes and fixes + +- inotify: don't ignore events for files that don't exist ([#260], [#470]) + + Previously the inotify watcher would call `os.Lstat()` to check if a file + still exists before emitting events. + + This was inconsistent with other platforms and resulted in inconsistent event + reporting (e.g. when a file is quickly removed and re-created), and generally + a source of confusion. It was added in 2013 to fix a memory leak that no + longer exists. + +- all: return `ErrNonExistentWatch` when `Remove()` is called on a path that's + not watched ([#460]) + +- inotify: replace epoll() with non-blocking inotify ([#434]) + + Non-blocking inotify was not generally available at the time this library was + written in 2014, but now it is. As a result, the minimum Linux version is + bumped from 2.6.27 to 2.6.32. This hugely simplifies the code and is faster. + +- kqueue: don't check for events every 100ms ([#480]) + + The watcher would wake up every 100ms, even when there was nothing to do. Now + it waits until there is something to do. + +- macos: retry opening files on EINTR ([#475]) + +- kqueue: skip unreadable files ([#479]) + + kqueue requires a file descriptor for every file in a directory; this would + fail if a file was unreadable by the current user. Now these files are simply + skipped. + +- windows: fix renaming a watched directory if the parent is also watched ([#370]) + +- windows: increase buffer size from 4K to 64K ([#485]) + +- windows: close file handle on Remove() ([#288]) + +- kqueue: put pathname in the error if watching a file fails ([#471]) + +- inotify, windows: calling Close() more than once could race ([#465]) + +- kqueue: improve Close() performance ([#233]) + +- all: various documentation additions and clarifications. + +[#233]: https://github.com/fsnotify/fsnotify/pull/233 +[#260]: https://github.com/fsnotify/fsnotify/pull/260 +[#288]: https://github.com/fsnotify/fsnotify/pull/288 +[#370]: https://github.com/fsnotify/fsnotify/pull/370 +[#434]: https://github.com/fsnotify/fsnotify/pull/434 +[#460]: https://github.com/fsnotify/fsnotify/pull/460 +[#463]: https://github.com/fsnotify/fsnotify/pull/463 +[#465]: https://github.com/fsnotify/fsnotify/pull/465 +[#470]: https://github.com/fsnotify/fsnotify/pull/470 +[#471]: https://github.com/fsnotify/fsnotify/pull/471 +[#475]: https://github.com/fsnotify/fsnotify/pull/475 +[#477]: https://github.com/fsnotify/fsnotify/pull/477 +[#479]: https://github.com/fsnotify/fsnotify/pull/479 +[#480]: https://github.com/fsnotify/fsnotify/pull/480 +[#485]: https://github.com/fsnotify/fsnotify/pull/485 + +## [1.5.4] - 2022-04-25 + +* Windows: add missing defer to `Watcher.WatchList` [#447](https://github.com/fsnotify/fsnotify/pull/447) +* go.mod: use latest x/sys [#444](https://github.com/fsnotify/fsnotify/pull/444) +* Fix compilation for OpenBSD [#443](https://github.com/fsnotify/fsnotify/pull/443) + +## [1.5.3] - 2022-04-22 + +* This version is retracted. An incorrect branch is published accidentally [#445](https://github.com/fsnotify/fsnotify/issues/445) + +## [1.5.2] - 2022-04-21 + +* Add a feature to return the directories and files that are being monitored [#374](https://github.com/fsnotify/fsnotify/pull/374) +* Fix potential crash on windows if `raw.FileNameLength` exceeds `syscall.MAX_PATH` [#361](https://github.com/fsnotify/fsnotify/pull/361) +* Allow build on unsupported GOOS [#424](https://github.com/fsnotify/fsnotify/pull/424) +* Don't set `poller.fd` twice in `newFdPoller` [#406](https://github.com/fsnotify/fsnotify/pull/406) +* fix go vet warnings: call to `(*T).Fatalf` from a non-test goroutine [#416](https://github.com/fsnotify/fsnotify/pull/416) + +## [1.5.1] - 2021-08-24 + +* Revert Add AddRaw to not follow symlinks [#394](https://github.com/fsnotify/fsnotify/pull/394) + +## [1.5.0] - 2021-08-20 + +* Go: Increase minimum required version to Go 1.12 [#381](https://github.com/fsnotify/fsnotify/pull/381) +* Feature: Add AddRaw method which does not follow symlinks when adding a watch [#289](https://github.com/fsnotify/fsnotify/pull/298) +* Windows: Follow symlinks by default like on all other systems [#289](https://github.com/fsnotify/fsnotify/pull/289) +* CI: Use GitHub Actions for CI and cover go 1.12-1.17 + [#378](https://github.com/fsnotify/fsnotify/pull/378) + [#381](https://github.com/fsnotify/fsnotify/pull/381) + [#385](https://github.com/fsnotify/fsnotify/pull/385) +* Go 1.14+: Fix unsafe pointer conversion [#325](https://github.com/fsnotify/fsnotify/pull/325) + +## [1.4.9] - 2020-03-11 + +* Move example usage to the readme #329. This may resolve #328. + +## [1.4.8] - 2020-03-10 + +* CI: test more go versions (@nathany 1d13583d846ea9d66dcabbfefbfb9d8e6fb05216) +* Tests: Queued inotify events could have been read by the test before max_queued_events was hit (@matthias-stone #265) +* Tests: t.Fatalf -> t.Errorf in go routines (@gdey #266) +* CI: Less verbosity (@nathany #267) +* Tests: Darwin: Exchangedata is deprecated on 10.13 (@nathany #267) +* Tests: Check if channels are closed in the example (@alexeykazakov #244) +* CI: Only run golint on latest version of go and fix issues (@cpuguy83 #284) +* CI: Add windows to travis matrix (@cpuguy83 #284) +* Docs: Remover appveyor badge (@nathany 11844c0959f6fff69ba325d097fce35bd85a8e93) +* Linux: create epoll and pipe fds with close-on-exec (@JohannesEbke #219) +* Linux: open files with close-on-exec (@linxiulei #273) +* Docs: Plan to support fanotify (@nathany ab058b44498e8b7566a799372a39d150d9ea0119 ) +* Project: Add go.mod (@nathany #309) +* Project: Revise editor config (@nathany #309) +* Project: Update copyright for 2019 (@nathany #309) +* CI: Drop go1.8 from CI matrix (@nathany #309) +* Docs: Updating the FAQ section for supportability with NFS & FUSE filesystems (@Pratik32 4bf2d1fec78374803a39307bfb8d340688f4f28e ) + +## [1.4.7] - 2018-01-09 * BSD/macOS: Fix possible deadlock on closing the watcher on kqueue (thanks @nhooyr and @glycerine) * Tests: Fix missing verb on format string (thanks @rchiossi) @@ -10,62 +234,62 @@ * Linux: Properly handle inotify's IN_Q_OVERFLOW event (thanks @zeldovich) * Docs: replace references to OS X with macOS -## v1.4.2 / 2016-10-10 +## [1.4.2] - 2016-10-10 * Linux: use InotifyInit1 with IN_CLOEXEC to stop leaking a file descriptor to a child process when using fork/exec [#178](https://github.com/fsnotify/fsnotify/pull/178) (thanks @pattyshack) -## v1.4.1 / 2016-10-04 +## [1.4.1] - 2016-10-04 * Fix flaky inotify stress test on Linux [#177](https://github.com/fsnotify/fsnotify/pull/177) (thanks @pattyshack) -## v1.4.0 / 2016-10-01 +## [1.4.0] - 2016-10-01 * add a String() method to Event.Op [#165](https://github.com/fsnotify/fsnotify/pull/165) (thanks @oozie) -## v1.3.1 / 2016-06-28 +## [1.3.1] - 2016-06-28 * Windows: fix for double backslash when watching the root of a drive [#151](https://github.com/fsnotify/fsnotify/issues/151) (thanks @brunoqc) -## v1.3.0 / 2016-04-19 +## [1.3.0] - 2016-04-19 * Support linux/arm64 by [patching](https://go-review.googlesource.com/#/c/21971/) x/sys/unix and switching to to it from syscall (thanks @suihkulokki) [#135](https://github.com/fsnotify/fsnotify/pull/135) -## v1.2.10 / 2016-03-02 +## [1.2.10] - 2016-03-02 * Fix golint errors in windows.go [#121](https://github.com/fsnotify/fsnotify/pull/121) (thanks @tiffanyfj) -## v1.2.9 / 2016-01-13 +## [1.2.9] - 2016-01-13 kqueue: Fix logic for CREATE after REMOVE [#111](https://github.com/fsnotify/fsnotify/pull/111) (thanks @bep) -## v1.2.8 / 2015-12-17 +## [1.2.8] - 2015-12-17 * kqueue: fix race condition in Close [#105](https://github.com/fsnotify/fsnotify/pull/105) (thanks @djui for reporting the issue and @ppknap for writing a failing test) * inotify: fix race in test * enable race detection for continuous integration (Linux, Mac, Windows) -## v1.2.5 / 2015-10-17 +## [1.2.5] - 2015-10-17 * inotify: use epoll_create1 for arm64 support (requires Linux 2.6.27 or later) [#100](https://github.com/fsnotify/fsnotify/pull/100) (thanks @suihkulokki) * inotify: fix path leaks [#73](https://github.com/fsnotify/fsnotify/pull/73) (thanks @chamaken) * kqueue: watch for rename events on subdirectories [#83](https://github.com/fsnotify/fsnotify/pull/83) (thanks @guotie) * kqueue: avoid infinite loops from symlinks cycles [#101](https://github.com/fsnotify/fsnotify/pull/101) (thanks @illicitonion) -## v1.2.1 / 2015-10-14 +## [1.2.1] - 2015-10-14 * kqueue: don't watch named pipes [#98](https://github.com/fsnotify/fsnotify/pull/98) (thanks @evanphx) -## v1.2.0 / 2015-02-08 +## [1.2.0] - 2015-02-08 * inotify: use epoll to wake up readEvents [#66](https://github.com/fsnotify/fsnotify/pull/66) (thanks @PieterD) * inotify: closing watcher should now always shut down goroutine [#63](https://github.com/fsnotify/fsnotify/pull/63) (thanks @PieterD) * kqueue: close kqueue after removing watches, fixes [#59](https://github.com/fsnotify/fsnotify/issues/59) -## v1.1.1 / 2015-02-05 +## [1.1.1] - 2015-02-05 * inotify: Retry read on EINTR [#61](https://github.com/fsnotify/fsnotify/issues/61) (thanks @PieterD) -## v1.1.0 / 2014-12-12 +## [1.1.0] - 2014-12-12 * kqueue: rework internals [#43](https://github.com/fsnotify/fsnotify/pull/43) * add low-level functions @@ -77,22 +301,22 @@ kqueue: Fix logic for CREATE after REMOVE [#111](https://github.com/fsnotify/fsn * kqueue: fix regression in rework causing subdirectories to be watched [#48](https://github.com/fsnotify/fsnotify/issues/48) * kqueue: cleanup internal watch before sending remove event [#51](https://github.com/fsnotify/fsnotify/issues/51) -## v1.0.4 / 2014-09-07 +## [1.0.4] - 2014-09-07 * kqueue: add dragonfly to the build tags. * Rename source code files, rearrange code so exported APIs are at the top. * Add done channel to example code. [#37](https://github.com/fsnotify/fsnotify/pull/37) (thanks @chenyukang) -## v1.0.3 / 2014-08-19 +## [1.0.3] - 2014-08-19 * [Fix] Windows MOVED_TO now translates to Create like on BSD and Linux. [#36](https://github.com/fsnotify/fsnotify/issues/36) -## v1.0.2 / 2014-08-17 +## [1.0.2] - 2014-08-17 * [Fix] Missing create events on macOS. [#14](https://github.com/fsnotify/fsnotify/issues/14) (thanks @zhsso) * [Fix] Make ./path and path equivalent. (thanks @zhsso) -## v1.0.0 / 2014-08-15 +## [1.0.0] - 2014-08-15 * [API] Remove AddWatch on Windows, use Add. * Improve documentation for exported identifiers. [#30](https://github.com/fsnotify/fsnotify/issues/30) @@ -146,51 +370,51 @@ kqueue: Fix logic for CREATE after REMOVE [#111](https://github.com/fsnotify/fsn * no tests for the current implementation * not fully implemented on Windows [#93](https://github.com/howeyc/fsnotify/issues/93#issuecomment-39285195) -## v0.9.3 / 2014-12-31 +## [0.9.3] - 2014-12-31 * kqueue: cleanup internal watch before sending remove event [#51](https://github.com/fsnotify/fsnotify/issues/51) -## v0.9.2 / 2014-08-17 +## [0.9.2] - 2014-08-17 * [Backport] Fix missing create events on macOS. [#14](https://github.com/fsnotify/fsnotify/issues/14) (thanks @zhsso) -## v0.9.1 / 2014-06-12 +## [0.9.1] - 2014-06-12 * Fix data race on kevent buffer (thanks @tilaks) [#98](https://github.com/howeyc/fsnotify/pull/98) -## v0.9.0 / 2014-01-17 +## [0.9.0] - 2014-01-17 * IsAttrib() for events that only concern a file's metadata [#79][] (thanks @abustany) * [Fix] kqueue: fix deadlock [#77][] (thanks @cespare) * [NOTICE] Development has moved to `code.google.com/p/go.exp/fsnotify` in preparation for inclusion in the Go standard library. -## v0.8.12 / 2013-11-13 +## [0.8.12] - 2013-11-13 * [API] Remove FD_SET and friends from Linux adapter -## v0.8.11 / 2013-11-02 +## [0.8.11] - 2013-11-02 * [Doc] Add Changelog [#72][] (thanks @nathany) * [Doc] Spotlight and double modify events on macOS [#62][] (reported by @paulhammond) -## v0.8.10 / 2013-10-19 +## [0.8.10] - 2013-10-19 * [Fix] kqueue: remove file watches when parent directory is removed [#71][] (reported by @mdwhatcott) * [Fix] kqueue: race between Close and readEvents [#70][] (reported by @bernerdschaefer) * [Doc] specify OS-specific limits in README (thanks @debrando) -## v0.8.9 / 2013-09-08 +## [0.8.9] - 2013-09-08 * [Doc] Contributing (thanks @nathany) * [Doc] update package path in example code [#63][] (thanks @paulhammond) * [Doc] GoCI badge in README (Linux only) [#60][] * [Doc] Cross-platform testing with Vagrant [#59][] (thanks @nathany) -## v0.8.8 / 2013-06-17 +## [0.8.8] - 2013-06-17 * [Fix] Windows: handle `ERROR_MORE_DATA` on Windows [#49][] (thanks @jbowtie) -## v0.8.7 / 2013-06-03 +## [0.8.7] - 2013-06-03 * [API] Make syscall flags internal * [Fix] inotify: ignore event changes @@ -198,74 +422,74 @@ kqueue: Fix logic for CREATE after REMOVE [#111](https://github.com/fsnotify/fsn * [Fix] tests on Windows * lower case error messages -## v0.8.6 / 2013-05-23 +## [0.8.6] - 2013-05-23 * kqueue: Use EVT_ONLY flag on Darwin * [Doc] Update README with full example -## v0.8.5 / 2013-05-09 +## [0.8.5] - 2013-05-09 * [Fix] inotify: allow monitoring of "broken" symlinks (thanks @tsg) -## v0.8.4 / 2013-04-07 +## [0.8.4] - 2013-04-07 * [Fix] kqueue: watch all file events [#40][] (thanks @ChrisBuchholz) -## v0.8.3 / 2013-03-13 +## [0.8.3] - 2013-03-13 * [Fix] inoitfy/kqueue memory leak [#36][] (reported by @nbkolchin) * [Fix] kqueue: use fsnFlags for watching a directory [#33][] (reported by @nbkolchin) -## v0.8.2 / 2013-02-07 +## [0.8.2] - 2013-02-07 * [Doc] add Authors * [Fix] fix data races for map access [#29][] (thanks @fsouza) -## v0.8.1 / 2013-01-09 +## [0.8.1] - 2013-01-09 * [Fix] Windows path separators * [Doc] BSD License -## v0.8.0 / 2012-11-09 +## [0.8.0] - 2012-11-09 * kqueue: directory watching improvements (thanks @vmirage) * inotify: add `IN_MOVED_TO` [#25][] (requested by @cpisto) * [Fix] kqueue: deleting watched directory [#24][] (reported by @jakerr) -## v0.7.4 / 2012-10-09 +## [0.7.4] - 2012-10-09 * [Fix] inotify: fixes from https://codereview.appspot.com/5418045/ (ugorji) * [Fix] kqueue: preserve watch flags when watching for delete [#21][] (reported by @robfig) * [Fix] kqueue: watch the directory even if it isn't a new watch (thanks @robfig) * [Fix] kqueue: modify after recreation of file -## v0.7.3 / 2012-09-27 +## [0.7.3] - 2012-09-27 * [Fix] kqueue: watch with an existing folder inside the watched folder (thanks @vmirage) * [Fix] kqueue: no longer get duplicate CREATE events -## v0.7.2 / 2012-09-01 +## [0.7.2] - 2012-09-01 * kqueue: events for created directories -## v0.7.1 / 2012-07-14 +## [0.7.1] - 2012-07-14 * [Fix] for renaming files -## v0.7.0 / 2012-07-02 +## [0.7.0] - 2012-07-02 * [Feature] FSNotify flags * [Fix] inotify: Added file name back to event path -## v0.6.0 / 2012-06-06 +## [0.6.0] - 2012-06-06 * kqueue: watch files after directory created (thanks @tmc) -## v0.5.1 / 2012-05-22 +## [0.5.1] - 2012-05-22 * [Fix] inotify: remove all watches before Close() -## v0.5.0 / 2012-05-03 +## [0.5.0] - 2012-05-03 * [API] kqueue: return errors during watch instead of sending over channel * kqueue: match symlink behavior on Linux @@ -273,22 +497,22 @@ kqueue: Fix logic for CREATE after REMOVE [#111](https://github.com/fsnotify/fsn * [Fix] kqueue: handle EINTR (reported by @robfig) * [Doc] Godoc example [#1][] (thanks @davecheney) -## v0.4.0 / 2012-03-30 +## [0.4.0] - 2012-03-30 * Go 1 released: build with go tool * [Feature] Windows support using winfsnotify * Windows does not have attribute change notifications * Roll attribute notifications into IsModify -## v0.3.0 / 2012-02-19 +## [0.3.0] - 2012-02-19 * kqueue: add files when watch directory -## v0.2.0 / 2011-12-30 +## [0.2.0] - 2011-12-30 * update to latest Go weekly code -## v0.1.0 / 2011-10-19 +## [0.1.0] - 2011-10-19 * kqueue: add watch on file creation to match inotify * kqueue: create file event diff --git a/vendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md b/vendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md index 828a60b2..ea379759 100644 --- a/vendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md +++ b/vendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md @@ -1,77 +1,26 @@ -# Contributing +Thank you for your interest in contributing to fsnotify! We try to review and +merge PRs in a reasonable timeframe, but please be aware that: -## Issues +- To avoid "wasted" work, please discus changes on the issue tracker first. You + can just send PRs, but they may end up being rejected for one reason or the + other. -* Request features and report bugs using the [GitHub Issue Tracker](https://github.com/fsnotify/fsnotify/issues). -* Please indicate the platform you are using fsnotify on. -* A code example to reproduce the problem is appreciated. +- fsnotify is a cross-platform library, and changes must work reasonably well on + all supported platforms. -## Pull Requests +- Changes will need to be compatible; old code should still compile, and the + runtime behaviour can't change in ways that are likely to lead to problems for + users. -### Contributor License Agreement +Testing +------- +Just `go test ./...` runs all the tests; the CI runs this on all supported +platforms. Testing different platforms locally can be done with something like +[goon] or [Vagrant], but this isn't super-easy to set up at the moment. -fsnotify is derived from code in the [golang.org/x/exp](https://godoc.org/golang.org/x/exp) package and it may be included [in the standard library](https://github.com/fsnotify/fsnotify/issues/1) in the future. Therefore fsnotify carries the same [LICENSE](https://github.com/fsnotify/fsnotify/blob/master/LICENSE) as Go. Contributors retain their copyright, so you need to fill out a short form before we can accept your contribution: [Google Individual Contributor License Agreement](https://developers.google.com/open-source/cla/individual). +Use the `-short` flag to make the "stress test" run faster. -Please indicate that you have signed the CLA in your pull request. -### How fsnotify is Developed - -* Development is done on feature branches. -* Tests are run on BSD, Linux, macOS and Windows. -* Pull requests are reviewed and [applied to master][am] using [hub][]. - * Maintainers may modify or squash commits rather than asking contributors to. -* To issue a new release, the maintainers will: - * Update the CHANGELOG - * Tag a version, which will become available through gopkg.in. - -### How to Fork - -For smooth sailing, always use the original import path. Installing with `go get` makes this easy. - -1. Install from GitHub (`go get -u github.com/fsnotify/fsnotify`) -2. Create your feature branch (`git checkout -b my-new-feature`) -3. Ensure everything works and the tests pass (see below) -4. Commit your changes (`git commit -am 'Add some feature'`) - -Contribute upstream: - -1. Fork fsnotify on GitHub -2. Add your remote (`git remote add fork git@github.com:mycompany/repo.git`) -3. Push to the branch (`git push fork my-new-feature`) -4. Create a new Pull Request on GitHub - -This workflow is [thoroughly explained by Katrina Owen](https://splice.com/blog/contributing-open-source-git-repositories-go/). - -### Testing - -fsnotify uses build tags to compile different code on Linux, BSD, macOS, and Windows. - -Before doing a pull request, please do your best to test your changes on multiple platforms, and list which platforms you were able/unable to test on. - -To aid in cross-platform testing there is a Vagrantfile for Linux and BSD. - -* Install [Vagrant](http://www.vagrantup.com/) and [VirtualBox](https://www.virtualbox.org/) -* Setup [Vagrant Gopher](https://github.com/nathany/vagrant-gopher) in your `src` folder. -* Run `vagrant up` from the project folder. You can also setup just one box with `vagrant up linux` or `vagrant up bsd` (note: the BSD box doesn't support Windows hosts at this time, and NFS may prompt for your host OS password) -* Once setup, you can run the test suite on a given OS with a single command `vagrant ssh linux -c 'cd fsnotify/fsnotify; go test'`. -* When you're done, you will want to halt or destroy the Vagrant boxes. - -Notice: fsnotify file system events won't trigger in shared folders. The tests get around this limitation by using the /tmp directory. - -Right now there is no equivalent solution for Windows and macOS, but there are Windows VMs [freely available from Microsoft](http://www.modern.ie/en-us/virtualization-tools#downloads). - -### Maintainers - -Help maintaining fsnotify is welcome. To be a maintainer: - -* Submit a pull request and sign the CLA as above. -* You must be able to run the test suite on Mac, Windows, Linux and BSD. - -To keep master clean, the fsnotify project uses the "apply mail" workflow outlined in Nathaniel Talbott's post ["Merge pull request" Considered Harmful][am]. This requires installing [hub][]. - -All code changes should be internal pull requests. - -Releases are tagged using [Semantic Versioning](http://semver.org/). - -[hub]: https://github.com/github/hub -[am]: http://blog.spreedly.com/2014/06/24/merge-pull-request-considered-harmful/#.VGa5yZPF_Zs +[goon]: https://github.com/arp242/goon +[Vagrant]: https://www.vagrantup.com/ +[integration_test.go]: /integration_test.go diff --git a/vendor/github.com/fsnotify/fsnotify/LICENSE b/vendor/github.com/fsnotify/fsnotify/LICENSE index e180c8fb..fb03ade7 100644 --- a/vendor/github.com/fsnotify/fsnotify/LICENSE +++ b/vendor/github.com/fsnotify/fsnotify/LICENSE @@ -1,28 +1,25 @@ -Copyright (c) 2012 The Go Authors. All rights reserved. -Copyright (c) 2012-2019 fsnotify Authors. All rights reserved. +Copyright © 2012 The Go Authors. All rights reserved. +Copyright © fsnotify Authors. All rights reserved. -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. +* Neither the name of Google Inc. nor the names of its contributors may be used + to endorse or promote products derived from this software without specific + prior written permission. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/fsnotify/fsnotify/README.md b/vendor/github.com/fsnotify/fsnotify/README.md index b2629e52..e480733d 100644 --- a/vendor/github.com/fsnotify/fsnotify/README.md +++ b/vendor/github.com/fsnotify/fsnotify/README.md @@ -1,130 +1,184 @@ -# File system notifications for Go +fsnotify is a Go library to provide cross-platform filesystem notifications on +Windows, Linux, macOS, BSD, and illumos. -[![GoDoc](https://godoc.org/github.com/fsnotify/fsnotify?status.svg)](https://godoc.org/github.com/fsnotify/fsnotify) [![Go Report Card](https://goreportcard.com/badge/github.com/fsnotify/fsnotify)](https://goreportcard.com/report/github.com/fsnotify/fsnotify) +Go 1.17 or newer is required; the full documentation is at +https://pkg.go.dev/github.com/fsnotify/fsnotify -fsnotify utilizes [golang.org/x/sys](https://godoc.org/golang.org/x/sys) rather than `syscall` from the standard library. Ensure you have the latest version installed by running: +--- -```console -go get -u golang.org/x/sys/... -``` - -Cross platform: Windows, Linux, BSD and macOS. - -| Adapter | OS | Status | -| --------------------- | -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- | -| inotify | Linux 2.6.27 or later, Android\* | Supported [![Build Status](https://travis-ci.org/fsnotify/fsnotify.svg?branch=master)](https://travis-ci.org/fsnotify/fsnotify) | -| kqueue | BSD, macOS, iOS\* | Supported [![Build Status](https://travis-ci.org/fsnotify/fsnotify.svg?branch=master)](https://travis-ci.org/fsnotify/fsnotify) | -| ReadDirectoryChangesW | Windows | Supported [![Build Status](https://travis-ci.org/fsnotify/fsnotify.svg?branch=master)](https://travis-ci.org/fsnotify/fsnotify) | -| FSEvents | macOS | [Planned](https://github.com/fsnotify/fsnotify/issues/11) | -| FEN | Solaris 11 | [In Progress](https://github.com/fsnotify/fsnotify/issues/12) | -| fanotify | Linux 2.6.37+ | [Planned](https://github.com/fsnotify/fsnotify/issues/114) | -| USN Journals | Windows | [Maybe](https://github.com/fsnotify/fsnotify/issues/53) | -| Polling | *All* | [Maybe](https://github.com/fsnotify/fsnotify/issues/9) | - -\* Android and iOS are untested. +Platform support: -Please see [the documentation](https://godoc.org/github.com/fsnotify/fsnotify) and consult the [FAQ](#faq) for usage information. +| Backend | OS | Status | +| :-------------------- | :--------- | :------------------------------------------------------------------------ | +| inotify | Linux | Supported | +| kqueue | BSD, macOS | Supported | +| ReadDirectoryChangesW | Windows | Supported | +| FEN | illumos | Supported | +| fanotify | Linux 5.9+ | [Not yet](https://github.com/fsnotify/fsnotify/issues/114) | +| AHAFS | AIX | [aix branch]; experimental due to lack of maintainer and test environment | +| FSEvents | macOS | [Needs support in x/sys/unix][fsevents] | +| USN Journals | Windows | [Needs support in x/sys/windows][usn] | +| Polling | *All* | [Not yet](https://github.com/fsnotify/fsnotify/issues/9) | -## API stability +Linux and illumos should include Android and Solaris, but these are currently +untested. -fsnotify is a fork of [howeyc/fsnotify](https://godoc.org/github.com/howeyc/fsnotify) with a new API as of v1.0. The API is based on [this design document](http://goo.gl/MrYxyA). +[fsevents]: https://github.com/fsnotify/fsnotify/issues/11#issuecomment-1279133120 +[usn]: https://github.com/fsnotify/fsnotify/issues/53#issuecomment-1279829847 +[aix branch]: https://github.com/fsnotify/fsnotify/issues/353#issuecomment-1284590129 -All [releases](https://github.com/fsnotify/fsnotify/releases) are tagged based on [Semantic Versioning](http://semver.org/). Further API changes are [planned](https://github.com/fsnotify/fsnotify/milestones), and will be tagged with a new major revision number. - -Go 1.6 supports dependencies located in the `vendor/` folder. Unless you are creating a library, it is recommended that you copy fsnotify into `vendor/github.com/fsnotify/fsnotify` within your project, and likewise for `golang.org/x/sys`. - -## Usage +Usage +----- +A basic example: ```go package main import ( - "log" + "log" - "github.com/fsnotify/fsnotify" + "github.com/fsnotify/fsnotify" ) func main() { - watcher, err := fsnotify.NewWatcher() - if err != nil { - log.Fatal(err) - } - defer watcher.Close() - - done := make(chan bool) - go func() { - for { - select { - case event, ok := <-watcher.Events: - if !ok { - return - } - log.Println("event:", event) - if event.Op&fsnotify.Write == fsnotify.Write { - log.Println("modified file:", event.Name) - } - case err, ok := <-watcher.Errors: - if !ok { - return - } - log.Println("error:", err) - } - } - }() - - err = watcher.Add("/tmp/foo") - if err != nil { - log.Fatal(err) - } - <-done + // Create new watcher. + watcher, err := fsnotify.NewWatcher() + if err != nil { + log.Fatal(err) + } + defer watcher.Close() + + // Start listening for events. + go func() { + for { + select { + case event, ok := <-watcher.Events: + if !ok { + return + } + log.Println("event:", event) + if event.Has(fsnotify.Write) { + log.Println("modified file:", event.Name) + } + case err, ok := <-watcher.Errors: + if !ok { + return + } + log.Println("error:", err) + } + } + }() + + // Add a path. + err = watcher.Add("/tmp") + if err != nil { + log.Fatal(err) + } + + // Block main goroutine forever. + <-make(chan struct{}) } ``` -## Contributing +Some more examples can be found in [cmd/fsnotify](cmd/fsnotify), which can be +run with: + + % go run ./cmd/fsnotify + +Further detailed documentation can be found in godoc: +https://pkg.go.dev/github.com/fsnotify/fsnotify -Please refer to [CONTRIBUTING][] before opening an issue or pull request. +FAQ +--- +### Will a file still be watched when it's moved to another directory? +No, not unless you are watching the location it was moved to. -## Example +### Are subdirectories watched? +No, you must add watches for any directory you want to watch (a recursive +watcher is on the roadmap: [#18]). -See [example_test.go](https://github.com/fsnotify/fsnotify/blob/master/example_test.go). +[#18]: https://github.com/fsnotify/fsnotify/issues/18 -## FAQ +### Do I have to watch the Error and Event channels in a goroutine? +Yes. You can read both channels in the same goroutine using `select` (you don't +need a separate goroutine for both channels; see the example). -**When a file is moved to another directory is it still being watched?** +### Why don't notifications work with NFS, SMB, FUSE, /proc, or /sys? +fsnotify requires support from underlying OS to work. The current NFS and SMB +protocols does not provide network level support for file notifications, and +neither do the /proc and /sys virtual filesystems. -No (it shouldn't be, unless you are watching where it was moved to). +This could be fixed with a polling watcher ([#9]), but it's not yet implemented. -**When I watch a directory, are all subdirectories watched as well?** +[#9]: https://github.com/fsnotify/fsnotify/issues/9 -No, you must add watches for any directory you want to watch (a recursive watcher is on the roadmap [#18][]). +### Why do I get many Chmod events? +Some programs may generate a lot of attribute changes; for example Spotlight on +macOS, anti-virus programs, backup applications, and some others are known to do +this. As a rule, it's typically best to ignore Chmod events. They're often not +useful, and tend to cause problems. -**Do I have to watch the Error and Event channels in a separate goroutine?** +Spotlight indexing on macOS can result in multiple events (see [#15]). A +temporary workaround is to add your folder(s) to the *Spotlight Privacy +settings* until we have a native FSEvents implementation (see [#11]). -As of now, yes. Looking into making this single-thread friendly (see [howeyc #7][#7]) +[#11]: https://github.com/fsnotify/fsnotify/issues/11 +[#15]: https://github.com/fsnotify/fsnotify/issues/15 -**Why am I receiving multiple events for the same file on OS X?** +### Watching a file doesn't work well +Watching individual files (rather than directories) is generally not recommended +as many programs (especially editors) update files atomically: it will write to +a temporary file which is then moved to to destination, overwriting the original +(or some variant thereof). The watcher on the original file is now lost, as that +no longer exists. -Spotlight indexing on OS X can result in multiple events (see [howeyc #62][#62]). A temporary workaround is to add your folder(s) to the *Spotlight Privacy settings* until we have a native FSEvents implementation (see [#11][]). +The upshot of this is that a power failure or crash won't leave a half-written +file. -**How many files can be watched at once?** +Watch the parent directory and use `Event.Name` to filter out files you're not +interested in. There is an example of this in `cmd/fsnotify/file.go`. -There are OS-specific limits as to how many watches can be created: -* Linux: /proc/sys/fs/inotify/max_user_watches contains the limit, reaching this limit results in a "no space left on device" error. -* BSD / OSX: sysctl variables "kern.maxfiles" and "kern.maxfilesperproc", reaching these limits results in a "too many open files" error. +Platform-specific notes +----------------------- +### Linux +When a file is removed a REMOVE event won't be emitted until all file +descriptors are closed; it will emit a CHMOD instead: -**Why don't notifications work with NFS filesystems or filesystem in userspace (FUSE)?** + fp := os.Open("file") + os.Remove("file") // CHMOD + fp.Close() // REMOVE -fsnotify requires support from underlying OS to work. The current NFS protocol does not provide network level support for file notifications. +This is the event that inotify sends, so not much can be changed about this. -[#62]: https://github.com/howeyc/fsnotify/issues/62 -[#18]: https://github.com/fsnotify/fsnotify/issues/18 -[#11]: https://github.com/fsnotify/fsnotify/issues/11 -[#7]: https://github.com/howeyc/fsnotify/issues/7 +The `fs.inotify.max_user_watches` sysctl variable specifies the upper limit for +the number of watches per user, and `fs.inotify.max_user_instances` specifies +the maximum number of inotify instances per user. Every Watcher you create is an +"instance", and every path you add is a "watch". + +These are also exposed in `/proc` as `/proc/sys/fs/inotify/max_user_watches` and +`/proc/sys/fs/inotify/max_user_instances` + +To increase them you can use `sysctl` or write the value to proc file: + + # The default values on Linux 5.18 + sysctl fs.inotify.max_user_watches=124983 + sysctl fs.inotify.max_user_instances=128 + +To make the changes persist on reboot edit `/etc/sysctl.conf` or +`/usr/lib/sysctl.d/50-default.conf` (details differ per Linux distro; check your +distro's documentation): -[contributing]: https://github.com/fsnotify/fsnotify/blob/master/CONTRIBUTING.md + fs.inotify.max_user_watches=124983 + fs.inotify.max_user_instances=128 -## Related Projects +Reaching the limit will result in a "no space left on device" or "too many open +files" error. -* [notify](https://github.com/rjeczalik/notify) -* [fsevents](https://github.com/fsnotify/fsevents) +### kqueue (macOS, all BSD systems) +kqueue requires opening a file descriptor for every file that's being watched; +so if you're watching a directory with five files then that's six file +descriptors. You will run in to your system's "max open files" limit faster on +these platforms. +The sysctl variables `kern.maxfiles` and `kern.maxfilesperproc` can be used to +control the maximum number of open files. diff --git a/vendor/github.com/fsnotify/fsnotify/backend_fen.go b/vendor/github.com/fsnotify/fsnotify/backend_fen.go new file mode 100644 index 00000000..28497f1d --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/backend_fen.go @@ -0,0 +1,640 @@ +//go:build solaris +// +build solaris + +// Note: the documentation on the Watcher type and methods is generated from +// mkdoc.zsh + +package fsnotify + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "sync" + + "golang.org/x/sys/unix" +) + +// Watcher watches a set of paths, delivering events on a channel. +// +// A watcher should not be copied (e.g. pass it by pointer, rather than by +// value). +// +// # Linux notes +// +// When a file is removed a Remove event won't be emitted until all file +// descriptors are closed, and deletes will always emit a Chmod. For example: +// +// fp := os.Open("file") +// os.Remove("file") // Triggers Chmod +// fp.Close() // Triggers Remove +// +// This is the event that inotify sends, so not much can be changed about this. +// +// The fs.inotify.max_user_watches sysctl variable specifies the upper limit +// for the number of watches per user, and fs.inotify.max_user_instances +// specifies the maximum number of inotify instances per user. Every Watcher you +// create is an "instance", and every path you add is a "watch". +// +// These are also exposed in /proc as /proc/sys/fs/inotify/max_user_watches and +// /proc/sys/fs/inotify/max_user_instances +// +// To increase them you can use sysctl or write the value to the /proc file: +// +// # Default values on Linux 5.18 +// sysctl fs.inotify.max_user_watches=124983 +// sysctl fs.inotify.max_user_instances=128 +// +// To make the changes persist on reboot edit /etc/sysctl.conf or +// /usr/lib/sysctl.d/50-default.conf (details differ per Linux distro; check +// your distro's documentation): +// +// fs.inotify.max_user_watches=124983 +// fs.inotify.max_user_instances=128 +// +// Reaching the limit will result in a "no space left on device" or "too many open +// files" error. +// +// # kqueue notes (macOS, BSD) +// +// kqueue requires opening a file descriptor for every file that's being watched; +// so if you're watching a directory with five files then that's six file +// descriptors. You will run in to your system's "max open files" limit faster on +// these platforms. +// +// The sysctl variables kern.maxfiles and kern.maxfilesperproc can be used to +// control the maximum number of open files, as well as /etc/login.conf on BSD +// systems. +// +// # Windows notes +// +// Paths can be added as "C:\path\to\dir", but forward slashes +// ("C:/path/to/dir") will also work. +// +// When a watched directory is removed it will always send an event for the +// directory itself, but may not send events for all files in that directory. +// Sometimes it will send events for all times, sometimes it will send no +// events, and often only for some files. +// +// The default ReadDirectoryChangesW() buffer size is 64K, which is the largest +// value that is guaranteed to work with SMB filesystems. If you have many +// events in quick succession this may not be enough, and you will have to use +// [WithBufferSize] to increase the value. +type Watcher struct { + // Events sends the filesystem change events. + // + // fsnotify can send the following events; a "path" here can refer to a + // file, directory, symbolic link, or special file like a FIFO. + // + // fsnotify.Create A new path was created; this may be followed by one + // or more Write events if data also gets written to a + // file. + // + // fsnotify.Remove A path was removed. + // + // fsnotify.Rename A path was renamed. A rename is always sent with the + // old path as Event.Name, and a Create event will be + // sent with the new name. Renames are only sent for + // paths that are currently watched; e.g. moving an + // unmonitored file into a monitored directory will + // show up as just a Create. Similarly, renaming a file + // to outside a monitored directory will show up as + // only a Rename. + // + // fsnotify.Write A file or named pipe was written to. A Truncate will + // also trigger a Write. A single "write action" + // initiated by the user may show up as one or multiple + // writes, depending on when the system syncs things to + // disk. For example when compiling a large Go program + // you may get hundreds of Write events, and you may + // want to wait until you've stopped receiving them + // (see the dedup example in cmd/fsnotify). + // + // Some systems may send Write event for directories + // when the directory content changes. + // + // fsnotify.Chmod Attributes were changed. On Linux this is also sent + // when a file is removed (or more accurately, when a + // link to an inode is removed). On kqueue it's sent + // when a file is truncated. On Windows it's never + // sent. + Events chan Event + + // Errors sends any errors. + // + // ErrEventOverflow is used to indicate there are too many events: + // + // - inotify: There are too many queued events (fs.inotify.max_queued_events sysctl) + // - windows: The buffer size is too small; WithBufferSize() can be used to increase it. + // - kqueue, fen: Not used. + Errors chan error + + mu sync.Mutex + port *unix.EventPort + done chan struct{} // Channel for sending a "quit message" to the reader goroutine + dirs map[string]struct{} // Explicitly watched directories + watches map[string]struct{} // Explicitly watched non-directories +} + +// NewWatcher creates a new Watcher. +func NewWatcher() (*Watcher, error) { + return NewBufferedWatcher(0) +} + +// NewBufferedWatcher creates a new Watcher with a buffered Watcher.Events +// channel. +// +// The main use case for this is situations with a very large number of events +// where the kernel buffer size can't be increased (e.g. due to lack of +// permissions). An unbuffered Watcher will perform better for almost all use +// cases, and whenever possible you will be better off increasing the kernel +// buffers instead of adding a large userspace buffer. +func NewBufferedWatcher(sz uint) (*Watcher, error) { + w := &Watcher{ + Events: make(chan Event, sz), + Errors: make(chan error), + dirs: make(map[string]struct{}), + watches: make(map[string]struct{}), + done: make(chan struct{}), + } + + var err error + w.port, err = unix.NewEventPort() + if err != nil { + return nil, fmt.Errorf("fsnotify.NewWatcher: %w", err) + } + + go w.readEvents() + return w, nil +} + +// sendEvent attempts to send an event to the user, returning true if the event +// was put in the channel successfully and false if the watcher has been closed. +func (w *Watcher) sendEvent(name string, op Op) (sent bool) { + select { + case w.Events <- Event{Name: name, Op: op}: + return true + case <-w.done: + return false + } +} + +// sendError attempts to send an error to the user, returning true if the error +// was put in the channel successfully and false if the watcher has been closed. +func (w *Watcher) sendError(err error) (sent bool) { + select { + case w.Errors <- err: + return true + case <-w.done: + return false + } +} + +func (w *Watcher) isClosed() bool { + select { + case <-w.done: + return true + default: + return false + } +} + +// Close removes all watches and closes the Events channel. +func (w *Watcher) Close() error { + // Take the lock used by associateFile to prevent lingering events from + // being processed after the close + w.mu.Lock() + defer w.mu.Unlock() + if w.isClosed() { + return nil + } + close(w.done) + return w.port.Close() +} + +// Add starts monitoring the path for changes. +// +// A path can only be watched once; watching it more than once is a no-op and will +// not return an error. Paths that do not yet exist on the filesystem cannot be +// watched. +// +// A watch will be automatically removed if the watched path is deleted or +// renamed. The exception is the Windows backend, which doesn't remove the +// watcher on renames. +// +// Notifications on network filesystems (NFS, SMB, FUSE, etc.) or special +// filesystems (/proc, /sys, etc.) generally don't work. +// +// Returns [ErrClosed] if [Watcher.Close] was called. +// +// See [Watcher.AddWith] for a version that allows adding options. +// +// # Watching directories +// +// All files in a directory are monitored, including new files that are created +// after the watcher is started. Subdirectories are not watched (i.e. it's +// non-recursive). +// +// # Watching files +// +// Watching individual files (rather than directories) is generally not +// recommended as many programs (especially editors) update files atomically: it +// will write to a temporary file which is then moved to to destination, +// overwriting the original (or some variant thereof). The watcher on the +// original file is now lost, as that no longer exists. +// +// The upshot of this is that a power failure or crash won't leave a +// half-written file. +// +// Watch the parent directory and use Event.Name to filter out files you're not +// interested in. There is an example of this in cmd/fsnotify/file.go. +func (w *Watcher) Add(name string) error { return w.AddWith(name) } + +// AddWith is like [Watcher.Add], but allows adding options. When using Add() +// the defaults described below are used. +// +// Possible options are: +// +// - [WithBufferSize] sets the buffer size for the Windows backend; no-op on +// other platforms. The default is 64K (65536 bytes). +func (w *Watcher) AddWith(name string, opts ...addOpt) error { + if w.isClosed() { + return ErrClosed + } + if w.port.PathIsWatched(name) { + return nil + } + + _ = getOptions(opts...) + + // Currently we resolve symlinks that were explicitly requested to be + // watched. Otherwise we would use LStat here. + stat, err := os.Stat(name) + if err != nil { + return err + } + + // Associate all files in the directory. + if stat.IsDir() { + err := w.handleDirectory(name, stat, true, w.associateFile) + if err != nil { + return err + } + + w.mu.Lock() + w.dirs[name] = struct{}{} + w.mu.Unlock() + return nil + } + + err = w.associateFile(name, stat, true) + if err != nil { + return err + } + + w.mu.Lock() + w.watches[name] = struct{}{} + w.mu.Unlock() + return nil +} + +// Remove stops monitoring the path for changes. +// +// Directories are always removed non-recursively. For example, if you added +// /tmp/dir and /tmp/dir/subdir then you will need to remove both. +// +// Removing a path that has not yet been added returns [ErrNonExistentWatch]. +// +// Returns nil if [Watcher.Close] was called. +func (w *Watcher) Remove(name string) error { + if w.isClosed() { + return nil + } + if !w.port.PathIsWatched(name) { + return fmt.Errorf("%w: %s", ErrNonExistentWatch, name) + } + + // The user has expressed an intent. Immediately remove this name from + // whichever watch list it might be in. If it's not in there the delete + // doesn't cause harm. + w.mu.Lock() + delete(w.watches, name) + delete(w.dirs, name) + w.mu.Unlock() + + stat, err := os.Stat(name) + if err != nil { + return err + } + + // Remove associations for every file in the directory. + if stat.IsDir() { + err := w.handleDirectory(name, stat, false, w.dissociateFile) + if err != nil { + return err + } + return nil + } + + err = w.port.DissociatePath(name) + if err != nil { + return err + } + + return nil +} + +// readEvents contains the main loop that runs in a goroutine watching for events. +func (w *Watcher) readEvents() { + // If this function returns, the watcher has been closed and we can close + // these channels + defer func() { + close(w.Errors) + close(w.Events) + }() + + pevents := make([]unix.PortEvent, 8) + for { + count, err := w.port.Get(pevents, 1, nil) + if err != nil && err != unix.ETIME { + // Interrupted system call (count should be 0) ignore and continue + if errors.Is(err, unix.EINTR) && count == 0 { + continue + } + // Get failed because we called w.Close() + if errors.Is(err, unix.EBADF) && w.isClosed() { + return + } + // There was an error not caused by calling w.Close() + if !w.sendError(err) { + return + } + } + + p := pevents[:count] + for _, pevent := range p { + if pevent.Source != unix.PORT_SOURCE_FILE { + // Event from unexpected source received; should never happen. + if !w.sendError(errors.New("Event from unexpected source received")) { + return + } + continue + } + + err = w.handleEvent(&pevent) + if err != nil { + if !w.sendError(err) { + return + } + } + } + } +} + +func (w *Watcher) handleDirectory(path string, stat os.FileInfo, follow bool, handler func(string, os.FileInfo, bool) error) error { + files, err := os.ReadDir(path) + if err != nil { + return err + } + + // Handle all children of the directory. + for _, entry := range files { + finfo, err := entry.Info() + if err != nil { + return err + } + err = handler(filepath.Join(path, finfo.Name()), finfo, false) + if err != nil { + return err + } + } + + // And finally handle the directory itself. + return handler(path, stat, follow) +} + +// handleEvent might need to emit more than one fsnotify event if the events +// bitmap matches more than one event type (e.g. the file was both modified and +// had the attributes changed between when the association was created and the +// when event was returned) +func (w *Watcher) handleEvent(event *unix.PortEvent) error { + var ( + events = event.Events + path = event.Path + fmode = event.Cookie.(os.FileMode) + reRegister = true + ) + + w.mu.Lock() + _, watchedDir := w.dirs[path] + _, watchedPath := w.watches[path] + w.mu.Unlock() + isWatched := watchedDir || watchedPath + + if events&unix.FILE_DELETE != 0 { + if !w.sendEvent(path, Remove) { + return nil + } + reRegister = false + } + if events&unix.FILE_RENAME_FROM != 0 { + if !w.sendEvent(path, Rename) { + return nil + } + // Don't keep watching the new file name + reRegister = false + } + if events&unix.FILE_RENAME_TO != 0 { + // We don't report a Rename event for this case, because Rename events + // are interpreted as referring to the _old_ name of the file, and in + // this case the event would refer to the new name of the file. This + // type of rename event is not supported by fsnotify. + + // inotify reports a Remove event in this case, so we simulate this + // here. + if !w.sendEvent(path, Remove) { + return nil + } + // Don't keep watching the file that was removed + reRegister = false + } + + // The file is gone, nothing left to do. + if !reRegister { + if watchedDir { + w.mu.Lock() + delete(w.dirs, path) + w.mu.Unlock() + } + if watchedPath { + w.mu.Lock() + delete(w.watches, path) + w.mu.Unlock() + } + return nil + } + + // If we didn't get a deletion the file still exists and we're going to have + // to watch it again. Let's Stat it now so that we can compare permissions + // and have what we need to continue watching the file + + stat, err := os.Lstat(path) + if err != nil { + // This is unexpected, but we should still emit an event. This happens + // most often on "rm -r" of a subdirectory inside a watched directory We + // get a modify event of something happening inside, but by the time we + // get here, the sudirectory is already gone. Clearly we were watching + // this path but now it is gone. Let's tell the user that it was + // removed. + if !w.sendEvent(path, Remove) { + return nil + } + // Suppress extra write events on removed directories; they are not + // informative and can be confusing. + return nil + } + + // resolve symlinks that were explicitly watched as we would have at Add() + // time. this helps suppress spurious Chmod events on watched symlinks + if isWatched { + stat, err = os.Stat(path) + if err != nil { + // The symlink still exists, but the target is gone. Report the + // Remove similar to above. + if !w.sendEvent(path, Remove) { + return nil + } + // Don't return the error + } + } + + if events&unix.FILE_MODIFIED != 0 { + if fmode.IsDir() { + if watchedDir { + if err := w.updateDirectory(path); err != nil { + return err + } + } else { + if !w.sendEvent(path, Write) { + return nil + } + } + } else { + if !w.sendEvent(path, Write) { + return nil + } + } + } + if events&unix.FILE_ATTRIB != 0 && stat != nil { + // Only send Chmod if perms changed + if stat.Mode().Perm() != fmode.Perm() { + if !w.sendEvent(path, Chmod) { + return nil + } + } + } + + if stat != nil { + // If we get here, it means we've hit an event above that requires us to + // continue watching the file or directory + return w.associateFile(path, stat, isWatched) + } + return nil +} + +func (w *Watcher) updateDirectory(path string) error { + // The directory was modified, so we must find unwatched entities and watch + // them. If something was removed from the directory, nothing will happen, + // as everything else should still be watched. + files, err := os.ReadDir(path) + if err != nil { + return err + } + + for _, entry := range files { + path := filepath.Join(path, entry.Name()) + if w.port.PathIsWatched(path) { + continue + } + + finfo, err := entry.Info() + if err != nil { + return err + } + err = w.associateFile(path, finfo, false) + if err != nil { + if !w.sendError(err) { + return nil + } + } + if !w.sendEvent(path, Create) { + return nil + } + } + return nil +} + +func (w *Watcher) associateFile(path string, stat os.FileInfo, follow bool) error { + if w.isClosed() { + return ErrClosed + } + // This is primarily protecting the call to AssociatePath but it is + // important and intentional that the call to PathIsWatched is also + // protected by this mutex. Without this mutex, AssociatePath has been seen + // to error out that the path is already associated. + w.mu.Lock() + defer w.mu.Unlock() + + if w.port.PathIsWatched(path) { + // Remove the old association in favor of this one If we get ENOENT, + // then while the x/sys/unix wrapper still thought that this path was + // associated, the underlying event port did not. This call will have + // cleared up that discrepancy. The most likely cause is that the event + // has fired but we haven't processed it yet. + err := w.port.DissociatePath(path) + if err != nil && err != unix.ENOENT { + return err + } + } + // FILE_NOFOLLOW means we watch symlinks themselves rather than their + // targets. + events := unix.FILE_MODIFIED | unix.FILE_ATTRIB | unix.FILE_NOFOLLOW + if follow { + // We *DO* follow symlinks for explicitly watched entries. + events = unix.FILE_MODIFIED | unix.FILE_ATTRIB + } + return w.port.AssociatePath(path, stat, + events, + stat.Mode()) +} + +func (w *Watcher) dissociateFile(path string, stat os.FileInfo, unused bool) error { + if !w.port.PathIsWatched(path) { + return nil + } + return w.port.DissociatePath(path) +} + +// WatchList returns all paths explicitly added with [Watcher.Add] (and are not +// yet removed). +// +// Returns nil if [Watcher.Close] was called. +func (w *Watcher) WatchList() []string { + if w.isClosed() { + return nil + } + + w.mu.Lock() + defer w.mu.Unlock() + + entries := make([]string, 0, len(w.watches)+len(w.dirs)) + for pathname := range w.dirs { + entries = append(entries, pathname) + } + for pathname := range w.watches { + entries = append(entries, pathname) + } + + return entries +} diff --git a/vendor/github.com/fsnotify/fsnotify/backend_inotify.go b/vendor/github.com/fsnotify/fsnotify/backend_inotify.go new file mode 100644 index 00000000..921c1c1e --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/backend_inotify.go @@ -0,0 +1,594 @@ +//go:build linux && !appengine +// +build linux,!appengine + +// Note: the documentation on the Watcher type and methods is generated from +// mkdoc.zsh + +package fsnotify + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "sync" + "unsafe" + + "golang.org/x/sys/unix" +) + +// Watcher watches a set of paths, delivering events on a channel. +// +// A watcher should not be copied (e.g. pass it by pointer, rather than by +// value). +// +// # Linux notes +// +// When a file is removed a Remove event won't be emitted until all file +// descriptors are closed, and deletes will always emit a Chmod. For example: +// +// fp := os.Open("file") +// os.Remove("file") // Triggers Chmod +// fp.Close() // Triggers Remove +// +// This is the event that inotify sends, so not much can be changed about this. +// +// The fs.inotify.max_user_watches sysctl variable specifies the upper limit +// for the number of watches per user, and fs.inotify.max_user_instances +// specifies the maximum number of inotify instances per user. Every Watcher you +// create is an "instance", and every path you add is a "watch". +// +// These are also exposed in /proc as /proc/sys/fs/inotify/max_user_watches and +// /proc/sys/fs/inotify/max_user_instances +// +// To increase them you can use sysctl or write the value to the /proc file: +// +// # Default values on Linux 5.18 +// sysctl fs.inotify.max_user_watches=124983 +// sysctl fs.inotify.max_user_instances=128 +// +// To make the changes persist on reboot edit /etc/sysctl.conf or +// /usr/lib/sysctl.d/50-default.conf (details differ per Linux distro; check +// your distro's documentation): +// +// fs.inotify.max_user_watches=124983 +// fs.inotify.max_user_instances=128 +// +// Reaching the limit will result in a "no space left on device" or "too many open +// files" error. +// +// # kqueue notes (macOS, BSD) +// +// kqueue requires opening a file descriptor for every file that's being watched; +// so if you're watching a directory with five files then that's six file +// descriptors. You will run in to your system's "max open files" limit faster on +// these platforms. +// +// The sysctl variables kern.maxfiles and kern.maxfilesperproc can be used to +// control the maximum number of open files, as well as /etc/login.conf on BSD +// systems. +// +// # Windows notes +// +// Paths can be added as "C:\path\to\dir", but forward slashes +// ("C:/path/to/dir") will also work. +// +// When a watched directory is removed it will always send an event for the +// directory itself, but may not send events for all files in that directory. +// Sometimes it will send events for all times, sometimes it will send no +// events, and often only for some files. +// +// The default ReadDirectoryChangesW() buffer size is 64K, which is the largest +// value that is guaranteed to work with SMB filesystems. If you have many +// events in quick succession this may not be enough, and you will have to use +// [WithBufferSize] to increase the value. +type Watcher struct { + // Events sends the filesystem change events. + // + // fsnotify can send the following events; a "path" here can refer to a + // file, directory, symbolic link, or special file like a FIFO. + // + // fsnotify.Create A new path was created; this may be followed by one + // or more Write events if data also gets written to a + // file. + // + // fsnotify.Remove A path was removed. + // + // fsnotify.Rename A path was renamed. A rename is always sent with the + // old path as Event.Name, and a Create event will be + // sent with the new name. Renames are only sent for + // paths that are currently watched; e.g. moving an + // unmonitored file into a monitored directory will + // show up as just a Create. Similarly, renaming a file + // to outside a monitored directory will show up as + // only a Rename. + // + // fsnotify.Write A file or named pipe was written to. A Truncate will + // also trigger a Write. A single "write action" + // initiated by the user may show up as one or multiple + // writes, depending on when the system syncs things to + // disk. For example when compiling a large Go program + // you may get hundreds of Write events, and you may + // want to wait until you've stopped receiving them + // (see the dedup example in cmd/fsnotify). + // + // Some systems may send Write event for directories + // when the directory content changes. + // + // fsnotify.Chmod Attributes were changed. On Linux this is also sent + // when a file is removed (or more accurately, when a + // link to an inode is removed). On kqueue it's sent + // when a file is truncated. On Windows it's never + // sent. + Events chan Event + + // Errors sends any errors. + // + // ErrEventOverflow is used to indicate there are too many events: + // + // - inotify: There are too many queued events (fs.inotify.max_queued_events sysctl) + // - windows: The buffer size is too small; WithBufferSize() can be used to increase it. + // - kqueue, fen: Not used. + Errors chan error + + // Store fd here as os.File.Read() will no longer return on close after + // calling Fd(). See: https://github.com/golang/go/issues/26439 + fd int + inotifyFile *os.File + watches *watches + done chan struct{} // Channel for sending a "quit message" to the reader goroutine + closeMu sync.Mutex + doneResp chan struct{} // Channel to respond to Close +} + +type ( + watches struct { + mu sync.RWMutex + wd map[uint32]*watch // wd → watch + path map[string]uint32 // pathname → wd + } + watch struct { + wd uint32 // Watch descriptor (as returned by the inotify_add_watch() syscall) + flags uint32 // inotify flags of this watch (see inotify(7) for the list of valid flags) + path string // Watch path. + } +) + +func newWatches() *watches { + return &watches{ + wd: make(map[uint32]*watch), + path: make(map[string]uint32), + } +} + +func (w *watches) len() int { + w.mu.RLock() + defer w.mu.RUnlock() + return len(w.wd) +} + +func (w *watches) add(ww *watch) { + w.mu.Lock() + defer w.mu.Unlock() + w.wd[ww.wd] = ww + w.path[ww.path] = ww.wd +} + +func (w *watches) remove(wd uint32) { + w.mu.Lock() + defer w.mu.Unlock() + delete(w.path, w.wd[wd].path) + delete(w.wd, wd) +} + +func (w *watches) removePath(path string) (uint32, bool) { + w.mu.Lock() + defer w.mu.Unlock() + + wd, ok := w.path[path] + if !ok { + return 0, false + } + + delete(w.path, path) + delete(w.wd, wd) + + return wd, true +} + +func (w *watches) byPath(path string) *watch { + w.mu.RLock() + defer w.mu.RUnlock() + return w.wd[w.path[path]] +} + +func (w *watches) byWd(wd uint32) *watch { + w.mu.RLock() + defer w.mu.RUnlock() + return w.wd[wd] +} + +func (w *watches) updatePath(path string, f func(*watch) (*watch, error)) error { + w.mu.Lock() + defer w.mu.Unlock() + + var existing *watch + wd, ok := w.path[path] + if ok { + existing = w.wd[wd] + } + + upd, err := f(existing) + if err != nil { + return err + } + if upd != nil { + w.wd[upd.wd] = upd + w.path[upd.path] = upd.wd + + if upd.wd != wd { + delete(w.wd, wd) + } + } + + return nil +} + +// NewWatcher creates a new Watcher. +func NewWatcher() (*Watcher, error) { + return NewBufferedWatcher(0) +} + +// NewBufferedWatcher creates a new Watcher with a buffered Watcher.Events +// channel. +// +// The main use case for this is situations with a very large number of events +// where the kernel buffer size can't be increased (e.g. due to lack of +// permissions). An unbuffered Watcher will perform better for almost all use +// cases, and whenever possible you will be better off increasing the kernel +// buffers instead of adding a large userspace buffer. +func NewBufferedWatcher(sz uint) (*Watcher, error) { + // Need to set nonblocking mode for SetDeadline to work, otherwise blocking + // I/O operations won't terminate on close. + fd, errno := unix.InotifyInit1(unix.IN_CLOEXEC | unix.IN_NONBLOCK) + if fd == -1 { + return nil, errno + } + + w := &Watcher{ + fd: fd, + inotifyFile: os.NewFile(uintptr(fd), ""), + watches: newWatches(), + Events: make(chan Event, sz), + Errors: make(chan error), + done: make(chan struct{}), + doneResp: make(chan struct{}), + } + + go w.readEvents() + return w, nil +} + +// Returns true if the event was sent, or false if watcher is closed. +func (w *Watcher) sendEvent(e Event) bool { + select { + case w.Events <- e: + return true + case <-w.done: + return false + } +} + +// Returns true if the error was sent, or false if watcher is closed. +func (w *Watcher) sendError(err error) bool { + select { + case w.Errors <- err: + return true + case <-w.done: + return false + } +} + +func (w *Watcher) isClosed() bool { + select { + case <-w.done: + return true + default: + return false + } +} + +// Close removes all watches and closes the Events channel. +func (w *Watcher) Close() error { + w.closeMu.Lock() + if w.isClosed() { + w.closeMu.Unlock() + return nil + } + close(w.done) + w.closeMu.Unlock() + + // Causes any blocking reads to return with an error, provided the file + // still supports deadline operations. + err := w.inotifyFile.Close() + if err != nil { + return err + } + + // Wait for goroutine to close + <-w.doneResp + + return nil +} + +// Add starts monitoring the path for changes. +// +// A path can only be watched once; watching it more than once is a no-op and will +// not return an error. Paths that do not yet exist on the filesystem cannot be +// watched. +// +// A watch will be automatically removed if the watched path is deleted or +// renamed. The exception is the Windows backend, which doesn't remove the +// watcher on renames. +// +// Notifications on network filesystems (NFS, SMB, FUSE, etc.) or special +// filesystems (/proc, /sys, etc.) generally don't work. +// +// Returns [ErrClosed] if [Watcher.Close] was called. +// +// See [Watcher.AddWith] for a version that allows adding options. +// +// # Watching directories +// +// All files in a directory are monitored, including new files that are created +// after the watcher is started. Subdirectories are not watched (i.e. it's +// non-recursive). +// +// # Watching files +// +// Watching individual files (rather than directories) is generally not +// recommended as many programs (especially editors) update files atomically: it +// will write to a temporary file which is then moved to to destination, +// overwriting the original (or some variant thereof). The watcher on the +// original file is now lost, as that no longer exists. +// +// The upshot of this is that a power failure or crash won't leave a +// half-written file. +// +// Watch the parent directory and use Event.Name to filter out files you're not +// interested in. There is an example of this in cmd/fsnotify/file.go. +func (w *Watcher) Add(name string) error { return w.AddWith(name) } + +// AddWith is like [Watcher.Add], but allows adding options. When using Add() +// the defaults described below are used. +// +// Possible options are: +// +// - [WithBufferSize] sets the buffer size for the Windows backend; no-op on +// other platforms. The default is 64K (65536 bytes). +func (w *Watcher) AddWith(name string, opts ...addOpt) error { + if w.isClosed() { + return ErrClosed + } + + name = filepath.Clean(name) + _ = getOptions(opts...) + + var flags uint32 = unix.IN_MOVED_TO | unix.IN_MOVED_FROM | + unix.IN_CREATE | unix.IN_ATTRIB | unix.IN_MODIFY | + unix.IN_MOVE_SELF | unix.IN_DELETE | unix.IN_DELETE_SELF + + return w.watches.updatePath(name, func(existing *watch) (*watch, error) { + if existing != nil { + flags |= existing.flags | unix.IN_MASK_ADD + } + + wd, err := unix.InotifyAddWatch(w.fd, name, flags) + if wd == -1 { + return nil, err + } + + if existing == nil { + return &watch{ + wd: uint32(wd), + path: name, + flags: flags, + }, nil + } + + existing.wd = uint32(wd) + existing.flags = flags + return existing, nil + }) +} + +// Remove stops monitoring the path for changes. +// +// Directories are always removed non-recursively. For example, if you added +// /tmp/dir and /tmp/dir/subdir then you will need to remove both. +// +// Removing a path that has not yet been added returns [ErrNonExistentWatch]. +// +// Returns nil if [Watcher.Close] was called. +func (w *Watcher) Remove(name string) error { + if w.isClosed() { + return nil + } + return w.remove(filepath.Clean(name)) +} + +func (w *Watcher) remove(name string) error { + wd, ok := w.watches.removePath(name) + if !ok { + return fmt.Errorf("%w: %s", ErrNonExistentWatch, name) + } + + success, errno := unix.InotifyRmWatch(w.fd, wd) + if success == -1 { + // TODO: Perhaps it's not helpful to return an error here in every case; + // The only two possible errors are: + // + // - EBADF, which happens when w.fd is not a valid file descriptor + // of any kind. + // - EINVAL, which is when fd is not an inotify descriptor or wd + // is not a valid watch descriptor. Watch descriptors are + // invalidated when they are removed explicitly or implicitly; + // explicitly by inotify_rm_watch, implicitly when the file they + // are watching is deleted. + return errno + } + return nil +} + +// WatchList returns all paths explicitly added with [Watcher.Add] (and are not +// yet removed). +// +// Returns nil if [Watcher.Close] was called. +func (w *Watcher) WatchList() []string { + if w.isClosed() { + return nil + } + + entries := make([]string, 0, w.watches.len()) + w.watches.mu.RLock() + for pathname := range w.watches.path { + entries = append(entries, pathname) + } + w.watches.mu.RUnlock() + + return entries +} + +// readEvents reads from the inotify file descriptor, converts the +// received events into Event objects and sends them via the Events channel +func (w *Watcher) readEvents() { + defer func() { + close(w.doneResp) + close(w.Errors) + close(w.Events) + }() + + var ( + buf [unix.SizeofInotifyEvent * 4096]byte // Buffer for a maximum of 4096 raw events + errno error // Syscall errno + ) + for { + // See if we have been closed. + if w.isClosed() { + return + } + + n, err := w.inotifyFile.Read(buf[:]) + switch { + case errors.Unwrap(err) == os.ErrClosed: + return + case err != nil: + if !w.sendError(err) { + return + } + continue + } + + if n < unix.SizeofInotifyEvent { + var err error + if n == 0 { + err = io.EOF // If EOF is received. This should really never happen. + } else if n < 0 { + err = errno // If an error occurred while reading. + } else { + err = errors.New("notify: short read in readEvents()") // Read was too short. + } + if !w.sendError(err) { + return + } + continue + } + + var offset uint32 + // We don't know how many events we just read into the buffer + // While the offset points to at least one whole event... + for offset <= uint32(n-unix.SizeofInotifyEvent) { + var ( + // Point "raw" to the event in the buffer + raw = (*unix.InotifyEvent)(unsafe.Pointer(&buf[offset])) + mask = uint32(raw.Mask) + nameLen = uint32(raw.Len) + ) + + if mask&unix.IN_Q_OVERFLOW != 0 { + if !w.sendError(ErrEventOverflow) { + return + } + } + + // If the event happened to the watched directory or the watched file, the kernel + // doesn't append the filename to the event, but we would like to always fill the + // the "Name" field with a valid filename. We retrieve the path of the watch from + // the "paths" map. + watch := w.watches.byWd(uint32(raw.Wd)) + + // inotify will automatically remove the watch on deletes; just need + // to clean our state here. + if watch != nil && mask&unix.IN_DELETE_SELF == unix.IN_DELETE_SELF { + w.watches.remove(watch.wd) + } + // We can't really update the state when a watched path is moved; + // only IN_MOVE_SELF is sent and not IN_MOVED_{FROM,TO}. So remove + // the watch. + if watch != nil && mask&unix.IN_MOVE_SELF == unix.IN_MOVE_SELF { + err := w.remove(watch.path) + if err != nil && !errors.Is(err, ErrNonExistentWatch) { + if !w.sendError(err) { + return + } + } + } + + var name string + if watch != nil { + name = watch.path + } + if nameLen > 0 { + // Point "bytes" at the first byte of the filename + bytes := (*[unix.PathMax]byte)(unsafe.Pointer(&buf[offset+unix.SizeofInotifyEvent]))[:nameLen:nameLen] + // The filename is padded with NULL bytes. TrimRight() gets rid of those. + name += "/" + strings.TrimRight(string(bytes[0:nameLen]), "\000") + } + + event := w.newEvent(name, mask) + + // Send the events that are not ignored on the events channel + if mask&unix.IN_IGNORED == 0 { + if !w.sendEvent(event) { + return + } + } + + // Move to the next event in the buffer + offset += unix.SizeofInotifyEvent + nameLen + } + } +} + +// newEvent returns an platform-independent Event based on an inotify mask. +func (w *Watcher) newEvent(name string, mask uint32) Event { + e := Event{Name: name} + if mask&unix.IN_CREATE == unix.IN_CREATE || mask&unix.IN_MOVED_TO == unix.IN_MOVED_TO { + e.Op |= Create + } + if mask&unix.IN_DELETE_SELF == unix.IN_DELETE_SELF || mask&unix.IN_DELETE == unix.IN_DELETE { + e.Op |= Remove + } + if mask&unix.IN_MODIFY == unix.IN_MODIFY { + e.Op |= Write + } + if mask&unix.IN_MOVE_SELF == unix.IN_MOVE_SELF || mask&unix.IN_MOVED_FROM == unix.IN_MOVED_FROM { + e.Op |= Rename + } + if mask&unix.IN_ATTRIB == unix.IN_ATTRIB { + e.Op |= Chmod + } + return e +} diff --git a/vendor/github.com/fsnotify/fsnotify/backend_kqueue.go b/vendor/github.com/fsnotify/fsnotify/backend_kqueue.go new file mode 100644 index 00000000..063a0915 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/backend_kqueue.go @@ -0,0 +1,782 @@ +//go:build freebsd || openbsd || netbsd || dragonfly || darwin +// +build freebsd openbsd netbsd dragonfly darwin + +// Note: the documentation on the Watcher type and methods is generated from +// mkdoc.zsh + +package fsnotify + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "sync" + + "golang.org/x/sys/unix" +) + +// Watcher watches a set of paths, delivering events on a channel. +// +// A watcher should not be copied (e.g. pass it by pointer, rather than by +// value). +// +// # Linux notes +// +// When a file is removed a Remove event won't be emitted until all file +// descriptors are closed, and deletes will always emit a Chmod. For example: +// +// fp := os.Open("file") +// os.Remove("file") // Triggers Chmod +// fp.Close() // Triggers Remove +// +// This is the event that inotify sends, so not much can be changed about this. +// +// The fs.inotify.max_user_watches sysctl variable specifies the upper limit +// for the number of watches per user, and fs.inotify.max_user_instances +// specifies the maximum number of inotify instances per user. Every Watcher you +// create is an "instance", and every path you add is a "watch". +// +// These are also exposed in /proc as /proc/sys/fs/inotify/max_user_watches and +// /proc/sys/fs/inotify/max_user_instances +// +// To increase them you can use sysctl or write the value to the /proc file: +// +// # Default values on Linux 5.18 +// sysctl fs.inotify.max_user_watches=124983 +// sysctl fs.inotify.max_user_instances=128 +// +// To make the changes persist on reboot edit /etc/sysctl.conf or +// /usr/lib/sysctl.d/50-default.conf (details differ per Linux distro; check +// your distro's documentation): +// +// fs.inotify.max_user_watches=124983 +// fs.inotify.max_user_instances=128 +// +// Reaching the limit will result in a "no space left on device" or "too many open +// files" error. +// +// # kqueue notes (macOS, BSD) +// +// kqueue requires opening a file descriptor for every file that's being watched; +// so if you're watching a directory with five files then that's six file +// descriptors. You will run in to your system's "max open files" limit faster on +// these platforms. +// +// The sysctl variables kern.maxfiles and kern.maxfilesperproc can be used to +// control the maximum number of open files, as well as /etc/login.conf on BSD +// systems. +// +// # Windows notes +// +// Paths can be added as "C:\path\to\dir", but forward slashes +// ("C:/path/to/dir") will also work. +// +// When a watched directory is removed it will always send an event for the +// directory itself, but may not send events for all files in that directory. +// Sometimes it will send events for all times, sometimes it will send no +// events, and often only for some files. +// +// The default ReadDirectoryChangesW() buffer size is 64K, which is the largest +// value that is guaranteed to work with SMB filesystems. If you have many +// events in quick succession this may not be enough, and you will have to use +// [WithBufferSize] to increase the value. +type Watcher struct { + // Events sends the filesystem change events. + // + // fsnotify can send the following events; a "path" here can refer to a + // file, directory, symbolic link, or special file like a FIFO. + // + // fsnotify.Create A new path was created; this may be followed by one + // or more Write events if data also gets written to a + // file. + // + // fsnotify.Remove A path was removed. + // + // fsnotify.Rename A path was renamed. A rename is always sent with the + // old path as Event.Name, and a Create event will be + // sent with the new name. Renames are only sent for + // paths that are currently watched; e.g. moving an + // unmonitored file into a monitored directory will + // show up as just a Create. Similarly, renaming a file + // to outside a monitored directory will show up as + // only a Rename. + // + // fsnotify.Write A file or named pipe was written to. A Truncate will + // also trigger a Write. A single "write action" + // initiated by the user may show up as one or multiple + // writes, depending on when the system syncs things to + // disk. For example when compiling a large Go program + // you may get hundreds of Write events, and you may + // want to wait until you've stopped receiving them + // (see the dedup example in cmd/fsnotify). + // + // Some systems may send Write event for directories + // when the directory content changes. + // + // fsnotify.Chmod Attributes were changed. On Linux this is also sent + // when a file is removed (or more accurately, when a + // link to an inode is removed). On kqueue it's sent + // when a file is truncated. On Windows it's never + // sent. + Events chan Event + + // Errors sends any errors. + // + // ErrEventOverflow is used to indicate there are too many events: + // + // - inotify: There are too many queued events (fs.inotify.max_queued_events sysctl) + // - windows: The buffer size is too small; WithBufferSize() can be used to increase it. + // - kqueue, fen: Not used. + Errors chan error + + done chan struct{} + kq int // File descriptor (as returned by the kqueue() syscall). + closepipe [2]int // Pipe used for closing. + mu sync.Mutex // Protects access to watcher data + watches map[string]int // Watched file descriptors (key: path). + watchesByDir map[string]map[int]struct{} // Watched file descriptors indexed by the parent directory (key: dirname(path)). + userWatches map[string]struct{} // Watches added with Watcher.Add() + dirFlags map[string]uint32 // Watched directories to fflags used in kqueue. + paths map[int]pathInfo // File descriptors to path names for processing kqueue events. + fileExists map[string]struct{} // Keep track of if we know this file exists (to stop duplicate create events). + isClosed bool // Set to true when Close() is first called +} + +type pathInfo struct { + name string + isDir bool +} + +// NewWatcher creates a new Watcher. +func NewWatcher() (*Watcher, error) { + return NewBufferedWatcher(0) +} + +// NewBufferedWatcher creates a new Watcher with a buffered Watcher.Events +// channel. +// +// The main use case for this is situations with a very large number of events +// where the kernel buffer size can't be increased (e.g. due to lack of +// permissions). An unbuffered Watcher will perform better for almost all use +// cases, and whenever possible you will be better off increasing the kernel +// buffers instead of adding a large userspace buffer. +func NewBufferedWatcher(sz uint) (*Watcher, error) { + kq, closepipe, err := newKqueue() + if err != nil { + return nil, err + } + + w := &Watcher{ + kq: kq, + closepipe: closepipe, + watches: make(map[string]int), + watchesByDir: make(map[string]map[int]struct{}), + dirFlags: make(map[string]uint32), + paths: make(map[int]pathInfo), + fileExists: make(map[string]struct{}), + userWatches: make(map[string]struct{}), + Events: make(chan Event, sz), + Errors: make(chan error), + done: make(chan struct{}), + } + + go w.readEvents() + return w, nil +} + +// newKqueue creates a new kernel event queue and returns a descriptor. +// +// This registers a new event on closepipe, which will trigger an event when +// it's closed. This way we can use kevent() without timeout/polling; without +// the closepipe, it would block forever and we wouldn't be able to stop it at +// all. +func newKqueue() (kq int, closepipe [2]int, err error) { + kq, err = unix.Kqueue() + if kq == -1 { + return kq, closepipe, err + } + + // Register the close pipe. + err = unix.Pipe(closepipe[:]) + if err != nil { + unix.Close(kq) + return kq, closepipe, err + } + + // Register changes to listen on the closepipe. + changes := make([]unix.Kevent_t, 1) + // SetKevent converts int to the platform-specific types. + unix.SetKevent(&changes[0], closepipe[0], unix.EVFILT_READ, + unix.EV_ADD|unix.EV_ENABLE|unix.EV_ONESHOT) + + ok, err := unix.Kevent(kq, changes, nil, nil) + if ok == -1 { + unix.Close(kq) + unix.Close(closepipe[0]) + unix.Close(closepipe[1]) + return kq, closepipe, err + } + return kq, closepipe, nil +} + +// Returns true if the event was sent, or false if watcher is closed. +func (w *Watcher) sendEvent(e Event) bool { + select { + case w.Events <- e: + return true + case <-w.done: + return false + } +} + +// Returns true if the error was sent, or false if watcher is closed. +func (w *Watcher) sendError(err error) bool { + select { + case w.Errors <- err: + return true + case <-w.done: + return false + } +} + +// Close removes all watches and closes the Events channel. +func (w *Watcher) Close() error { + w.mu.Lock() + if w.isClosed { + w.mu.Unlock() + return nil + } + w.isClosed = true + + // copy paths to remove while locked + pathsToRemove := make([]string, 0, len(w.watches)) + for name := range w.watches { + pathsToRemove = append(pathsToRemove, name) + } + w.mu.Unlock() // Unlock before calling Remove, which also locks + for _, name := range pathsToRemove { + w.Remove(name) + } + + // Send "quit" message to the reader goroutine. + unix.Close(w.closepipe[1]) + close(w.done) + + return nil +} + +// Add starts monitoring the path for changes. +// +// A path can only be watched once; watching it more than once is a no-op and will +// not return an error. Paths that do not yet exist on the filesystem cannot be +// watched. +// +// A watch will be automatically removed if the watched path is deleted or +// renamed. The exception is the Windows backend, which doesn't remove the +// watcher on renames. +// +// Notifications on network filesystems (NFS, SMB, FUSE, etc.) or special +// filesystems (/proc, /sys, etc.) generally don't work. +// +// Returns [ErrClosed] if [Watcher.Close] was called. +// +// See [Watcher.AddWith] for a version that allows adding options. +// +// # Watching directories +// +// All files in a directory are monitored, including new files that are created +// after the watcher is started. Subdirectories are not watched (i.e. it's +// non-recursive). +// +// # Watching files +// +// Watching individual files (rather than directories) is generally not +// recommended as many programs (especially editors) update files atomically: it +// will write to a temporary file which is then moved to to destination, +// overwriting the original (or some variant thereof). The watcher on the +// original file is now lost, as that no longer exists. +// +// The upshot of this is that a power failure or crash won't leave a +// half-written file. +// +// Watch the parent directory and use Event.Name to filter out files you're not +// interested in. There is an example of this in cmd/fsnotify/file.go. +func (w *Watcher) Add(name string) error { return w.AddWith(name) } + +// AddWith is like [Watcher.Add], but allows adding options. When using Add() +// the defaults described below are used. +// +// Possible options are: +// +// - [WithBufferSize] sets the buffer size for the Windows backend; no-op on +// other platforms. The default is 64K (65536 bytes). +func (w *Watcher) AddWith(name string, opts ...addOpt) error { + _ = getOptions(opts...) + + w.mu.Lock() + w.userWatches[name] = struct{}{} + w.mu.Unlock() + _, err := w.addWatch(name, noteAllEvents) + return err +} + +// Remove stops monitoring the path for changes. +// +// Directories are always removed non-recursively. For example, if you added +// /tmp/dir and /tmp/dir/subdir then you will need to remove both. +// +// Removing a path that has not yet been added returns [ErrNonExistentWatch]. +// +// Returns nil if [Watcher.Close] was called. +func (w *Watcher) Remove(name string) error { + return w.remove(name, true) +} + +func (w *Watcher) remove(name string, unwatchFiles bool) error { + name = filepath.Clean(name) + w.mu.Lock() + if w.isClosed { + w.mu.Unlock() + return nil + } + watchfd, ok := w.watches[name] + w.mu.Unlock() + if !ok { + return fmt.Errorf("%w: %s", ErrNonExistentWatch, name) + } + + err := w.register([]int{watchfd}, unix.EV_DELETE, 0) + if err != nil { + return err + } + + unix.Close(watchfd) + + w.mu.Lock() + isDir := w.paths[watchfd].isDir + delete(w.watches, name) + delete(w.userWatches, name) + + parentName := filepath.Dir(name) + delete(w.watchesByDir[parentName], watchfd) + + if len(w.watchesByDir[parentName]) == 0 { + delete(w.watchesByDir, parentName) + } + + delete(w.paths, watchfd) + delete(w.dirFlags, name) + delete(w.fileExists, name) + w.mu.Unlock() + + // Find all watched paths that are in this directory that are not external. + if unwatchFiles && isDir { + var pathsToRemove []string + w.mu.Lock() + for fd := range w.watchesByDir[name] { + path := w.paths[fd] + if _, ok := w.userWatches[path.name]; !ok { + pathsToRemove = append(pathsToRemove, path.name) + } + } + w.mu.Unlock() + for _, name := range pathsToRemove { + // Since these are internal, not much sense in propagating error to + // the user, as that will just confuse them with an error about a + // path they did not explicitly watch themselves. + w.Remove(name) + } + } + return nil +} + +// WatchList returns all paths explicitly added with [Watcher.Add] (and are not +// yet removed). +// +// Returns nil if [Watcher.Close] was called. +func (w *Watcher) WatchList() []string { + w.mu.Lock() + defer w.mu.Unlock() + if w.isClosed { + return nil + } + + entries := make([]string, 0, len(w.userWatches)) + for pathname := range w.userWatches { + entries = append(entries, pathname) + } + + return entries +} + +// Watch all events (except NOTE_EXTEND, NOTE_LINK, NOTE_REVOKE) +const noteAllEvents = unix.NOTE_DELETE | unix.NOTE_WRITE | unix.NOTE_ATTRIB | unix.NOTE_RENAME + +// addWatch adds name to the watched file set; the flags are interpreted as +// described in kevent(2). +// +// Returns the real path to the file which was added, with symlinks resolved. +func (w *Watcher) addWatch(name string, flags uint32) (string, error) { + var isDir bool + name = filepath.Clean(name) + + w.mu.Lock() + if w.isClosed { + w.mu.Unlock() + return "", ErrClosed + } + watchfd, alreadyWatching := w.watches[name] + // We already have a watch, but we can still override flags. + if alreadyWatching { + isDir = w.paths[watchfd].isDir + } + w.mu.Unlock() + + if !alreadyWatching { + fi, err := os.Lstat(name) + if err != nil { + return "", err + } + + // Don't watch sockets or named pipes + if (fi.Mode()&os.ModeSocket == os.ModeSocket) || (fi.Mode()&os.ModeNamedPipe == os.ModeNamedPipe) { + return "", nil + } + + // Follow Symlinks. + if fi.Mode()&os.ModeSymlink == os.ModeSymlink { + link, err := os.Readlink(name) + if err != nil { + // Return nil because Linux can add unresolvable symlinks to the + // watch list without problems, so maintain consistency with + // that. There will be no file events for broken symlinks. + // TODO: more specific check; returns os.PathError; ENOENT? + return "", nil + } + + w.mu.Lock() + _, alreadyWatching = w.watches[link] + w.mu.Unlock() + + if alreadyWatching { + // Add to watches so we don't get spurious Create events later + // on when we diff the directories. + w.watches[name] = 0 + w.fileExists[name] = struct{}{} + return link, nil + } + + name = link + fi, err = os.Lstat(name) + if err != nil { + return "", nil + } + } + + // Retry on EINTR; open() can return EINTR in practice on macOS. + // See #354, and Go issues 11180 and 39237. + for { + watchfd, err = unix.Open(name, openMode, 0) + if err == nil { + break + } + if errors.Is(err, unix.EINTR) { + continue + } + + return "", err + } + + isDir = fi.IsDir() + } + + err := w.register([]int{watchfd}, unix.EV_ADD|unix.EV_CLEAR|unix.EV_ENABLE, flags) + if err != nil { + unix.Close(watchfd) + return "", err + } + + if !alreadyWatching { + w.mu.Lock() + parentName := filepath.Dir(name) + w.watches[name] = watchfd + + watchesByDir, ok := w.watchesByDir[parentName] + if !ok { + watchesByDir = make(map[int]struct{}, 1) + w.watchesByDir[parentName] = watchesByDir + } + watchesByDir[watchfd] = struct{}{} + w.paths[watchfd] = pathInfo{name: name, isDir: isDir} + w.mu.Unlock() + } + + if isDir { + // Watch the directory if it has not been watched before, or if it was + // watched before, but perhaps only a NOTE_DELETE (watchDirectoryFiles) + w.mu.Lock() + + watchDir := (flags&unix.NOTE_WRITE) == unix.NOTE_WRITE && + (!alreadyWatching || (w.dirFlags[name]&unix.NOTE_WRITE) != unix.NOTE_WRITE) + // Store flags so this watch can be updated later + w.dirFlags[name] = flags + w.mu.Unlock() + + if watchDir { + if err := w.watchDirectoryFiles(name); err != nil { + return "", err + } + } + } + return name, nil +} + +// readEvents reads from kqueue and converts the received kevents into +// Event values that it sends down the Events channel. +func (w *Watcher) readEvents() { + defer func() { + close(w.Events) + close(w.Errors) + _ = unix.Close(w.kq) + unix.Close(w.closepipe[0]) + }() + + eventBuffer := make([]unix.Kevent_t, 10) + for closed := false; !closed; { + kevents, err := w.read(eventBuffer) + // EINTR is okay, the syscall was interrupted before timeout expired. + if err != nil && err != unix.EINTR { + if !w.sendError(fmt.Errorf("fsnotify.readEvents: %w", err)) { + closed = true + } + continue + } + + // Flush the events we received to the Events channel + for _, kevent := range kevents { + var ( + watchfd = int(kevent.Ident) + mask = uint32(kevent.Fflags) + ) + + // Shut down the loop when the pipe is closed, but only after all + // other events have been processed. + if watchfd == w.closepipe[0] { + closed = true + continue + } + + w.mu.Lock() + path := w.paths[watchfd] + w.mu.Unlock() + + event := w.newEvent(path.name, mask) + + if event.Has(Rename) || event.Has(Remove) { + w.remove(event.Name, false) + w.mu.Lock() + delete(w.fileExists, event.Name) + w.mu.Unlock() + } + + if path.isDir && event.Has(Write) && !event.Has(Remove) { + w.sendDirectoryChangeEvents(event.Name) + } else { + if !w.sendEvent(event) { + closed = true + continue + } + } + + if event.Has(Remove) { + // Look for a file that may have overwritten this; for example, + // mv f1 f2 will delete f2, then create f2. + if path.isDir { + fileDir := filepath.Clean(event.Name) + w.mu.Lock() + _, found := w.watches[fileDir] + w.mu.Unlock() + if found { + err := w.sendDirectoryChangeEvents(fileDir) + if err != nil { + if !w.sendError(err) { + closed = true + } + } + } + } else { + filePath := filepath.Clean(event.Name) + if fi, err := os.Lstat(filePath); err == nil { + err := w.sendFileCreatedEventIfNew(filePath, fi) + if err != nil { + if !w.sendError(err) { + closed = true + } + } + } + } + } + } + } +} + +// newEvent returns an platform-independent Event based on kqueue Fflags. +func (w *Watcher) newEvent(name string, mask uint32) Event { + e := Event{Name: name} + if mask&unix.NOTE_DELETE == unix.NOTE_DELETE { + e.Op |= Remove + } + if mask&unix.NOTE_WRITE == unix.NOTE_WRITE { + e.Op |= Write + } + if mask&unix.NOTE_RENAME == unix.NOTE_RENAME { + e.Op |= Rename + } + if mask&unix.NOTE_ATTRIB == unix.NOTE_ATTRIB { + e.Op |= Chmod + } + // No point sending a write and delete event at the same time: if it's gone, + // then it's gone. + if e.Op.Has(Write) && e.Op.Has(Remove) { + e.Op &^= Write + } + return e +} + +// watchDirectoryFiles to mimic inotify when adding a watch on a directory +func (w *Watcher) watchDirectoryFiles(dirPath string) error { + // Get all files + files, err := os.ReadDir(dirPath) + if err != nil { + return err + } + + for _, f := range files { + path := filepath.Join(dirPath, f.Name()) + + fi, err := f.Info() + if err != nil { + return fmt.Errorf("%q: %w", path, err) + } + + cleanPath, err := w.internalWatch(path, fi) + if err != nil { + // No permission to read the file; that's not a problem: just skip. + // But do add it to w.fileExists to prevent it from being picked up + // as a "new" file later (it still shows up in the directory + // listing). + switch { + case errors.Is(err, unix.EACCES) || errors.Is(err, unix.EPERM): + cleanPath = filepath.Clean(path) + default: + return fmt.Errorf("%q: %w", path, err) + } + } + + w.mu.Lock() + w.fileExists[cleanPath] = struct{}{} + w.mu.Unlock() + } + + return nil +} + +// Search the directory for new files and send an event for them. +// +// This functionality is to have the BSD watcher match the inotify, which sends +// a create event for files created in a watched directory. +func (w *Watcher) sendDirectoryChangeEvents(dir string) error { + files, err := os.ReadDir(dir) + if err != nil { + // Directory no longer exists: we can ignore this safely. kqueue will + // still give us the correct events. + if errors.Is(err, os.ErrNotExist) { + return nil + } + return fmt.Errorf("fsnotify.sendDirectoryChangeEvents: %w", err) + } + + for _, f := range files { + fi, err := f.Info() + if err != nil { + return fmt.Errorf("fsnotify.sendDirectoryChangeEvents: %w", err) + } + + err = w.sendFileCreatedEventIfNew(filepath.Join(dir, fi.Name()), fi) + if err != nil { + // Don't need to send an error if this file isn't readable. + if errors.Is(err, unix.EACCES) || errors.Is(err, unix.EPERM) { + return nil + } + return fmt.Errorf("fsnotify.sendDirectoryChangeEvents: %w", err) + } + } + return nil +} + +// sendFileCreatedEvent sends a create event if the file isn't already being tracked. +func (w *Watcher) sendFileCreatedEventIfNew(filePath string, fi os.FileInfo) (err error) { + w.mu.Lock() + _, doesExist := w.fileExists[filePath] + w.mu.Unlock() + if !doesExist { + if !w.sendEvent(Event{Name: filePath, Op: Create}) { + return + } + } + + // like watchDirectoryFiles (but without doing another ReadDir) + filePath, err = w.internalWatch(filePath, fi) + if err != nil { + return err + } + + w.mu.Lock() + w.fileExists[filePath] = struct{}{} + w.mu.Unlock() + + return nil +} + +func (w *Watcher) internalWatch(name string, fi os.FileInfo) (string, error) { + if fi.IsDir() { + // mimic Linux providing delete events for subdirectories, but preserve + // the flags used if currently watching subdirectory + w.mu.Lock() + flags := w.dirFlags[name] + w.mu.Unlock() + + flags |= unix.NOTE_DELETE | unix.NOTE_RENAME + return w.addWatch(name, flags) + } + + // watch file to mimic Linux inotify + return w.addWatch(name, noteAllEvents) +} + +// Register events with the queue. +func (w *Watcher) register(fds []int, flags int, fflags uint32) error { + changes := make([]unix.Kevent_t, len(fds)) + for i, fd := range fds { + // SetKevent converts int to the platform-specific types. + unix.SetKevent(&changes[i], fd, unix.EVFILT_VNODE, flags) + changes[i].Fflags = fflags + } + + // Register the events. + success, err := unix.Kevent(w.kq, changes, nil, nil) + if success == -1 { + return err + } + return nil +} + +// read retrieves pending events, or waits until an event occurs. +func (w *Watcher) read(events []unix.Kevent_t) ([]unix.Kevent_t, error) { + n, err := unix.Kevent(w.kq, nil, events, nil) + if err != nil { + return nil, err + } + return events[0:n], nil +} diff --git a/vendor/github.com/fsnotify/fsnotify/backend_other.go b/vendor/github.com/fsnotify/fsnotify/backend_other.go new file mode 100644 index 00000000..d34a23c0 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/backend_other.go @@ -0,0 +1,205 @@ +//go:build appengine || (!darwin && !dragonfly && !freebsd && !openbsd && !linux && !netbsd && !solaris && !windows) +// +build appengine !darwin,!dragonfly,!freebsd,!openbsd,!linux,!netbsd,!solaris,!windows + +// Note: the documentation on the Watcher type and methods is generated from +// mkdoc.zsh + +package fsnotify + +import "errors" + +// Watcher watches a set of paths, delivering events on a channel. +// +// A watcher should not be copied (e.g. pass it by pointer, rather than by +// value). +// +// # Linux notes +// +// When a file is removed a Remove event won't be emitted until all file +// descriptors are closed, and deletes will always emit a Chmod. For example: +// +// fp := os.Open("file") +// os.Remove("file") // Triggers Chmod +// fp.Close() // Triggers Remove +// +// This is the event that inotify sends, so not much can be changed about this. +// +// The fs.inotify.max_user_watches sysctl variable specifies the upper limit +// for the number of watches per user, and fs.inotify.max_user_instances +// specifies the maximum number of inotify instances per user. Every Watcher you +// create is an "instance", and every path you add is a "watch". +// +// These are also exposed in /proc as /proc/sys/fs/inotify/max_user_watches and +// /proc/sys/fs/inotify/max_user_instances +// +// To increase them you can use sysctl or write the value to the /proc file: +// +// # Default values on Linux 5.18 +// sysctl fs.inotify.max_user_watches=124983 +// sysctl fs.inotify.max_user_instances=128 +// +// To make the changes persist on reboot edit /etc/sysctl.conf or +// /usr/lib/sysctl.d/50-default.conf (details differ per Linux distro; check +// your distro's documentation): +// +// fs.inotify.max_user_watches=124983 +// fs.inotify.max_user_instances=128 +// +// Reaching the limit will result in a "no space left on device" or "too many open +// files" error. +// +// # kqueue notes (macOS, BSD) +// +// kqueue requires opening a file descriptor for every file that's being watched; +// so if you're watching a directory with five files then that's six file +// descriptors. You will run in to your system's "max open files" limit faster on +// these platforms. +// +// The sysctl variables kern.maxfiles and kern.maxfilesperproc can be used to +// control the maximum number of open files, as well as /etc/login.conf on BSD +// systems. +// +// # Windows notes +// +// Paths can be added as "C:\path\to\dir", but forward slashes +// ("C:/path/to/dir") will also work. +// +// When a watched directory is removed it will always send an event for the +// directory itself, but may not send events for all files in that directory. +// Sometimes it will send events for all times, sometimes it will send no +// events, and often only for some files. +// +// The default ReadDirectoryChangesW() buffer size is 64K, which is the largest +// value that is guaranteed to work with SMB filesystems. If you have many +// events in quick succession this may not be enough, and you will have to use +// [WithBufferSize] to increase the value. +type Watcher struct { + // Events sends the filesystem change events. + // + // fsnotify can send the following events; a "path" here can refer to a + // file, directory, symbolic link, or special file like a FIFO. + // + // fsnotify.Create A new path was created; this may be followed by one + // or more Write events if data also gets written to a + // file. + // + // fsnotify.Remove A path was removed. + // + // fsnotify.Rename A path was renamed. A rename is always sent with the + // old path as Event.Name, and a Create event will be + // sent with the new name. Renames are only sent for + // paths that are currently watched; e.g. moving an + // unmonitored file into a monitored directory will + // show up as just a Create. Similarly, renaming a file + // to outside a monitored directory will show up as + // only a Rename. + // + // fsnotify.Write A file or named pipe was written to. A Truncate will + // also trigger a Write. A single "write action" + // initiated by the user may show up as one or multiple + // writes, depending on when the system syncs things to + // disk. For example when compiling a large Go program + // you may get hundreds of Write events, and you may + // want to wait until you've stopped receiving them + // (see the dedup example in cmd/fsnotify). + // + // Some systems may send Write event for directories + // when the directory content changes. + // + // fsnotify.Chmod Attributes were changed. On Linux this is also sent + // when a file is removed (or more accurately, when a + // link to an inode is removed). On kqueue it's sent + // when a file is truncated. On Windows it's never + // sent. + Events chan Event + + // Errors sends any errors. + // + // ErrEventOverflow is used to indicate there are too many events: + // + // - inotify: There are too many queued events (fs.inotify.max_queued_events sysctl) + // - windows: The buffer size is too small; WithBufferSize() can be used to increase it. + // - kqueue, fen: Not used. + Errors chan error +} + +// NewWatcher creates a new Watcher. +func NewWatcher() (*Watcher, error) { + return nil, errors.New("fsnotify not supported on the current platform") +} + +// NewBufferedWatcher creates a new Watcher with a buffered Watcher.Events +// channel. +// +// The main use case for this is situations with a very large number of events +// where the kernel buffer size can't be increased (e.g. due to lack of +// permissions). An unbuffered Watcher will perform better for almost all use +// cases, and whenever possible you will be better off increasing the kernel +// buffers instead of adding a large userspace buffer. +func NewBufferedWatcher(sz uint) (*Watcher, error) { return NewWatcher() } + +// Close removes all watches and closes the Events channel. +func (w *Watcher) Close() error { return nil } + +// WatchList returns all paths explicitly added with [Watcher.Add] (and are not +// yet removed). +// +// Returns nil if [Watcher.Close] was called. +func (w *Watcher) WatchList() []string { return nil } + +// Add starts monitoring the path for changes. +// +// A path can only be watched once; watching it more than once is a no-op and will +// not return an error. Paths that do not yet exist on the filesystem cannot be +// watched. +// +// A watch will be automatically removed if the watched path is deleted or +// renamed. The exception is the Windows backend, which doesn't remove the +// watcher on renames. +// +// Notifications on network filesystems (NFS, SMB, FUSE, etc.) or special +// filesystems (/proc, /sys, etc.) generally don't work. +// +// Returns [ErrClosed] if [Watcher.Close] was called. +// +// See [Watcher.AddWith] for a version that allows adding options. +// +// # Watching directories +// +// All files in a directory are monitored, including new files that are created +// after the watcher is started. Subdirectories are not watched (i.e. it's +// non-recursive). +// +// # Watching files +// +// Watching individual files (rather than directories) is generally not +// recommended as many programs (especially editors) update files atomically: it +// will write to a temporary file which is then moved to to destination, +// overwriting the original (or some variant thereof). The watcher on the +// original file is now lost, as that no longer exists. +// +// The upshot of this is that a power failure or crash won't leave a +// half-written file. +// +// Watch the parent directory and use Event.Name to filter out files you're not +// interested in. There is an example of this in cmd/fsnotify/file.go. +func (w *Watcher) Add(name string) error { return nil } + +// AddWith is like [Watcher.Add], but allows adding options. When using Add() +// the defaults described below are used. +// +// Possible options are: +// +// - [WithBufferSize] sets the buffer size for the Windows backend; no-op on +// other platforms. The default is 64K (65536 bytes). +func (w *Watcher) AddWith(name string, opts ...addOpt) error { return nil } + +// Remove stops monitoring the path for changes. +// +// Directories are always removed non-recursively. For example, if you added +// /tmp/dir and /tmp/dir/subdir then you will need to remove both. +// +// Removing a path that has not yet been added returns [ErrNonExistentWatch]. +// +// Returns nil if [Watcher.Close] was called. +func (w *Watcher) Remove(name string) error { return nil } diff --git a/vendor/github.com/fsnotify/fsnotify/backend_windows.go b/vendor/github.com/fsnotify/fsnotify/backend_windows.go new file mode 100644 index 00000000..9bc91e5d --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/backend_windows.go @@ -0,0 +1,827 @@ +//go:build windows +// +build windows + +// Windows backend based on ReadDirectoryChangesW() +// +// https://learn.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-readdirectorychangesw +// +// Note: the documentation on the Watcher type and methods is generated from +// mkdoc.zsh + +package fsnotify + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" + "sync" + "unsafe" + + "golang.org/x/sys/windows" +) + +// Watcher watches a set of paths, delivering events on a channel. +// +// A watcher should not be copied (e.g. pass it by pointer, rather than by +// value). +// +// # Linux notes +// +// When a file is removed a Remove event won't be emitted until all file +// descriptors are closed, and deletes will always emit a Chmod. For example: +// +// fp := os.Open("file") +// os.Remove("file") // Triggers Chmod +// fp.Close() // Triggers Remove +// +// This is the event that inotify sends, so not much can be changed about this. +// +// The fs.inotify.max_user_watches sysctl variable specifies the upper limit +// for the number of watches per user, and fs.inotify.max_user_instances +// specifies the maximum number of inotify instances per user. Every Watcher you +// create is an "instance", and every path you add is a "watch". +// +// These are also exposed in /proc as /proc/sys/fs/inotify/max_user_watches and +// /proc/sys/fs/inotify/max_user_instances +// +// To increase them you can use sysctl or write the value to the /proc file: +// +// # Default values on Linux 5.18 +// sysctl fs.inotify.max_user_watches=124983 +// sysctl fs.inotify.max_user_instances=128 +// +// To make the changes persist on reboot edit /etc/sysctl.conf or +// /usr/lib/sysctl.d/50-default.conf (details differ per Linux distro; check +// your distro's documentation): +// +// fs.inotify.max_user_watches=124983 +// fs.inotify.max_user_instances=128 +// +// Reaching the limit will result in a "no space left on device" or "too many open +// files" error. +// +// # kqueue notes (macOS, BSD) +// +// kqueue requires opening a file descriptor for every file that's being watched; +// so if you're watching a directory with five files then that's six file +// descriptors. You will run in to your system's "max open files" limit faster on +// these platforms. +// +// The sysctl variables kern.maxfiles and kern.maxfilesperproc can be used to +// control the maximum number of open files, as well as /etc/login.conf on BSD +// systems. +// +// # Windows notes +// +// Paths can be added as "C:\path\to\dir", but forward slashes +// ("C:/path/to/dir") will also work. +// +// When a watched directory is removed it will always send an event for the +// directory itself, but may not send events for all files in that directory. +// Sometimes it will send events for all times, sometimes it will send no +// events, and often only for some files. +// +// The default ReadDirectoryChangesW() buffer size is 64K, which is the largest +// value that is guaranteed to work with SMB filesystems. If you have many +// events in quick succession this may not be enough, and you will have to use +// [WithBufferSize] to increase the value. +type Watcher struct { + // Events sends the filesystem change events. + // + // fsnotify can send the following events; a "path" here can refer to a + // file, directory, symbolic link, or special file like a FIFO. + // + // fsnotify.Create A new path was created; this may be followed by one + // or more Write events if data also gets written to a + // file. + // + // fsnotify.Remove A path was removed. + // + // fsnotify.Rename A path was renamed. A rename is always sent with the + // old path as Event.Name, and a Create event will be + // sent with the new name. Renames are only sent for + // paths that are currently watched; e.g. moving an + // unmonitored file into a monitored directory will + // show up as just a Create. Similarly, renaming a file + // to outside a monitored directory will show up as + // only a Rename. + // + // fsnotify.Write A file or named pipe was written to. A Truncate will + // also trigger a Write. A single "write action" + // initiated by the user may show up as one or multiple + // writes, depending on when the system syncs things to + // disk. For example when compiling a large Go program + // you may get hundreds of Write events, and you may + // want to wait until you've stopped receiving them + // (see the dedup example in cmd/fsnotify). + // + // Some systems may send Write event for directories + // when the directory content changes. + // + // fsnotify.Chmod Attributes were changed. On Linux this is also sent + // when a file is removed (or more accurately, when a + // link to an inode is removed). On kqueue it's sent + // when a file is truncated. On Windows it's never + // sent. + Events chan Event + + // Errors sends any errors. + // + // ErrEventOverflow is used to indicate there are too many events: + // + // - inotify: There are too many queued events (fs.inotify.max_queued_events sysctl) + // - windows: The buffer size is too small; WithBufferSize() can be used to increase it. + // - kqueue, fen: Not used. + Errors chan error + + port windows.Handle // Handle to completion port + input chan *input // Inputs to the reader are sent on this channel + quit chan chan<- error + + mu sync.Mutex // Protects access to watches, closed + watches watchMap // Map of watches (key: i-number) + closed bool // Set to true when Close() is first called +} + +// NewWatcher creates a new Watcher. +func NewWatcher() (*Watcher, error) { + return NewBufferedWatcher(50) +} + +// NewBufferedWatcher creates a new Watcher with a buffered Watcher.Events +// channel. +// +// The main use case for this is situations with a very large number of events +// where the kernel buffer size can't be increased (e.g. due to lack of +// permissions). An unbuffered Watcher will perform better for almost all use +// cases, and whenever possible you will be better off increasing the kernel +// buffers instead of adding a large userspace buffer. +func NewBufferedWatcher(sz uint) (*Watcher, error) { + port, err := windows.CreateIoCompletionPort(windows.InvalidHandle, 0, 0, 0) + if err != nil { + return nil, os.NewSyscallError("CreateIoCompletionPort", err) + } + w := &Watcher{ + port: port, + watches: make(watchMap), + input: make(chan *input, 1), + Events: make(chan Event, sz), + Errors: make(chan error), + quit: make(chan chan<- error, 1), + } + go w.readEvents() + return w, nil +} + +func (w *Watcher) isClosed() bool { + w.mu.Lock() + defer w.mu.Unlock() + return w.closed +} + +func (w *Watcher) sendEvent(name string, mask uint64) bool { + if mask == 0 { + return false + } + + event := w.newEvent(name, uint32(mask)) + select { + case ch := <-w.quit: + w.quit <- ch + case w.Events <- event: + } + return true +} + +// Returns true if the error was sent, or false if watcher is closed. +func (w *Watcher) sendError(err error) bool { + select { + case w.Errors <- err: + return true + case <-w.quit: + } + return false +} + +// Close removes all watches and closes the Events channel. +func (w *Watcher) Close() error { + if w.isClosed() { + return nil + } + + w.mu.Lock() + w.closed = true + w.mu.Unlock() + + // Send "quit" message to the reader goroutine + ch := make(chan error) + w.quit <- ch + if err := w.wakeupReader(); err != nil { + return err + } + return <-ch +} + +// Add starts monitoring the path for changes. +// +// A path can only be watched once; watching it more than once is a no-op and will +// not return an error. Paths that do not yet exist on the filesystem cannot be +// watched. +// +// A watch will be automatically removed if the watched path is deleted or +// renamed. The exception is the Windows backend, which doesn't remove the +// watcher on renames. +// +// Notifications on network filesystems (NFS, SMB, FUSE, etc.) or special +// filesystems (/proc, /sys, etc.) generally don't work. +// +// Returns [ErrClosed] if [Watcher.Close] was called. +// +// See [Watcher.AddWith] for a version that allows adding options. +// +// # Watching directories +// +// All files in a directory are monitored, including new files that are created +// after the watcher is started. Subdirectories are not watched (i.e. it's +// non-recursive). +// +// # Watching files +// +// Watching individual files (rather than directories) is generally not +// recommended as many programs (especially editors) update files atomically: it +// will write to a temporary file which is then moved to to destination, +// overwriting the original (or some variant thereof). The watcher on the +// original file is now lost, as that no longer exists. +// +// The upshot of this is that a power failure or crash won't leave a +// half-written file. +// +// Watch the parent directory and use Event.Name to filter out files you're not +// interested in. There is an example of this in cmd/fsnotify/file.go. +func (w *Watcher) Add(name string) error { return w.AddWith(name) } + +// AddWith is like [Watcher.Add], but allows adding options. When using Add() +// the defaults described below are used. +// +// Possible options are: +// +// - [WithBufferSize] sets the buffer size for the Windows backend; no-op on +// other platforms. The default is 64K (65536 bytes). +func (w *Watcher) AddWith(name string, opts ...addOpt) error { + if w.isClosed() { + return ErrClosed + } + + with := getOptions(opts...) + if with.bufsize < 4096 { + return fmt.Errorf("fsnotify.WithBufferSize: buffer size cannot be smaller than 4096 bytes") + } + + in := &input{ + op: opAddWatch, + path: filepath.Clean(name), + flags: sysFSALLEVENTS, + reply: make(chan error), + bufsize: with.bufsize, + } + w.input <- in + if err := w.wakeupReader(); err != nil { + return err + } + return <-in.reply +} + +// Remove stops monitoring the path for changes. +// +// Directories are always removed non-recursively. For example, if you added +// /tmp/dir and /tmp/dir/subdir then you will need to remove both. +// +// Removing a path that has not yet been added returns [ErrNonExistentWatch]. +// +// Returns nil if [Watcher.Close] was called. +func (w *Watcher) Remove(name string) error { + if w.isClosed() { + return nil + } + + in := &input{ + op: opRemoveWatch, + path: filepath.Clean(name), + reply: make(chan error), + } + w.input <- in + if err := w.wakeupReader(); err != nil { + return err + } + return <-in.reply +} + +// WatchList returns all paths explicitly added with [Watcher.Add] (and are not +// yet removed). +// +// Returns nil if [Watcher.Close] was called. +func (w *Watcher) WatchList() []string { + if w.isClosed() { + return nil + } + + w.mu.Lock() + defer w.mu.Unlock() + + entries := make([]string, 0, len(w.watches)) + for _, entry := range w.watches { + for _, watchEntry := range entry { + entries = append(entries, watchEntry.path) + } + } + + return entries +} + +// These options are from the old golang.org/x/exp/winfsnotify, where you could +// add various options to the watch. This has long since been removed. +// +// The "sys" in the name is misleading as they're not part of any "system". +// +// This should all be removed at some point, and just use windows.FILE_NOTIFY_* +const ( + sysFSALLEVENTS = 0xfff + sysFSCREATE = 0x100 + sysFSDELETE = 0x200 + sysFSDELETESELF = 0x400 + sysFSMODIFY = 0x2 + sysFSMOVE = 0xc0 + sysFSMOVEDFROM = 0x40 + sysFSMOVEDTO = 0x80 + sysFSMOVESELF = 0x800 + sysFSIGNORED = 0x8000 +) + +func (w *Watcher) newEvent(name string, mask uint32) Event { + e := Event{Name: name} + if mask&sysFSCREATE == sysFSCREATE || mask&sysFSMOVEDTO == sysFSMOVEDTO { + e.Op |= Create + } + if mask&sysFSDELETE == sysFSDELETE || mask&sysFSDELETESELF == sysFSDELETESELF { + e.Op |= Remove + } + if mask&sysFSMODIFY == sysFSMODIFY { + e.Op |= Write + } + if mask&sysFSMOVE == sysFSMOVE || mask&sysFSMOVESELF == sysFSMOVESELF || mask&sysFSMOVEDFROM == sysFSMOVEDFROM { + e.Op |= Rename + } + return e +} + +const ( + opAddWatch = iota + opRemoveWatch +) + +const ( + provisional uint64 = 1 << (32 + iota) +) + +type input struct { + op int + path string + flags uint32 + bufsize int + reply chan error +} + +type inode struct { + handle windows.Handle + volume uint32 + index uint64 +} + +type watch struct { + ov windows.Overlapped + ino *inode // i-number + recurse bool // Recursive watch? + path string // Directory path + mask uint64 // Directory itself is being watched with these notify flags + names map[string]uint64 // Map of names being watched and their notify flags + rename string // Remembers the old name while renaming a file + buf []byte // buffer, allocated later +} + +type ( + indexMap map[uint64]*watch + watchMap map[uint32]indexMap +) + +func (w *Watcher) wakeupReader() error { + err := windows.PostQueuedCompletionStatus(w.port, 0, 0, nil) + if err != nil { + return os.NewSyscallError("PostQueuedCompletionStatus", err) + } + return nil +} + +func (w *Watcher) getDir(pathname string) (dir string, err error) { + attr, err := windows.GetFileAttributes(windows.StringToUTF16Ptr(pathname)) + if err != nil { + return "", os.NewSyscallError("GetFileAttributes", err) + } + if attr&windows.FILE_ATTRIBUTE_DIRECTORY != 0 { + dir = pathname + } else { + dir, _ = filepath.Split(pathname) + dir = filepath.Clean(dir) + } + return +} + +func (w *Watcher) getIno(path string) (ino *inode, err error) { + h, err := windows.CreateFile(windows.StringToUTF16Ptr(path), + windows.FILE_LIST_DIRECTORY, + windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE|windows.FILE_SHARE_DELETE, + nil, windows.OPEN_EXISTING, + windows.FILE_FLAG_BACKUP_SEMANTICS|windows.FILE_FLAG_OVERLAPPED, 0) + if err != nil { + return nil, os.NewSyscallError("CreateFile", err) + } + + var fi windows.ByHandleFileInformation + err = windows.GetFileInformationByHandle(h, &fi) + if err != nil { + windows.CloseHandle(h) + return nil, os.NewSyscallError("GetFileInformationByHandle", err) + } + ino = &inode{ + handle: h, + volume: fi.VolumeSerialNumber, + index: uint64(fi.FileIndexHigh)<<32 | uint64(fi.FileIndexLow), + } + return ino, nil +} + +// Must run within the I/O thread. +func (m watchMap) get(ino *inode) *watch { + if i := m[ino.volume]; i != nil { + return i[ino.index] + } + return nil +} + +// Must run within the I/O thread. +func (m watchMap) set(ino *inode, watch *watch) { + i := m[ino.volume] + if i == nil { + i = make(indexMap) + m[ino.volume] = i + } + i[ino.index] = watch +} + +// Must run within the I/O thread. +func (w *Watcher) addWatch(pathname string, flags uint64, bufsize int) error { + //pathname, recurse := recursivePath(pathname) + recurse := false + + dir, err := w.getDir(pathname) + if err != nil { + return err + } + + ino, err := w.getIno(dir) + if err != nil { + return err + } + w.mu.Lock() + watchEntry := w.watches.get(ino) + w.mu.Unlock() + if watchEntry == nil { + _, err := windows.CreateIoCompletionPort(ino.handle, w.port, 0, 0) + if err != nil { + windows.CloseHandle(ino.handle) + return os.NewSyscallError("CreateIoCompletionPort", err) + } + watchEntry = &watch{ + ino: ino, + path: dir, + names: make(map[string]uint64), + recurse: recurse, + buf: make([]byte, bufsize), + } + w.mu.Lock() + w.watches.set(ino, watchEntry) + w.mu.Unlock() + flags |= provisional + } else { + windows.CloseHandle(ino.handle) + } + if pathname == dir { + watchEntry.mask |= flags + } else { + watchEntry.names[filepath.Base(pathname)] |= flags + } + + err = w.startRead(watchEntry) + if err != nil { + return err + } + + if pathname == dir { + watchEntry.mask &= ^provisional + } else { + watchEntry.names[filepath.Base(pathname)] &= ^provisional + } + return nil +} + +// Must run within the I/O thread. +func (w *Watcher) remWatch(pathname string) error { + pathname, recurse := recursivePath(pathname) + + dir, err := w.getDir(pathname) + if err != nil { + return err + } + ino, err := w.getIno(dir) + if err != nil { + return err + } + + w.mu.Lock() + watch := w.watches.get(ino) + w.mu.Unlock() + + if recurse && !watch.recurse { + return fmt.Errorf("can't use \\... with non-recursive watch %q", pathname) + } + + err = windows.CloseHandle(ino.handle) + if err != nil { + w.sendError(os.NewSyscallError("CloseHandle", err)) + } + if watch == nil { + return fmt.Errorf("%w: %s", ErrNonExistentWatch, pathname) + } + if pathname == dir { + w.sendEvent(watch.path, watch.mask&sysFSIGNORED) + watch.mask = 0 + } else { + name := filepath.Base(pathname) + w.sendEvent(filepath.Join(watch.path, name), watch.names[name]&sysFSIGNORED) + delete(watch.names, name) + } + + return w.startRead(watch) +} + +// Must run within the I/O thread. +func (w *Watcher) deleteWatch(watch *watch) { + for name, mask := range watch.names { + if mask&provisional == 0 { + w.sendEvent(filepath.Join(watch.path, name), mask&sysFSIGNORED) + } + delete(watch.names, name) + } + if watch.mask != 0 { + if watch.mask&provisional == 0 { + w.sendEvent(watch.path, watch.mask&sysFSIGNORED) + } + watch.mask = 0 + } +} + +// Must run within the I/O thread. +func (w *Watcher) startRead(watch *watch) error { + err := windows.CancelIo(watch.ino.handle) + if err != nil { + w.sendError(os.NewSyscallError("CancelIo", err)) + w.deleteWatch(watch) + } + mask := w.toWindowsFlags(watch.mask) + for _, m := range watch.names { + mask |= w.toWindowsFlags(m) + } + if mask == 0 { + err := windows.CloseHandle(watch.ino.handle) + if err != nil { + w.sendError(os.NewSyscallError("CloseHandle", err)) + } + w.mu.Lock() + delete(w.watches[watch.ino.volume], watch.ino.index) + w.mu.Unlock() + return nil + } + + // We need to pass the array, rather than the slice. + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&watch.buf)) + rdErr := windows.ReadDirectoryChanges(watch.ino.handle, + (*byte)(unsafe.Pointer(hdr.Data)), uint32(hdr.Len), + watch.recurse, mask, nil, &watch.ov, 0) + if rdErr != nil { + err := os.NewSyscallError("ReadDirectoryChanges", rdErr) + if rdErr == windows.ERROR_ACCESS_DENIED && watch.mask&provisional == 0 { + // Watched directory was probably removed + w.sendEvent(watch.path, watch.mask&sysFSDELETESELF) + err = nil + } + w.deleteWatch(watch) + w.startRead(watch) + return err + } + return nil +} + +// readEvents reads from the I/O completion port, converts the +// received events into Event objects and sends them via the Events channel. +// Entry point to the I/O thread. +func (w *Watcher) readEvents() { + var ( + n uint32 + key uintptr + ov *windows.Overlapped + ) + runtime.LockOSThread() + + for { + // This error is handled after the watch == nil check below. + qErr := windows.GetQueuedCompletionStatus(w.port, &n, &key, &ov, windows.INFINITE) + + watch := (*watch)(unsafe.Pointer(ov)) + if watch == nil { + select { + case ch := <-w.quit: + w.mu.Lock() + var indexes []indexMap + for _, index := range w.watches { + indexes = append(indexes, index) + } + w.mu.Unlock() + for _, index := range indexes { + for _, watch := range index { + w.deleteWatch(watch) + w.startRead(watch) + } + } + + err := windows.CloseHandle(w.port) + if err != nil { + err = os.NewSyscallError("CloseHandle", err) + } + close(w.Events) + close(w.Errors) + ch <- err + return + case in := <-w.input: + switch in.op { + case opAddWatch: + in.reply <- w.addWatch(in.path, uint64(in.flags), in.bufsize) + case opRemoveWatch: + in.reply <- w.remWatch(in.path) + } + default: + } + continue + } + + switch qErr { + case nil: + // No error + case windows.ERROR_MORE_DATA: + if watch == nil { + w.sendError(errors.New("ERROR_MORE_DATA has unexpectedly null lpOverlapped buffer")) + } else { + // The i/o succeeded but the buffer is full. + // In theory we should be building up a full packet. + // In practice we can get away with just carrying on. + n = uint32(unsafe.Sizeof(watch.buf)) + } + case windows.ERROR_ACCESS_DENIED: + // Watched directory was probably removed + w.sendEvent(watch.path, watch.mask&sysFSDELETESELF) + w.deleteWatch(watch) + w.startRead(watch) + continue + case windows.ERROR_OPERATION_ABORTED: + // CancelIo was called on this handle + continue + default: + w.sendError(os.NewSyscallError("GetQueuedCompletionPort", qErr)) + continue + } + + var offset uint32 + for { + if n == 0 { + w.sendError(ErrEventOverflow) + break + } + + // Point "raw" to the event in the buffer + raw := (*windows.FileNotifyInformation)(unsafe.Pointer(&watch.buf[offset])) + + // Create a buf that is the size of the path name + size := int(raw.FileNameLength / 2) + var buf []uint16 + // TODO: Use unsafe.Slice in Go 1.17; https://stackoverflow.com/questions/51187973 + sh := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + sh.Data = uintptr(unsafe.Pointer(&raw.FileName)) + sh.Len = size + sh.Cap = size + name := windows.UTF16ToString(buf) + fullname := filepath.Join(watch.path, name) + + var mask uint64 + switch raw.Action { + case windows.FILE_ACTION_REMOVED: + mask = sysFSDELETESELF + case windows.FILE_ACTION_MODIFIED: + mask = sysFSMODIFY + case windows.FILE_ACTION_RENAMED_OLD_NAME: + watch.rename = name + case windows.FILE_ACTION_RENAMED_NEW_NAME: + // Update saved path of all sub-watches. + old := filepath.Join(watch.path, watch.rename) + w.mu.Lock() + for _, watchMap := range w.watches { + for _, ww := range watchMap { + if strings.HasPrefix(ww.path, old) { + ww.path = filepath.Join(fullname, strings.TrimPrefix(ww.path, old)) + } + } + } + w.mu.Unlock() + + if watch.names[watch.rename] != 0 { + watch.names[name] |= watch.names[watch.rename] + delete(watch.names, watch.rename) + mask = sysFSMOVESELF + } + } + + sendNameEvent := func() { + w.sendEvent(fullname, watch.names[name]&mask) + } + if raw.Action != windows.FILE_ACTION_RENAMED_NEW_NAME { + sendNameEvent() + } + if raw.Action == windows.FILE_ACTION_REMOVED { + w.sendEvent(fullname, watch.names[name]&sysFSIGNORED) + delete(watch.names, name) + } + + w.sendEvent(fullname, watch.mask&w.toFSnotifyFlags(raw.Action)) + if raw.Action == windows.FILE_ACTION_RENAMED_NEW_NAME { + fullname = filepath.Join(watch.path, watch.rename) + sendNameEvent() + } + + // Move to the next event in the buffer + if raw.NextEntryOffset == 0 { + break + } + offset += raw.NextEntryOffset + + // Error! + if offset >= n { + //lint:ignore ST1005 Windows should be capitalized + w.sendError(errors.New( + "Windows system assumed buffer larger than it is, events have likely been missed")) + break + } + } + + if err := w.startRead(watch); err != nil { + w.sendError(err) + } + } +} + +func (w *Watcher) toWindowsFlags(mask uint64) uint32 { + var m uint32 + if mask&sysFSMODIFY != 0 { + m |= windows.FILE_NOTIFY_CHANGE_LAST_WRITE + } + if mask&(sysFSMOVE|sysFSCREATE|sysFSDELETE) != 0 { + m |= windows.FILE_NOTIFY_CHANGE_FILE_NAME | windows.FILE_NOTIFY_CHANGE_DIR_NAME + } + return m +} + +func (w *Watcher) toFSnotifyFlags(action uint32) uint64 { + switch action { + case windows.FILE_ACTION_ADDED: + return sysFSCREATE + case windows.FILE_ACTION_REMOVED: + return sysFSDELETE + case windows.FILE_ACTION_MODIFIED: + return sysFSMODIFY + case windows.FILE_ACTION_RENAMED_OLD_NAME: + return sysFSMOVEDFROM + case windows.FILE_ACTION_RENAMED_NEW_NAME: + return sysFSMOVEDTO + } + return 0 +} diff --git a/vendor/github.com/fsnotify/fsnotify/fen.go b/vendor/github.com/fsnotify/fsnotify/fen.go deleted file mode 100644 index ced39cb8..00000000 --- a/vendor/github.com/fsnotify/fsnotify/fen.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build solaris - -package fsnotify - -import ( - "errors" -) - -// Watcher watches a set of files, delivering events to a channel. -type Watcher struct { - Events chan Event - Errors chan error -} - -// NewWatcher establishes a new watcher with the underlying OS and begins waiting for events. -func NewWatcher() (*Watcher, error) { - return nil, errors.New("FEN based watcher not yet supported for fsnotify\n") -} - -// Close removes all watches and closes the events channel. -func (w *Watcher) Close() error { - return nil -} - -// Add starts watching the named file or directory (non-recursively). -func (w *Watcher) Add(name string) error { - return nil -} - -// Remove stops watching the the named file or directory (non-recursively). -func (w *Watcher) Remove(name string) error { - return nil -} diff --git a/vendor/github.com/fsnotify/fsnotify/fsnotify.go b/vendor/github.com/fsnotify/fsnotify/fsnotify.go index 89cab046..24c99cc4 100644 --- a/vendor/github.com/fsnotify/fsnotify/fsnotify.go +++ b/vendor/github.com/fsnotify/fsnotify/fsnotify.go @@ -1,68 +1,146 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build !plan9 - -// Package fsnotify provides a platform-independent interface for file system notifications. +// Package fsnotify provides a cross-platform interface for file system +// notifications. +// +// Currently supported systems: +// +// Linux 2.6.32+ via inotify +// BSD, macOS via kqueue +// Windows via ReadDirectoryChangesW +// illumos via FEN package fsnotify import ( - "bytes" "errors" "fmt" + "path/filepath" + "strings" ) -// Event represents a single file system notification. +// Event represents a file system notification. type Event struct { - Name string // Relative path to the file or directory. - Op Op // File operation that triggered the event. + // Path to the file or directory. + // + // Paths are relative to the input; for example with Add("dir") the Name + // will be set to "dir/file" if you create that file, but if you use + // Add("/path/to/dir") it will be "/path/to/dir/file". + Name string + + // File operation that triggered the event. + // + // This is a bitmask and some systems may send multiple operations at once. + // Use the Event.Has() method instead of comparing with ==. + Op Op } // Op describes a set of file operations. type Op uint32 -// These are the generalized file operations that can trigger a notification. +// The operations fsnotify can trigger; see the documentation on [Watcher] for a +// full description, and check them with [Event.Has]. const ( + // A new pathname was created. Create Op = 1 << iota + + // The pathname was written to; this does *not* mean the write has finished, + // and a write can be followed by more writes. Write + + // The path was removed; any watches on it will be removed. Some "remove" + // operations may trigger a Rename if the file is actually moved (for + // example "remove to trash" is often a rename). Remove + + // The path was renamed to something else; any watched on it will be + // removed. Rename + + // File attributes were changed. + // + // It's generally not recommended to take action on this event, as it may + // get triggered very frequently by some software. For example, Spotlight + // indexing on macOS, anti-virus software, backup software, etc. Chmod ) -func (op Op) String() string { - // Use a buffer for efficient string concatenation - var buffer bytes.Buffer +// Common errors that can be reported. +var ( + ErrNonExistentWatch = errors.New("fsnotify: can't remove non-existent watch") + ErrEventOverflow = errors.New("fsnotify: queue or buffer overflow") + ErrClosed = errors.New("fsnotify: watcher already closed") +) - if op&Create == Create { - buffer.WriteString("|CREATE") +func (o Op) String() string { + var b strings.Builder + if o.Has(Create) { + b.WriteString("|CREATE") } - if op&Remove == Remove { - buffer.WriteString("|REMOVE") + if o.Has(Remove) { + b.WriteString("|REMOVE") } - if op&Write == Write { - buffer.WriteString("|WRITE") + if o.Has(Write) { + b.WriteString("|WRITE") } - if op&Rename == Rename { - buffer.WriteString("|RENAME") + if o.Has(Rename) { + b.WriteString("|RENAME") } - if op&Chmod == Chmod { - buffer.WriteString("|CHMOD") + if o.Has(Chmod) { + b.WriteString("|CHMOD") } - if buffer.Len() == 0 { - return "" + if b.Len() == 0 { + return "[no events]" } - return buffer.String()[1:] // Strip leading pipe + return b.String()[1:] } -// String returns a string representation of the event in the form -// "file: REMOVE|WRITE|..." +// Has reports if this operation has the given operation. +func (o Op) Has(h Op) bool { return o&h != 0 } + +// Has reports if this event has the given operation. +func (e Event) Has(op Op) bool { return e.Op.Has(op) } + +// String returns a string representation of the event with their path. func (e Event) String() string { - return fmt.Sprintf("%q: %s", e.Name, e.Op.String()) + return fmt.Sprintf("%-13s %q", e.Op.String(), e.Name) } -// Common errors that can be reported by a watcher -var ( - ErrEventOverflow = errors.New("fsnotify queue overflow") +type ( + addOpt func(opt *withOpts) + withOpts struct { + bufsize int + } ) + +var defaultOpts = withOpts{ + bufsize: 65536, // 64K +} + +func getOptions(opts ...addOpt) withOpts { + with := defaultOpts + for _, o := range opts { + o(&with) + } + return with +} + +// WithBufferSize sets the [ReadDirectoryChangesW] buffer size. +// +// This only has effect on Windows systems, and is a no-op for other backends. +// +// The default value is 64K (65536 bytes) which is the highest value that works +// on all filesystems and should be enough for most applications, but if you +// have a large burst of events it may not be enough. You can increase it if +// you're hitting "queue or buffer overflow" errors ([ErrEventOverflow]). +// +// [ReadDirectoryChangesW]: https://learn.microsoft.com/en-gb/windows/win32/api/winbase/nf-winbase-readdirectorychangesw +func WithBufferSize(bytes int) addOpt { + return func(opt *withOpts) { opt.bufsize = bytes } +} + +// Check if this path is recursive (ends with "/..." or "\..."), and return the +// path with the /... stripped. +func recursivePath(path string) (string, bool) { + if filepath.Base(path) == "..." { + return filepath.Dir(path), true + } + return path, false +} diff --git a/vendor/github.com/fsnotify/fsnotify/inotify.go b/vendor/github.com/fsnotify/fsnotify/inotify.go deleted file mode 100644 index d9fd1b88..00000000 --- a/vendor/github.com/fsnotify/fsnotify/inotify.go +++ /dev/null @@ -1,337 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build linux - -package fsnotify - -import ( - "errors" - "fmt" - "io" - "os" - "path/filepath" - "strings" - "sync" - "unsafe" - - "golang.org/x/sys/unix" -) - -// Watcher watches a set of files, delivering events to a channel. -type Watcher struct { - Events chan Event - Errors chan error - mu sync.Mutex // Map access - fd int - poller *fdPoller - watches map[string]*watch // Map of inotify watches (key: path) - paths map[int]string // Map of watched paths (key: watch descriptor) - done chan struct{} // Channel for sending a "quit message" to the reader goroutine - doneResp chan struct{} // Channel to respond to Close -} - -// NewWatcher establishes a new watcher with the underlying OS and begins waiting for events. -func NewWatcher() (*Watcher, error) { - // Create inotify fd - fd, errno := unix.InotifyInit1(unix.IN_CLOEXEC) - if fd == -1 { - return nil, errno - } - // Create epoll - poller, err := newFdPoller(fd) - if err != nil { - unix.Close(fd) - return nil, err - } - w := &Watcher{ - fd: fd, - poller: poller, - watches: make(map[string]*watch), - paths: make(map[int]string), - Events: make(chan Event), - Errors: make(chan error), - done: make(chan struct{}), - doneResp: make(chan struct{}), - } - - go w.readEvents() - return w, nil -} - -func (w *Watcher) isClosed() bool { - select { - case <-w.done: - return true - default: - return false - } -} - -// Close removes all watches and closes the events channel. -func (w *Watcher) Close() error { - if w.isClosed() { - return nil - } - - // Send 'close' signal to goroutine, and set the Watcher to closed. - close(w.done) - - // Wake up goroutine - w.poller.wake() - - // Wait for goroutine to close - <-w.doneResp - - return nil -} - -// Add starts watching the named file or directory (non-recursively). -func (w *Watcher) Add(name string) error { - name = filepath.Clean(name) - if w.isClosed() { - return errors.New("inotify instance already closed") - } - - const agnosticEvents = unix.IN_MOVED_TO | unix.IN_MOVED_FROM | - unix.IN_CREATE | unix.IN_ATTRIB | unix.IN_MODIFY | - unix.IN_MOVE_SELF | unix.IN_DELETE | unix.IN_DELETE_SELF - - var flags uint32 = agnosticEvents - - w.mu.Lock() - defer w.mu.Unlock() - watchEntry := w.watches[name] - if watchEntry != nil { - flags |= watchEntry.flags | unix.IN_MASK_ADD - } - wd, errno := unix.InotifyAddWatch(w.fd, name, flags) - if wd == -1 { - return errno - } - - if watchEntry == nil { - w.watches[name] = &watch{wd: uint32(wd), flags: flags} - w.paths[wd] = name - } else { - watchEntry.wd = uint32(wd) - watchEntry.flags = flags - } - - return nil -} - -// Remove stops watching the named file or directory (non-recursively). -func (w *Watcher) Remove(name string) error { - name = filepath.Clean(name) - - // Fetch the watch. - w.mu.Lock() - defer w.mu.Unlock() - watch, ok := w.watches[name] - - // Remove it from inotify. - if !ok { - return fmt.Errorf("can't remove non-existent inotify watch for: %s", name) - } - - // We successfully removed the watch if InotifyRmWatch doesn't return an - // error, we need to clean up our internal state to ensure it matches - // inotify's kernel state. - delete(w.paths, int(watch.wd)) - delete(w.watches, name) - - // inotify_rm_watch will return EINVAL if the file has been deleted; - // the inotify will already have been removed. - // watches and pathes are deleted in ignoreLinux() implicitly and asynchronously - // by calling inotify_rm_watch() below. e.g. readEvents() goroutine receives IN_IGNORE - // so that EINVAL means that the wd is being rm_watch()ed or its file removed - // by another thread and we have not received IN_IGNORE event. - success, errno := unix.InotifyRmWatch(w.fd, watch.wd) - if success == -1 { - // TODO: Perhaps it's not helpful to return an error here in every case. - // the only two possible errors are: - // EBADF, which happens when w.fd is not a valid file descriptor of any kind. - // EINVAL, which is when fd is not an inotify descriptor or wd is not a valid watch descriptor. - // Watch descriptors are invalidated when they are removed explicitly or implicitly; - // explicitly by inotify_rm_watch, implicitly when the file they are watching is deleted. - return errno - } - - return nil -} - -type watch struct { - wd uint32 // Watch descriptor (as returned by the inotify_add_watch() syscall) - flags uint32 // inotify flags of this watch (see inotify(7) for the list of valid flags) -} - -// readEvents reads from the inotify file descriptor, converts the -// received events into Event objects and sends them via the Events channel -func (w *Watcher) readEvents() { - var ( - buf [unix.SizeofInotifyEvent * 4096]byte // Buffer for a maximum of 4096 raw events - n int // Number of bytes read with read() - errno error // Syscall errno - ok bool // For poller.wait - ) - - defer close(w.doneResp) - defer close(w.Errors) - defer close(w.Events) - defer unix.Close(w.fd) - defer w.poller.close() - - for { - // See if we have been closed. - if w.isClosed() { - return - } - - ok, errno = w.poller.wait() - if errno != nil { - select { - case w.Errors <- errno: - case <-w.done: - return - } - continue - } - - if !ok { - continue - } - - n, errno = unix.Read(w.fd, buf[:]) - // If a signal interrupted execution, see if we've been asked to close, and try again. - // http://man7.org/linux/man-pages/man7/signal.7.html : - // "Before Linux 3.8, reads from an inotify(7) file descriptor were not restartable" - if errno == unix.EINTR { - continue - } - - // unix.Read might have been woken up by Close. If so, we're done. - if w.isClosed() { - return - } - - if n < unix.SizeofInotifyEvent { - var err error - if n == 0 { - // If EOF is received. This should really never happen. - err = io.EOF - } else if n < 0 { - // If an error occurred while reading. - err = errno - } else { - // Read was too short. - err = errors.New("notify: short read in readEvents()") - } - select { - case w.Errors <- err: - case <-w.done: - return - } - continue - } - - var offset uint32 - // We don't know how many events we just read into the buffer - // While the offset points to at least one whole event... - for offset <= uint32(n-unix.SizeofInotifyEvent) { - // Point "raw" to the event in the buffer - raw := (*unix.InotifyEvent)(unsafe.Pointer(&buf[offset])) - - mask := uint32(raw.Mask) - nameLen := uint32(raw.Len) - - if mask&unix.IN_Q_OVERFLOW != 0 { - select { - case w.Errors <- ErrEventOverflow: - case <-w.done: - return - } - } - - // If the event happened to the watched directory or the watched file, the kernel - // doesn't append the filename to the event, but we would like to always fill the - // the "Name" field with a valid filename. We retrieve the path of the watch from - // the "paths" map. - w.mu.Lock() - name, ok := w.paths[int(raw.Wd)] - // IN_DELETE_SELF occurs when the file/directory being watched is removed. - // This is a sign to clean up the maps, otherwise we are no longer in sync - // with the inotify kernel state which has already deleted the watch - // automatically. - if ok && mask&unix.IN_DELETE_SELF == unix.IN_DELETE_SELF { - delete(w.paths, int(raw.Wd)) - delete(w.watches, name) - } - w.mu.Unlock() - - if nameLen > 0 { - // Point "bytes" at the first byte of the filename - bytes := (*[unix.PathMax]byte)(unsafe.Pointer(&buf[offset+unix.SizeofInotifyEvent])) - // The filename is padded with NULL bytes. TrimRight() gets rid of those. - name += "/" + strings.TrimRight(string(bytes[0:nameLen]), "\000") - } - - event := newEvent(name, mask) - - // Send the events that are not ignored on the events channel - if !event.ignoreLinux(mask) { - select { - case w.Events <- event: - case <-w.done: - return - } - } - - // Move to the next event in the buffer - offset += unix.SizeofInotifyEvent + nameLen - } - } -} - -// Certain types of events can be "ignored" and not sent over the Events -// channel. Such as events marked ignore by the kernel, or MODIFY events -// against files that do not exist. -func (e *Event) ignoreLinux(mask uint32) bool { - // Ignore anything the inotify API says to ignore - if mask&unix.IN_IGNORED == unix.IN_IGNORED { - return true - } - - // If the event is not a DELETE or RENAME, the file must exist. - // Otherwise the event is ignored. - // *Note*: this was put in place because it was seen that a MODIFY - // event was sent after the DELETE. This ignores that MODIFY and - // assumes a DELETE will come or has come if the file doesn't exist. - if !(e.Op&Remove == Remove || e.Op&Rename == Rename) { - _, statErr := os.Lstat(e.Name) - return os.IsNotExist(statErr) - } - return false -} - -// newEvent returns an platform-independent Event based on an inotify mask. -func newEvent(name string, mask uint32) Event { - e := Event{Name: name} - if mask&unix.IN_CREATE == unix.IN_CREATE || mask&unix.IN_MOVED_TO == unix.IN_MOVED_TO { - e.Op |= Create - } - if mask&unix.IN_DELETE_SELF == unix.IN_DELETE_SELF || mask&unix.IN_DELETE == unix.IN_DELETE { - e.Op |= Remove - } - if mask&unix.IN_MODIFY == unix.IN_MODIFY { - e.Op |= Write - } - if mask&unix.IN_MOVE_SELF == unix.IN_MOVE_SELF || mask&unix.IN_MOVED_FROM == unix.IN_MOVED_FROM { - e.Op |= Rename - } - if mask&unix.IN_ATTRIB == unix.IN_ATTRIB { - e.Op |= Chmod - } - return e -} diff --git a/vendor/github.com/fsnotify/fsnotify/inotify_poller.go b/vendor/github.com/fsnotify/fsnotify/inotify_poller.go deleted file mode 100644 index b33f2b4d..00000000 --- a/vendor/github.com/fsnotify/fsnotify/inotify_poller.go +++ /dev/null @@ -1,187 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build linux - -package fsnotify - -import ( - "errors" - - "golang.org/x/sys/unix" -) - -type fdPoller struct { - fd int // File descriptor (as returned by the inotify_init() syscall) - epfd int // Epoll file descriptor - pipe [2]int // Pipe for waking up -} - -func emptyPoller(fd int) *fdPoller { - poller := new(fdPoller) - poller.fd = fd - poller.epfd = -1 - poller.pipe[0] = -1 - poller.pipe[1] = -1 - return poller -} - -// Create a new inotify poller. -// This creates an inotify handler, and an epoll handler. -func newFdPoller(fd int) (*fdPoller, error) { - var errno error - poller := emptyPoller(fd) - defer func() { - if errno != nil { - poller.close() - } - }() - poller.fd = fd - - // Create epoll fd - poller.epfd, errno = unix.EpollCreate1(unix.EPOLL_CLOEXEC) - if poller.epfd == -1 { - return nil, errno - } - // Create pipe; pipe[0] is the read end, pipe[1] the write end. - errno = unix.Pipe2(poller.pipe[:], unix.O_NONBLOCK|unix.O_CLOEXEC) - if errno != nil { - return nil, errno - } - - // Register inotify fd with epoll - event := unix.EpollEvent{ - Fd: int32(poller.fd), - Events: unix.EPOLLIN, - } - errno = unix.EpollCtl(poller.epfd, unix.EPOLL_CTL_ADD, poller.fd, &event) - if errno != nil { - return nil, errno - } - - // Register pipe fd with epoll - event = unix.EpollEvent{ - Fd: int32(poller.pipe[0]), - Events: unix.EPOLLIN, - } - errno = unix.EpollCtl(poller.epfd, unix.EPOLL_CTL_ADD, poller.pipe[0], &event) - if errno != nil { - return nil, errno - } - - return poller, nil -} - -// Wait using epoll. -// Returns true if something is ready to be read, -// false if there is not. -func (poller *fdPoller) wait() (bool, error) { - // 3 possible events per fd, and 2 fds, makes a maximum of 6 events. - // I don't know whether epoll_wait returns the number of events returned, - // or the total number of events ready. - // I decided to catch both by making the buffer one larger than the maximum. - events := make([]unix.EpollEvent, 7) - for { - n, errno := unix.EpollWait(poller.epfd, events, -1) - if n == -1 { - if errno == unix.EINTR { - continue - } - return false, errno - } - if n == 0 { - // If there are no events, try again. - continue - } - if n > 6 { - // This should never happen. More events were returned than should be possible. - return false, errors.New("epoll_wait returned more events than I know what to do with") - } - ready := events[:n] - epollhup := false - epollerr := false - epollin := false - for _, event := range ready { - if event.Fd == int32(poller.fd) { - if event.Events&unix.EPOLLHUP != 0 { - // This should not happen, but if it does, treat it as a wakeup. - epollhup = true - } - if event.Events&unix.EPOLLERR != 0 { - // If an error is waiting on the file descriptor, we should pretend - // something is ready to read, and let unix.Read pick up the error. - epollerr = true - } - if event.Events&unix.EPOLLIN != 0 { - // There is data to read. - epollin = true - } - } - if event.Fd == int32(poller.pipe[0]) { - if event.Events&unix.EPOLLHUP != 0 { - // Write pipe descriptor was closed, by us. This means we're closing down the - // watcher, and we should wake up. - } - if event.Events&unix.EPOLLERR != 0 { - // If an error is waiting on the pipe file descriptor. - // This is an absolute mystery, and should never ever happen. - return false, errors.New("Error on the pipe descriptor.") - } - if event.Events&unix.EPOLLIN != 0 { - // This is a regular wakeup, so we have to clear the buffer. - err := poller.clearWake() - if err != nil { - return false, err - } - } - } - } - - if epollhup || epollerr || epollin { - return true, nil - } - return false, nil - } -} - -// Close the write end of the poller. -func (poller *fdPoller) wake() error { - buf := make([]byte, 1) - n, errno := unix.Write(poller.pipe[1], buf) - if n == -1 { - if errno == unix.EAGAIN { - // Buffer is full, poller will wake. - return nil - } - return errno - } - return nil -} - -func (poller *fdPoller) clearWake() error { - // You have to be woken up a LOT in order to get to 100! - buf := make([]byte, 100) - n, errno := unix.Read(poller.pipe[0], buf) - if n == -1 { - if errno == unix.EAGAIN { - // Buffer is empty, someone else cleared our wake. - return nil - } - return errno - } - return nil -} - -// Close all poller file descriptors, but not the one passed to it. -func (poller *fdPoller) close() { - if poller.pipe[1] != -1 { - unix.Close(poller.pipe[1]) - } - if poller.pipe[0] != -1 { - unix.Close(poller.pipe[0]) - } - if poller.epfd != -1 { - unix.Close(poller.epfd) - } -} diff --git a/vendor/github.com/fsnotify/fsnotify/kqueue.go b/vendor/github.com/fsnotify/fsnotify/kqueue.go deleted file mode 100644 index 86e76a3d..00000000 --- a/vendor/github.com/fsnotify/fsnotify/kqueue.go +++ /dev/null @@ -1,521 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build freebsd openbsd netbsd dragonfly darwin - -package fsnotify - -import ( - "errors" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "sync" - "time" - - "golang.org/x/sys/unix" -) - -// Watcher watches a set of files, delivering events to a channel. -type Watcher struct { - Events chan Event - Errors chan error - done chan struct{} // Channel for sending a "quit message" to the reader goroutine - - kq int // File descriptor (as returned by the kqueue() syscall). - - mu sync.Mutex // Protects access to watcher data - watches map[string]int // Map of watched file descriptors (key: path). - externalWatches map[string]bool // Map of watches added by user of the library. - dirFlags map[string]uint32 // Map of watched directories to fflags used in kqueue. - paths map[int]pathInfo // Map file descriptors to path names for processing kqueue events. - fileExists map[string]bool // Keep track of if we know this file exists (to stop duplicate create events). - isClosed bool // Set to true when Close() is first called -} - -type pathInfo struct { - name string - isDir bool -} - -// NewWatcher establishes a new watcher with the underlying OS and begins waiting for events. -func NewWatcher() (*Watcher, error) { - kq, err := kqueue() - if err != nil { - return nil, err - } - - w := &Watcher{ - kq: kq, - watches: make(map[string]int), - dirFlags: make(map[string]uint32), - paths: make(map[int]pathInfo), - fileExists: make(map[string]bool), - externalWatches: make(map[string]bool), - Events: make(chan Event), - Errors: make(chan error), - done: make(chan struct{}), - } - - go w.readEvents() - return w, nil -} - -// Close removes all watches and closes the events channel. -func (w *Watcher) Close() error { - w.mu.Lock() - if w.isClosed { - w.mu.Unlock() - return nil - } - w.isClosed = true - - // copy paths to remove while locked - var pathsToRemove = make([]string, 0, len(w.watches)) - for name := range w.watches { - pathsToRemove = append(pathsToRemove, name) - } - w.mu.Unlock() - // unlock before calling Remove, which also locks - - for _, name := range pathsToRemove { - w.Remove(name) - } - - // send a "quit" message to the reader goroutine - close(w.done) - - return nil -} - -// Add starts watching the named file or directory (non-recursively). -func (w *Watcher) Add(name string) error { - w.mu.Lock() - w.externalWatches[name] = true - w.mu.Unlock() - _, err := w.addWatch(name, noteAllEvents) - return err -} - -// Remove stops watching the the named file or directory (non-recursively). -func (w *Watcher) Remove(name string) error { - name = filepath.Clean(name) - w.mu.Lock() - watchfd, ok := w.watches[name] - w.mu.Unlock() - if !ok { - return fmt.Errorf("can't remove non-existent kevent watch for: %s", name) - } - - const registerRemove = unix.EV_DELETE - if err := register(w.kq, []int{watchfd}, registerRemove, 0); err != nil { - return err - } - - unix.Close(watchfd) - - w.mu.Lock() - isDir := w.paths[watchfd].isDir - delete(w.watches, name) - delete(w.paths, watchfd) - delete(w.dirFlags, name) - w.mu.Unlock() - - // Find all watched paths that are in this directory that are not external. - if isDir { - var pathsToRemove []string - w.mu.Lock() - for _, path := range w.paths { - wdir, _ := filepath.Split(path.name) - if filepath.Clean(wdir) == name { - if !w.externalWatches[path.name] { - pathsToRemove = append(pathsToRemove, path.name) - } - } - } - w.mu.Unlock() - for _, name := range pathsToRemove { - // Since these are internal, not much sense in propagating error - // to the user, as that will just confuse them with an error about - // a path they did not explicitly watch themselves. - w.Remove(name) - } - } - - return nil -} - -// Watch all events (except NOTE_EXTEND, NOTE_LINK, NOTE_REVOKE) -const noteAllEvents = unix.NOTE_DELETE | unix.NOTE_WRITE | unix.NOTE_ATTRIB | unix.NOTE_RENAME - -// keventWaitTime to block on each read from kevent -var keventWaitTime = durationToTimespec(100 * time.Millisecond) - -// addWatch adds name to the watched file set. -// The flags are interpreted as described in kevent(2). -// Returns the real path to the file which was added, if any, which may be different from the one passed in the case of symlinks. -func (w *Watcher) addWatch(name string, flags uint32) (string, error) { - var isDir bool - // Make ./name and name equivalent - name = filepath.Clean(name) - - w.mu.Lock() - if w.isClosed { - w.mu.Unlock() - return "", errors.New("kevent instance already closed") - } - watchfd, alreadyWatching := w.watches[name] - // We already have a watch, but we can still override flags. - if alreadyWatching { - isDir = w.paths[watchfd].isDir - } - w.mu.Unlock() - - if !alreadyWatching { - fi, err := os.Lstat(name) - if err != nil { - return "", err - } - - // Don't watch sockets. - if fi.Mode()&os.ModeSocket == os.ModeSocket { - return "", nil - } - - // Don't watch named pipes. - if fi.Mode()&os.ModeNamedPipe == os.ModeNamedPipe { - return "", nil - } - - // Follow Symlinks - // Unfortunately, Linux can add bogus symlinks to watch list without - // issue, and Windows can't do symlinks period (AFAIK). To maintain - // consistency, we will act like everything is fine. There will simply - // be no file events for broken symlinks. - // Hence the returns of nil on errors. - if fi.Mode()&os.ModeSymlink == os.ModeSymlink { - name, err = filepath.EvalSymlinks(name) - if err != nil { - return "", nil - } - - w.mu.Lock() - _, alreadyWatching = w.watches[name] - w.mu.Unlock() - - if alreadyWatching { - return name, nil - } - - fi, err = os.Lstat(name) - if err != nil { - return "", nil - } - } - - watchfd, err = unix.Open(name, openMode, 0700) - if watchfd == -1 { - return "", err - } - - isDir = fi.IsDir() - } - - const registerAdd = unix.EV_ADD | unix.EV_CLEAR | unix.EV_ENABLE - if err := register(w.kq, []int{watchfd}, registerAdd, flags); err != nil { - unix.Close(watchfd) - return "", err - } - - if !alreadyWatching { - w.mu.Lock() - w.watches[name] = watchfd - w.paths[watchfd] = pathInfo{name: name, isDir: isDir} - w.mu.Unlock() - } - - if isDir { - // Watch the directory if it has not been watched before, - // or if it was watched before, but perhaps only a NOTE_DELETE (watchDirectoryFiles) - w.mu.Lock() - - watchDir := (flags&unix.NOTE_WRITE) == unix.NOTE_WRITE && - (!alreadyWatching || (w.dirFlags[name]&unix.NOTE_WRITE) != unix.NOTE_WRITE) - // Store flags so this watch can be updated later - w.dirFlags[name] = flags - w.mu.Unlock() - - if watchDir { - if err := w.watchDirectoryFiles(name); err != nil { - return "", err - } - } - } - return name, nil -} - -// readEvents reads from kqueue and converts the received kevents into -// Event values that it sends down the Events channel. -func (w *Watcher) readEvents() { - eventBuffer := make([]unix.Kevent_t, 10) - -loop: - for { - // See if there is a message on the "done" channel - select { - case <-w.done: - break loop - default: - } - - // Get new events - kevents, err := read(w.kq, eventBuffer, &keventWaitTime) - // EINTR is okay, the syscall was interrupted before timeout expired. - if err != nil && err != unix.EINTR { - select { - case w.Errors <- err: - case <-w.done: - break loop - } - continue - } - - // Flush the events we received to the Events channel - for len(kevents) > 0 { - kevent := &kevents[0] - watchfd := int(kevent.Ident) - mask := uint32(kevent.Fflags) - w.mu.Lock() - path := w.paths[watchfd] - w.mu.Unlock() - event := newEvent(path.name, mask) - - if path.isDir && !(event.Op&Remove == Remove) { - // Double check to make sure the directory exists. This can happen when - // we do a rm -fr on a recursively watched folders and we receive a - // modification event first but the folder has been deleted and later - // receive the delete event - if _, err := os.Lstat(event.Name); os.IsNotExist(err) { - // mark is as delete event - event.Op |= Remove - } - } - - if event.Op&Rename == Rename || event.Op&Remove == Remove { - w.Remove(event.Name) - w.mu.Lock() - delete(w.fileExists, event.Name) - w.mu.Unlock() - } - - if path.isDir && event.Op&Write == Write && !(event.Op&Remove == Remove) { - w.sendDirectoryChangeEvents(event.Name) - } else { - // Send the event on the Events channel. - select { - case w.Events <- event: - case <-w.done: - break loop - } - } - - if event.Op&Remove == Remove { - // Look for a file that may have overwritten this. - // For example, mv f1 f2 will delete f2, then create f2. - if path.isDir { - fileDir := filepath.Clean(event.Name) - w.mu.Lock() - _, found := w.watches[fileDir] - w.mu.Unlock() - if found { - // make sure the directory exists before we watch for changes. When we - // do a recursive watch and perform rm -fr, the parent directory might - // have gone missing, ignore the missing directory and let the - // upcoming delete event remove the watch from the parent directory. - if _, err := os.Lstat(fileDir); err == nil { - w.sendDirectoryChangeEvents(fileDir) - } - } - } else { - filePath := filepath.Clean(event.Name) - if fileInfo, err := os.Lstat(filePath); err == nil { - w.sendFileCreatedEventIfNew(filePath, fileInfo) - } - } - } - - // Move to next event - kevents = kevents[1:] - } - } - - // cleanup - err := unix.Close(w.kq) - if err != nil { - // only way the previous loop breaks is if w.done was closed so we need to async send to w.Errors. - select { - case w.Errors <- err: - default: - } - } - close(w.Events) - close(w.Errors) -} - -// newEvent returns an platform-independent Event based on kqueue Fflags. -func newEvent(name string, mask uint32) Event { - e := Event{Name: name} - if mask&unix.NOTE_DELETE == unix.NOTE_DELETE { - e.Op |= Remove - } - if mask&unix.NOTE_WRITE == unix.NOTE_WRITE { - e.Op |= Write - } - if mask&unix.NOTE_RENAME == unix.NOTE_RENAME { - e.Op |= Rename - } - if mask&unix.NOTE_ATTRIB == unix.NOTE_ATTRIB { - e.Op |= Chmod - } - return e -} - -func newCreateEvent(name string) Event { - return Event{Name: name, Op: Create} -} - -// watchDirectoryFiles to mimic inotify when adding a watch on a directory -func (w *Watcher) watchDirectoryFiles(dirPath string) error { - // Get all files - files, err := ioutil.ReadDir(dirPath) - if err != nil { - return err - } - - for _, fileInfo := range files { - filePath := filepath.Join(dirPath, fileInfo.Name()) - filePath, err = w.internalWatch(filePath, fileInfo) - if err != nil { - return err - } - - w.mu.Lock() - w.fileExists[filePath] = true - w.mu.Unlock() - } - - return nil -} - -// sendDirectoryEvents searches the directory for newly created files -// and sends them over the event channel. This functionality is to have -// the BSD version of fsnotify match Linux inotify which provides a -// create event for files created in a watched directory. -func (w *Watcher) sendDirectoryChangeEvents(dirPath string) { - // Get all files - files, err := ioutil.ReadDir(dirPath) - if err != nil { - select { - case w.Errors <- err: - case <-w.done: - return - } - } - - // Search for new files - for _, fileInfo := range files { - filePath := filepath.Join(dirPath, fileInfo.Name()) - err := w.sendFileCreatedEventIfNew(filePath, fileInfo) - - if err != nil { - return - } - } -} - -// sendFileCreatedEvent sends a create event if the file isn't already being tracked. -func (w *Watcher) sendFileCreatedEventIfNew(filePath string, fileInfo os.FileInfo) (err error) { - w.mu.Lock() - _, doesExist := w.fileExists[filePath] - w.mu.Unlock() - if !doesExist { - // Send create event - select { - case w.Events <- newCreateEvent(filePath): - case <-w.done: - return - } - } - - // like watchDirectoryFiles (but without doing another ReadDir) - filePath, err = w.internalWatch(filePath, fileInfo) - if err != nil { - return err - } - - w.mu.Lock() - w.fileExists[filePath] = true - w.mu.Unlock() - - return nil -} - -func (w *Watcher) internalWatch(name string, fileInfo os.FileInfo) (string, error) { - if fileInfo.IsDir() { - // mimic Linux providing delete events for subdirectories - // but preserve the flags used if currently watching subdirectory - w.mu.Lock() - flags := w.dirFlags[name] - w.mu.Unlock() - - flags |= unix.NOTE_DELETE | unix.NOTE_RENAME - return w.addWatch(name, flags) - } - - // watch file to mimic Linux inotify - return w.addWatch(name, noteAllEvents) -} - -// kqueue creates a new kernel event queue and returns a descriptor. -func kqueue() (kq int, err error) { - kq, err = unix.Kqueue() - if kq == -1 { - return kq, err - } - return kq, nil -} - -// register events with the queue -func register(kq int, fds []int, flags int, fflags uint32) error { - changes := make([]unix.Kevent_t, len(fds)) - - for i, fd := range fds { - // SetKevent converts int to the platform-specific types: - unix.SetKevent(&changes[i], fd, unix.EVFILT_VNODE, flags) - changes[i].Fflags = fflags - } - - // register the events - success, err := unix.Kevent(kq, changes, nil, nil) - if success == -1 { - return err - } - return nil -} - -// read retrieves pending events, or waits until an event occurs. -// A timeout of nil blocks indefinitely, while 0 polls the queue. -func read(kq int, events []unix.Kevent_t, timeout *unix.Timespec) ([]unix.Kevent_t, error) { - n, err := unix.Kevent(kq, nil, events, timeout) - if err != nil { - return nil, err - } - return events[0:n], nil -} - -// durationToTimespec prepares a timeout value -func durationToTimespec(d time.Duration) unix.Timespec { - return unix.NsecToTimespec(d.Nanoseconds()) -} diff --git a/vendor/github.com/fsnotify/fsnotify/mkdoc.zsh b/vendor/github.com/fsnotify/fsnotify/mkdoc.zsh new file mode 100644 index 00000000..99012ae6 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/mkdoc.zsh @@ -0,0 +1,259 @@ +#!/usr/bin/env zsh +[ "${ZSH_VERSION:-}" = "" ] && echo >&2 "Only works with zsh" && exit 1 +setopt err_exit no_unset pipefail extended_glob + +# Simple script to update the godoc comments on all watchers so you don't need +# to update the same comment 5 times. + +watcher=$(</tmp/x + print -r -- $cmt >>/tmp/x + tail -n+$(( end + 1 )) $file >>/tmp/x + mv /tmp/x $file + done +} + +set-cmt '^type Watcher struct ' $watcher +set-cmt '^func NewWatcher(' $new +set-cmt '^func NewBufferedWatcher(' $newbuffered +set-cmt '^func (w \*Watcher) Add(' $add +set-cmt '^func (w \*Watcher) AddWith(' $addwith +set-cmt '^func (w \*Watcher) Remove(' $remove +set-cmt '^func (w \*Watcher) Close(' $close +set-cmt '^func (w \*Watcher) WatchList(' $watchlist +set-cmt '^[[:space:]]*Events *chan Event$' $events +set-cmt '^[[:space:]]*Errors *chan error$' $errors diff --git a/vendor/github.com/fsnotify/fsnotify/open_mode_bsd.go b/vendor/github.com/fsnotify/fsnotify/open_mode_bsd.go deleted file mode 100644 index 2306c462..00000000 --- a/vendor/github.com/fsnotify/fsnotify/open_mode_bsd.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build freebsd openbsd netbsd dragonfly - -package fsnotify - -import "golang.org/x/sys/unix" - -const openMode = unix.O_NONBLOCK | unix.O_RDONLY | unix.O_CLOEXEC diff --git a/vendor/github.com/fsnotify/fsnotify/open_mode_darwin.go b/vendor/github.com/fsnotify/fsnotify/open_mode_darwin.go deleted file mode 100644 index 870c4d6d..00000000 --- a/vendor/github.com/fsnotify/fsnotify/open_mode_darwin.go +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build darwin - -package fsnotify - -import "golang.org/x/sys/unix" - -// note: this constant is not defined on BSD -const openMode = unix.O_EVTONLY | unix.O_CLOEXEC diff --git a/vendor/github.com/fsnotify/fsnotify/system_bsd.go b/vendor/github.com/fsnotify/fsnotify/system_bsd.go new file mode 100644 index 00000000..4322b0b8 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/system_bsd.go @@ -0,0 +1,8 @@ +//go:build freebsd || openbsd || netbsd || dragonfly +// +build freebsd openbsd netbsd dragonfly + +package fsnotify + +import "golang.org/x/sys/unix" + +const openMode = unix.O_NONBLOCK | unix.O_RDONLY | unix.O_CLOEXEC diff --git a/vendor/github.com/fsnotify/fsnotify/system_darwin.go b/vendor/github.com/fsnotify/fsnotify/system_darwin.go new file mode 100644 index 00000000..5da5ffa7 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/system_darwin.go @@ -0,0 +1,9 @@ +//go:build darwin +// +build darwin + +package fsnotify + +import "golang.org/x/sys/unix" + +// note: this constant is not defined on BSD +const openMode = unix.O_EVTONLY | unix.O_CLOEXEC diff --git a/vendor/github.com/fsnotify/fsnotify/windows.go b/vendor/github.com/fsnotify/fsnotify/windows.go deleted file mode 100644 index 09436f31..00000000 --- a/vendor/github.com/fsnotify/fsnotify/windows.go +++ /dev/null @@ -1,561 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build windows - -package fsnotify - -import ( - "errors" - "fmt" - "os" - "path/filepath" - "runtime" - "sync" - "syscall" - "unsafe" -) - -// Watcher watches a set of files, delivering events to a channel. -type Watcher struct { - Events chan Event - Errors chan error - isClosed bool // Set to true when Close() is first called - mu sync.Mutex // Map access - port syscall.Handle // Handle to completion port - watches watchMap // Map of watches (key: i-number) - input chan *input // Inputs to the reader are sent on this channel - quit chan chan<- error -} - -// NewWatcher establishes a new watcher with the underlying OS and begins waiting for events. -func NewWatcher() (*Watcher, error) { - port, e := syscall.CreateIoCompletionPort(syscall.InvalidHandle, 0, 0, 0) - if e != nil { - return nil, os.NewSyscallError("CreateIoCompletionPort", e) - } - w := &Watcher{ - port: port, - watches: make(watchMap), - input: make(chan *input, 1), - Events: make(chan Event, 50), - Errors: make(chan error), - quit: make(chan chan<- error, 1), - } - go w.readEvents() - return w, nil -} - -// Close removes all watches and closes the events channel. -func (w *Watcher) Close() error { - if w.isClosed { - return nil - } - w.isClosed = true - - // Send "quit" message to the reader goroutine - ch := make(chan error) - w.quit <- ch - if err := w.wakeupReader(); err != nil { - return err - } - return <-ch -} - -// Add starts watching the named file or directory (non-recursively). -func (w *Watcher) Add(name string) error { - if w.isClosed { - return errors.New("watcher already closed") - } - in := &input{ - op: opAddWatch, - path: filepath.Clean(name), - flags: sysFSALLEVENTS, - reply: make(chan error), - } - w.input <- in - if err := w.wakeupReader(); err != nil { - return err - } - return <-in.reply -} - -// Remove stops watching the the named file or directory (non-recursively). -func (w *Watcher) Remove(name string) error { - in := &input{ - op: opRemoveWatch, - path: filepath.Clean(name), - reply: make(chan error), - } - w.input <- in - if err := w.wakeupReader(); err != nil { - return err - } - return <-in.reply -} - -const ( - // Options for AddWatch - sysFSONESHOT = 0x80000000 - sysFSONLYDIR = 0x1000000 - - // Events - sysFSACCESS = 0x1 - sysFSALLEVENTS = 0xfff - sysFSATTRIB = 0x4 - sysFSCLOSE = 0x18 - sysFSCREATE = 0x100 - sysFSDELETE = 0x200 - sysFSDELETESELF = 0x400 - sysFSMODIFY = 0x2 - sysFSMOVE = 0xc0 - sysFSMOVEDFROM = 0x40 - sysFSMOVEDTO = 0x80 - sysFSMOVESELF = 0x800 - - // Special events - sysFSIGNORED = 0x8000 - sysFSQOVERFLOW = 0x4000 -) - -func newEvent(name string, mask uint32) Event { - e := Event{Name: name} - if mask&sysFSCREATE == sysFSCREATE || mask&sysFSMOVEDTO == sysFSMOVEDTO { - e.Op |= Create - } - if mask&sysFSDELETE == sysFSDELETE || mask&sysFSDELETESELF == sysFSDELETESELF { - e.Op |= Remove - } - if mask&sysFSMODIFY == sysFSMODIFY { - e.Op |= Write - } - if mask&sysFSMOVE == sysFSMOVE || mask&sysFSMOVESELF == sysFSMOVESELF || mask&sysFSMOVEDFROM == sysFSMOVEDFROM { - e.Op |= Rename - } - if mask&sysFSATTRIB == sysFSATTRIB { - e.Op |= Chmod - } - return e -} - -const ( - opAddWatch = iota - opRemoveWatch -) - -const ( - provisional uint64 = 1 << (32 + iota) -) - -type input struct { - op int - path string - flags uint32 - reply chan error -} - -type inode struct { - handle syscall.Handle - volume uint32 - index uint64 -} - -type watch struct { - ov syscall.Overlapped - ino *inode // i-number - path string // Directory path - mask uint64 // Directory itself is being watched with these notify flags - names map[string]uint64 // Map of names being watched and their notify flags - rename string // Remembers the old name while renaming a file - buf [4096]byte -} - -type indexMap map[uint64]*watch -type watchMap map[uint32]indexMap - -func (w *Watcher) wakeupReader() error { - e := syscall.PostQueuedCompletionStatus(w.port, 0, 0, nil) - if e != nil { - return os.NewSyscallError("PostQueuedCompletionStatus", e) - } - return nil -} - -func getDir(pathname string) (dir string, err error) { - attr, e := syscall.GetFileAttributes(syscall.StringToUTF16Ptr(pathname)) - if e != nil { - return "", os.NewSyscallError("GetFileAttributes", e) - } - if attr&syscall.FILE_ATTRIBUTE_DIRECTORY != 0 { - dir = pathname - } else { - dir, _ = filepath.Split(pathname) - dir = filepath.Clean(dir) - } - return -} - -func getIno(path string) (ino *inode, err error) { - h, e := syscall.CreateFile(syscall.StringToUTF16Ptr(path), - syscall.FILE_LIST_DIRECTORY, - syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE, - nil, syscall.OPEN_EXISTING, - syscall.FILE_FLAG_BACKUP_SEMANTICS|syscall.FILE_FLAG_OVERLAPPED, 0) - if e != nil { - return nil, os.NewSyscallError("CreateFile", e) - } - var fi syscall.ByHandleFileInformation - if e = syscall.GetFileInformationByHandle(h, &fi); e != nil { - syscall.CloseHandle(h) - return nil, os.NewSyscallError("GetFileInformationByHandle", e) - } - ino = &inode{ - handle: h, - volume: fi.VolumeSerialNumber, - index: uint64(fi.FileIndexHigh)<<32 | uint64(fi.FileIndexLow), - } - return ino, nil -} - -// Must run within the I/O thread. -func (m watchMap) get(ino *inode) *watch { - if i := m[ino.volume]; i != nil { - return i[ino.index] - } - return nil -} - -// Must run within the I/O thread. -func (m watchMap) set(ino *inode, watch *watch) { - i := m[ino.volume] - if i == nil { - i = make(indexMap) - m[ino.volume] = i - } - i[ino.index] = watch -} - -// Must run within the I/O thread. -func (w *Watcher) addWatch(pathname string, flags uint64) error { - dir, err := getDir(pathname) - if err != nil { - return err - } - if flags&sysFSONLYDIR != 0 && pathname != dir { - return nil - } - ino, err := getIno(dir) - if err != nil { - return err - } - w.mu.Lock() - watchEntry := w.watches.get(ino) - w.mu.Unlock() - if watchEntry == nil { - if _, e := syscall.CreateIoCompletionPort(ino.handle, w.port, 0, 0); e != nil { - syscall.CloseHandle(ino.handle) - return os.NewSyscallError("CreateIoCompletionPort", e) - } - watchEntry = &watch{ - ino: ino, - path: dir, - names: make(map[string]uint64), - } - w.mu.Lock() - w.watches.set(ino, watchEntry) - w.mu.Unlock() - flags |= provisional - } else { - syscall.CloseHandle(ino.handle) - } - if pathname == dir { - watchEntry.mask |= flags - } else { - watchEntry.names[filepath.Base(pathname)] |= flags - } - if err = w.startRead(watchEntry); err != nil { - return err - } - if pathname == dir { - watchEntry.mask &= ^provisional - } else { - watchEntry.names[filepath.Base(pathname)] &= ^provisional - } - return nil -} - -// Must run within the I/O thread. -func (w *Watcher) remWatch(pathname string) error { - dir, err := getDir(pathname) - if err != nil { - return err - } - ino, err := getIno(dir) - if err != nil { - return err - } - w.mu.Lock() - watch := w.watches.get(ino) - w.mu.Unlock() - if watch == nil { - return fmt.Errorf("can't remove non-existent watch for: %s", pathname) - } - if pathname == dir { - w.sendEvent(watch.path, watch.mask&sysFSIGNORED) - watch.mask = 0 - } else { - name := filepath.Base(pathname) - w.sendEvent(filepath.Join(watch.path, name), watch.names[name]&sysFSIGNORED) - delete(watch.names, name) - } - return w.startRead(watch) -} - -// Must run within the I/O thread. -func (w *Watcher) deleteWatch(watch *watch) { - for name, mask := range watch.names { - if mask&provisional == 0 { - w.sendEvent(filepath.Join(watch.path, name), mask&sysFSIGNORED) - } - delete(watch.names, name) - } - if watch.mask != 0 { - if watch.mask&provisional == 0 { - w.sendEvent(watch.path, watch.mask&sysFSIGNORED) - } - watch.mask = 0 - } -} - -// Must run within the I/O thread. -func (w *Watcher) startRead(watch *watch) error { - if e := syscall.CancelIo(watch.ino.handle); e != nil { - w.Errors <- os.NewSyscallError("CancelIo", e) - w.deleteWatch(watch) - } - mask := toWindowsFlags(watch.mask) - for _, m := range watch.names { - mask |= toWindowsFlags(m) - } - if mask == 0 { - if e := syscall.CloseHandle(watch.ino.handle); e != nil { - w.Errors <- os.NewSyscallError("CloseHandle", e) - } - w.mu.Lock() - delete(w.watches[watch.ino.volume], watch.ino.index) - w.mu.Unlock() - return nil - } - e := syscall.ReadDirectoryChanges(watch.ino.handle, &watch.buf[0], - uint32(unsafe.Sizeof(watch.buf)), false, mask, nil, &watch.ov, 0) - if e != nil { - err := os.NewSyscallError("ReadDirectoryChanges", e) - if e == syscall.ERROR_ACCESS_DENIED && watch.mask&provisional == 0 { - // Watched directory was probably removed - if w.sendEvent(watch.path, watch.mask&sysFSDELETESELF) { - if watch.mask&sysFSONESHOT != 0 { - watch.mask = 0 - } - } - err = nil - } - w.deleteWatch(watch) - w.startRead(watch) - return err - } - return nil -} - -// readEvents reads from the I/O completion port, converts the -// received events into Event objects and sends them via the Events channel. -// Entry point to the I/O thread. -func (w *Watcher) readEvents() { - var ( - n, key uint32 - ov *syscall.Overlapped - ) - runtime.LockOSThread() - - for { - e := syscall.GetQueuedCompletionStatus(w.port, &n, &key, &ov, syscall.INFINITE) - watch := (*watch)(unsafe.Pointer(ov)) - - if watch == nil { - select { - case ch := <-w.quit: - w.mu.Lock() - var indexes []indexMap - for _, index := range w.watches { - indexes = append(indexes, index) - } - w.mu.Unlock() - for _, index := range indexes { - for _, watch := range index { - w.deleteWatch(watch) - w.startRead(watch) - } - } - var err error - if e := syscall.CloseHandle(w.port); e != nil { - err = os.NewSyscallError("CloseHandle", e) - } - close(w.Events) - close(w.Errors) - ch <- err - return - case in := <-w.input: - switch in.op { - case opAddWatch: - in.reply <- w.addWatch(in.path, uint64(in.flags)) - case opRemoveWatch: - in.reply <- w.remWatch(in.path) - } - default: - } - continue - } - - switch e { - case syscall.ERROR_MORE_DATA: - if watch == nil { - w.Errors <- errors.New("ERROR_MORE_DATA has unexpectedly null lpOverlapped buffer") - } else { - // The i/o succeeded but the buffer is full. - // In theory we should be building up a full packet. - // In practice we can get away with just carrying on. - n = uint32(unsafe.Sizeof(watch.buf)) - } - case syscall.ERROR_ACCESS_DENIED: - // Watched directory was probably removed - w.sendEvent(watch.path, watch.mask&sysFSDELETESELF) - w.deleteWatch(watch) - w.startRead(watch) - continue - case syscall.ERROR_OPERATION_ABORTED: - // CancelIo was called on this handle - continue - default: - w.Errors <- os.NewSyscallError("GetQueuedCompletionPort", e) - continue - case nil: - } - - var offset uint32 - for { - if n == 0 { - w.Events <- newEvent("", sysFSQOVERFLOW) - w.Errors <- errors.New("short read in readEvents()") - break - } - - // Point "raw" to the event in the buffer - raw := (*syscall.FileNotifyInformation)(unsafe.Pointer(&watch.buf[offset])) - buf := (*[syscall.MAX_PATH]uint16)(unsafe.Pointer(&raw.FileName)) - name := syscall.UTF16ToString(buf[:raw.FileNameLength/2]) - fullname := filepath.Join(watch.path, name) - - var mask uint64 - switch raw.Action { - case syscall.FILE_ACTION_REMOVED: - mask = sysFSDELETESELF - case syscall.FILE_ACTION_MODIFIED: - mask = sysFSMODIFY - case syscall.FILE_ACTION_RENAMED_OLD_NAME: - watch.rename = name - case syscall.FILE_ACTION_RENAMED_NEW_NAME: - if watch.names[watch.rename] != 0 { - watch.names[name] |= watch.names[watch.rename] - delete(watch.names, watch.rename) - mask = sysFSMOVESELF - } - } - - sendNameEvent := func() { - if w.sendEvent(fullname, watch.names[name]&mask) { - if watch.names[name]&sysFSONESHOT != 0 { - delete(watch.names, name) - } - } - } - if raw.Action != syscall.FILE_ACTION_RENAMED_NEW_NAME { - sendNameEvent() - } - if raw.Action == syscall.FILE_ACTION_REMOVED { - w.sendEvent(fullname, watch.names[name]&sysFSIGNORED) - delete(watch.names, name) - } - if w.sendEvent(fullname, watch.mask&toFSnotifyFlags(raw.Action)) { - if watch.mask&sysFSONESHOT != 0 { - watch.mask = 0 - } - } - if raw.Action == syscall.FILE_ACTION_RENAMED_NEW_NAME { - fullname = filepath.Join(watch.path, watch.rename) - sendNameEvent() - } - - // Move to the next event in the buffer - if raw.NextEntryOffset == 0 { - break - } - offset += raw.NextEntryOffset - - // Error! - if offset >= n { - w.Errors <- errors.New("Windows system assumed buffer larger than it is, events have likely been missed.") - break - } - } - - if err := w.startRead(watch); err != nil { - w.Errors <- err - } - } -} - -func (w *Watcher) sendEvent(name string, mask uint64) bool { - if mask == 0 { - return false - } - event := newEvent(name, uint32(mask)) - select { - case ch := <-w.quit: - w.quit <- ch - case w.Events <- event: - } - return true -} - -func toWindowsFlags(mask uint64) uint32 { - var m uint32 - if mask&sysFSACCESS != 0 { - m |= syscall.FILE_NOTIFY_CHANGE_LAST_ACCESS - } - if mask&sysFSMODIFY != 0 { - m |= syscall.FILE_NOTIFY_CHANGE_LAST_WRITE - } - if mask&sysFSATTRIB != 0 { - m |= syscall.FILE_NOTIFY_CHANGE_ATTRIBUTES - } - if mask&(sysFSMOVE|sysFSCREATE|sysFSDELETE) != 0 { - m |= syscall.FILE_NOTIFY_CHANGE_FILE_NAME | syscall.FILE_NOTIFY_CHANGE_DIR_NAME - } - return m -} - -func toFSnotifyFlags(action uint32) uint64 { - switch action { - case syscall.FILE_ACTION_ADDED: - return sysFSCREATE - case syscall.FILE_ACTION_REMOVED: - return sysFSDELETE - case syscall.FILE_ACTION_MODIFIED: - return sysFSMODIFY - case syscall.FILE_ACTION_RENAMED_OLD_NAME: - return sysFSMOVEDFROM - case syscall.FILE_ACTION_RENAMED_NEW_NAME: - return sysFSMOVEDTO - } - return 0 -} diff --git a/vendor/github.com/getsentry/sentry-go/.codecov.yml b/vendor/github.com/getsentry/sentry-go/.codecov.yml new file mode 100644 index 00000000..df0b2681 --- /dev/null +++ b/vendor/github.com/getsentry/sentry-go/.codecov.yml @@ -0,0 +1,13 @@ +codecov: + # across + notify: + # Do not notify until at least this number of reports have been uploaded + # from the CI pipeline. We normally have more than that number, but 6 + # should be enough to get a first notification. + after_n_builds: 6 +coverage: + status: + project: + default: + # Do not fail the commit status if the coverage was reduced up to this value + threshold: 0.5% diff --git a/vendor/github.com/getsentry/sentry-go/Makefile b/vendor/github.com/getsentry/sentry-go/Makefile new file mode 100644 index 00000000..89523ac0 --- /dev/null +++ b/vendor/github.com/getsentry/sentry-go/Makefile @@ -0,0 +1,83 @@ +.DEFAULT_GOAL := help + +MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) +MKFILE_DIR := $(dir $(MKFILE_PATH)) +ALL_GO_MOD_DIRS := $(shell find . -type f -name 'go.mod' -exec dirname {} \; | sort) +GO = go +TIMEOUT = 300 + +# Parse Makefile and display the help +help: ## Show help + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' +.PHONY: help + +build: ## Build everything + for dir in $(ALL_GO_MOD_DIRS); do \ + cd "$${dir}"; \ + echo ">>> Running 'go build' for module: $${dir}"; \ + go build ./...; \ + done; +.PHONY: build + +### Tests (inspired by https://github.com/open-telemetry/opentelemetry-go/blob/main/Makefile) +TEST_TARGETS := test-short test-verbose test-race +test-race: ARGS=-race +test-short: ARGS=-short +test-verbose: ARGS=-v -race +$(TEST_TARGETS): test +test: $(ALL_GO_MOD_DIRS:%=test/%) ## Run tests +test/%: DIR=$* +test/%: + @echo ">>> Running tests for module: $(DIR)" + @# We use '-count=1' to disable test caching. + (cd $(DIR) && $(GO) test -count=1 -timeout $(TIMEOUT)s $(ARGS) ./...) +.PHONY: $(TEST_TARGETS) test + +# Coverage +COVERAGE_MODE = atomic +COVERAGE_PROFILE = coverage.out +COVERAGE_REPORT_DIR = .coverage +COVERAGE_REPORT_DIR_ABS = "$(MKFILE_DIR)/$(COVERAGE_REPORT_DIR)" +$(COVERAGE_REPORT_DIR): + mkdir -p $(COVERAGE_REPORT_DIR) +clean-report-dir: $(COVERAGE_REPORT_DIR) + test $(COVERAGE_REPORT_DIR) && rm -f $(COVERAGE_REPORT_DIR)/* +test-coverage: $(COVERAGE_REPORT_DIR) clean-report-dir ## Test with coverage enabled + set -e ; \ + for dir in $(ALL_GO_MOD_DIRS); do \ + echo ">>> Running tests with coverage for module: $${dir}"; \ + DIR_ABS=$$(python -c 'import os, sys; print(os.path.realpath(sys.argv[1]))' $${dir}) ; \ + REPORT_NAME=$$(basename $${DIR_ABS}); \ + (cd "$${dir}" && \ + $(GO) test -count=1 -timeout $(TIMEOUT)s -coverpkg=./... -covermode=$(COVERAGE_MODE) -coverprofile="$(COVERAGE_PROFILE)" ./... && \ + cp $(COVERAGE_PROFILE) "$(COVERAGE_REPORT_DIR_ABS)/$${REPORT_NAME}_$(COVERAGE_PROFILE)" && \ + $(GO) tool cover -html=$(COVERAGE_PROFILE) -o coverage.html); \ + done; +.PHONY: test-coverage clean-report-dir + +mod-tidy: ## Check go.mod tidiness + set -e ; \ + for dir in $(ALL_GO_MOD_DIRS); do \ + cd "$${dir}"; \ + echo ">>> Running 'go mod tidy' for module: $${dir}"; \ + go mod tidy -go=1.18 -compat=1.18; \ + done; \ + git diff --exit-code; +.PHONY: mod-tidy + +vet: ## Run "go vet" + set -e ; \ + for dir in $(ALL_GO_MOD_DIRS); do \ + cd "$${dir}"; \ + echo ">>> Running 'go vet' for module: $${dir}"; \ + go vet ./...; \ + done; +.PHONY: vet + +lint: ## Lint (using "golangci-lint") + golangci-lint run +.PHONY: lint + +fmt: ## Format all Go files + gofmt -l -w -s . +.PHONY: fmt diff --git a/vendor/github.com/getsentry/sentry-go/check_in.go b/vendor/github.com/getsentry/sentry-go/check_in.go new file mode 100644 index 00000000..d41ba3d2 --- /dev/null +++ b/vendor/github.com/getsentry/sentry-go/check_in.go @@ -0,0 +1,117 @@ +package sentry + +import "time" + +type CheckInStatus string + +const ( + CheckInStatusInProgress CheckInStatus = "in_progress" + CheckInStatusOK CheckInStatus = "ok" + CheckInStatusError CheckInStatus = "error" +) + +type checkInScheduleType string + +const ( + checkInScheduleTypeCrontab checkInScheduleType = "crontab" + checkInScheduleTypeInterval checkInScheduleType = "interval" +) + +type MonitorSchedule interface { + // scheduleType is a private method that must be implemented for monitor schedule + // implementation. It should never be called. This method is made for having + // specific private implementation of MonitorSchedule interface. + scheduleType() checkInScheduleType +} + +type crontabSchedule struct { + Type string `json:"type"` + Value string `json:"value"` +} + +func (c crontabSchedule) scheduleType() checkInScheduleType { + return checkInScheduleTypeCrontab +} + +// CrontabSchedule defines the MonitorSchedule with a cron format. +// Example: "8 * * * *". +func CrontabSchedule(scheduleString string) MonitorSchedule { + return crontabSchedule{ + Type: string(checkInScheduleTypeCrontab), + Value: scheduleString, + } +} + +type intervalSchedule struct { + Type string `json:"type"` + Value int64 `json:"value"` + Unit string `json:"unit"` +} + +func (i intervalSchedule) scheduleType() checkInScheduleType { + return checkInScheduleTypeInterval +} + +type MonitorScheduleUnit string + +const ( + MonitorScheduleUnitMinute MonitorScheduleUnit = "minute" + MonitorScheduleUnitHour MonitorScheduleUnit = "hour" + MonitorScheduleUnitDay MonitorScheduleUnit = "day" + MonitorScheduleUnitWeek MonitorScheduleUnit = "week" + MonitorScheduleUnitMonth MonitorScheduleUnit = "month" + MonitorScheduleUnitYear MonitorScheduleUnit = "year" +) + +// IntervalSchedule defines the MonitorSchedule with an interval format. +// +// Example: +// +// IntervalSchedule(1, sentry.MonitorScheduleUnitDay) +func IntervalSchedule(value int64, unit MonitorScheduleUnit) MonitorSchedule { + return intervalSchedule{ + Type: string(checkInScheduleTypeInterval), + Value: value, + Unit: string(unit), + } +} + +type MonitorConfig struct { //nolint: maligned // prefer readability over optimal memory layout + Schedule MonitorSchedule `json:"schedule,omitempty"` + // The allowed margin of minutes after the expected check-in time that + // the monitor will not be considered missed for. + CheckInMargin int64 `json:"checkin_margin,omitempty"` + // The allowed duration in minutes that the monitor may be `in_progress` + // for before being considered failed due to timeout. + MaxRuntime int64 `json:"max_runtime,omitempty"` + // A tz database string representing the timezone which the monitor's execution schedule is in. + // See: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones + Timezone string `json:"timezone,omitempty"` +} + +type CheckIn struct { //nolint: maligned // prefer readability over optimal memory layout + // Check-In ID (unique and client generated) + ID EventID `json:"check_in_id"` + // The distinct slug of the monitor. + MonitorSlug string `json:"monitor_slug"` + // The status of the check-in. + Status CheckInStatus `json:"status"` + // The duration of the check-in. Will only take effect if the status is ok or error. + Duration time.Duration `json:"duration,omitempty"` +} + +// serializedCheckIn is used by checkInMarshalJSON method on Event struct. +// See https://develop.sentry.dev/sdk/check-ins/ +type serializedCheckIn struct { //nolint: maligned + // Check-In ID (unique and client generated). + CheckInID string `json:"check_in_id"` + // The distinct slug of the monitor. + MonitorSlug string `json:"monitor_slug"` + // The status of the check-in. + Status CheckInStatus `json:"status"` + // The duration of the check-in in seconds. Will only take effect if the status is ok or error. + Duration float64 `json:"duration,omitempty"` + Release string `json:"release,omitempty"` + Environment string `json:"environment,omitempty"` + MonitorConfig *MonitorConfig `json:"monitor_config,omitempty"` +} diff --git a/vendor/github.com/getsentry/sentry-go/dynamic_sampling_context.go b/vendor/github.com/getsentry/sentry-go/dynamic_sampling_context.go new file mode 100644 index 00000000..36507260 --- /dev/null +++ b/vendor/github.com/getsentry/sentry-go/dynamic_sampling_context.go @@ -0,0 +1,123 @@ +package sentry + +import ( + "strconv" + "strings" + + "github.com/getsentry/sentry-go/internal/otel/baggage" +) + +const ( + sentryPrefix = "sentry-" +) + +// DynamicSamplingContext holds information about the current event that can be used to make dynamic sampling decisions. +type DynamicSamplingContext struct { + Entries map[string]string + Frozen bool +} + +func DynamicSamplingContextFromHeader(header []byte) (DynamicSamplingContext, error) { + bag, err := baggage.Parse(string(header)) + if err != nil { + return DynamicSamplingContext{}, err + } + + entries := map[string]string{} + for _, member := range bag.Members() { + // We only store baggage members if their key starts with "sentry-". + if k, v := member.Key(), member.Value(); strings.HasPrefix(k, sentryPrefix) { + entries[strings.TrimPrefix(k, sentryPrefix)] = v + } + } + + return DynamicSamplingContext{ + Entries: entries, + // If there's at least one Sentry value, we consider the DSC frozen + Frozen: len(entries) > 0, + }, nil +} + +func DynamicSamplingContextFromTransaction(span *Span) DynamicSamplingContext { + entries := map[string]string{} + + hub := hubFromContext(span.Context()) + scope := hub.Scope() + client := hub.Client() + + if client == nil || scope == nil { + return DynamicSamplingContext{ + Entries: map[string]string{}, + Frozen: false, + } + } + + if traceID := span.TraceID.String(); traceID != "" { + entries["trace_id"] = traceID + } + if sampleRate := span.sampleRate; sampleRate != 0 { + entries["sample_rate"] = strconv.FormatFloat(sampleRate, 'f', -1, 64) + } + + if dsn := client.dsn; dsn != nil { + if publicKey := dsn.publicKey; publicKey != "" { + entries["public_key"] = publicKey + } + } + if release := client.options.Release; release != "" { + entries["release"] = release + } + if environment := client.options.Environment; environment != "" { + entries["environment"] = environment + } + + // Only include the transaction name if it's of good quality (not empty and not SourceURL) + if span.Source != "" && span.Source != SourceURL { + if span.IsTransaction() { + entries["transaction"] = span.Name + } + } + + if userSegment := scope.user.Segment; userSegment != "" { + entries["user_segment"] = userSegment + } + + if span.Sampled.Bool() { + entries["sampled"] = "true" + } else { + entries["sampled"] = "false" + } + + return DynamicSamplingContext{ + Entries: entries, + Frozen: true, + } +} + +func (d DynamicSamplingContext) HasEntries() bool { + return len(d.Entries) > 0 +} + +func (d DynamicSamplingContext) IsFrozen() bool { + return d.Frozen +} + +func (d DynamicSamplingContext) String() string { + members := []baggage.Member{} + for k, entry := range d.Entries { + member, err := baggage.NewMember(sentryPrefix+k, entry) + if err != nil { + continue + } + members = append(members, member) + } + if len(members) > 0 { + baggage, err := baggage.New(members...) + if err != nil { + return "" + } + return baggage.String() + } + + return "" +} diff --git a/vendor/github.com/getsentry/sentry-go/internal/otel/baggage/README.md b/vendor/github.com/getsentry/sentry-go/internal/otel/baggage/README.md new file mode 100644 index 00000000..2718f314 --- /dev/null +++ b/vendor/github.com/getsentry/sentry-go/internal/otel/baggage/README.md @@ -0,0 +1,12 @@ +## Why do we have this "otel/baggage" folder? + +The root sentry-go SDK (namely, the Dynamic Sampling functionality) needs an implementation of the [baggage spec](https://www.w3.org/TR/baggage/). +For that reason, we've taken the existing baggage implementation from the [opentelemetry-go](https://github.com/open-telemetry/opentelemetry-go/) repository, and fixed a few things that in our opinion were violating the specification. + +These issues are: +1. Baggage string value `one%20two` should be properly parsed as "one two" +1. Baggage string value `one+two` should be parsed as "one+two" +1. Go string value "one two" should be encoded as `one%20two` (percent encoding), and NOT as `one+two` (URL query encoding). +1. Go string value "1=1" might be encoded as `1=1`, because the spec says: "Note, value MAY contain any number of the equal sign (=) characters. Parsers MUST NOT assume that the equal sign is only used to separate key and value.". `1%3D1` is also valid, but to simplify the implementation we're not doing it. + +Changes were made in this PR: https://github.com/getsentry/sentry-go/pull/568 diff --git a/vendor/github.com/getsentry/sentry-go/internal/otel/baggage/baggage.go b/vendor/github.com/getsentry/sentry-go/internal/otel/baggage/baggage.go new file mode 100644 index 00000000..18065550 --- /dev/null +++ b/vendor/github.com/getsentry/sentry-go/internal/otel/baggage/baggage.go @@ -0,0 +1,604 @@ +// Adapted from https://github.com/open-telemetry/opentelemetry-go/blob/c21b6b6bb31a2f74edd06e262f1690f3f6ea3d5c/baggage/baggage.go +// +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package baggage + +import ( + "errors" + "fmt" + "net/url" + "regexp" + "strings" + "unicode/utf8" + + "github.com/getsentry/sentry-go/internal/otel/baggage/internal/baggage" +) + +const ( + maxMembers = 180 + maxBytesPerMembers = 4096 + maxBytesPerBaggageString = 8192 + + listDelimiter = "," + keyValueDelimiter = "=" + propertyDelimiter = ";" + + keyDef = `([\x21\x23-\x27\x2A\x2B\x2D\x2E\x30-\x39\x41-\x5a\x5e-\x7a\x7c\x7e]+)` + valueDef = `([\x21\x23-\x2b\x2d-\x3a\x3c-\x5B\x5D-\x7e]*)` + keyValueDef = `\s*` + keyDef + `\s*` + keyValueDelimiter + `\s*` + valueDef + `\s*` +) + +var ( + keyRe = regexp.MustCompile(`^` + keyDef + `$`) + valueRe = regexp.MustCompile(`^` + valueDef + `$`) + propertyRe = regexp.MustCompile(`^(?:\s*` + keyDef + `\s*|` + keyValueDef + `)$`) +) + +var ( + errInvalidKey = errors.New("invalid key") + errInvalidValue = errors.New("invalid value") + errInvalidProperty = errors.New("invalid baggage list-member property") + errInvalidMember = errors.New("invalid baggage list-member") + errMemberNumber = errors.New("too many list-members in baggage-string") + errMemberBytes = errors.New("list-member too large") + errBaggageBytes = errors.New("baggage-string too large") +) + +// Property is an additional metadata entry for a baggage list-member. +type Property struct { + key, value string + + // hasValue indicates if a zero-value value means the property does not + // have a value or if it was the zero-value. + hasValue bool + + // hasData indicates whether the created property contains data or not. + // Properties that do not contain data are invalid with no other check + // required. + hasData bool +} + +// NewKeyProperty returns a new Property for key. +// +// If key is invalid, an error will be returned. +func NewKeyProperty(key string) (Property, error) { + if !keyRe.MatchString(key) { + return newInvalidProperty(), fmt.Errorf("%w: %q", errInvalidKey, key) + } + + p := Property{key: key, hasData: true} + return p, nil +} + +// NewKeyValueProperty returns a new Property for key with value. +// +// If key or value are invalid, an error will be returned. +func NewKeyValueProperty(key, value string) (Property, error) { + if !keyRe.MatchString(key) { + return newInvalidProperty(), fmt.Errorf("%w: %q", errInvalidKey, key) + } + if !valueRe.MatchString(value) { + return newInvalidProperty(), fmt.Errorf("%w: %q", errInvalidValue, value) + } + + p := Property{ + key: key, + value: value, + hasValue: true, + hasData: true, + } + return p, nil +} + +func newInvalidProperty() Property { + return Property{} +} + +// parseProperty attempts to decode a Property from the passed string. It +// returns an error if the input is invalid according to the W3C Baggage +// specification. +func parseProperty(property string) (Property, error) { + if property == "" { + return newInvalidProperty(), nil + } + + match := propertyRe.FindStringSubmatch(property) + if len(match) != 4 { + return newInvalidProperty(), fmt.Errorf("%w: %q", errInvalidProperty, property) + } + + p := Property{hasData: true} + if match[1] != "" { + p.key = match[1] + } else { + p.key = match[2] + p.value = match[3] + p.hasValue = true + } + + return p, nil +} + +// validate ensures p conforms to the W3C Baggage specification, returning an +// error otherwise. +func (p Property) validate() error { + errFunc := func(err error) error { + return fmt.Errorf("invalid property: %w", err) + } + + if !p.hasData { + return errFunc(fmt.Errorf("%w: %q", errInvalidProperty, p)) + } + + if !keyRe.MatchString(p.key) { + return errFunc(fmt.Errorf("%w: %q", errInvalidKey, p.key)) + } + if p.hasValue && !valueRe.MatchString(p.value) { + return errFunc(fmt.Errorf("%w: %q", errInvalidValue, p.value)) + } + if !p.hasValue && p.value != "" { + return errFunc(errors.New("inconsistent value")) + } + return nil +} + +// Key returns the Property key. +func (p Property) Key() string { + return p.key +} + +// Value returns the Property value. Additionally, a boolean value is returned +// indicating if the returned value is the empty if the Property has a value +// that is empty or if the value is not set. +func (p Property) Value() (string, bool) { + return p.value, p.hasValue +} + +// String encodes Property into a string compliant with the W3C Baggage +// specification. +func (p Property) String() string { + if p.hasValue { + return fmt.Sprintf("%s%s%v", p.key, keyValueDelimiter, p.value) + } + return p.key +} + +type properties []Property + +func fromInternalProperties(iProps []baggage.Property) properties { + if len(iProps) == 0 { + return nil + } + + props := make(properties, len(iProps)) + for i, p := range iProps { + props[i] = Property{ + key: p.Key, + value: p.Value, + hasValue: p.HasValue, + } + } + return props +} + +func (p properties) asInternal() []baggage.Property { + if len(p) == 0 { + return nil + } + + iProps := make([]baggage.Property, len(p)) + for i, prop := range p { + iProps[i] = baggage.Property{ + Key: prop.key, + Value: prop.value, + HasValue: prop.hasValue, + } + } + return iProps +} + +func (p properties) Copy() properties { + if len(p) == 0 { + return nil + } + + props := make(properties, len(p)) + copy(props, p) + return props +} + +// validate ensures each Property in p conforms to the W3C Baggage +// specification, returning an error otherwise. +func (p properties) validate() error { + for _, prop := range p { + if err := prop.validate(); err != nil { + return err + } + } + return nil +} + +// String encodes properties into a string compliant with the W3C Baggage +// specification. +func (p properties) String() string { + props := make([]string, len(p)) + for i, prop := range p { + props[i] = prop.String() + } + return strings.Join(props, propertyDelimiter) +} + +// Member is a list-member of a baggage-string as defined by the W3C Baggage +// specification. +type Member struct { + key, value string + properties properties + + // hasData indicates whether the created property contains data or not. + // Properties that do not contain data are invalid with no other check + // required. + hasData bool +} + +// NewMember returns a new Member from the passed arguments. The key will be +// used directly while the value will be url decoded after validation. An error +// is returned if the created Member would be invalid according to the W3C +// Baggage specification. +func NewMember(key, value string, props ...Property) (Member, error) { + m := Member{ + key: key, + value: value, + properties: properties(props).Copy(), + hasData: true, + } + if err := m.validate(); err != nil { + return newInvalidMember(), err + } + //// NOTE(anton): I don't think we need to unescape here + // decodedValue, err := url.PathUnescape(value) + // if err != nil { + // return newInvalidMember(), fmt.Errorf("%w: %q", errInvalidValue, value) + // } + // m.value = decodedValue + return m, nil +} + +func newInvalidMember() Member { + return Member{} +} + +// parseMember attempts to decode a Member from the passed string. It returns +// an error if the input is invalid according to the W3C Baggage +// specification. +func parseMember(member string) (Member, error) { + if n := len(member); n > maxBytesPerMembers { + return newInvalidMember(), fmt.Errorf("%w: %d", errMemberBytes, n) + } + + var ( + key, value string + props properties + ) + + parts := strings.SplitN(member, propertyDelimiter, 2) + switch len(parts) { + case 2: + // Parse the member properties. + for _, pStr := range strings.Split(parts[1], propertyDelimiter) { + p, err := parseProperty(pStr) + if err != nil { + return newInvalidMember(), err + } + props = append(props, p) + } + fallthrough + case 1: + // Parse the member key/value pair. + + // Take into account a value can contain equal signs (=). + kv := strings.SplitN(parts[0], keyValueDelimiter, 2) + if len(kv) != 2 { + return newInvalidMember(), fmt.Errorf("%w: %q", errInvalidMember, member) + } + // "Leading and trailing whitespaces are allowed but MUST be trimmed + // when converting the header into a data structure." + key = strings.TrimSpace(kv[0]) + value = strings.TrimSpace(kv[1]) + var err error + if !keyRe.MatchString(key) { + return newInvalidMember(), fmt.Errorf("%w: %q", errInvalidKey, key) + } + if !valueRe.MatchString(value) { + return newInvalidMember(), fmt.Errorf("%w: %q", errInvalidValue, value) + } + decodedValue, err := url.PathUnescape(value) + if err != nil { + return newInvalidMember(), fmt.Errorf("%w: %q", err, value) + } + value = decodedValue + default: + // This should never happen unless a developer has changed the string + // splitting somehow. Panic instead of failing silently and allowing + // the bug to slip past the CI checks. + panic("failed to parse baggage member") + } + + return Member{key: key, value: value, properties: props, hasData: true}, nil +} + +// validate ensures m conforms to the W3C Baggage specification. +// A key is just an ASCII string, but a value must be URL encoded UTF-8, +// returning an error otherwise. +func (m Member) validate() error { + if !m.hasData { + return fmt.Errorf("%w: %q", errInvalidMember, m) + } + + if !keyRe.MatchString(m.key) { + return fmt.Errorf("%w: %q", errInvalidKey, m.key) + } + //// NOTE(anton): IMO it's too early to validate the value here. + // if !valueRe.MatchString(m.value) { + // return fmt.Errorf("%w: %q", errInvalidValue, m.value) + // } + return m.properties.validate() +} + +// Key returns the Member key. +func (m Member) Key() string { return m.key } + +// Value returns the Member value. +func (m Member) Value() string { return m.value } + +// Properties returns a copy of the Member properties. +func (m Member) Properties() []Property { return m.properties.Copy() } + +// String encodes Member into a string compliant with the W3C Baggage +// specification. +func (m Member) String() string { + // A key is just an ASCII string, but a value is URL encoded UTF-8. + s := fmt.Sprintf("%s%s%s", m.key, keyValueDelimiter, percentEncodeValue(m.value)) + if len(m.properties) > 0 { + s = fmt.Sprintf("%s%s%s", s, propertyDelimiter, m.properties.String()) + } + return s +} + +// percentEncodeValue encodes the baggage value, using percent-encoding for +// disallowed octets. +func percentEncodeValue(s string) string { + const upperhex = "0123456789ABCDEF" + var sb strings.Builder + + for byteIndex, width := 0, 0; byteIndex < len(s); byteIndex += width { + runeValue, w := utf8.DecodeRuneInString(s[byteIndex:]) + width = w + char := string(runeValue) + if valueRe.MatchString(char) && char != "%" { + // The character is returned as is, no need to percent-encode + sb.WriteString(char) + } else { + // We need to percent-encode each byte of the multi-octet character + for j := 0; j < width; j++ { + b := s[byteIndex+j] + sb.WriteByte('%') + // Bitwise operations are inspired by "net/url" + sb.WriteByte(upperhex[b>>4]) + sb.WriteByte(upperhex[b&15]) + } + } + } + return sb.String() +} + +// Baggage is a list of baggage members representing the baggage-string as +// defined by the W3C Baggage specification. +type Baggage struct { //nolint:golint + list baggage.List +} + +// New returns a new valid Baggage. It returns an error if it results in a +// Baggage exceeding limits set in that specification. +// +// It expects all the provided members to have already been validated. +func New(members ...Member) (Baggage, error) { + if len(members) == 0 { + return Baggage{}, nil + } + + b := make(baggage.List) + for _, m := range members { + if !m.hasData { + return Baggage{}, errInvalidMember + } + + // OpenTelemetry resolves duplicates by last-one-wins. + b[m.key] = baggage.Item{ + Value: m.value, + Properties: m.properties.asInternal(), + } + } + + // Check member numbers after deduplication. + if len(b) > maxMembers { + return Baggage{}, errMemberNumber + } + + bag := Baggage{b} + if n := len(bag.String()); n > maxBytesPerBaggageString { + return Baggage{}, fmt.Errorf("%w: %d", errBaggageBytes, n) + } + + return bag, nil +} + +// Parse attempts to decode a baggage-string from the passed string. It +// returns an error if the input is invalid according to the W3C Baggage +// specification. +// +// If there are duplicate list-members contained in baggage, the last one +// defined (reading left-to-right) will be the only one kept. This diverges +// from the W3C Baggage specification which allows duplicate list-members, but +// conforms to the OpenTelemetry Baggage specification. +func Parse(bStr string) (Baggage, error) { + if bStr == "" { + return Baggage{}, nil + } + + if n := len(bStr); n > maxBytesPerBaggageString { + return Baggage{}, fmt.Errorf("%w: %d", errBaggageBytes, n) + } + + b := make(baggage.List) + for _, memberStr := range strings.Split(bStr, listDelimiter) { + m, err := parseMember(memberStr) + if err != nil { + return Baggage{}, err + } + // OpenTelemetry resolves duplicates by last-one-wins. + b[m.key] = baggage.Item{ + Value: m.value, + Properties: m.properties.asInternal(), + } + } + + // OpenTelemetry does not allow for duplicate list-members, but the W3C + // specification does. Now that we have deduplicated, ensure the baggage + // does not exceed list-member limits. + if len(b) > maxMembers { + return Baggage{}, errMemberNumber + } + + return Baggage{b}, nil +} + +// Member returns the baggage list-member identified by key. +// +// If there is no list-member matching the passed key the returned Member will +// be a zero-value Member. +// The returned member is not validated, as we assume the validation happened +// when it was added to the Baggage. +func (b Baggage) Member(key string) Member { + v, ok := b.list[key] + if !ok { + // We do not need to worry about distinguishing between the situation + // where a zero-valued Member is included in the Baggage because a + // zero-valued Member is invalid according to the W3C Baggage + // specification (it has an empty key). + return newInvalidMember() + } + + return Member{ + key: key, + value: v.Value, + properties: fromInternalProperties(v.Properties), + hasData: true, + } +} + +// Members returns all the baggage list-members. +// The order of the returned list-members does not have significance. +// +// The returned members are not validated, as we assume the validation happened +// when they were added to the Baggage. +func (b Baggage) Members() []Member { + if len(b.list) == 0 { + return nil + } + + members := make([]Member, 0, len(b.list)) + for k, v := range b.list { + members = append(members, Member{ + key: k, + value: v.Value, + properties: fromInternalProperties(v.Properties), + hasData: true, + }) + } + return members +} + +// SetMember returns a copy the Baggage with the member included. If the +// baggage contains a Member with the same key the existing Member is +// replaced. +// +// If member is invalid according to the W3C Baggage specification, an error +// is returned with the original Baggage. +func (b Baggage) SetMember(member Member) (Baggage, error) { + if !member.hasData { + return b, errInvalidMember + } + + n := len(b.list) + if _, ok := b.list[member.key]; !ok { + n++ + } + list := make(baggage.List, n) + + for k, v := range b.list { + // Do not copy if we are just going to overwrite. + if k == member.key { + continue + } + list[k] = v + } + + list[member.key] = baggage.Item{ + Value: member.value, + Properties: member.properties.asInternal(), + } + + return Baggage{list: list}, nil +} + +// DeleteMember returns a copy of the Baggage with the list-member identified +// by key removed. +func (b Baggage) DeleteMember(key string) Baggage { + n := len(b.list) + if _, ok := b.list[key]; ok { + n-- + } + list := make(baggage.List, n) + + for k, v := range b.list { + if k == key { + continue + } + list[k] = v + } + + return Baggage{list: list} +} + +// Len returns the number of list-members in the Baggage. +func (b Baggage) Len() int { + return len(b.list) +} + +// String encodes Baggage into a string compliant with the W3C Baggage +// specification. The returned string will be invalid if the Baggage contains +// any invalid list-members. +func (b Baggage) String() string { + members := make([]string, 0, len(b.list)) + for k, v := range b.list { + members = append(members, Member{ + key: k, + value: v.Value, + properties: fromInternalProperties(v.Properties), + }.String()) + } + return strings.Join(members, listDelimiter) +} diff --git a/vendor/github.com/getsentry/sentry-go/internal/otel/baggage/internal/baggage/baggage.go b/vendor/github.com/getsentry/sentry-go/internal/otel/baggage/internal/baggage/baggage.go new file mode 100644 index 00000000..ea99ccbf --- /dev/null +++ b/vendor/github.com/getsentry/sentry-go/internal/otel/baggage/internal/baggage/baggage.go @@ -0,0 +1,45 @@ +// Adapted from https://github.com/open-telemetry/opentelemetry-go/blob/c21b6b6bb31a2f74edd06e262f1690f3f6ea3d5c/internal/baggage/baggage.go +// +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* +Package baggage provides base types and functionality to store and retrieve +baggage in Go context. This package exists because the OpenTracing bridge to +OpenTelemetry needs to synchronize state whenever baggage for a context is +modified and that context contains an OpenTracing span. If it were not for +this need this package would not need to exist and the +`go.opentelemetry.io/otel/baggage` package would be the singular place where +W3C baggage is handled. +*/ +package baggage + +// List is the collection of baggage members. The W3C allows for duplicates, +// but OpenTelemetry does not, therefore, this is represented as a map. +type List map[string]Item + +// Item is the value and metadata properties part of a list-member. +type Item struct { + Value string + Properties []Property +} + +// Property is a metadata entry for a list-member. +type Property struct { + Key, Value string + + // HasValue indicates if a zero-value value means the property does not + // have a value or if it was the zero-value. + HasValue bool +} diff --git a/vendor/github.com/getsentry/sentry-go/internal/traceparser/README.md b/vendor/github.com/getsentry/sentry-go/internal/traceparser/README.md new file mode 100644 index 00000000..78964587 --- /dev/null +++ b/vendor/github.com/getsentry/sentry-go/internal/traceparser/README.md @@ -0,0 +1,15 @@ +## Benchmark results + +``` +goos: windows +goarch: amd64 +pkg: github.com/getsentry/sentry-go/internal/trace +cpu: 12th Gen Intel(R) Core(TM) i7-12700K +BenchmarkEqualBytes-20 44323621 26.08 ns/op +BenchmarkStringEqual-20 60980257 18.27 ns/op +BenchmarkEqualPrefix-20 41369181 31.12 ns/op +BenchmarkFullParse-20 702012 1507 ns/op 1353.42 MB/s 1024 B/op 6 allocs/op +BenchmarkFramesIterator-20 1229971 969.3 ns/op 896 B/op 5 allocs/op +BenchmarkFramesReversedIterator-20 1271061 944.5 ns/op 896 B/op 5 allocs/op +BenchmarkSplitOnly-20 2250800 534.0 ns/op 3818.23 MB/s 128 B/op 1 allocs/op +``` diff --git a/vendor/github.com/getsentry/sentry-go/internal/traceparser/parser.go b/vendor/github.com/getsentry/sentry-go/internal/traceparser/parser.go new file mode 100644 index 00000000..8a7aab32 --- /dev/null +++ b/vendor/github.com/getsentry/sentry-go/internal/traceparser/parser.go @@ -0,0 +1,217 @@ +package traceparser + +import ( + "bytes" + "strconv" +) + +var blockSeparator = []byte("\n\n") +var lineSeparator = []byte("\n") + +// Parses multi-stacktrace text dump produced by runtime.Stack([]byte, all=true). +// The parser prioritizes performance but requires the input to be well-formed in order to return correct data. +// See https://github.com/golang/go/blob/go1.20.4/src/runtime/mprof.go#L1191 +func Parse(data []byte) TraceCollection { + var it = TraceCollection{} + if len(data) > 0 { + it.blocks = bytes.Split(data, blockSeparator) + } + return it +} + +type TraceCollection struct { + blocks [][]byte +} + +func (it TraceCollection) Length() int { + return len(it.blocks) +} + +// Returns the stacktrace item at the given index. +func (it *TraceCollection) Item(i int) Trace { + // The first item may have a leading data separator and the last one may have a trailing one. + // Note: Trim() doesn't make a copy for single-character cutset under 0x80. It will just slice the original. + var data []byte + switch { + case i == 0: + data = bytes.TrimLeft(it.blocks[i], "\n") + case i == len(it.blocks)-1: + data = bytes.TrimRight(it.blocks[i], "\n") + default: + data = it.blocks[i] + } + + var splitAt = bytes.IndexByte(data, '\n') + if splitAt < 0 { + return Trace{header: data} + } + + return Trace{ + header: data[:splitAt], + data: data[splitAt+1:], + } +} + +// Trace represents a single stacktrace block, identified by a Goroutine ID and a sequence of Frames. +type Trace struct { + header []byte + data []byte +} + +var goroutinePrefix = []byte("goroutine ") + +// GoID parses the Goroutine ID from the header. +func (t *Trace) GoID() (id uint64) { + if bytes.HasPrefix(t.header, goroutinePrefix) { + var line = t.header[len(goroutinePrefix):] + var splitAt = bytes.IndexByte(line, ' ') + if splitAt >= 0 { + id, _ = strconv.ParseUint(string(line[:splitAt]), 10, 64) + } + } + return id +} + +// UniqueIdentifier can be used as a map key to identify the trace. +func (t *Trace) UniqueIdentifier() []byte { + return t.data +} + +func (t *Trace) Frames() FrameIterator { + var lines = bytes.Split(t.data, lineSeparator) + return FrameIterator{lines: lines, i: 0, len: len(lines)} +} + +func (t *Trace) FramesReversed() ReverseFrameIterator { + var lines = bytes.Split(t.data, lineSeparator) + return ReverseFrameIterator{lines: lines, i: len(lines)} +} + +const framesElided = "...additional frames elided..." + +// FrameIterator iterates over stack frames. +type FrameIterator struct { + lines [][]byte + i int + len int +} + +// Next returns the next frame, or nil if there are none. +func (it *FrameIterator) Next() Frame { + return Frame{it.popLine(), it.popLine()} +} + +func (it *FrameIterator) popLine() []byte { + switch { + case it.i >= it.len: + return nil + case string(it.lines[it.i]) == framesElided: + it.i++ + return it.popLine() + default: + it.i++ + return it.lines[it.i-1] + } +} + +// HasNext return true if there are values to be read. +func (it *FrameIterator) HasNext() bool { + return it.i < it.len +} + +// LengthUpperBound returns the maximum number of elements this stacks may contain. +// The actual number may be lower because of elided frames. As such, the returned value +// cannot be used to iterate over the frames but may be used to reserve capacity. +func (it *FrameIterator) LengthUpperBound() int { + return it.len / 2 +} + +// ReverseFrameIterator iterates over stack frames in reverse order. +type ReverseFrameIterator struct { + lines [][]byte + i int +} + +// Next returns the next frame, or nil if there are none. +func (it *ReverseFrameIterator) Next() Frame { + var line2 = it.popLine() + return Frame{it.popLine(), line2} +} + +func (it *ReverseFrameIterator) popLine() []byte { + it.i-- + switch { + case it.i < 0: + return nil + case string(it.lines[it.i]) == framesElided: + return it.popLine() + default: + return it.lines[it.i] + } +} + +// HasNext return true if there are values to be read. +func (it *ReverseFrameIterator) HasNext() bool { + return it.i > 1 +} + +// LengthUpperBound returns the maximum number of elements this stacks may contain. +// The actual number may be lower because of elided frames. As such, the returned value +// cannot be used to iterate over the frames but may be used to reserve capacity. +func (it *ReverseFrameIterator) LengthUpperBound() int { + return len(it.lines) / 2 +} + +type Frame struct { + line1 []byte + line2 []byte +} + +// UniqueIdentifier can be used as a map key to identify the frame. +func (f *Frame) UniqueIdentifier() []byte { + // line2 contains file path, line number and program-counter offset from the beginning of a function + // e.g. C:/Users/name/scoop/apps/go/current/src/testing/testing.go:1906 +0x63a + return f.line2 +} + +var createdByPrefix = []byte("created by ") + +func (f *Frame) Func() []byte { + if bytes.HasPrefix(f.line1, createdByPrefix) { + // Since go1.21, the line ends with " in goroutine X", saying which goroutine created this one. + // We currently don't have use for that so just remove it. + var line = f.line1[len(createdByPrefix):] + var spaceAt = bytes.IndexByte(line, ' ') + if spaceAt < 0 { + return line + } + return line[:spaceAt] + } + + var end = bytes.LastIndexByte(f.line1, '(') + if end >= 0 { + return f.line1[:end] + } + + return f.line1 +} + +func (f *Frame) File() (path []byte, lineNumber int) { + var line = f.line2 + if len(line) > 0 && line[0] == '\t' { + line = line[1:] + } + + var splitAt = bytes.IndexByte(line, ' ') + if splitAt >= 0 { + line = line[:splitAt] + } + + splitAt = bytes.LastIndexByte(line, ':') + if splitAt < 0 { + return line, 0 + } + + lineNumber, _ = strconv.Atoi(string(line[splitAt+1:])) + return line[:splitAt], lineNumber +} diff --git a/vendor/github.com/getsentry/sentry-go/profile_sample.go b/vendor/github.com/getsentry/sentry-go/profile_sample.go new file mode 100644 index 00000000..65059872 --- /dev/null +++ b/vendor/github.com/getsentry/sentry-go/profile_sample.go @@ -0,0 +1,73 @@ +package sentry + +// Based on https://github.com/getsentry/vroom/blob/d11c26063e802d66b9a592c4010261746ca3dfa4/internal/sample/sample.go + +import ( + "time" +) + +type ( + profileDevice struct { + Architecture string `json:"architecture"` + Classification string `json:"classification"` + Locale string `json:"locale"` + Manufacturer string `json:"manufacturer"` + Model string `json:"model"` + } + + profileOS struct { + BuildNumber string `json:"build_number"` + Name string `json:"name"` + Version string `json:"version"` + } + + profileRuntime struct { + Name string `json:"name"` + Version string `json:"version"` + } + + profileSample struct { + ElapsedSinceStartNS uint64 `json:"elapsed_since_start_ns"` + StackID int `json:"stack_id"` + ThreadID uint64 `json:"thread_id"` + } + + profileThreadMetadata struct { + Name string `json:"name,omitempty"` + Priority int `json:"priority,omitempty"` + } + + profileStack []int + + profileTrace struct { + Frames []*Frame `json:"frames"` + Samples []profileSample `json:"samples"` + Stacks []profileStack `json:"stacks"` + ThreadMetadata map[uint64]*profileThreadMetadata `json:"thread_metadata"` + } + + profileInfo struct { + DebugMeta *DebugMeta `json:"debug_meta,omitempty"` + Device profileDevice `json:"device"` + Environment string `json:"environment,omitempty"` + EventID string `json:"event_id"` + OS profileOS `json:"os"` + Platform string `json:"platform"` + Release string `json:"release"` + Dist string `json:"dist"` + Runtime profileRuntime `json:"runtime"` + Timestamp time.Time `json:"timestamp"` + Trace *profileTrace `json:"profile"` + Transaction profileTransaction `json:"transaction"` + Version string `json:"version"` + } + + // see https://github.com/getsentry/vroom/blob/a91e39416723ec44fc54010257020eeaf9a77cbd/internal/transaction/transaction.go + profileTransaction struct { + ActiveThreadID uint64 `json:"active_thread_id"` + DurationNS uint64 `json:"duration_ns,omitempty"` + ID EventID `json:"id"` + Name string `json:"name"` + TraceID string `json:"trace_id"` + } +) diff --git a/vendor/github.com/getsentry/sentry-go/profiler.go b/vendor/github.com/getsentry/sentry-go/profiler.go new file mode 100644 index 00000000..c0b858cc --- /dev/null +++ b/vendor/github.com/getsentry/sentry-go/profiler.go @@ -0,0 +1,451 @@ +package sentry + +import ( + "container/ring" + "strconv" + + "runtime" + "sync" + "sync/atomic" + "time" + + "github.com/getsentry/sentry-go/internal/traceparser" +) + +// Start a profiler that collects samples continuously, with a buffer of up to 30 seconds. +// Later, you can collect a slice from this buffer, producing a Trace. +func startProfiling(startTime time.Time) profiler { + onProfilerStart() + + p := newProfiler(startTime) + + // Wait for the profiler to finish setting up before returning to the caller. + started := make(chan struct{}) + go p.run(started) + + if _, ok := <-started; ok { + return p + } + return nil +} + +type profiler interface { + // GetSlice returns a slice of the profiled data between the given times. + GetSlice(startTime, endTime time.Time) *profilerResult + Stop(wait bool) +} + +type profilerResult struct { + callerGoID uint64 + trace *profileTrace +} + +func getCurrentGoID() uint64 { + // We shouldn't panic but let's be super safe. + defer func() { + if err := recover(); err != nil { + Logger.Printf("Profiler panic in getCurrentGoID(): %v\n", err) + } + }() + + // Buffer to read the stack trace into. We should be good with a small buffer because we only need the first line. + var stacksBuffer = make([]byte, 100) + var n = runtime.Stack(stacksBuffer, false) + if n > 0 { + var traces = traceparser.Parse(stacksBuffer[0:n]) + if traces.Length() > 0 { + var trace = traces.Item(0) + return trace.GoID() + } + } + return 0 +} + +const profilerSamplingRateHz = 101 // 101 Hz; not 100 Hz because of the lockstep sampling (https://stackoverflow.com/a/45471031/1181370) +const profilerSamplingRate = time.Second / profilerSamplingRateHz +const stackBufferMaxGrowth = 512 * 1024 +const stackBufferLimit = 10 * 1024 * 1024 +const profilerRuntimeLimit = 30 // seconds + +type profileRecorder struct { + startTime time.Time + stopSignal chan struct{} + stopped int64 + mutex sync.RWMutex + testProfilerPanic int64 + + // Map from runtime.StackRecord.Stack0 to an index in stacks. + stackIndexes map[string]int + stacks []profileStack + newStacks []profileStack // New stacks created in the current interation. + stackKeyBuffer []byte + + // Map from runtime.Frame.PC to an index in frames. + frameIndexes map[string]int + frames []*Frame + newFrames []*Frame // New frames created in the current interation. + + // We keep a ring buffer of 30 seconds worth of samples, so that we can later slice it. + // Each bucket is a slice of samples all taken at the same time. + samplesBucketsHead *ring.Ring + + // Buffer to read current stacks - will grow automatically up to stackBufferLimit. + stacksBuffer []byte +} + +func newProfiler(startTime time.Time) *profileRecorder { + // Pre-allocate the profile trace for the currently active number of routines & 100 ms worth of samples. + // Other coefficients are just guesses of what might be a good starting point to avoid allocs on short runs. + return &profileRecorder{ + startTime: startTime, + stopSignal: make(chan struct{}, 1), + + stackIndexes: make(map[string]int, 32), + stacks: make([]profileStack, 0, 32), + newStacks: make([]profileStack, 0, 32), + + frameIndexes: make(map[string]int, 128), + frames: make([]*Frame, 0, 128), + newFrames: make([]*Frame, 0, 128), + + samplesBucketsHead: ring.New(profilerRuntimeLimit * profilerSamplingRateHz), + + // A buffer of 2 KiB per goroutine stack looks like a good starting point (empirically determined). + stacksBuffer: make([]byte, runtime.NumGoroutine()*2048), + } +} + +// This allows us to test whether panic during profiling are handled correctly and don't block execution. +// If the number is lower than 0, profilerGoroutine() will panic immedately. +// If the number is higher than 0, profiler.onTick() will panic when the given samples-set index is being collected. +var testProfilerPanic int64 +var profilerRunning int64 + +func (p *profileRecorder) run(started chan struct{}) { + // Code backup for manual test debugging: + // if !atomic.CompareAndSwapInt64(&profilerRunning, 0, 1) { + // panic("Only one profiler can be running at a time") + // } + + // We shouldn't panic but let's be super safe. + defer func() { + if err := recover(); err != nil { + Logger.Printf("Profiler panic in run(): %v\n", err) + } + atomic.StoreInt64(&testProfilerPanic, 0) + close(started) + p.stopSignal <- struct{}{} + atomic.StoreInt64(&p.stopped, 1) + atomic.StoreInt64(&profilerRunning, 0) + }() + + p.testProfilerPanic = atomic.LoadInt64(&testProfilerPanic) + if p.testProfilerPanic < 0 { + Logger.Printf("Profiler panicking during startup because testProfilerPanic == %v\n", p.testProfilerPanic) + panic("This is an expected panic in profilerGoroutine() during tests") + } + + // Collect the first sample immediately. + p.onTick() + + // Periodically collect stacks, starting after profilerSamplingRate has passed. + collectTicker := profilerTickerFactory(profilerSamplingRate) + defer collectTicker.Stop() + var tickerChannel = collectTicker.TickSource() + + started <- struct{}{} + + for { + select { + case <-tickerChannel: + p.onTick() + collectTicker.Ticked() + case <-p.stopSignal: + return + } + } +} + +func (p *profileRecorder) Stop(wait bool) { + if atomic.LoadInt64(&p.stopped) == 1 { + return + } + p.stopSignal <- struct{}{} + if wait { + <-p.stopSignal + } +} + +func (p *profileRecorder) GetSlice(startTime, endTime time.Time) *profilerResult { + // Unlikely edge cases - profiler wasn't running at all or the given times are invalid in relation to each other. + if p.startTime.After(endTime) || startTime.After(endTime) { + return nil + } + + var relativeStartNS = uint64(0) + if p.startTime.Before(startTime) { + relativeStartNS = uint64(startTime.Sub(p.startTime).Nanoseconds()) + } + var relativeEndNS = uint64(endTime.Sub(p.startTime).Nanoseconds()) + + samplesCount, bucketsReversed, trace := p.getBuckets(relativeStartNS, relativeEndNS) + if samplesCount == 0 { + return nil + } + + var result = &profilerResult{ + callerGoID: getCurrentGoID(), + trace: trace, + } + + trace.Samples = make([]profileSample, samplesCount) + trace.ThreadMetadata = make(map[uint64]*profileThreadMetadata, len(bucketsReversed[0].goIDs)) + var s = samplesCount - 1 + for _, bucket := range bucketsReversed { + var elapsedSinceStartNS = bucket.relativeTimeNS - relativeStartNS + for i, goID := range bucket.goIDs { + trace.Samples[s].ElapsedSinceStartNS = elapsedSinceStartNS + trace.Samples[s].ThreadID = goID + trace.Samples[s].StackID = bucket.stackIDs[i] + s-- + + if _, goroutineExists := trace.ThreadMetadata[goID]; !goroutineExists { + trace.ThreadMetadata[goID] = &profileThreadMetadata{ + Name: "Goroutine " + strconv.FormatUint(goID, 10), + } + } + } + } + + return result +} + +// Collect all buckets of samples in the given time range while holding a read lock. +func (p *profileRecorder) getBuckets(relativeStartNS, relativeEndNS uint64) (samplesCount int, buckets []*profileSamplesBucket, trace *profileTrace) { + p.mutex.RLock() + defer p.mutex.RUnlock() + + // sampleBucketsHead points at the last stored bucket so it's a good starting point to search backwards for the end. + var end = p.samplesBucketsHead + for end.Value != nil && end.Value.(*profileSamplesBucket).relativeTimeNS > relativeEndNS { + end = end.Prev() + } + + // Edge case - no items stored before the given endTime. + if end.Value == nil { + return 0, nil, nil + } + + { // Find the first item after the given startTime. + var start = end + var prevBucket *profileSamplesBucket + samplesCount = 0 + buckets = make([]*profileSamplesBucket, 0, int64((relativeEndNS-relativeStartNS)/uint64(profilerSamplingRate.Nanoseconds()))+1) + for start.Value != nil { + var bucket = start.Value.(*profileSamplesBucket) + + // If this bucket's time is before the requests start time, don't collect it (and stop iterating further). + if bucket.relativeTimeNS < relativeStartNS { + break + } + + // If this bucket time is greater than previous the bucket's time, we have exhausted the whole ring buffer + // before we were able to find the start time. That means the start time is not present and we must break. + // This happens if the slice duration exceeds the ring buffer capacity. + if prevBucket != nil && bucket.relativeTimeNS > prevBucket.relativeTimeNS { + break + } + + samplesCount += len(bucket.goIDs) + buckets = append(buckets, bucket) + + start = start.Prev() + prevBucket = bucket + } + } + + // Edge case - if the period requested was too short and we haven't collected enough samples. + if len(buckets) < 2 { + return 0, nil, nil + } + + trace = &profileTrace{ + Frames: p.frames, + Stacks: p.stacks, + } + return samplesCount, buckets, trace +} + +func (p *profileRecorder) onTick() { + elapsedNs := time.Since(p.startTime).Nanoseconds() + + if p.testProfilerPanic > 0 { + Logger.Printf("Profiler testProfilerPanic == %v\n", p.testProfilerPanic) + if p.testProfilerPanic == 1 { + Logger.Println("Profiler panicking onTick()") + panic("This is an expected panic in Profiler.OnTick() during tests") + } + p.testProfilerPanic-- + } + + records := p.collectRecords() + p.processRecords(uint64(elapsedNs), records) + + // Free up some memory if we don't need such a large buffer anymore. + if len(p.stacksBuffer) > len(records)*3 { + p.stacksBuffer = make([]byte, len(records)*3) + } +} + +func (p *profileRecorder) collectRecords() []byte { + for { + // Capture stacks for all existing goroutines. + // Note: runtime.GoroutineProfile() would be better but we can't use it at the moment because + // it doesn't give us `gid` for each routine, see https://github.com/golang/go/issues/59663 + n := runtime.Stack(p.stacksBuffer, true) + + // If we couldn't read everything, increase the buffer and try again. + if n >= len(p.stacksBuffer) && n < stackBufferLimit { + var newSize = n * 2 + if newSize > n+stackBufferMaxGrowth { + newSize = n + stackBufferMaxGrowth + } + if newSize > stackBufferLimit { + newSize = stackBufferLimit + } + p.stacksBuffer = make([]byte, newSize) + } else { + return p.stacksBuffer[0:n] + } + } +} + +func (p *profileRecorder) processRecords(elapsedNs uint64, stacksBuffer []byte) { + var traces = traceparser.Parse(stacksBuffer) + var length = traces.Length() + + // Shouldn't happen but let's be safe and don't store empty buckets. + if length == 0 { + return + } + + var bucket = &profileSamplesBucket{ + relativeTimeNS: elapsedNs, + stackIDs: make([]int, length), + goIDs: make([]uint64, length), + } + + // reset buffers + p.newFrames = p.newFrames[:0] + p.newStacks = p.newStacks[:0] + + for i := 0; i < length; i++ { + var stack = traces.Item(i) + bucket.stackIDs[i] = p.addStackTrace(stack) + bucket.goIDs[i] = stack.GoID() + } + + p.mutex.Lock() + defer p.mutex.Unlock() + + p.stacks = append(p.stacks, p.newStacks...) + p.frames = append(p.frames, p.newFrames...) + + p.samplesBucketsHead = p.samplesBucketsHead.Next() + p.samplesBucketsHead.Value = bucket +} + +func (p *profileRecorder) addStackTrace(capturedStack traceparser.Trace) int { + iter := capturedStack.Frames() + stack := make(profileStack, 0, iter.LengthUpperBound()) + + // Originally, we've used `capturedStack.UniqueIdentifier()` as a key but that was incorrect because it also + // contains function arguments and we want to group stacks by function name and file/line only. + // Instead, we need to parse frames and we use a list of their indexes as a key. + // We reuse the same buffer for each stack to avoid allocations; this is a hot spot. + var expectedBufferLen = cap(stack) * 5 // 4 bytes per frame + 1 byte for space + if cap(p.stackKeyBuffer) < expectedBufferLen { + p.stackKeyBuffer = make([]byte, 0, expectedBufferLen) + } else { + p.stackKeyBuffer = p.stackKeyBuffer[:0] + } + + for iter.HasNext() { + var frame = iter.Next() + if frameIndex := p.addFrame(frame); frameIndex >= 0 { + stack = append(stack, frameIndex) + + p.stackKeyBuffer = append(p.stackKeyBuffer, 0) // space + + // The following code is just like binary.AppendUvarint() which isn't yet available in Go 1.18. + x := uint64(frameIndex) + 1 + for x >= 0x80 { + p.stackKeyBuffer = append(p.stackKeyBuffer, byte(x)|0x80) + x >>= 7 + } + p.stackKeyBuffer = append(p.stackKeyBuffer, byte(x)) + } + } + + stackIndex, exists := p.stackIndexes[string(p.stackKeyBuffer)] + if !exists { + stackIndex = len(p.stacks) + len(p.newStacks) + p.newStacks = append(p.newStacks, stack) + p.stackIndexes[string(p.stackKeyBuffer)] = stackIndex + } + + return stackIndex +} + +func (p *profileRecorder) addFrame(capturedFrame traceparser.Frame) int { + // NOTE: Don't convert to string yet, it's expensive and compiler can avoid it when + // indexing into a map (only needs a copy when adding a new key to the map). + var key = capturedFrame.UniqueIdentifier() + + frameIndex, exists := p.frameIndexes[string(key)] + if !exists { + module, function := splitQualifiedFunctionName(string(capturedFrame.Func())) + file, line := capturedFrame.File() + frame := newFrame(module, function, string(file), line) + frameIndex = len(p.frames) + len(p.newFrames) + p.newFrames = append(p.newFrames, &frame) + p.frameIndexes[string(key)] = frameIndex + } + return frameIndex +} + +type profileSamplesBucket struct { + relativeTimeNS uint64 + stackIDs []int + goIDs []uint64 +} + +// A Ticker holds a channel that delivers “ticks” of a clock at intervals. +type profilerTicker interface { + // Stop turns off a ticker. After Stop, no more ticks will be sent. + Stop() + + // TickSource returns a read-only channel of ticks. + TickSource() <-chan time.Time + + // Ticked is called by the Profiler after a tick is processed to notify the ticker. Used for testing. + Ticked() +} + +type timeTicker struct { + *time.Ticker +} + +func (t *timeTicker) TickSource() <-chan time.Time { + return t.C +} + +func (t *timeTicker) Ticked() {} + +func profilerTickerFactoryDefault(d time.Duration) profilerTicker { + return &timeTicker{time.NewTicker(d)} +} + +// We allow overriding the ticker for tests. CI is terribly flaky +// because the time.Ticker doesn't guarantee regular ticks - they may come (a lot) later than the given interval. +var profilerTickerFactory = profilerTickerFactoryDefault diff --git a/vendor/github.com/getsentry/sentry-go/profiler_other.go b/vendor/github.com/getsentry/sentry-go/profiler_other.go new file mode 100644 index 00000000..fbb79b0c --- /dev/null +++ b/vendor/github.com/getsentry/sentry-go/profiler_other.go @@ -0,0 +1,5 @@ +//go:build !windows + +package sentry + +func onProfilerStart() {} diff --git a/vendor/github.com/getsentry/sentry-go/profiler_windows.go b/vendor/github.com/getsentry/sentry-go/profiler_windows.go new file mode 100644 index 00000000..33279824 --- /dev/null +++ b/vendor/github.com/getsentry/sentry-go/profiler_windows.go @@ -0,0 +1,24 @@ +package sentry + +import ( + "sync" + "syscall" +) + +// This works around the ticker resolution on Windows being ~15ms by default. +// See https://github.com/golang/go/issues/44343 +func setTimeTickerResolution() { + var winmmDLL = syscall.NewLazyDLL("winmm.dll") + if winmmDLL != nil { + var timeBeginPeriod = winmmDLL.NewProc("timeBeginPeriod") + if timeBeginPeriod != nil { + timeBeginPeriod.Call(uintptr(1)) + } + } +} + +var setupTickerResolutionOnce sync.Once + +func onProfilerStart() { + setupTickerResolutionOnce.Do(setTimeTickerResolution) +} diff --git a/vendor/github.com/getsentry/sentry-go/stacktrace_below_go1.20.go b/vendor/github.com/getsentry/sentry-go/stacktrace_below_go1.20.go new file mode 100644 index 00000000..f6fb8e1e --- /dev/null +++ b/vendor/github.com/getsentry/sentry-go/stacktrace_below_go1.20.go @@ -0,0 +1,15 @@ +//go:build !go1.20 + +package sentry + +import "strings" + +func isCompilerGeneratedSymbol(name string) bool { + // In versions of Go below 1.20 a prefix of "type." and "go." is a + // compiler-generated symbol that doesn't belong to any package. + // See variable reservedimports in cmd/compile/internal/gc/subr.go + if strings.HasPrefix(name, "go.") || strings.HasPrefix(name, "type.") { + return true + } + return false +} diff --git a/vendor/github.com/getsentry/sentry-go/stacktrace_go1.20.go b/vendor/github.com/getsentry/sentry-go/stacktrace_go1.20.go new file mode 100644 index 00000000..ff1cbf60 --- /dev/null +++ b/vendor/github.com/getsentry/sentry-go/stacktrace_go1.20.go @@ -0,0 +1,15 @@ +//go:build go1.20 + +package sentry + +import "strings" + +func isCompilerGeneratedSymbol(name string) bool { + // In versions of Go 1.20 and above a prefix of "type:" and "go:" is a + // compiler-generated symbol that doesn't belong to any package. + // See variable reservedimports in cmd/compile/internal/gc/subr.go + if strings.HasPrefix(name, "go:") || strings.HasPrefix(name, "type:") { + return true + } + return false +} diff --git a/vendor/github.com/getsentry/sentry-go/traces_profiler.go b/vendor/github.com/getsentry/sentry-go/traces_profiler.go new file mode 100644 index 00000000..2655fe84 --- /dev/null +++ b/vendor/github.com/getsentry/sentry-go/traces_profiler.go @@ -0,0 +1,90 @@ +package sentry + +import ( + "sync" + "time" +) + +// Checks whether the transaction should be profiled (according to ProfilesSampleRate) +// and starts a profiler if so. +func (span *Span) sampleTransactionProfile() { + var sampleRate = span.clientOptions().ProfilesSampleRate + switch { + case sampleRate < 0.0 || sampleRate > 1.0: + Logger.Printf("Skipping transaction profiling: ProfilesSampleRate out of range [0.0, 1.0]: %f\n", sampleRate) + case sampleRate == 0.0 || rng.Float64() >= sampleRate: + Logger.Printf("Skipping transaction profiling: ProfilesSampleRate is: %f\n", sampleRate) + default: + startProfilerOnce.Do(startGlobalProfiler) + if globalProfiler == nil { + Logger.Println("Skipping transaction profiling: the profiler couldn't be started") + } else { + span.collectProfile = collectTransactionProfile + } + } +} + +// transactionProfiler collects a profile for a given span. +type transactionProfiler func(span *Span) *profileInfo + +var startProfilerOnce sync.Once +var globalProfiler profiler + +func startGlobalProfiler() { + globalProfiler = startProfiling(time.Now()) +} + +func collectTransactionProfile(span *Span) *profileInfo { + result := globalProfiler.GetSlice(span.StartTime, span.EndTime) + if result == nil || result.trace == nil { + return nil + } + + info := &profileInfo{ + Version: "1", + EventID: uuid(), + // See https://github.com/getsentry/sentry-go/pull/626#discussion_r1204870340 for explanation why we use the Transaction time. + Timestamp: span.StartTime, + Trace: result.trace, + Transaction: profileTransaction{ + DurationNS: uint64(span.EndTime.Sub(span.StartTime).Nanoseconds()), + Name: span.Name, + TraceID: span.TraceID.String(), + }, + } + if len(info.Transaction.Name) == 0 { + // Name is required by Relay so use the operation name if the span name is empty. + info.Transaction.Name = span.Op + } + if result.callerGoID > 0 { + info.Transaction.ActiveThreadID = result.callerGoID + } + return info +} + +func (info *profileInfo) UpdateFromEvent(event *Event) { + info.Environment = event.Environment + info.Platform = event.Platform + info.Release = event.Release + info.Dist = event.Dist + info.Transaction.ID = event.EventID + + if runtimeContext, ok := event.Contexts["runtime"]; ok { + if value, ok := runtimeContext["name"]; !ok { + info.Runtime.Name = value.(string) + } + if value, ok := runtimeContext["version"]; !ok { + info.Runtime.Version = value.(string) + } + } + if osContext, ok := event.Contexts["os"]; ok { + if value, ok := osContext["name"]; !ok { + info.OS.Name = value.(string) + } + } + if deviceContext, ok := event.Contexts["device"]; ok { + if value, ok := deviceContext["arch"]; !ok { + info.Device.Architecture = value.(string) + } + } +} diff --git a/vendor/github.com/glycerine/go-unsnap-stream/.gitignore b/vendor/github.com/glycerine/go-unsnap-stream/.gitignore deleted file mode 100644 index 00268614..00000000 --- a/vendor/github.com/glycerine/go-unsnap-stream/.gitignore +++ /dev/null @@ -1,22 +0,0 @@ -# Compiled Object files, Static and Dynamic libs (Shared Objects) -*.o -*.a -*.so - -# Folders -_obj -_test - -# Architecture specific extensions/prefixes -*.[568vq] -[568vq].out - -*.cgo1.go -*.cgo2.c -_cgo_defun.c -_cgo_gotypes.go -_cgo_export.* - -_testmain.go - -*.exe diff --git a/vendor/github.com/glycerine/go-unsnap-stream/LICENSE b/vendor/github.com/glycerine/go-unsnap-stream/LICENSE deleted file mode 100644 index a441b993..00000000 --- a/vendor/github.com/glycerine/go-unsnap-stream/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT license. - -Copyright (c) 2014 the go-unsnap-stream authors. - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - diff --git a/vendor/github.com/glycerine/go-unsnap-stream/README.md b/vendor/github.com/glycerine/go-unsnap-stream/README.md deleted file mode 100644 index 932291f7..00000000 --- a/vendor/github.com/glycerine/go-unsnap-stream/README.md +++ /dev/null @@ -1,22 +0,0 @@ -go-unsnap-stream -================ - -This is a small golang library for decoding and encoding the snappy *streaming* format, specified here: https://github.com/google/snappy/blob/master/framing_format.txt - -Note that the *streaming or framing format* for snappy is different from snappy itself. Think of it as a train of boxcars: the streaming format breaks your data in chunks, applies snappy to each chunk alone, then puts a thin wrapper around the chunk, and sends it along in turn. You can begin decoding before receiving everything. And memory requirements for decoding are sane. - -Strangely, though the streaming format was first proposed in Go[1][2], it was never upated, and I could not locate any other library for Go that would handle the streaming/framed snappy format. Hence this implementation of the spec. There is a command line tool[3] that has a C implementation, but this is the only Go implementation that I am aware of. The reference for the framing/streaming spec seems to be the python implementation[4]. - -Update to the previous paragraph: Horray! Good news: Thanks to @nigeltao, we have since learned that the [github.com/golang/snappy](https://github.com/golang/snappy) package now provides the snappy streaming format too. Even though the type level descriptions are a little misleading because they don't mention that they are for the stream format, the [snappy package header documentation](https://godoc.org/github.com/golang/snappy) points out that the [snappy.Reader](https://godoc.org/github.com/golang/snappy#Reader) and [snappy.Writer](https://godoc.org/github.com/golang/snappy#Writer) types do indeed provide stream (vs block) handling. Although I have not benchmarked, you should probably prefer that package as it will likely be maintained more than I have time to devote, and also perhaps better integrated with the underlying snappy as they share the same repo. - -For binary compatibility with the [python implementation](https://pypi.python.org/pypi/python-snappy) in [4], one could use the C-snappy compressor/decompressor code directly; using github.com/dgryski/go-csnappy. In fact we did this for a while to verify byte-for-byte compatiblity, as the native Go implementation produces slightly different binary compression (still conformant with the standard of course), which made test-diffs harder, and some have complained about it being slower than the C. - -However, while the c-snappy was useful for checking compatibility, it introduced dependencies on external C libraries (both the c-snappy library and the C standard library). Our go binary executable that used the go-unsnap-stream library was no longer standalone, and deployment was painful if not impossible if the target had a different C standard library. So we've gone back to using the snappy-go implementation (entirely in Go) for ease of deployment. See the comments at the top of unsnap.go if you wish to use c-snappy instead. - -[1] https://groups.google.com/forum/#!msg/snappy-compression/qvLNe2cSH9s/R19oBC-p7g4J - -[2] https://codereview.appspot.com/5167058 - -[3] https://github.com/kubo/snzip - -[4] https://pypi.python.org/pypi/python-snappy diff --git a/vendor/github.com/glycerine/go-unsnap-stream/binary.dat b/vendor/github.com/glycerine/go-unsnap-stream/binary.dat deleted file mode 100644 index f31eee2e..00000000 Binary files a/vendor/github.com/glycerine/go-unsnap-stream/binary.dat and /dev/null differ diff --git a/vendor/github.com/glycerine/go-unsnap-stream/binary.dat.snappy b/vendor/github.com/glycerine/go-unsnap-stream/binary.dat.snappy deleted file mode 100644 index ed370242..00000000 Binary files a/vendor/github.com/glycerine/go-unsnap-stream/binary.dat.snappy and /dev/null differ diff --git a/vendor/github.com/glycerine/go-unsnap-stream/rbuf.go b/vendor/github.com/glycerine/go-unsnap-stream/rbuf.go deleted file mode 100644 index f771c392..00000000 --- a/vendor/github.com/glycerine/go-unsnap-stream/rbuf.go +++ /dev/null @@ -1,375 +0,0 @@ -package unsnap - -// copyright (c) 2014, Jason E. Aten -// license: MIT - -// Some text from the Golang standard library doc is adapted and -// reproduced in fragments below to document the expected behaviors -// of the interface functions Read()/Write()/ReadFrom()/WriteTo() that -// are implemented here. Those descriptions (see -// http://golang.org/pkg/io/#Reader for example) are -// copyright 2010 The Go Authors. - -import "io" - -// FixedSizeRingBuf: -// -// a fixed-size circular ring buffer. Yes, just what is says. -// -// We keep a pair of ping/pong buffers so that we can linearize -// the circular buffer into a contiguous slice if need be. -// -// For efficiency, a FixedSizeRingBuf may be vastly preferred to -// a bytes.Buffer. The ReadWithoutAdvance(), Advance(), and Adopt() -// methods are all non-standard methods written for speed. -// -// For an I/O heavy application, I have replaced bytes.Buffer with -// FixedSizeRingBuf and seen memory consumption go from 8GB to 25MB. -// Yes, that is a 300x reduction in memory footprint. Everything ran -// faster too. -// -// Note that Bytes(), while inescapable at times, is expensive: avoid -// it if possible. Instead it is better to use the FixedSizeRingBuf.Readable -// member to get the number of bytes available. Bytes() is expensive because -// it may copy the back and then the front of a wrapped buffer A[Use] -// into A[1-Use] in order to get a contiguous slice. If possible use ContigLen() -// first to get the size that can be read without copying, Read() that -// amount, and then Read() a second time -- to avoid the copy. - -type FixedSizeRingBuf struct { - A [2][]byte // a pair of ping/pong buffers. Only one is active. - Use int // which A buffer is in active use, 0 or 1 - N int // MaxViewInBytes, the size of A[0] and A[1] in bytes. - Beg int // start of data in A[Use] - Readable int // number of bytes available to read in A[Use] - - OneMade bool // lazily instantiate the [1] buffer. If we never call Bytes(), - // we may never need it. If OneMade is false, the Use must be = 0. -} - -func (b *FixedSizeRingBuf) Make2ndBuffer() { - if b.OneMade { - return - } - b.A[1] = make([]byte, b.N, b.N) - b.OneMade = true -} - -// get the length of the largest read that we can provide to a contiguous slice -// without an extra linearizing copy of all bytes internally. -func (b *FixedSizeRingBuf) ContigLen() int { - extent := b.Beg + b.Readable - firstContigLen := intMin(extent, b.N) - b.Beg - return firstContigLen -} - -func NewFixedSizeRingBuf(maxViewInBytes int) *FixedSizeRingBuf { - n := maxViewInBytes - r := &FixedSizeRingBuf{ - Use: 0, // 0 or 1, whichever is actually in use at the moment. - // If we are asked for Bytes() and we wrap, linearize into the other. - - N: n, - Beg: 0, - Readable: 0, - OneMade: false, - } - r.A[0] = make([]byte, n, n) - - // r.A[1] initialized lazily now. - - return r -} - -// from the standard library description of Bytes(): -// Bytes() returns a slice of the contents of the unread portion of the buffer. -// If the caller changes the contents of the -// returned slice, the contents of the buffer will change provided there -// are no intervening method calls on the Buffer. -// -func (b *FixedSizeRingBuf) Bytes() []byte { - - extent := b.Beg + b.Readable - if extent <= b.N { - // we fit contiguously in this buffer without wrapping to the other - return b.A[b.Use][b.Beg:(b.Beg + b.Readable)] - } - - // wrap into the other buffer - b.Make2ndBuffer() - - src := b.Use - dest := 1 - b.Use - - n := copy(b.A[dest], b.A[src][b.Beg:]) - n += copy(b.A[dest][n:], b.A[src][0:(extent%b.N)]) - - b.Use = dest - b.Beg = 0 - - return b.A[b.Use][:n] -} - -// Read(): -// -// from bytes.Buffer.Read(): Read reads the next len(p) bytes -// from the buffer or until the buffer is drained. The return -// value n is the number of bytes read. If the buffer has no data -// to return, err is io.EOF (unless len(p) is zero); otherwise it is nil. -// -// from the description of the Reader interface, -// http://golang.org/pkg/io/#Reader -// -/* -Reader is the interface that wraps the basic Read method. - -Read reads up to len(p) bytes into p. It returns the number -of bytes read (0 <= n <= len(p)) and any error encountered. -Even if Read returns n < len(p), it may use all of p as scratch -space during the call. If some data is available but not -len(p) bytes, Read conventionally returns what is available -instead of waiting for more. - -When Read encounters an error or end-of-file condition after -successfully reading n > 0 bytes, it returns the number of bytes -read. It may return the (non-nil) error from the same call or -return the error (and n == 0) from a subsequent call. An instance -of this general case is that a Reader returning a non-zero number -of bytes at the end of the input stream may return -either err == EOF or err == nil. The next Read should -return 0, EOF regardless. - -Callers should always process the n > 0 bytes returned before -considering the error err. Doing so correctly handles I/O errors -that happen after reading some bytes and also both of the -allowed EOF behaviors. - -Implementations of Read are discouraged from returning a zero -byte count with a nil error, and callers should treat that -situation as a no-op. -*/ -// - -func (b *FixedSizeRingBuf) Read(p []byte) (n int, err error) { - return b.ReadAndMaybeAdvance(p, true) -} - -// if you want to Read the data and leave it in the buffer, so as -// to peek ahead for example. -func (b *FixedSizeRingBuf) ReadWithoutAdvance(p []byte) (n int, err error) { - return b.ReadAndMaybeAdvance(p, false) -} - -func (b *FixedSizeRingBuf) ReadAndMaybeAdvance(p []byte, doAdvance bool) (n int, err error) { - if len(p) == 0 { - return 0, nil - } - if b.Readable == 0 { - return 0, io.EOF - } - extent := b.Beg + b.Readable - if extent <= b.N { - n += copy(p, b.A[b.Use][b.Beg:extent]) - } else { - n += copy(p, b.A[b.Use][b.Beg:b.N]) - if n < len(p) { - n += copy(p[n:], b.A[b.Use][0:(extent%b.N)]) - } - } - if doAdvance { - b.Advance(n) - } - return -} - -// -// Write writes len(p) bytes from p to the underlying data stream. -// It returns the number of bytes written from p (0 <= n <= len(p)) -// and any error encountered that caused the write to stop early. -// Write must return a non-nil error if it returns n < len(p). -// -func (b *FixedSizeRingBuf) Write(p []byte) (n int, err error) { - for { - if len(p) == 0 { - // nothing (left) to copy in; notice we shorten our - // local copy p (below) as we read from it. - return - } - - writeCapacity := b.N - b.Readable - if writeCapacity <= 0 { - // we are all full up already. - return n, io.ErrShortWrite - } - if len(p) > writeCapacity { - err = io.ErrShortWrite - // leave err set and - // keep going, write what we can. - } - - writeStart := (b.Beg + b.Readable) % b.N - - upperLim := intMin(writeStart+writeCapacity, b.N) - - k := copy(b.A[b.Use][writeStart:upperLim], p) - - n += k - b.Readable += k - p = p[k:] - - // we can fill from b.A[b.Use][0:something] from - // p's remainder, so loop - } -} - -// WriteTo and ReadFrom avoid intermediate allocation and copies. - -// WriteTo writes data to w until there's no more data to write -// or when an error occurs. The return value n is the number of -// bytes written. Any error encountered during the write is also returned. -func (b *FixedSizeRingBuf) WriteTo(w io.Writer) (n int64, err error) { - - if b.Readable == 0 { - return 0, io.EOF - } - - extent := b.Beg + b.Readable - firstWriteLen := intMin(extent, b.N) - b.Beg - secondWriteLen := b.Readable - firstWriteLen - if firstWriteLen > 0 { - m, e := w.Write(b.A[b.Use][b.Beg:(b.Beg + firstWriteLen)]) - n += int64(m) - b.Advance(m) - - if e != nil { - return n, e - } - // all bytes should have been written, by definition of - // Write method in io.Writer - if m != firstWriteLen { - return n, io.ErrShortWrite - } - } - if secondWriteLen > 0 { - m, e := w.Write(b.A[b.Use][0:secondWriteLen]) - n += int64(m) - b.Advance(m) - - if e != nil { - return n, e - } - // all bytes should have been written, by definition of - // Write method in io.Writer - if m != secondWriteLen { - return n, io.ErrShortWrite - } - } - - return n, nil -} - -// ReadFrom() reads data from r until EOF or error. The return value n -// is the number of bytes read. Any error except io.EOF encountered -// during the read is also returned. -func (b *FixedSizeRingBuf) ReadFrom(r io.Reader) (n int64, err error) { - for { - writeCapacity := b.N - b.Readable - if writeCapacity <= 0 { - // we are all full - return n, nil - } - writeStart := (b.Beg + b.Readable) % b.N - upperLim := intMin(writeStart+writeCapacity, b.N) - - m, e := r.Read(b.A[b.Use][writeStart:upperLim]) - n += int64(m) - b.Readable += m - if e == io.EOF { - return n, nil - } - if e != nil { - return n, e - } - } -} - -func (b *FixedSizeRingBuf) Reset() { - b.Beg = 0 - b.Readable = 0 - b.Use = 0 -} - -// Advance(): non-standard, but better than Next(), -// because we don't have to unwrap our buffer and pay the cpu time -// for the copy that unwrapping may need. -// Useful in conjuction/after ReadWithoutAdvance() above. -func (b *FixedSizeRingBuf) Advance(n int) { - if n <= 0 { - return - } - if n > b.Readable { - n = b.Readable - } - b.Readable -= n - b.Beg = (b.Beg + n) % b.N -} - -// Adopt(): non-standard. -// -// For efficiency's sake, (possibly) take ownership of -// already allocated slice offered in me. -// -// If me is large we will adopt it, and we will potentially then -// write to the me buffer. -// If we already have a bigger buffer, copy me into the existing -// buffer instead. -func (b *FixedSizeRingBuf) Adopt(me []byte) { - n := len(me) - if n > b.N { - b.A[0] = me - b.OneMade = false - b.N = n - b.Use = 0 - b.Beg = 0 - b.Readable = n - } else { - // we already have a larger buffer, reuse it. - copy(b.A[0], me) - b.Use = 0 - b.Beg = 0 - b.Readable = n - } -} - -func intMax(a, b int) int { - if a > b { - return a - } else { - return b - } -} - -func intMin(a, b int) int { - if a < b { - return a - } else { - return b - } -} - -// Get the (beg, end] indices of the tailing empty buffer of bytes slice that from that is free for writing. -// Note: not guaranteed to be zeroed. At all. -func (b *FixedSizeRingBuf) GetEndmostWritable() (beg int, end int) { - extent := b.Beg + b.Readable - if extent < b.N { - return extent, b.N - } - - return extent % b.N, b.Beg -} - -// Note: not guaranteed to be zeroed. -func (b *FixedSizeRingBuf) GetEndmostWritableSlice() []byte { - beg, e := b.GetEndmostWritable() - return b.A[b.Use][beg:e] -} diff --git a/vendor/github.com/glycerine/go-unsnap-stream/snap.go b/vendor/github.com/glycerine/go-unsnap-stream/snap.go deleted file mode 100644 index 12a8d40b..00000000 --- a/vendor/github.com/glycerine/go-unsnap-stream/snap.go +++ /dev/null @@ -1,100 +0,0 @@ -package unsnap - -import ( - "encoding/binary" - - // no c lib dependency - snappy "github.com/golang/snappy" - // or, use the C wrapper for speed - //snappy "github.com/dgryski/go-csnappy" -) - -// add Write() method for SnappyFile (see unsnap.go) - -// reference for snappy framing/streaming format: -// http://code.google.com/p/snappy/source/browse/trunk/framing_format.txt -// ?spec=svn68&r=71 - -// -// Write writes len(p) bytes from p to the underlying data stream. -// It returns the number of bytes written from p (0 <= n <= len(p)) and -// any error encountered that caused the write to stop early. Write -// must return a non-nil error if it returns n < len(p). -// -func (sf *SnappyFile) Write(p []byte) (n int, err error) { - - if sf.SnappyEncodeDecodeOff { - return sf.Writer.Write(p) - } - - if !sf.Writing { - panic("Writing on a read-only SnappyFile") - } - - // encoding in snappy can apparently go beyond the original size, beware. - // so our buffers must be sized 2*max snappy chunk => 2 * CHUNK_MAX(65536) - - sf.DecBuf.Reset() - sf.EncBuf.Reset() - - if !sf.HeaderChunkWritten { - sf.HeaderChunkWritten = true - _, err = sf.Writer.Write(SnappyStreamHeaderMagic) - if err != nil { - return - } - } - var chunk []byte - var chunk_type byte - var crc uint32 - - for len(p) > 0 { - - // chunk points to input p by default, unencoded input. - chunk = p[:IntMin(len(p), CHUNK_MAX)] - crc = masked_crc32c(chunk) - - writeme := chunk[:] - - // first write to EncBuf, as a temp, in case we want - // to discard and send uncompressed instead. - compressed_chunk := snappy.Encode(sf.EncBuf.GetEndmostWritableSlice(), chunk) - - if len(compressed_chunk) <= int((1-_COMPRESSION_THRESHOLD)*float64(len(chunk))) { - writeme = compressed_chunk - chunk_type = _COMPRESSED_CHUNK - } else { - // keep writeme pointing at original chunk (uncompressed) - chunk_type = _UNCOMPRESSED_CHUNK - } - - const crc32Sz = 4 - var tag32 uint32 = uint32(chunk_type) + (uint32(len(writeme)+crc32Sz) << 8) - - err = binary.Write(sf.Writer, binary.LittleEndian, tag32) - if err != nil { - return - } - - err = binary.Write(sf.Writer, binary.LittleEndian, crc) - if err != nil { - return - } - - _, err = sf.Writer.Write(writeme) - if err != nil { - return - } - - n += len(chunk) - p = p[len(chunk):] - } - return n, nil -} - -func IntMin(a int, b int) int { - if a < b { - return a - } - return b -} diff --git a/vendor/github.com/glycerine/go-unsnap-stream/unenc.txt b/vendor/github.com/glycerine/go-unsnap-stream/unenc.txt deleted file mode 100644 index 5f502793..00000000 --- a/vendor/github.com/glycerine/go-unsnap-stream/unenc.txt +++ /dev/null @@ -1 +0,0 @@ -hello_snappy diff --git a/vendor/github.com/glycerine/go-unsnap-stream/unenc.txt.snappy b/vendor/github.com/glycerine/go-unsnap-stream/unenc.txt.snappy deleted file mode 100644 index ba45ecd4..00000000 Binary files a/vendor/github.com/glycerine/go-unsnap-stream/unenc.txt.snappy and /dev/null differ diff --git a/vendor/github.com/glycerine/go-unsnap-stream/unsnap.go b/vendor/github.com/glycerine/go-unsnap-stream/unsnap.go deleted file mode 100644 index 0d33949e..00000000 --- a/vendor/github.com/glycerine/go-unsnap-stream/unsnap.go +++ /dev/null @@ -1,519 +0,0 @@ -package unsnap - -import ( - "bytes" - "encoding/binary" - "fmt" - "io" - "io/ioutil" - "os" - "strings" - - "hash/crc32" - - snappy "github.com/golang/snappy" - // The C library can be used, but this makes the binary dependent - // lots of extraneous c-libraries; it is no longer stand-alone. Yuck. - // - // Therefore we comment out the "dgryski/go-csnappy" path and use the - // "github.com/golang/snappy/snappy" above instead. If you are - // performance limited and can deal with distributing more libraries, - // then this is easy to swap. - // - // If you swap, note that some of the tests won't pass - // because snappy-go produces slightly different (but still - // conformant) encodings on some data. Here are bindings - // to the C-snappy: - // snappy "github.com/dgryski/go-csnappy" -) - -// SnappyFile: create a drop-in-replacement/wrapper for an *os.File that handles doing the unsnappification online as more is read from it - -type SnappyFile struct { - Fname string - - Reader io.Reader - Writer io.Writer - - // allow clients to substitute us for an os.File and just switch - // off compression if they don't want it. - SnappyEncodeDecodeOff bool // if true, we bypass straight to Filep - - EncBuf FixedSizeRingBuf // holds any extra that isn't yet returned, encoded - DecBuf FixedSizeRingBuf // holds any extra that isn't yet returned, decoded - - // for writing to stream-framed snappy - HeaderChunkWritten bool - - // Sanity check: we can only read, or only write, to one SnappyFile. - // EncBuf and DecBuf are used differently in each mode. Verify - // that we are consistent with this flag. - Writing bool -} - -var total int - -// for debugging, show state of buffers -func (f *SnappyFile) Dump() { - fmt.Printf("EncBuf has length %d and contents:\n%s\n", len(f.EncBuf.Bytes()), string(f.EncBuf.Bytes())) - fmt.Printf("DecBuf has length %d and contents:\n%s\n", len(f.DecBuf.Bytes()), string(f.DecBuf.Bytes())) -} - -func (f *SnappyFile) Read(p []byte) (n int, err error) { - - if f.SnappyEncodeDecodeOff { - return f.Reader.Read(p) - } - - if f.Writing { - panic("Reading on a write-only SnappyFile") - } - - // before we unencrypt more, try to drain the DecBuf first - n, _ = f.DecBuf.Read(p) - if n > 0 { - total += n - return n, nil - } - - //nEncRead, nDecAdded, err := UnsnapOneFrame(f.Filep, &f.EncBuf, &f.DecBuf, f.Fname) - _, _, err = UnsnapOneFrame(f.Reader, &f.EncBuf, &f.DecBuf, f.Fname) - if err != nil && err != io.EOF { - panic(err) - } - - n, _ = f.DecBuf.Read(p) - - if n > 0 { - total += n - return n, nil - } - if f.DecBuf.Readable == 0 { - if f.DecBuf.Readable == 0 && f.EncBuf.Readable == 0 { - // only now (when EncBuf is empty) can we give io.EOF. - // Any earlier, and we leave stuff un-decoded! - return 0, io.EOF - } - } - return 0, nil -} - -func Open(name string) (file *SnappyFile, err error) { - fp, err := os.Open(name) - if err != nil { - return nil, err - } - // encoding in snappy can apparently go beyond the original size, so - // we make our buffers big enough, 2*max snappy chunk => 2 * CHUNK_MAX(65536) - - snap := NewReader(fp) - snap.Fname = name - return snap, nil -} - -func NewReader(r io.Reader) *SnappyFile { - return &SnappyFile{ - Reader: r, - EncBuf: *NewFixedSizeRingBuf(CHUNK_MAX * 2), // buffer of snappy encoded bytes - DecBuf: *NewFixedSizeRingBuf(CHUNK_MAX * 2), // buffer of snapppy decoded bytes - Writing: false, - } -} - -func NewWriter(w io.Writer) *SnappyFile { - return &SnappyFile{ - Writer: w, - EncBuf: *NewFixedSizeRingBuf(65536), // on writing: temp for testing compression - DecBuf: *NewFixedSizeRingBuf(65536 * 2), // on writing: final buffer of snappy framed and encoded bytes - Writing: true, - } -} - -func Create(name string) (file *SnappyFile, err error) { - fp, err := os.Create(name) - if err != nil { - return nil, err - } - snap := NewWriter(fp) - snap.Fname = name - return snap, nil -} - -func (f *SnappyFile) Close() error { - if f.Writing { - wc, ok := f.Writer.(io.WriteCloser) - if ok { - return wc.Close() - } - return nil - } - rc, ok := f.Reader.(io.ReadCloser) - if ok { - return rc.Close() - } - return nil -} - -func (f *SnappyFile) Sync() error { - file, ok := f.Writer.(*os.File) - if ok { - return file.Sync() - } - return nil -} - -// for an increment of a frame at a time: -// read from r into encBuf (encBuf is still encoded, thus the name), and write unsnappified frames into outDecodedBuf -// the returned n: number of bytes read from the encrypted encBuf -func UnsnapOneFrame(r io.Reader, encBuf *FixedSizeRingBuf, outDecodedBuf *FixedSizeRingBuf, fname string) (nEnc int64, nDec int64, err error) { - // b, err := ioutil.ReadAll(r) - // if err != nil { - // panic(err) - // } - - nEnc = 0 - nDec = 0 - - // read up to 65536 bytes from r into encBuf, at least a snappy frame - nread, err := io.CopyN(encBuf, r, 65536) // returns nwrotebytes, err - nEnc += nread - if err != nil { - if err == io.EOF { - if nread == 0 { - if encBuf.Readable == 0 { - return nEnc, nDec, io.EOF - } - // else we have bytes in encBuf, so decode them! - err = nil - } else { - // continue below, processing the nread bytes - err = nil - } - } else { - // may be an odd already closed... don't panic on that - if strings.Contains(err.Error(), "file already closed") { - err = nil - } else { - panic(err) - } - } - } - - // flag for printing chunk size alignment messages - verbose := false - - const snappyStreamHeaderSz = 10 - const headerSz = 4 - const crc32Sz = 4 - // the magic 18 bytes accounts for the snappy streaming header and the first chunks size and checksum - // http://code.google.com/p/snappy/source/browse/trunk/framing_format.txt - - chunk := (*encBuf).Bytes() - - // however we exit, advance as - // defer func() { (*encBuf).Next(N) }() - - // 65536 is the max size of a snappy framed chunk. See - // http://code.google.com/p/snappy/source/browse/trunk/framing_format.txt:91 - // buf := make([]byte, 65536) - - // fmt.Printf("read from file, b is len:%d with value: %#v\n", len(b), b) - // fmt.Printf("read from file, bcut is len:%d with value: %#v\n", len(bcut), bcut) - - //fmt.Printf("raw bytes of chunksz are: %v\n", b[11:14]) - - fourbytes := make([]byte, 4) - chunkCount := 0 - - for nDec < 65536 { - if len(chunk) == 0 { - break - } - chunkCount++ - fourbytes[3] = 0 - copy(fourbytes, chunk[1:4]) - chunksz := binary.LittleEndian.Uint32(fourbytes) - chunk_type := chunk[0] - - switch true { - case chunk_type == 0xff: - { // stream identifier - - streamHeader := chunk[:snappyStreamHeaderSz] - if 0 != bytes.Compare(streamHeader, []byte{0xff, 0x06, 0x00, 0x00, 0x73, 0x4e, 0x61, 0x50, 0x70, 0x59}) { - panic("file had chunk starting with 0xff but then no magic snappy streaming protocol bytes, aborting.") - } else { - //fmt.Printf("got streaming snappy magic header just fine.\n") - } - chunk = chunk[snappyStreamHeaderSz:] - (*encBuf).Advance(snappyStreamHeaderSz) - nEnc += snappyStreamHeaderSz - continue - } - case chunk_type == 0x00: - { // compressed data - if verbose { - fmt.Fprintf(os.Stderr, "chunksz is %d while total bytes avail are: %d\n", int(chunksz), len(chunk)-4) - } - - crc := binary.LittleEndian.Uint32(chunk[headerSz:(headerSz + crc32Sz)]) - section := chunk[(headerSz + crc32Sz):(headerSz + chunksz)] - - dec, ok := snappy.Decode(nil, section) - if ok != nil { - // we've probably truncated a snappy frame at this point - // ok=snappy: corrupt input - // len(dec) == 0 - // - panic(fmt.Sprintf("could not decode snappy stream: '%s' and len dec=%d and ok=%v\n", fname, len(dec), ok)) - - // get back to caller with what we've got so far - return nEnc, nDec, nil - } - // fmt.Printf("ok, b is %#v , %#v\n", ok, dec) - - // spit out decoded text - // n, err := w.Write(dec) - //fmt.Printf("len(dec) = %d, outDecodedBuf.Readable=%d\n", len(dec), outDecodedBuf.Readable) - bnb := bytes.NewBuffer(dec) - n, err := io.Copy(outDecodedBuf, bnb) - if err != nil { - //fmt.Printf("got n=%d, err= %s ; when trying to io.Copy(outDecodedBuf: N=%d, Readable=%d)\n", n, err, outDecodedBuf.N, outDecodedBuf.Readable) - panic(err) - } - if n != int64(len(dec)) { - panic("could not write all bytes to outDecodedBuf") - } - nDec += n - - // verify the crc32 rotated checksum - m32 := masked_crc32c(dec) - if m32 != crc { - panic(fmt.Sprintf("crc32 masked failiure. expected: %v but got: %v", crc, m32)) - } else { - //fmt.Printf("\nchecksums match: %v == %v\n", crc, m32) - } - - // move to next header - inc := (headerSz + int(chunksz)) - chunk = chunk[inc:] - (*encBuf).Advance(inc) - nEnc += int64(inc) - continue - } - case chunk_type == 0x01: - { // uncompressed data - - //n, err := w.Write(chunk[(headerSz+crc32Sz):(headerSz + int(chunksz))]) - n, err := io.Copy(outDecodedBuf, bytes.NewBuffer(chunk[(headerSz+crc32Sz):(headerSz+int(chunksz))])) - if verbose { - //fmt.Printf("debug: n=%d err=%v chunksz=%d outDecodedBuf='%v'\n", n, err, chunksz, outDecodedBuf) - } - if err != nil { - panic(err) - } - if n != int64(chunksz-crc32Sz) { - panic("could not write all bytes to stdout") - } - nDec += n - - inc := (headerSz + int(chunksz)) - chunk = chunk[inc:] - (*encBuf).Advance(inc) - nEnc += int64(inc) - continue - } - case chunk_type == 0xfe: - fallthrough // padding, just skip it - case chunk_type >= 0x80 && chunk_type <= 0xfd: - { // Reserved skippable chunks - //fmt.Printf("\nin reserved skippable chunks, at nEnc=%v\n", nEnc) - inc := (headerSz + int(chunksz)) - chunk = chunk[inc:] - nEnc += int64(inc) - (*encBuf).Advance(inc) - continue - } - - default: - panic(fmt.Sprintf("unrecognized/unsupported chunk type %#v", chunk_type)) - } - - } // end for{} - - return nEnc, nDec, err - //return int64(N), nil -} - -// for whole file at once: -// -// receive on stdin a stream of bytes in the snappy-streaming framed -// format, defined here: http://code.google.com/p/snappy/source/browse/trunk/framing_format.txt -// Grab each frame, run it through the snappy decoder, and spit out -// each frame all joined back-to-back on stdout. -// -func Unsnappy(r io.Reader, w io.Writer) (err error) { - b, err := ioutil.ReadAll(r) - if err != nil { - panic(err) - } - - // flag for printing chunk size alignment messages - verbose := false - - const snappyStreamHeaderSz = 10 - const headerSz = 4 - const crc32Sz = 4 - // the magic 18 bytes accounts for the snappy streaming header and the first chunks size and checksum - // http://code.google.com/p/snappy/source/browse/trunk/framing_format.txt - - chunk := b[:] - - // 65536 is the max size of a snappy framed chunk. See - // http://code.google.com/p/snappy/source/browse/trunk/framing_format.txt:91 - //buf := make([]byte, 65536) - - // fmt.Printf("read from file, b is len:%d with value: %#v\n", len(b), b) - // fmt.Printf("read from file, bcut is len:%d with value: %#v\n", len(bcut), bcut) - - //fmt.Printf("raw bytes of chunksz are: %v\n", b[11:14]) - - fourbytes := make([]byte, 4) - chunkCount := 0 - - for { - if len(chunk) == 0 { - break - } - chunkCount++ - fourbytes[3] = 0 - copy(fourbytes, chunk[1:4]) - chunksz := binary.LittleEndian.Uint32(fourbytes) - chunk_type := chunk[0] - - switch true { - case chunk_type == 0xff: - { // stream identifier - - streamHeader := chunk[:snappyStreamHeaderSz] - if 0 != bytes.Compare(streamHeader, []byte{0xff, 0x06, 0x00, 0x00, 0x73, 0x4e, 0x61, 0x50, 0x70, 0x59}) { - panic("file had chunk starting with 0xff but then no magic snappy streaming protocol bytes, aborting.") - } else { - //fmt.Printf("got streaming snappy magic header just fine.\n") - } - chunk = chunk[snappyStreamHeaderSz:] - continue - } - case chunk_type == 0x00: - { // compressed data - if verbose { - fmt.Fprintf(os.Stderr, "chunksz is %d while total bytes avail are: %d\n", int(chunksz), len(chunk)-4) - } - - //crc := binary.LittleEndian.Uint32(chunk[headerSz:(headerSz + crc32Sz)]) - section := chunk[(headerSz + crc32Sz):(headerSz + chunksz)] - - dec, ok := snappy.Decode(nil, section) - if ok != nil { - panic("could not decode snappy stream") - } - // fmt.Printf("ok, b is %#v , %#v\n", ok, dec) - - // spit out decoded text - n, err := w.Write(dec) - if err != nil { - panic(err) - } - if n != len(dec) { - panic("could not write all bytes to stdout") - } - - // TODO: verify the crc32 rotated checksum? - - // move to next header - chunk = chunk[(headerSz + int(chunksz)):] - continue - } - case chunk_type == 0x01: - { // uncompressed data - - //crc := binary.LittleEndian.Uint32(chunk[headerSz:(headerSz + crc32Sz)]) - section := chunk[(headerSz + crc32Sz):(headerSz + chunksz)] - - n, err := w.Write(section) - if err != nil { - panic(err) - } - if n != int(chunksz-crc32Sz) { - panic("could not write all bytes to stdout") - } - - chunk = chunk[(headerSz + int(chunksz)):] - continue - } - case chunk_type == 0xfe: - fallthrough // padding, just skip it - case chunk_type >= 0x80 && chunk_type <= 0xfd: - { // Reserved skippable chunks - chunk = chunk[(headerSz + int(chunksz)):] - continue - } - - default: - panic(fmt.Sprintf("unrecognized/unsupported chunk type %#v", chunk_type)) - } - - } // end for{} - - return nil -} - -// 0xff 0x06 0x00 0x00 sNaPpY -var SnappyStreamHeaderMagic = []byte{0xff, 0x06, 0x00, 0x00, 0x73, 0x4e, 0x61, 0x50, 0x70, 0x59} - -const CHUNK_MAX = 65536 -const _STREAM_TO_STREAM_BLOCK_SIZE = CHUNK_MAX -const _STREAM_IDENTIFIER = `sNaPpY` -const _COMPRESSED_CHUNK = 0x00 -const _UNCOMPRESSED_CHUNK = 0x01 -const _IDENTIFIER_CHUNK = 0xff -const _RESERVED_UNSKIPPABLE0 = 0x02 // chunk ranges are [inclusive, exclusive) -const _RESERVED_UNSKIPPABLE1 = 0x80 -const _RESERVED_SKIPPABLE0 = 0x80 -const _RESERVED_SKIPPABLE1 = 0xff - -// the minimum percent of bytes compression must save to be enabled in automatic -// mode -const _COMPRESSION_THRESHOLD = .125 - -var crctab *crc32.Table - -func init() { - crctab = crc32.MakeTable(crc32.Castagnoli) // this is correct table, matches the crc32c.c code used by python -} - -func masked_crc32c(data []byte) uint32 { - - // see the framing format specification, http://code.google.com/p/snappy/source/browse/trunk/framing_format.txt - var crc uint32 = crc32.Checksum(data, crctab) - return (uint32((crc>>15)|(crc<<17)) + 0xa282ead8) -} - -func ReadSnappyStreamCompressedFile(filename string) ([]byte, error) { - - snappyFile, err := Open(filename) - if err != nil { - return []byte{}, err - } - - var bb bytes.Buffer - _, err = bb.ReadFrom(snappyFile) - if err == io.EOF { - err = nil - } - if err != nil { - panic(err) - } - - return bb.Bytes(), err -} diff --git a/vendor/github.com/gobwas/ws/Makefile b/vendor/github.com/gobwas/ws/Makefile index 8f727393..6d89f78c 100644 --- a/vendor/github.com/gobwas/ws/Makefile +++ b/vendor/github.com/gobwas/ws/Makefile @@ -24,7 +24,7 @@ test: go test -coverprofile=ws.coverage . go test -coverprofile=wsutil.coverage ./wsutil go test -coverprofile=wsfalte.coverage ./wsflate - # No statemenets to cover in ./tests (there are only tests). + # No statements to cover in ./tests (there are only tests). go test ./tests cover: bin/gocovmerge test autobahn diff --git a/vendor/github.com/gobwas/ws/dialer.go b/vendor/github.com/gobwas/ws/dialer.go index d35dc14b..e66678e4 100644 --- a/vendor/github.com/gobwas/ws/dialer.go +++ b/vendor/github.com/gobwas/ws/dialer.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "net" + "net/http" "net/url" "strconv" "strings" @@ -86,6 +87,13 @@ type Dialer struct { // land. Header HandshakeHeader + // Host is an optional string that could be used to specify the host during + // HTTP upgrade request by setting 'Host' header. + // + // Default value is an empty string, which results in setting 'Host' header + // equal to the URL hostname given to Dialer.Dial(). + Host string + // OnStatusError is the callback that will be called after receiving non // "101 Continue" HTTP response status. It receives an io.Reader object // representing server response bytes. That is, it gives ability to parse @@ -145,7 +153,7 @@ type Dialer struct { func (d Dialer) Dial(ctx context.Context, urlstr string) (conn net.Conn, br *bufio.Reader, hs Handshake, err error) { u, err := url.ParseRequestURI(urlstr) if err != nil { - return + return nil, nil, hs, err } // Prepare context to dial with. Initially it is the same as original, but @@ -163,7 +171,7 @@ func (d Dialer) Dial(ctx context.Context, urlstr string) (conn net.Conn, br *buf } } if conn, err = d.dial(dialctx, u); err != nil { - return + return conn, nil, hs, err } defer func() { if err != nil { @@ -189,7 +197,7 @@ func (d Dialer) Dial(ctx context.Context, urlstr string) (conn net.Conn, br *buf br, hs, err = d.Upgrade(conn, u) - return + return conn, br, hs, err } var ( @@ -204,7 +212,7 @@ func tlsDefaultConfig() *tls.Config { return &tlsEmptyConfig } -func hostport(host string, defaultPort string) (hostname, addr string) { +func hostport(host, defaultPort string) (hostname, addr string) { var ( colon = strings.LastIndexByte(host, ':') bracket = strings.IndexByte(host, ']') @@ -228,7 +236,7 @@ func (d Dialer) dial(ctx context.Context, u *url.URL) (conn net.Conn, err error) hostname, addr := hostport(u.Host, ":443") conn, err = dial(ctx, "tcp", addr) if err != nil { - return + return nil, err } tlsClient := d.TLSClient if tlsClient == nil { @@ -241,7 +249,7 @@ func (d Dialer) dial(ctx context.Context, u *url.URL) (conn net.Conn, err error) if wrap := d.WrapConn; wrap != nil { conn = wrap(conn) } - return + return conn, err } func (d Dialer) tlsClient(conn net.Conn, hostname string) net.Conn { @@ -309,30 +317,30 @@ func (d Dialer) Upgrade(conn io.ReadWriter, u *url.URL) (br *bufio.Reader, hs Ha nonce := make([]byte, nonceSize) initNonce(nonce) - httpWriteUpgradeRequest(bw, u, nonce, d.Protocols, d.Extensions, d.Header) - if err = bw.Flush(); err != nil { - return + httpWriteUpgradeRequest(bw, u, nonce, d.Protocols, d.Extensions, d.Header, d.Host) + if err := bw.Flush(); err != nil { + return br, hs, err } // Read HTTP status line like "HTTP/1.1 101 Switching Protocols". sl, err := readLine(br) if err != nil { - return + return br, hs, err } // Begin validation of the response. // See https://tools.ietf.org/html/rfc6455#section-4.2.2 // Parse request line data like HTTP version, uri and method. resp, err := httpParseResponseLine(sl) if err != nil { - return + return br, hs, err } // Even if RFC says "1.1 or higher" without mentioning the part of the // version, we apply it only to minor part. if resp.major != 1 || resp.minor < 1 { err = ErrHandshakeBadProtocol - return + return br, hs, err } - if resp.status != 101 { + if resp.status != http.StatusSwitchingProtocols { err = StatusError(resp.status) if onStatusError := d.OnStatusError; onStatusError != nil { // Invoke callback with multireader of status-line bytes br. @@ -344,7 +352,7 @@ func (d Dialer) Upgrade(conn io.ReadWriter, u *url.URL) (br *bufio.Reader, hs Ha ), ) } - return + return br, hs, err } // If response status is 101 then we expect all technical headers to be // valid. If not, then we stop processing response without giving user @@ -355,7 +363,7 @@ func (d Dialer) Upgrade(conn io.ReadWriter, u *url.URL) (br *bufio.Reader, hs Ha line, e := readLine(br) if e != nil { err = e - return + return br, hs, err } if len(line) == 0 { // Blank line, no more lines to read. @@ -365,7 +373,7 @@ func (d Dialer) Upgrade(conn io.ReadWriter, u *url.URL) (br *bufio.Reader, hs Ha k, v, ok := httpParseHeaderLine(line) if !ok { err = ErrMalformedResponse - return + return br, hs, err } switch btsToString(k) { @@ -373,7 +381,7 @@ func (d Dialer) Upgrade(conn io.ReadWriter, u *url.URL) (br *bufio.Reader, hs Ha headerSeen |= headerSeenUpgrade if !bytes.Equal(v, specHeaderValueUpgrade) && !bytes.EqualFold(v, specHeaderValueUpgrade) { err = ErrHandshakeBadUpgrade - return + return br, hs, err } case headerConnectionCanonical: @@ -384,14 +392,14 @@ func (d Dialer) Upgrade(conn io.ReadWriter, u *url.URL) (br *bufio.Reader, hs Ha // multiple token. But in response it must contains exactly one. if !bytes.Equal(v, specHeaderValueConnection) && !bytes.EqualFold(v, specHeaderValueConnection) { err = ErrHandshakeBadConnection - return + return br, hs, err } case headerSecAcceptCanonical: headerSeen |= headerSeenSecAccept if !checkAcceptFromNonce(v, nonce) { err = ErrHandshakeBadSecAccept - return + return br, hs, err } case headerSecProtocolCanonical: @@ -409,20 +417,20 @@ func (d Dialer) Upgrade(conn io.ReadWriter, u *url.URL) (br *bufio.Reader, hs Ha // Server echoed subprotocol that is not present in client // requested protocols. err = ErrHandshakeBadSubProtocol - return + return br, hs, err } case headerSecExtensionsCanonical: hs.Extensions, err = matchSelectedExtensions(v, d.Extensions, hs.Extensions) if err != nil { - return + return br, hs, err } default: if onHeader := d.OnHeader; onHeader != nil { if e := onHeader(k, v); e != nil { err = e - return + return br, hs, err } } } @@ -439,7 +447,7 @@ func (d Dialer) Upgrade(conn io.ReadWriter, u *url.URL) (br *bufio.Reader, hs Ha panic("unknown headers state") } } - return + return br, hs, err } // PutReader returns bufio.Reader instance to the inner reuse pool. @@ -484,8 +492,10 @@ func matchSelectedExtensions(selected []byte, wanted, received []httphead.Option if bytes.Equal(option.Name, want.Name) { // Check parsed extension to be present in client // requested extensions. We move matched extension - // from client list to avoid allocation. - received = append(received, option) + // from client list to avoid allocation of httphead.Option.Name, + // httphead.Option.Parameters have to be copied from the header + want.Parameters, _ = option.Parameters.Copy(make([]byte, option.Parameters.Size())) + received = append(received, want) return true } } diff --git a/vendor/github.com/gobwas/ws/dialer_tls_go18.go b/vendor/github.com/gobwas/ws/dialer_tls_go18.go index a6704d51..5589ee5e 100644 --- a/vendor/github.com/gobwas/ws/dialer_tls_go18.go +++ b/vendor/github.com/gobwas/ws/dialer_tls_go18.go @@ -1,3 +1,4 @@ +//go:build go1.8 // +build go1.8 package ws diff --git a/vendor/github.com/gobwas/ws/doc.go b/vendor/github.com/gobwas/ws/doc.go index c9d57915..0118ce2c 100644 --- a/vendor/github.com/gobwas/ws/doc.go +++ b/vendor/github.com/gobwas/ws/doc.go @@ -11,70 +11,70 @@ Upgrade to WebSocket (or WebSocket handshake) can be done in two ways. The first way is to use `net/http` server: - http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - conn, _, _, err := ws.UpgradeHTTP(r, w) - }) + http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + conn, _, _, err := ws.UpgradeHTTP(r, w) + }) The second and much more efficient way is so-called "zero-copy upgrade". It avoids redundant allocations and copying of not used headers or other request data. User decides by himself which data should be copied. - ln, err := net.Listen("tcp", ":8080") - if err != nil { - // handle error - } + ln, err := net.Listen("tcp", ":8080") + if err != nil { + // handle error + } - conn, err := ln.Accept() - if err != nil { - // handle error - } + conn, err := ln.Accept() + if err != nil { + // handle error + } - handshake, err := ws.Upgrade(conn) - if err != nil { - // handle error - } + handshake, err := ws.Upgrade(conn) + if err != nil { + // handle error + } For customization details see `ws.Upgrader` documentation. After WebSocket handshake you can work with connection in multiple ways. That is, `ws` does not force the only one way of how to work with WebSocket: - header, err := ws.ReadHeader(conn) - if err != nil { - // handle err - } + header, err := ws.ReadHeader(conn) + if err != nil { + // handle err + } - buf := make([]byte, header.Length) - _, err := io.ReadFull(conn, buf) - if err != nil { - // handle err - } + buf := make([]byte, header.Length) + _, err := io.ReadFull(conn, buf) + if err != nil { + // handle err + } - resp := ws.NewBinaryFrame([]byte("hello, world!")) - if err := ws.WriteFrame(conn, frame); err != nil { - // handle err - } + resp := ws.NewBinaryFrame([]byte("hello, world!")) + if err := ws.WriteFrame(conn, frame); err != nil { + // handle err + } As you can see, it stream friendly: - const N = 42 + const N = 42 - ws.WriteHeader(ws.Header{ - Fin: true, - Length: N, - OpCode: ws.OpBinary, - }) + ws.WriteHeader(ws.Header{ + Fin: true, + Length: N, + OpCode: ws.OpBinary, + }) - io.CopyN(conn, rand.Reader, N) + io.CopyN(conn, rand.Reader, N) Or: - header, err := ws.ReadHeader(conn) - if err != nil { - // handle err - } + header, err := ws.ReadHeader(conn) + if err != nil { + // handle err + } - io.CopyN(ioutil.Discard, conn, header.Length) + io.CopyN(ioutil.Discard, conn, header.Length) For more info see the documentation. */ diff --git a/vendor/github.com/gobwas/ws/frame.go b/vendor/github.com/gobwas/ws/frame.go index a4b9ddb3..ae10144e 100644 --- a/vendor/github.com/gobwas/ws/frame.go +++ b/vendor/github.com/gobwas/ws/frame.go @@ -225,7 +225,7 @@ func RsvBits(rsv byte) (r1, r2, r3 bool) { r1 = rsv&bit5 != 0 r2 = rsv&bit6 != 0 r3 = rsv&bit7 != 0 - return + return r1, r2, r3 } // Frame represents websocket frame. @@ -378,7 +378,7 @@ func MaskFrameInPlaceWith(f Frame, m [4]byte) Frame { // NewMask creates new random mask. func NewMask() (ret [4]byte) { binary.BigEndian.PutUint32(ret[:], rand.Uint32()) - return + return ret } // CompileFrame returns byte representation of given frame. @@ -388,7 +388,7 @@ func CompileFrame(f Frame) (bts []byte, err error) { buf := bytes.NewBuffer(make([]byte, 0, 16)) err = WriteFrame(buf, f) bts = buf.Bytes() - return + return bts, err } // MustCompileFrame is like CompileFrame but panics if frame can not be diff --git a/vendor/github.com/gobwas/ws/hijack_go119.go b/vendor/github.com/gobwas/ws/hijack_go119.go new file mode 100644 index 00000000..6ac556c1 --- /dev/null +++ b/vendor/github.com/gobwas/ws/hijack_go119.go @@ -0,0 +1,18 @@ +//go:build !go1.20 +// +build !go1.20 + +package ws + +import ( + "bufio" + "net" + "net/http" +) + +func hijack(w http.ResponseWriter) (net.Conn, *bufio.ReadWriter, error) { + hj, ok := w.(http.Hijacker) + if ok { + return hj.Hijack() + } + return nil, nil, ErrNotHijacker +} diff --git a/vendor/github.com/gobwas/ws/hijack_go120.go b/vendor/github.com/gobwas/ws/hijack_go120.go new file mode 100644 index 00000000..e67b439d --- /dev/null +++ b/vendor/github.com/gobwas/ws/hijack_go120.go @@ -0,0 +1,19 @@ +//go:build go1.20 +// +build go1.20 + +package ws + +import ( + "bufio" + "errors" + "net" + "net/http" +) + +func hijack(w http.ResponseWriter) (net.Conn, *bufio.ReadWriter, error) { + conn, rw, err := http.NewResponseController(w).Hijack() + if errors.Is(err, http.ErrNotSupported) { + return nil, nil, ErrNotHijacker + } + return conn, rw, err +} diff --git a/vendor/github.com/gobwas/ws/http.go b/vendor/github.com/gobwas/ws/http.go index 7d7175a2..a3a682d3 100644 --- a/vendor/github.com/gobwas/ws/http.go +++ b/vendor/github.com/gobwas/ws/http.go @@ -5,7 +5,6 @@ import ( "bytes" "io" "net/http" - "net/textproto" "net/url" "strconv" @@ -38,7 +37,8 @@ var ( textTailErrUpgradeRequired = errorText(ErrHandshakeUpgradeRequired) ) -var ( +const ( + // Every new header must be added to TestHeaderNames test. headerHost = "Host" headerUpgrade = "Upgrade" headerConnection = "Connection" @@ -48,14 +48,14 @@ var ( headerSecKey = "Sec-WebSocket-Key" headerSecAccept = "Sec-WebSocket-Accept" - headerHostCanonical = textproto.CanonicalMIMEHeaderKey(headerHost) - headerUpgradeCanonical = textproto.CanonicalMIMEHeaderKey(headerUpgrade) - headerConnectionCanonical = textproto.CanonicalMIMEHeaderKey(headerConnection) - headerSecVersionCanonical = textproto.CanonicalMIMEHeaderKey(headerSecVersion) - headerSecProtocolCanonical = textproto.CanonicalMIMEHeaderKey(headerSecProtocol) - headerSecExtensionsCanonical = textproto.CanonicalMIMEHeaderKey(headerSecExtensions) - headerSecKeyCanonical = textproto.CanonicalMIMEHeaderKey(headerSecKey) - headerSecAcceptCanonical = textproto.CanonicalMIMEHeaderKey(headerSecAccept) + headerHostCanonical = headerHost + headerUpgradeCanonical = headerUpgrade + headerConnectionCanonical = headerConnection + headerSecVersionCanonical = "Sec-Websocket-Version" + headerSecProtocolCanonical = "Sec-Websocket-Protocol" + headerSecExtensionsCanonical = "Sec-Websocket-Extensions" + headerSecKeyCanonical = "Sec-Websocket-Key" + headerSecAcceptCanonical = "Sec-Websocket-Accept" ) var ( @@ -91,10 +91,8 @@ func httpParseRequestLine(line []byte) (req httpRequestLine, err error) { req.major, req.minor, ok = httpParseVersion(proto) if !ok { err = ErrMalformedRequest - return } - - return + return req, err } func httpParseResponseLine(line []byte) (resp httpResponseLine, err error) { @@ -128,25 +126,25 @@ func httpParseVersion(bts []byte) (major, minor int, ok bool) { case bytes.Equal(bts, httpVersion1_1): return 1, 1, true case len(bts) < 8: - return + return 0, 0, false case !bytes.Equal(bts[:5], httpVersionPrefix): - return + return 0, 0, false } bts = bts[5:] dot := bytes.IndexByte(bts, '.') if dot == -1 { - return + return 0, 0, false } var err error major, err = asciiToInt(bts[:dot]) if err != nil { - return + return major, 0, false } minor, err = asciiToInt(bts[dot+1:]) if err != nil { - return + return major, minor, false } return major, minor, true @@ -157,7 +155,7 @@ func httpParseVersion(bts []byte) (major, minor int, ok bool) { func httpParseHeaderLine(line []byte) (k, v []byte, ok bool) { colon := bytes.IndexByte(line, ':') if colon == -1 { - return + return nil, nil, false } k = btrim(line[:colon]) @@ -198,8 +196,9 @@ func strSelectProtocol(h string, check func(string) bool) (ret string, ok bool) } return true }) - return + return ret, ok } + func btsSelectProtocol(h []byte, check func([]byte) bool) (ret string, ok bool) { var selected []byte ok = httphead.ScanTokens(h, func(v []byte) bool { @@ -212,7 +211,7 @@ func btsSelectProtocol(h []byte, check func([]byte) bool) (ret string, ok bool) if ok && selected != nil { return string(selected), true } - return + return ret, ok } func btsSelectExtensions(h []byte, selected []httphead.Option, check func(httphead.Option) bool) ([]httphead.Option, bool) { @@ -287,12 +286,16 @@ func httpWriteUpgradeRequest( protocols []string, extensions []httphead.Option, header HandshakeHeader, + host string, ) { bw.WriteString("GET ") bw.WriteString(u.RequestURI()) bw.WriteString(" HTTP/1.1\r\n") - httpWriteHeader(bw, headerHost, u.Host) + if host == "" { + host = u.Host + } + httpWriteHeader(bw, headerHost, host) httpWriteHeaderBts(bw, headerUpgrade, specHeaderValueUpgrade) httpWriteHeaderBts(bw, headerConnection, specHeaderValueConnection) diff --git a/vendor/github.com/gobwas/ws/nonce.go b/vendor/github.com/gobwas/ws/nonce.go index e694da7c..7b0edd97 100644 --- a/vendor/github.com/gobwas/ws/nonce.go +++ b/vendor/github.com/gobwas/ws/nonce.go @@ -65,8 +65,6 @@ func initAcceptFromNonce(accept, nonce []byte) { sum := sha1.Sum(p) base64.StdEncoding.Encode(accept, sum[:]) - - return } func writeAccept(bw *bufio.Writer, nonce []byte) (int, error) { diff --git a/vendor/github.com/gobwas/ws/read.go b/vendor/github.com/gobwas/ws/read.go index bc653e46..1771816a 100644 --- a/vendor/github.com/gobwas/ws/read.go +++ b/vendor/github.com/gobwas/ws/read.go @@ -24,7 +24,7 @@ func ReadHeader(r io.Reader) (h Header, err error) { // Prepare to hold first 2 bytes to choose size of next read. _, err = io.ReadFull(r, bts) if err != nil { - return + return h, err } h.Fin = bts[0]&bit0 != 0 @@ -51,11 +51,11 @@ func ReadHeader(r io.Reader) (h Header, err error) { default: err = ErrHeaderLengthUnexpected - return + return h, err } if extra == 0 { - return + return h, err } // Increase len of bts to extra bytes need to read. @@ -63,7 +63,7 @@ func ReadHeader(r io.Reader) (h Header, err error) { bts = bts[:extra] _, err = io.ReadFull(r, bts) if err != nil { - return + return h, err } switch { @@ -74,7 +74,7 @@ func ReadHeader(r io.Reader) (h Header, err error) { case length == 127: if bts[0]&0x80 != 0 { err = ErrHeaderLengthMSB - return + return h, err } h.Length = int64(binary.BigEndian.Uint64(bts[:8])) bts = bts[8:] @@ -84,7 +84,7 @@ func ReadHeader(r io.Reader) (h Header, err error) { copy(h.Mask[:], bts) } - return + return h, nil } // ReadFrame reads a frame from r. @@ -95,7 +95,7 @@ func ReadHeader(r io.Reader) (h Header, err error) { func ReadFrame(r io.Reader) (f Frame, err error) { f.Header, err = ReadHeader(r) if err != nil { - return + return f, err } if f.Header.Length > 0 { @@ -105,7 +105,7 @@ func ReadFrame(r io.Reader) (f Frame, err error) { _, err = io.ReadFull(r, f.Payload) } - return + return f, err } // MustReadFrame is like ReadFrame but panics if frame can not be read. @@ -128,20 +128,20 @@ func ParseCloseFrameData(payload []byte) (code StatusCode, reason string) { // In other words, we ignoring this rule [RFC6455:7.1.5]: // If this Close control frame contains no status code, _The WebSocket // Connection Close Code_ is considered to be 1005. - return + return code, reason } code = StatusCode(binary.BigEndian.Uint16(payload)) reason = string(payload[2:]) - return + return code, reason } // ParseCloseFrameDataUnsafe is like ParseCloseFrameData except the thing // that it does not copies payload bytes into reason, but prepares unsafe cast. func ParseCloseFrameDataUnsafe(payload []byte) (code StatusCode, reason string) { if len(payload) < 2 { - return + return code, reason } code = StatusCode(binary.BigEndian.Uint16(payload)) reason = btsToString(payload[2:]) - return + return code, reason } diff --git a/vendor/github.com/gobwas/ws/server.go b/vendor/github.com/gobwas/ws/server.go index ce530ae5..863bb229 100644 --- a/vendor/github.com/gobwas/ws/server.go +++ b/vendor/github.com/gobwas/ws/server.go @@ -24,11 +24,11 @@ const ( var ( ErrHandshakeBadProtocol = RejectConnectionError( RejectionStatus(http.StatusHTTPVersionNotSupported), - RejectionReason(fmt.Sprintf("handshake error: bad HTTP protocol version")), + RejectionReason("handshake error: bad HTTP protocol version"), ) ErrHandshakeBadMethod = RejectConnectionError( RejectionStatus(http.StatusMethodNotAllowed), - RejectionReason(fmt.Sprintf("handshake error: bad HTTP request method")), + RejectionReason("handshake error: bad HTTP request method"), ) ErrHandshakeBadHost = RejectConnectionError( RejectionStatus(http.StatusBadRequest), @@ -130,7 +130,7 @@ type HTTPUpgrader struct { // list requested by client. If this field is set, then the all matched // extensions are sent to a client as negotiated. // - // DEPRECATED. Use Negotiate instead. + // Deprecated: use Negotiate instead. Extension func(httphead.Option) bool // Negotiate is the callback that is used to negotiate extensions from @@ -155,15 +155,10 @@ type HTTPUpgrader struct { func (u HTTPUpgrader) Upgrade(r *http.Request, w http.ResponseWriter) (conn net.Conn, rw *bufio.ReadWriter, hs Handshake, err error) { // Hijack connection first to get the ability to write rejection errors the // same way as in Upgrader. - hj, ok := w.(http.Hijacker) - if ok { - conn, rw, err = hj.Hijack() - } else { - err = ErrNotHijacker - } + conn, rw, err = hijack(w) if err != nil { httpError(w, err.Error(), http.StatusInternalServerError) - return + return conn, rw, hs, err } // See https://tools.ietf.org/html/rfc6455#section-4.1 @@ -262,7 +257,7 @@ func (u HTTPUpgrader) Upgrade(r *http.Request, w http.ResponseWriter) (conn net. // Do not store Flush() error to not override already existing one. _ = rw.Writer.Flush() } - return + return conn, rw, hs, err } // Upgrader contains options for upgrading connection to websocket. @@ -311,7 +306,7 @@ type Upgrader struct { // header fields it wishes to use, with the first options listed being most // preferable." // - // DEPRECATED. Use Negotiate instead. + // Deprecated: use Negotiate instead. Extension func(httphead.Option) bool // ExtensionCustom allow user to parse Sec-WebSocket-Extensions header @@ -451,12 +446,12 @@ func (u Upgrader) Upgrade(conn io.ReadWriter) (hs Handshake, err error) { // Read HTTP request line like "GET /ws HTTP/1.1". rl, err := readLine(br) if err != nil { - return + return hs, err } // Parse request line data like HTTP version, uri and method. req, err := httpParseRequestLine(rl) if err != nil { - return + return hs, err } // Prepare stack-based handshake header list. @@ -549,7 +544,7 @@ func (u Upgrader) Upgrade(conn io.ReadWriter) (hs Handshake, err error) { if len(v) != nonceSize { err = ErrHandshakeBadSecKey } else { - copy(nonce[:], v) + copy(nonce, v) } case headerSecProtocolCanonical: @@ -640,13 +635,13 @@ func (u Upgrader) Upgrade(conn io.ReadWriter) (hs Handshake, err error) { httpWriteResponseError(bw, err, code, header.WriteTo) // Do not store Flush() error to not override already existing one. _ = bw.Flush() - return + return hs, err } httpWriteResponseUpgrade(bw, nonce, hs, header.WriteTo) err = bw.Flush() - return + return hs, err } type handshakeHeader [2]HandshakeHeader diff --git a/vendor/github.com/gobwas/ws/util.go b/vendor/github.com/gobwas/ws/util.go index 67ad906e..1dd5aa60 100644 --- a/vendor/github.com/gobwas/ws/util.go +++ b/vendor/github.com/gobwas/ws/util.go @@ -4,8 +4,6 @@ import ( "bufio" "bytes" "fmt" - "reflect" - "unsafe" "github.com/gobwas/httphead" ) @@ -41,19 +39,6 @@ func SelectEqual(v string) func(string) bool { } } -func strToBytes(str string) (bts []byte) { - s := (*reflect.StringHeader)(unsafe.Pointer(&str)) - b := (*reflect.SliceHeader)(unsafe.Pointer(&bts)) - b.Data = s.Data - b.Len = s.Len - b.Cap = s.Len - return -} - -func btsToString(bts []byte) (str string) { - return *(*string)(unsafe.Pointer(&bts)) -} - // asciiToInt converts bytes to int. func asciiToInt(bts []byte) (ret int, err error) { // ASCII numbers all start with the high-order bits 0011. @@ -73,7 +58,7 @@ func asciiToInt(bts []byte) (ret int, err error) { } // pow for integers implementation. -// See Donald Knuth, The Art of Computer Programming, Volume 2, Section 4.6.3 +// See Donald Knuth, The Art of Computer Programming, Volume 2, Section 4.6.3. func pow(a, b int) int { p := 1 for b > 0 { @@ -116,7 +101,7 @@ func btsHasToken(header, token []byte) (has bool) { has = bytes.EqualFold(v, token) return !has }) - return + return has } const ( diff --git a/vendor/github.com/gobwas/ws/util_purego.go b/vendor/github.com/gobwas/ws/util_purego.go new file mode 100644 index 00000000..449b3fdf --- /dev/null +++ b/vendor/github.com/gobwas/ws/util_purego.go @@ -0,0 +1,12 @@ +//go:build purego +// +build purego + +package ws + +func strToBytes(str string) (bts []byte) { + return []byte(str) +} + +func btsToString(bts []byte) (str string) { + return string(bts) +} diff --git a/vendor/github.com/gobwas/ws/util_unsafe.go b/vendor/github.com/gobwas/ws/util_unsafe.go new file mode 100644 index 00000000..b732297c --- /dev/null +++ b/vendor/github.com/gobwas/ws/util_unsafe.go @@ -0,0 +1,22 @@ +//go:build !purego +// +build !purego + +package ws + +import ( + "reflect" + "unsafe" +) + +func strToBytes(str string) (bts []byte) { + s := (*reflect.StringHeader)(unsafe.Pointer(&str)) + b := (*reflect.SliceHeader)(unsafe.Pointer(&bts)) + b.Data = s.Data + b.Len = s.Len + b.Cap = s.Len + return bts +} + +func btsToString(bts []byte) (str string) { + return *(*string)(unsafe.Pointer(&bts)) +} diff --git a/vendor/github.com/gobwas/ws/wsutil/cipher.go b/vendor/github.com/gobwas/ws/wsutil/cipher.go index f234be73..bc25064f 100644 --- a/vendor/github.com/gobwas/ws/wsutil/cipher.go +++ b/vendor/github.com/gobwas/ws/wsutil/cipher.go @@ -34,7 +34,7 @@ func (c *CipherReader) Read(p []byte) (n int, err error) { n, err = c.r.Read(p) ws.Cipher(p[:n], c.mask, c.pos) c.pos += n - return + return n, err } // CipherWriter implements io.Writer that applies xor-cipher to the bytes @@ -68,5 +68,5 @@ func (c *CipherWriter) Write(p []byte) (n int, err error) { n, err = c.w.Write(cp) c.pos += n - return + return n, err } diff --git a/vendor/github.com/gobwas/ws/wsutil/dialer.go b/vendor/github.com/gobwas/ws/wsutil/dialer.go index 91c03d51..4f8788fb 100644 --- a/vendor/github.com/gobwas/ws/wsutil/dialer.go +++ b/vendor/github.com/gobwas/ws/wsutil/dialer.go @@ -113,6 +113,7 @@ type rwConn struct { func (rwc rwConn) Read(p []byte) (int, error) { return rwc.r.Read(p) } + func (rwc rwConn) Write(p []byte) (int, error) { return rwc.w.Write(p) } diff --git a/vendor/github.com/gobwas/ws/wsutil/extenstion.go b/vendor/github.com/gobwas/ws/wsutil/extenstion.go new file mode 100644 index 00000000..6e1ebbf4 --- /dev/null +++ b/vendor/github.com/gobwas/ws/wsutil/extenstion.go @@ -0,0 +1,31 @@ +package wsutil + +import "github.com/gobwas/ws" + +// RecvExtension is an interface for clearing fragment header RSV bits. +type RecvExtension interface { + UnsetBits(ws.Header) (ws.Header, error) +} + +// RecvExtensionFunc is an adapter to allow the use of ordinary functions as +// RecvExtension. +type RecvExtensionFunc func(ws.Header) (ws.Header, error) + +// BitsRecv implements RecvExtension. +func (fn RecvExtensionFunc) UnsetBits(h ws.Header) (ws.Header, error) { + return fn(h) +} + +// SendExtension is an interface for setting fragment header RSV bits. +type SendExtension interface { + SetBits(ws.Header) (ws.Header, error) +} + +// SendExtensionFunc is an adapter to allow the use of ordinary functions as +// SendExtension. +type SendExtensionFunc func(ws.Header) (ws.Header, error) + +// BitsSend implements SendExtension. +func (fn SendExtensionFunc) SetBits(h ws.Header) (ws.Header, error) { + return fn(h) +} diff --git a/vendor/github.com/gobwas/ws/wsutil/handler.go b/vendor/github.com/gobwas/ws/wsutil/handler.go index abb7cb73..44fd360e 100644 --- a/vendor/github.com/gobwas/ws/wsutil/handler.go +++ b/vendor/github.com/gobwas/ws/wsutil/handler.go @@ -199,7 +199,7 @@ func (c ControlHandler) HandleClose(h ws.Header) error { if err != nil { return err } - if err = w.Flush(); err != nil { + if err := w.Flush(); err != nil { return err } return ClosedError{ diff --git a/vendor/github.com/gobwas/ws/wsutil/helper.go b/vendor/github.com/gobwas/ws/wsutil/helper.go index 411d68d7..231760bc 100644 --- a/vendor/github.com/gobwas/ws/wsutil/helper.go +++ b/vendor/github.com/gobwas/ws/wsutil/helper.go @@ -64,14 +64,14 @@ func ReadMessage(r io.Reader, s ws.State, m []Message) ([]Message, error) { // ReadClientMessage reads next message from r, considering that caller // represents server side. -// It is a shortcut for ReadMessage(r, ws.StateServerSide, m) +// It is a shortcut for ReadMessage(r, ws.StateServerSide, m). func ReadClientMessage(r io.Reader, m []Message) ([]Message, error) { return ReadMessage(r, ws.StateServerSide, m) } // ReadServerMessage reads next message from r, considering that caller // represents client side. -// It is a shortcut for ReadMessage(r, ws.StateClientSide, m) +// It is a shortcut for ReadMessage(r, ws.StateClientSide, m). func ReadServerMessage(r io.Reader, m []Message) ([]Message, error) { return ReadMessage(r, ws.StateClientSide, m) } diff --git a/vendor/github.com/gobwas/ws/wsutil/reader.go b/vendor/github.com/gobwas/ws/wsutil/reader.go index 33a22910..f2710af6 100644 --- a/vendor/github.com/gobwas/ws/wsutil/reader.go +++ b/vendor/github.com/gobwas/ws/wsutil/reader.go @@ -1,6 +1,7 @@ package wsutil import ( + "encoding/binary" "errors" "io" "io/ioutil" @@ -56,10 +57,12 @@ type Reader struct { OnContinuation FrameHandlerFunc OnIntermediate FrameHandlerFunc - opCode ws.OpCode // Used to store message op code on fragmentation. - frame io.Reader // Used to as frame reader. - raw io.LimitedReader // Used to discard frames without cipher. - utf8 UTF8Reader // Used to check UTF8 sequences if CheckUTF8 is true. + opCode ws.OpCode // Used to store message op code on fragmentation. + frame io.Reader // Used to as frame reader. + raw io.LimitedReader // Used to discard frames without cipher. + utf8 UTF8Reader // Used to check UTF8 sequences if CheckUTF8 is true. + tmp [ws.MaxHeaderSize - 2]byte // Used for reading headers. + cr *CipherReader // Used by NextFrame() to unmask frame payload. } // NewReader creates new frame reader that reads from r keeping given state to @@ -111,7 +114,7 @@ func (r *Reader) Read(p []byte) (n int, err error) { n, err = r.frame.Read(p) if err != nil && err != io.EOF { - return + return n, err } if err == nil && r.raw.N != 0 { return n, nil @@ -137,7 +140,7 @@ func (r *Reader) Read(p []byte) (n int, err error) { err = io.EOF } - return + return n, err } // Discard discards current message unread bytes. @@ -165,7 +168,7 @@ func (r *Reader) Discard() (err error) { // Note that next NextFrame() call must be done after receiving or discarding // all current message bytes. func (r *Reader) NextFrame() (hdr ws.Header, err error) { - hdr, err = ws.ReadHeader(r.Source) + hdr, err = r.readHeader(r.Source) if err == io.EOF && r.fragmented() { // If we are in fragmented state EOF means that is was totally // unexpected. @@ -196,7 +199,12 @@ func (r *Reader) NextFrame() (hdr ws.Header, err error) { frame := io.Reader(&r.raw) if hdr.Masked { - frame = NewCipherReader(frame, hdr.Mask) + if r.cr == nil { + r.cr = NewCipherReader(frame, hdr.Mask) + } else { + r.cr.Reset(frame, hdr.Mask) + } + frame = r.cr } for _, x := range r.Extensions { @@ -215,7 +223,7 @@ func (r *Reader) NextFrame() (hdr ws.Header, err error) { // Ensure that src is empty. _, err = io.Copy(ioutil.Discard, &r.raw) } - return + return hdr, err } } else { r.opCode = hdr.OpCode @@ -240,7 +248,7 @@ func (r *Reader) NextFrame() (hdr ws.Header, err error) { r.State = r.State.Set(ws.StateFragmented) } - return + return hdr, err } func (r *Reader) fragmented() bool { @@ -261,6 +269,82 @@ func (r *Reader) reset() { r.opCode = 0 } +// readHeader reads a frame header from in. +func (r *Reader) readHeader(in io.Reader) (h ws.Header, err error) { + // Make slice of bytes with capacity 12 that could hold any header. + // + // The maximum header size is 14, but due to the 2 hop reads, + // after first hop that reads first 2 constant bytes, we could reuse 2 bytes. + // So 14 - 2 = 12. + bts := r.tmp[:2] + + // Prepare to hold first 2 bytes to choose size of next read. + _, err = io.ReadFull(in, bts) + if err != nil { + return h, err + } + const bit0 = 0x80 + + h.Fin = bts[0]&bit0 != 0 + h.Rsv = (bts[0] & 0x70) >> 4 + h.OpCode = ws.OpCode(bts[0] & 0x0f) + + var extra int + + if bts[1]&bit0 != 0 { + h.Masked = true + extra += 4 + } + + length := bts[1] & 0x7f + switch { + case length < 126: + h.Length = int64(length) + + case length == 126: + extra += 2 + + case length == 127: + extra += 8 + + default: + err = ws.ErrHeaderLengthUnexpected + return h, err + } + + if extra == 0 { + return h, err + } + + // Increase len of bts to extra bytes need to read. + // Overwrite first 2 bytes that was read before. + bts = bts[:extra] + _, err = io.ReadFull(in, bts) + if err != nil { + return h, err + } + + switch { + case length == 126: + h.Length = int64(binary.BigEndian.Uint16(bts[:2])) + bts = bts[2:] + + case length == 127: + if bts[0]&0x80 != 0 { + err = ws.ErrHeaderLengthMSB + return h, err + } + h.Length = int64(binary.BigEndian.Uint64(bts[:8])) + bts = bts[8:] + } + + if h.Masked { + copy(h.Mask[:], bts) + } + + return h, nil +} + // NextReader prepares next message read from r. It returns header that // describes the message and io.Reader to read message's payload. It returns // non-nil error when it is not possible to read message's initial frame. diff --git a/vendor/github.com/gobwas/ws/wsutil/utf8.go b/vendor/github.com/gobwas/ws/wsutil/utf8.go index d877be0b..b8dc7264 100644 --- a/vendor/github.com/gobwas/ws/wsutil/utf8.go +++ b/vendor/github.com/gobwas/ws/wsutil/utf8.go @@ -65,7 +65,7 @@ func (u *UTF8Reader) Read(p []byte) (n int, err error) { u.state, u.codep = s, c u.accepted = accepted - return + return n, err } // Valid checks current reader state. It returns true if all read bytes are diff --git a/vendor/github.com/gobwas/ws/wsutil/wsutil.go b/vendor/github.com/gobwas/ws/wsutil/wsutil.go index ffd43367..86211f3e 100644 --- a/vendor/github.com/gobwas/ws/wsutil/wsutil.go +++ b/vendor/github.com/gobwas/ws/wsutil/wsutil.go @@ -3,54 +3,54 @@ Package wsutil provides utilities for working with WebSocket protocol. Overview: - // Read masked text message from peer and check utf8 encoding. - header, err := ws.ReadHeader(conn) - if err != nil { - // handle err - } - - // Prepare to read payload. - r := io.LimitReader(conn, header.Length) - r = wsutil.NewCipherReader(r, header.Mask) - r = wsutil.NewUTF8Reader(r) - - payload, err := ioutil.ReadAll(r) - if err != nil { - // handle err - } + // Read masked text message from peer and check utf8 encoding. + header, err := ws.ReadHeader(conn) + if err != nil { + // handle err + } + + // Prepare to read payload. + r := io.LimitReader(conn, header.Length) + r = wsutil.NewCipherReader(r, header.Mask) + r = wsutil.NewUTF8Reader(r) + + payload, err := ioutil.ReadAll(r) + if err != nil { + // handle err + } You could get the same behavior using just `wsutil.Reader`: - r := wsutil.Reader{ - Source: conn, - CheckUTF8: true, - } + r := wsutil.Reader{ + Source: conn, + CheckUTF8: true, + } - payload, err := ioutil.ReadAll(r) - if err != nil { - // handle err - } + payload, err := ioutil.ReadAll(r) + if err != nil { + // handle err + } Or even simplest: - payload, err := wsutil.ReadClientText(conn) - if err != nil { - // handle err - } + payload, err := wsutil.ReadClientText(conn) + if err != nil { + // handle err + } Package is also exports tools for buffered writing: - // Create buffered writer, that will buffer output bytes and send them as - // 128-length fragments (with exception on large writes, see the doc). - writer := wsutil.NewWriterSize(conn, ws.StateServerSide, ws.OpText, 128) - - _, err := io.CopyN(writer, rand.Reader, 100) - if err == nil { - err = writer.Flush() - } - if err != nil { - // handle error - } + // Create buffered writer, that will buffer output bytes and send them as + // 128-length fragments (with exception on large writes, see the doc). + writer := wsutil.NewWriterSize(conn, ws.StateServerSide, ws.OpText, 128) + + _, err := io.CopyN(writer, rand.Reader, 100) + if err == nil { + err = writer.Flush() + } + if err != nil { + // handle error + } For more utils and helpers see the documentation. */ diff --git a/vendor/github.com/golang/geo/LICENSE b/vendor/github.com/golang/geo/LICENSE new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/vendor/github.com/golang/geo/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/golang/geo/r1/doc.go b/vendor/github.com/golang/geo/r1/doc.go new file mode 100644 index 00000000..c6b65c0e --- /dev/null +++ b/vendor/github.com/golang/geo/r1/doc.go @@ -0,0 +1,20 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* +Package r1 implements types and functions for working with geometry in ℝ¹. + +See ../s2 for a more detailed overview. +*/ +package r1 diff --git a/vendor/github.com/golang/geo/r1/interval.go b/vendor/github.com/golang/geo/r1/interval.go new file mode 100644 index 00000000..48ea5198 --- /dev/null +++ b/vendor/github.com/golang/geo/r1/interval.go @@ -0,0 +1,177 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package r1 + +import ( + "fmt" + "math" +) + +// Interval represents a closed interval on ℝ. +// Zero-length intervals (where Lo == Hi) represent single points. +// If Lo > Hi then the interval is empty. +type Interval struct { + Lo, Hi float64 +} + +// EmptyInterval returns an empty interval. +func EmptyInterval() Interval { return Interval{1, 0} } + +// IntervalFromPoint returns an interval representing a single point. +func IntervalFromPoint(p float64) Interval { return Interval{p, p} } + +// IsEmpty reports whether the interval is empty. +func (i Interval) IsEmpty() bool { return i.Lo > i.Hi } + +// Equal returns true iff the interval contains the same points as oi. +func (i Interval) Equal(oi Interval) bool { + return i == oi || i.IsEmpty() && oi.IsEmpty() +} + +// Center returns the midpoint of the interval. +// It is undefined for empty intervals. +func (i Interval) Center() float64 { return 0.5 * (i.Lo + i.Hi) } + +// Length returns the length of the interval. +// The length of an empty interval is negative. +func (i Interval) Length() float64 { return i.Hi - i.Lo } + +// Contains returns true iff the interval contains p. +func (i Interval) Contains(p float64) bool { return i.Lo <= p && p <= i.Hi } + +// ContainsInterval returns true iff the interval contains oi. +func (i Interval) ContainsInterval(oi Interval) bool { + if oi.IsEmpty() { + return true + } + return i.Lo <= oi.Lo && oi.Hi <= i.Hi +} + +// InteriorContains returns true iff the interval strictly contains p. +func (i Interval) InteriorContains(p float64) bool { + return i.Lo < p && p < i.Hi +} + +// InteriorContainsInterval returns true iff the interval strictly contains oi. +func (i Interval) InteriorContainsInterval(oi Interval) bool { + if oi.IsEmpty() { + return true + } + return i.Lo < oi.Lo && oi.Hi < i.Hi +} + +// Intersects returns true iff the interval contains any points in common with oi. +func (i Interval) Intersects(oi Interval) bool { + if i.Lo <= oi.Lo { + return oi.Lo <= i.Hi && oi.Lo <= oi.Hi // oi.Lo ∈ i and oi is not empty + } + return i.Lo <= oi.Hi && i.Lo <= i.Hi // i.Lo ∈ oi and i is not empty +} + +// InteriorIntersects returns true iff the interior of the interval contains any points in common with oi, including the latter's boundary. +func (i Interval) InteriorIntersects(oi Interval) bool { + return oi.Lo < i.Hi && i.Lo < oi.Hi && i.Lo < i.Hi && oi.Lo <= oi.Hi +} + +// Intersection returns the interval containing all points common to i and j. +func (i Interval) Intersection(j Interval) Interval { + // Empty intervals do not need to be special-cased. + return Interval{ + Lo: math.Max(i.Lo, j.Lo), + Hi: math.Min(i.Hi, j.Hi), + } +} + +// AddPoint returns the interval expanded so that it contains the given point. +func (i Interval) AddPoint(p float64) Interval { + if i.IsEmpty() { + return Interval{p, p} + } + if p < i.Lo { + return Interval{p, i.Hi} + } + if p > i.Hi { + return Interval{i.Lo, p} + } + return i +} + +// ClampPoint returns the closest point in the interval to the given point "p". +// The interval must be non-empty. +func (i Interval) ClampPoint(p float64) float64 { + return math.Max(i.Lo, math.Min(i.Hi, p)) +} + +// Expanded returns an interval that has been expanded on each side by margin. +// If margin is negative, then the function shrinks the interval on +// each side by margin instead. The resulting interval may be empty. Any +// expansion of an empty interval remains empty. +func (i Interval) Expanded(margin float64) Interval { + if i.IsEmpty() { + return i + } + return Interval{i.Lo - margin, i.Hi + margin} +} + +// Union returns the smallest interval that contains this interval and the given interval. +func (i Interval) Union(other Interval) Interval { + if i.IsEmpty() { + return other + } + if other.IsEmpty() { + return i + } + return Interval{math.Min(i.Lo, other.Lo), math.Max(i.Hi, other.Hi)} +} + +func (i Interval) String() string { return fmt.Sprintf("[%.7f, %.7f]", i.Lo, i.Hi) } + +const ( + // epsilon is a small number that represents a reasonable level of noise between two + // values that can be considered to be equal. + epsilon = 1e-15 + // dblEpsilon is a smaller number for values that require more precision. + // This is the C++ DBL_EPSILON equivalent. + dblEpsilon = 2.220446049250313e-16 +) + +// ApproxEqual reports whether the interval can be transformed into the +// given interval by moving each endpoint a small distance. +// The empty interval is considered to be positioned arbitrarily on the +// real line, so any interval with a small enough length will match +// the empty interval. +func (i Interval) ApproxEqual(other Interval) bool { + if i.IsEmpty() { + return other.Length() <= 2*epsilon + } + if other.IsEmpty() { + return i.Length() <= 2*epsilon + } + return math.Abs(other.Lo-i.Lo) <= epsilon && + math.Abs(other.Hi-i.Hi) <= epsilon +} + +// DirectedHausdorffDistance returns the Hausdorff distance to the given interval. For two +// intervals x and y, this distance is defined as +// h(x, y) = max_{p in x} min_{q in y} d(p, q). +func (i Interval) DirectedHausdorffDistance(other Interval) float64 { + if i.IsEmpty() { + return 0 + } + if other.IsEmpty() { + return math.Inf(1) + } + return math.Max(0, math.Max(i.Hi-other.Hi, other.Lo-i.Lo)) +} diff --git a/vendor/github.com/golang/geo/r2/doc.go b/vendor/github.com/golang/geo/r2/doc.go new file mode 100644 index 00000000..05b15554 --- /dev/null +++ b/vendor/github.com/golang/geo/r2/doc.go @@ -0,0 +1,20 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* +Package r2 implements types and functions for working with geometry in ℝ². + +See package s2 for a more detailed overview. +*/ +package r2 diff --git a/vendor/github.com/golang/geo/r2/rect.go b/vendor/github.com/golang/geo/r2/rect.go new file mode 100644 index 00000000..495545bb --- /dev/null +++ b/vendor/github.com/golang/geo/r2/rect.go @@ -0,0 +1,255 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package r2 + +import ( + "fmt" + "math" + + "github.com/golang/geo/r1" +) + +// Point represents a point in ℝ². +type Point struct { + X, Y float64 +} + +// Add returns the sum of p and op. +func (p Point) Add(op Point) Point { return Point{p.X + op.X, p.Y + op.Y} } + +// Sub returns the difference of p and op. +func (p Point) Sub(op Point) Point { return Point{p.X - op.X, p.Y - op.Y} } + +// Mul returns the scalar product of p and m. +func (p Point) Mul(m float64) Point { return Point{m * p.X, m * p.Y} } + +// Ortho returns a counterclockwise orthogonal point with the same norm. +func (p Point) Ortho() Point { return Point{-p.Y, p.X} } + +// Dot returns the dot product between p and op. +func (p Point) Dot(op Point) float64 { return p.X*op.X + p.Y*op.Y } + +// Cross returns the cross product of p and op. +func (p Point) Cross(op Point) float64 { return p.X*op.Y - p.Y*op.X } + +// Norm returns the vector's norm. +func (p Point) Norm() float64 { return math.Hypot(p.X, p.Y) } + +// Normalize returns a unit point in the same direction as p. +func (p Point) Normalize() Point { + if p.X == 0 && p.Y == 0 { + return p + } + return p.Mul(1 / p.Norm()) +} + +func (p Point) String() string { return fmt.Sprintf("(%.12f, %.12f)", p.X, p.Y) } + +// Rect represents a closed axis-aligned rectangle in the (x,y) plane. +type Rect struct { + X, Y r1.Interval +} + +// RectFromPoints constructs a rect that contains the given points. +func RectFromPoints(pts ...Point) Rect { + // Because the default value on interval is 0,0, we need to manually + // define the interval from the first point passed in as our starting + // interval, otherwise we end up with the case of passing in + // Point{0.2, 0.3} and getting the starting Rect of {0, 0.2}, {0, 0.3} + // instead of the Rect {0.2, 0.2}, {0.3, 0.3} which is not correct. + if len(pts) == 0 { + return Rect{} + } + + r := Rect{ + X: r1.Interval{Lo: pts[0].X, Hi: pts[0].X}, + Y: r1.Interval{Lo: pts[0].Y, Hi: pts[0].Y}, + } + + for _, p := range pts[1:] { + r = r.AddPoint(p) + } + return r +} + +// RectFromCenterSize constructs a rectangle with the given center and size. +// Both dimensions of size must be non-negative. +func RectFromCenterSize(center, size Point) Rect { + return Rect{ + r1.Interval{Lo: center.X - size.X/2, Hi: center.X + size.X/2}, + r1.Interval{Lo: center.Y - size.Y/2, Hi: center.Y + size.Y/2}, + } +} + +// EmptyRect constructs the canonical empty rectangle. Use IsEmpty() to test +// for empty rectangles, since they have more than one representation. A Rect{} +// is not the same as the EmptyRect. +func EmptyRect() Rect { + return Rect{r1.EmptyInterval(), r1.EmptyInterval()} +} + +// IsValid reports whether the rectangle is valid. +// This requires the width to be empty iff the height is empty. +func (r Rect) IsValid() bool { + return r.X.IsEmpty() == r.Y.IsEmpty() +} + +// IsEmpty reports whether the rectangle is empty. +func (r Rect) IsEmpty() bool { + return r.X.IsEmpty() +} + +// Vertices returns all four vertices of the rectangle. Vertices are returned in +// CCW direction starting with the lower left corner. +func (r Rect) Vertices() [4]Point { + return [4]Point{ + {r.X.Lo, r.Y.Lo}, + {r.X.Hi, r.Y.Lo}, + {r.X.Hi, r.Y.Hi}, + {r.X.Lo, r.Y.Hi}, + } +} + +// VertexIJ returns the vertex in direction i along the X-axis (0=left, 1=right) and +// direction j along the Y-axis (0=down, 1=up). +func (r Rect) VertexIJ(i, j int) Point { + x := r.X.Lo + if i == 1 { + x = r.X.Hi + } + y := r.Y.Lo + if j == 1 { + y = r.Y.Hi + } + return Point{x, y} +} + +// Lo returns the low corner of the rect. +func (r Rect) Lo() Point { + return Point{r.X.Lo, r.Y.Lo} +} + +// Hi returns the high corner of the rect. +func (r Rect) Hi() Point { + return Point{r.X.Hi, r.Y.Hi} +} + +// Center returns the center of the rectangle in (x,y)-space +func (r Rect) Center() Point { + return Point{r.X.Center(), r.Y.Center()} +} + +// Size returns the width and height of this rectangle in (x,y)-space. Empty +// rectangles have a negative width and height. +func (r Rect) Size() Point { + return Point{r.X.Length(), r.Y.Length()} +} + +// ContainsPoint reports whether the rectangle contains the given point. +// Rectangles are closed regions, i.e. they contain their boundary. +func (r Rect) ContainsPoint(p Point) bool { + return r.X.Contains(p.X) && r.Y.Contains(p.Y) +} + +// InteriorContainsPoint returns true iff the given point is contained in the interior +// of the region (i.e. the region excluding its boundary). +func (r Rect) InteriorContainsPoint(p Point) bool { + return r.X.InteriorContains(p.X) && r.Y.InteriorContains(p.Y) +} + +// Contains reports whether the rectangle contains the given rectangle. +func (r Rect) Contains(other Rect) bool { + return r.X.ContainsInterval(other.X) && r.Y.ContainsInterval(other.Y) +} + +// InteriorContains reports whether the interior of this rectangle contains all of the +// points of the given other rectangle (including its boundary). +func (r Rect) InteriorContains(other Rect) bool { + return r.X.InteriorContainsInterval(other.X) && r.Y.InteriorContainsInterval(other.Y) +} + +// Intersects reports whether this rectangle and the other rectangle have any points in common. +func (r Rect) Intersects(other Rect) bool { + return r.X.Intersects(other.X) && r.Y.Intersects(other.Y) +} + +// InteriorIntersects reports whether the interior of this rectangle intersects +// any point (including the boundary) of the given other rectangle. +func (r Rect) InteriorIntersects(other Rect) bool { + return r.X.InteriorIntersects(other.X) && r.Y.InteriorIntersects(other.Y) +} + +// AddPoint expands the rectangle to include the given point. The rectangle is +// expanded by the minimum amount possible. +func (r Rect) AddPoint(p Point) Rect { + return Rect{r.X.AddPoint(p.X), r.Y.AddPoint(p.Y)} +} + +// AddRect expands the rectangle to include the given rectangle. This is the +// same as replacing the rectangle by the union of the two rectangles, but +// is more efficient. +func (r Rect) AddRect(other Rect) Rect { + return Rect{r.X.Union(other.X), r.Y.Union(other.Y)} +} + +// ClampPoint returns the closest point in the rectangle to the given point. +// The rectangle must be non-empty. +func (r Rect) ClampPoint(p Point) Point { + return Point{r.X.ClampPoint(p.X), r.Y.ClampPoint(p.Y)} +} + +// Expanded returns a rectangle that has been expanded in the x-direction +// by margin.X, and in y-direction by margin.Y. If either margin is empty, +// then shrink the interval on the corresponding sides instead. The resulting +// rectangle may be empty. Any expansion of an empty rectangle remains empty. +func (r Rect) Expanded(margin Point) Rect { + xx := r.X.Expanded(margin.X) + yy := r.Y.Expanded(margin.Y) + if xx.IsEmpty() || yy.IsEmpty() { + return EmptyRect() + } + return Rect{xx, yy} +} + +// ExpandedByMargin returns a Rect that has been expanded by the amount on all sides. +func (r Rect) ExpandedByMargin(margin float64) Rect { + return r.Expanded(Point{margin, margin}) +} + +// Union returns the smallest rectangle containing the union of this rectangle and +// the given rectangle. +func (r Rect) Union(other Rect) Rect { + return Rect{r.X.Union(other.X), r.Y.Union(other.Y)} +} + +// Intersection returns the smallest rectangle containing the intersection of this +// rectangle and the given rectangle. +func (r Rect) Intersection(other Rect) Rect { + xx := r.X.Intersection(other.X) + yy := r.Y.Intersection(other.Y) + if xx.IsEmpty() || yy.IsEmpty() { + return EmptyRect() + } + + return Rect{xx, yy} +} + +// ApproxEqual returns true if the x- and y-intervals of the two rectangles are +// the same up to the given tolerance. +func (r Rect) ApproxEqual(r2 Rect) bool { + return r.X.ApproxEqual(r2.X) && r.Y.ApproxEqual(r2.Y) +} + +func (r Rect) String() string { return fmt.Sprintf("[Lo%s, Hi%s]", r.Lo(), r.Hi()) } diff --git a/vendor/github.com/golang/geo/r3/doc.go b/vendor/github.com/golang/geo/r3/doc.go new file mode 100644 index 00000000..1eb4710c --- /dev/null +++ b/vendor/github.com/golang/geo/r3/doc.go @@ -0,0 +1,20 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* +Package r3 implements types and functions for working with geometry in ℝ³. + +See ../s2 for a more detailed overview. +*/ +package r3 diff --git a/vendor/github.com/golang/geo/r3/precisevector.go b/vendor/github.com/golang/geo/r3/precisevector.go new file mode 100644 index 00000000..b13393db --- /dev/null +++ b/vendor/github.com/golang/geo/r3/precisevector.go @@ -0,0 +1,198 @@ +// Copyright 2016 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package r3 + +import ( + "fmt" + "math/big" +) + +const ( + // prec is the number of bits of precision to use for the Float values. + // To keep things simple, we use the maximum allowable precision on big + // values. This allows us to handle all values we expect in the s2 library. + prec = big.MaxPrec +) + +// define some commonly referenced values. +var ( + precise0 = precInt(0) + precise1 = precInt(1) +) + +// precStr wraps the conversion from a string into a big.Float. For results that +// actually can be represented exactly, this should only be used on values that +// are integer multiples of integer powers of 2. +func precStr(s string) *big.Float { + // Explicitly ignoring the bool return for this usage. + f, _ := new(big.Float).SetPrec(prec).SetString(s) + return f +} + +func precInt(i int64) *big.Float { + return new(big.Float).SetPrec(prec).SetInt64(i) +} + +func precFloat(f float64) *big.Float { + return new(big.Float).SetPrec(prec).SetFloat64(f) +} + +func precAdd(a, b *big.Float) *big.Float { + return new(big.Float).SetPrec(prec).Add(a, b) +} + +func precSub(a, b *big.Float) *big.Float { + return new(big.Float).SetPrec(prec).Sub(a, b) +} + +func precMul(a, b *big.Float) *big.Float { + return new(big.Float).SetPrec(prec).Mul(a, b) +} + +// PreciseVector represents a point in ℝ³ using high-precision values. +// Note that this is NOT a complete implementation because there are some +// operations that Vector supports that are not feasible with arbitrary precision +// math. (e.g., methods that need division like Normalize, or methods needing a +// square root operation such as Norm) +type PreciseVector struct { + X, Y, Z *big.Float +} + +// PreciseVectorFromVector creates a high precision vector from the given Vector. +func PreciseVectorFromVector(v Vector) PreciseVector { + return NewPreciseVector(v.X, v.Y, v.Z) +} + +// NewPreciseVector creates a high precision vector from the given floating point values. +func NewPreciseVector(x, y, z float64) PreciseVector { + return PreciseVector{ + X: precFloat(x), + Y: precFloat(y), + Z: precFloat(z), + } +} + +// Vector returns this precise vector converted to a Vector. +func (v PreciseVector) Vector() Vector { + // The accuracy flag is ignored on these conversions back to float64. + x, _ := v.X.Float64() + y, _ := v.Y.Float64() + z, _ := v.Z.Float64() + return Vector{x, y, z}.Normalize() +} + +// Equal reports whether v and ov are equal. +func (v PreciseVector) Equal(ov PreciseVector) bool { + return v.X.Cmp(ov.X) == 0 && v.Y.Cmp(ov.Y) == 0 && v.Z.Cmp(ov.Z) == 0 +} + +func (v PreciseVector) String() string { + return fmt.Sprintf("(%10g, %10g, %10g)", v.X, v.Y, v.Z) +} + +// Norm2 returns the square of the norm. +func (v PreciseVector) Norm2() *big.Float { return v.Dot(v) } + +// IsUnit reports whether this vector is of unit length. +func (v PreciseVector) IsUnit() bool { + return v.Norm2().Cmp(precise1) == 0 +} + +// Abs returns the vector with nonnegative components. +func (v PreciseVector) Abs() PreciseVector { + return PreciseVector{ + X: new(big.Float).Abs(v.X), + Y: new(big.Float).Abs(v.Y), + Z: new(big.Float).Abs(v.Z), + } +} + +// Add returns the standard vector sum of v and ov. +func (v PreciseVector) Add(ov PreciseVector) PreciseVector { + return PreciseVector{ + X: precAdd(v.X, ov.X), + Y: precAdd(v.Y, ov.Y), + Z: precAdd(v.Z, ov.Z), + } +} + +// Sub returns the standard vector difference of v and ov. +func (v PreciseVector) Sub(ov PreciseVector) PreciseVector { + return PreciseVector{ + X: precSub(v.X, ov.X), + Y: precSub(v.Y, ov.Y), + Z: precSub(v.Z, ov.Z), + } +} + +// Mul returns the standard scalar product of v and f. +func (v PreciseVector) Mul(f *big.Float) PreciseVector { + return PreciseVector{ + X: precMul(v.X, f), + Y: precMul(v.Y, f), + Z: precMul(v.Z, f), + } +} + +// MulByFloat64 returns the standard scalar product of v and f. +func (v PreciseVector) MulByFloat64(f float64) PreciseVector { + return v.Mul(precFloat(f)) +} + +// Dot returns the standard dot product of v and ov. +func (v PreciseVector) Dot(ov PreciseVector) *big.Float { + return precAdd(precMul(v.X, ov.X), precAdd(precMul(v.Y, ov.Y), precMul(v.Z, ov.Z))) +} + +// Cross returns the standard cross product of v and ov. +func (v PreciseVector) Cross(ov PreciseVector) PreciseVector { + return PreciseVector{ + X: precSub(precMul(v.Y, ov.Z), precMul(v.Z, ov.Y)), + Y: precSub(precMul(v.Z, ov.X), precMul(v.X, ov.Z)), + Z: precSub(precMul(v.X, ov.Y), precMul(v.Y, ov.X)), + } +} + +// LargestComponent returns the axis that represents the largest component in this vector. +func (v PreciseVector) LargestComponent() Axis { + t := v.Abs() + + if t.X.Cmp(t.Y) > 0 { + if t.X.Cmp(t.Z) > 0 { + return XAxis + } + return ZAxis + } + if t.Y.Cmp(t.Z) > 0 { + return YAxis + } + return ZAxis +} + +// SmallestComponent returns the axis that represents the smallest component in this vector. +func (v PreciseVector) SmallestComponent() Axis { + t := v.Abs() + + if t.X.Cmp(t.Y) < 0 { + if t.X.Cmp(t.Z) < 0 { + return XAxis + } + return ZAxis + } + if t.Y.Cmp(t.Z) < 0 { + return YAxis + } + return ZAxis +} diff --git a/vendor/github.com/golang/geo/r3/vector.go b/vendor/github.com/golang/geo/r3/vector.go new file mode 100644 index 00000000..ccda622f --- /dev/null +++ b/vendor/github.com/golang/geo/r3/vector.go @@ -0,0 +1,183 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package r3 + +import ( + "fmt" + "math" + + "github.com/golang/geo/s1" +) + +// Vector represents a point in ℝ³. +type Vector struct { + X, Y, Z float64 +} + +// ApproxEqual reports whether v and ov are equal within a small epsilon. +func (v Vector) ApproxEqual(ov Vector) bool { + const epsilon = 1e-16 + return math.Abs(v.X-ov.X) < epsilon && math.Abs(v.Y-ov.Y) < epsilon && math.Abs(v.Z-ov.Z) < epsilon +} + +func (v Vector) String() string { return fmt.Sprintf("(%0.24f, %0.24f, %0.24f)", v.X, v.Y, v.Z) } + +// Norm returns the vector's norm. +func (v Vector) Norm() float64 { return math.Sqrt(v.Dot(v)) } + +// Norm2 returns the square of the norm. +func (v Vector) Norm2() float64 { return v.Dot(v) } + +// Normalize returns a unit vector in the same direction as v. +func (v Vector) Normalize() Vector { + n2 := v.Norm2() + if n2 == 0 { + return Vector{0, 0, 0} + } + return v.Mul(1 / math.Sqrt(n2)) +} + +// IsUnit returns whether this vector is of approximately unit length. +func (v Vector) IsUnit() bool { + const epsilon = 5e-14 + return math.Abs(v.Norm2()-1) <= epsilon +} + +// Abs returns the vector with nonnegative components. +func (v Vector) Abs() Vector { return Vector{math.Abs(v.X), math.Abs(v.Y), math.Abs(v.Z)} } + +// Add returns the standard vector sum of v and ov. +func (v Vector) Add(ov Vector) Vector { return Vector{v.X + ov.X, v.Y + ov.Y, v.Z + ov.Z} } + +// Sub returns the standard vector difference of v and ov. +func (v Vector) Sub(ov Vector) Vector { return Vector{v.X - ov.X, v.Y - ov.Y, v.Z - ov.Z} } + +// Mul returns the standard scalar product of v and m. +func (v Vector) Mul(m float64) Vector { return Vector{m * v.X, m * v.Y, m * v.Z} } + +// Dot returns the standard dot product of v and ov. +func (v Vector) Dot(ov Vector) float64 { return v.X*ov.X + v.Y*ov.Y + v.Z*ov.Z } + +// Cross returns the standard cross product of v and ov. +func (v Vector) Cross(ov Vector) Vector { + return Vector{ + v.Y*ov.Z - v.Z*ov.Y, + v.Z*ov.X - v.X*ov.Z, + v.X*ov.Y - v.Y*ov.X, + } +} + +// Distance returns the Euclidean distance between v and ov. +func (v Vector) Distance(ov Vector) float64 { return v.Sub(ov).Norm() } + +// Angle returns the angle between v and ov. +func (v Vector) Angle(ov Vector) s1.Angle { + return s1.Angle(math.Atan2(v.Cross(ov).Norm(), v.Dot(ov))) * s1.Radian +} + +// Axis enumerates the 3 axes of ℝ³. +type Axis int + +// The three axes of ℝ³. +const ( + XAxis Axis = iota + YAxis + ZAxis +) + +// Ortho returns a unit vector that is orthogonal to v. +// Ortho(-v) = -Ortho(v) for all v. +func (v Vector) Ortho() Vector { + ov := Vector{0.012, 0.0053, 0.00457} + switch v.LargestComponent() { + case XAxis: + ov.Z = 1 + case YAxis: + ov.X = 1 + default: + ov.Y = 1 + } + return v.Cross(ov).Normalize() +} + +// LargestComponent returns the axis that represents the largest component in this vector. +func (v Vector) LargestComponent() Axis { + t := v.Abs() + + if t.X > t.Y { + if t.X > t.Z { + return XAxis + } + return ZAxis + } + if t.Y > t.Z { + return YAxis + } + return ZAxis +} + +// SmallestComponent returns the axis that represents the smallest component in this vector. +func (v Vector) SmallestComponent() Axis { + t := v.Abs() + + if t.X < t.Y { + if t.X < t.Z { + return XAxis + } + return ZAxis + } + if t.Y < t.Z { + return YAxis + } + return ZAxis +} + +// Cmp compares v and ov lexicographically and returns: +// +// -1 if v < ov +// 0 if v == ov +// +1 if v > ov +// +// This method is based on C++'s std::lexicographical_compare. Two entities +// are compared element by element with the given operator. The first mismatch +// defines which is less (or greater) than the other. If both have equivalent +// values they are lexicographically equal. +func (v Vector) Cmp(ov Vector) int { + if v.X < ov.X { + return -1 + } + if v.X > ov.X { + return 1 + } + + // First elements were the same, try the next. + if v.Y < ov.Y { + return -1 + } + if v.Y > ov.Y { + return 1 + } + + // Second elements were the same return the final compare. + if v.Z < ov.Z { + return -1 + } + if v.Z > ov.Z { + return 1 + } + + // Both are equal + return 0 +} diff --git a/vendor/github.com/golang/geo/s1/angle.go b/vendor/github.com/golang/geo/s1/angle.go new file mode 100644 index 00000000..747b23de --- /dev/null +++ b/vendor/github.com/golang/geo/s1/angle.go @@ -0,0 +1,120 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s1 + +import ( + "math" + "strconv" +) + +// Angle represents a 1D angle. The internal representation is a double precision +// value in radians, so conversion to and from radians is exact. +// Conversions between E5, E6, E7, and Degrees are not always +// exact. For example, Degrees(3.1) is different from E6(3100000) or E7(31000000). +// +// The following conversions between degrees and radians are exact: +// +// Degree*180 == Radian*math.Pi +// Degree*(180/n) == Radian*(math.Pi/n) for n == 0..8 +// +// These identities hold when the arguments are scaled up or down by any power +// of 2. Some similar identities are also true, for example, +// +// Degree*60 == Radian*(math.Pi/3) +// +// But be aware that this type of identity does not hold in general. For example, +// +// Degree*3 != Radian*(math.Pi/60) +// +// Similarly, the conversion to radians means that (Angle(x)*Degree).Degrees() +// does not always equal x. For example, +// +// (Angle(45*n)*Degree).Degrees() == 45*n for n == 0..8 +// +// but +// +// (60*Degree).Degrees() != 60 +// +// When testing for equality, you should allow for numerical errors (ApproxEqual) +// or convert to discrete E5/E6/E7 values first. +type Angle float64 + +// Angle units. +const ( + Radian Angle = 1 + Degree = (math.Pi / 180) * Radian + + E5 = 1e-5 * Degree + E6 = 1e-6 * Degree + E7 = 1e-7 * Degree +) + +// Radians returns the angle in radians. +func (a Angle) Radians() float64 { return float64(a) } + +// Degrees returns the angle in degrees. +func (a Angle) Degrees() float64 { return float64(a / Degree) } + +// round returns the value rounded to nearest as an int32. +// This does not match C++ exactly for the case of x.5. +func round(val float64) int32 { + if val < 0 { + return int32(val - 0.5) + } + return int32(val + 0.5) +} + +// InfAngle returns an angle larger than any finite angle. +func InfAngle() Angle { + return Angle(math.Inf(1)) +} + +// isInf reports whether this Angle is infinite. +func (a Angle) isInf() bool { + return math.IsInf(float64(a), 0) +} + +// E5 returns the angle in hundred thousandths of degrees. +func (a Angle) E5() int32 { return round(a.Degrees() * 1e5) } + +// E6 returns the angle in millionths of degrees. +func (a Angle) E6() int32 { return round(a.Degrees() * 1e6) } + +// E7 returns the angle in ten millionths of degrees. +func (a Angle) E7() int32 { return round(a.Degrees() * 1e7) } + +// Abs returns the absolute value of the angle. +func (a Angle) Abs() Angle { return Angle(math.Abs(float64(a))) } + +// Normalized returns an equivalent angle in (-π, π]. +func (a Angle) Normalized() Angle { + rad := math.Remainder(float64(a), 2*math.Pi) + if rad <= -math.Pi { + rad = math.Pi + } + return Angle(rad) +} + +func (a Angle) String() string { + return strconv.FormatFloat(a.Degrees(), 'f', 7, 64) // like "%.7f" +} + +// ApproxEqual reports whether the two angles are the same up to a small tolerance. +func (a Angle) ApproxEqual(other Angle) bool { + return math.Abs(float64(a)-float64(other)) <= epsilon +} + +// BUG(dsymonds): The major differences from the C++ version are: +// - no unsigned E5/E6/E7 methods diff --git a/vendor/github.com/golang/geo/s1/chordangle.go b/vendor/github.com/golang/geo/s1/chordangle.go new file mode 100644 index 00000000..77d71648 --- /dev/null +++ b/vendor/github.com/golang/geo/s1/chordangle.go @@ -0,0 +1,320 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s1 + +import ( + "math" +) + +// ChordAngle represents the angle subtended by a chord (i.e., the straight +// line segment connecting two points on the sphere). Its representation +// makes it very efficient for computing and comparing distances, but unlike +// Angle it is only capable of representing angles between 0 and π radians. +// Generally, ChordAngle should only be used in loops where many angles need +// to be calculated and compared. Otherwise it is simpler to use Angle. +// +// ChordAngle loses some accuracy as the angle approaches π radians. +// There are several different ways to measure this error, including the +// representational error (i.e., how accurately ChordAngle can represent +// angles near π radians), the conversion error (i.e., how much precision is +// lost when an Angle is converted to an ChordAngle), and the measurement +// error (i.e., how accurate the ChordAngle(a, b) constructor is when the +// points A and B are separated by angles close to π radians). All of these +// errors differ by a small constant factor. +// +// For the measurement error (which is the largest of these errors and also +// the most important in practice), let the angle between A and B be (π - x) +// radians, i.e. A and B are within "x" radians of being antipodal. The +// corresponding chord length is +// +// r = 2 * sin((π - x) / 2) = 2 * cos(x / 2) +// +// For values of x not close to π the relative error in the squared chord +// length is at most 4.5 * dblEpsilon (see MaxPointError below). +// The relative error in "r" is thus at most 2.25 * dblEpsilon ~= 5e-16. To +// convert this error into an equivalent angle, we have +// +// |dr / dx| = sin(x / 2) +// +// and therefore +// +// |dx| = dr / sin(x / 2) +// = 5e-16 * (2 * cos(x / 2)) / sin(x / 2) +// = 1e-15 / tan(x / 2) +// +// The maximum error is attained when +// +// x = |dx| +// = 1e-15 / tan(x / 2) +// ~= 1e-15 / (x / 2) +// ~= sqrt(2e-15) +// +// In summary, the measurement error for an angle (π - x) is at most +// +// dx = min(1e-15 / tan(x / 2), sqrt(2e-15)) +// (~= min(2e-15 / x, sqrt(2e-15)) when x is small) +// +// On the Earth's surface (assuming a radius of 6371km), this corresponds to +// the following worst-case measurement errors: +// +// Accuracy: Unless antipodal to within: +// --------- --------------------------- +// 6.4 nanometers 10,000 km (90 degrees) +// 1 micrometer 81.2 kilometers +// 1 millimeter 81.2 meters +// 1 centimeter 8.12 meters +// 28.5 centimeters 28.5 centimeters +// +// The representational and conversion errors referred to earlier are somewhat +// smaller than this. For example, maximum distance between adjacent +// representable ChordAngle values is only 13.5 cm rather than 28.5 cm. To +// see this, observe that the closest representable value to r^2 = 4 is +// r^2 = 4 * (1 - dblEpsilon / 2). Thus r = 2 * (1 - dblEpsilon / 4) and +// the angle between these two representable values is +// +// x = 2 * acos(r / 2) +// = 2 * acos(1 - dblEpsilon / 4) +// ~= 2 * asin(sqrt(dblEpsilon / 2) +// ~= sqrt(2 * dblEpsilon) +// ~= 2.1e-8 +// +// which is 13.5 cm on the Earth's surface. +// +// The worst case rounding error occurs when the value halfway between these +// two representable values is rounded up to 4. This halfway value is +// r^2 = (4 * (1 - dblEpsilon / 4)), thus r = 2 * (1 - dblEpsilon / 8) and +// the worst case rounding error is +// +// x = 2 * acos(r / 2) +// = 2 * acos(1 - dblEpsilon / 8) +// ~= 2 * asin(sqrt(dblEpsilon / 4) +// ~= sqrt(dblEpsilon) +// ~= 1.5e-8 +// +// which is 9.5 cm on the Earth's surface. +type ChordAngle float64 + +const ( + // NegativeChordAngle represents a chord angle smaller than the zero angle. + // The only valid operations on a NegativeChordAngle are comparisons, + // Angle conversions, and Successor/Predecessor. + NegativeChordAngle = ChordAngle(-1) + + // RightChordAngle represents a chord angle of 90 degrees (a "right angle"). + RightChordAngle = ChordAngle(2) + + // StraightChordAngle represents a chord angle of 180 degrees (a "straight angle"). + // This is the maximum finite chord angle. + StraightChordAngle = ChordAngle(4) + + // maxLength2 is the square of the maximum length allowed in a ChordAngle. + maxLength2 = 4.0 +) + +// ChordAngleFromAngle returns a ChordAngle from the given Angle. +func ChordAngleFromAngle(a Angle) ChordAngle { + if a < 0 { + return NegativeChordAngle + } + if a.isInf() { + return InfChordAngle() + } + l := 2 * math.Sin(0.5*math.Min(math.Pi, a.Radians())) + return ChordAngle(l * l) +} + +// ChordAngleFromSquaredLength returns a ChordAngle from the squared chord length. +// Note that the argument is automatically clamped to a maximum of 4 to +// handle possible roundoff errors. The argument must be non-negative. +func ChordAngleFromSquaredLength(length2 float64) ChordAngle { + if length2 > maxLength2 { + return StraightChordAngle + } + return ChordAngle(length2) +} + +// Expanded returns a new ChordAngle that has been adjusted by the given error +// bound (which can be positive or negative). Error should be the value +// returned by either MaxPointError or MaxAngleError. For example: +// a := ChordAngleFromPoints(x, y) +// a1 := a.Expanded(a.MaxPointError()) +func (c ChordAngle) Expanded(e float64) ChordAngle { + // If the angle is special, don't change it. Otherwise clamp it to the valid range. + if c.isSpecial() { + return c + } + return ChordAngle(math.Max(0.0, math.Min(maxLength2, float64(c)+e))) +} + +// Angle converts this ChordAngle to an Angle. +func (c ChordAngle) Angle() Angle { + if c < 0 { + return -1 * Radian + } + if c.isInf() { + return InfAngle() + } + return Angle(2 * math.Asin(0.5*math.Sqrt(float64(c)))) +} + +// InfChordAngle returns a chord angle larger than any finite chord angle. +// The only valid operations on an InfChordAngle are comparisons, Angle +// conversions, and Successor/Predecessor. +func InfChordAngle() ChordAngle { + return ChordAngle(math.Inf(1)) +} + +// isInf reports whether this ChordAngle is infinite. +func (c ChordAngle) isInf() bool { + return math.IsInf(float64(c), 1) +} + +// isSpecial reports whether this ChordAngle is one of the special cases. +func (c ChordAngle) isSpecial() bool { + return c < 0 || c.isInf() +} + +// isValid reports whether this ChordAngle is valid or not. +func (c ChordAngle) isValid() bool { + return (c >= 0 && c <= maxLength2) || c.isSpecial() +} + +// Successor returns the smallest representable ChordAngle larger than this one. +// This can be used to convert a "<" comparison to a "<=" comparison. +// +// Note the following special cases: +// NegativeChordAngle.Successor == 0 +// StraightChordAngle.Successor == InfChordAngle +// InfChordAngle.Successor == InfChordAngle +func (c ChordAngle) Successor() ChordAngle { + if c >= maxLength2 { + return InfChordAngle() + } + if c < 0 { + return 0 + } + return ChordAngle(math.Nextafter(float64(c), 10.0)) +} + +// Predecessor returns the largest representable ChordAngle less than this one. +// +// Note the following special cases: +// InfChordAngle.Predecessor == StraightChordAngle +// ChordAngle(0).Predecessor == NegativeChordAngle +// NegativeChordAngle.Predecessor == NegativeChordAngle +func (c ChordAngle) Predecessor() ChordAngle { + if c <= 0 { + return NegativeChordAngle + } + if c > maxLength2 { + return StraightChordAngle + } + + return ChordAngle(math.Nextafter(float64(c), -10.0)) +} + +// MaxPointError returns the maximum error size for a ChordAngle constructed +// from 2 Points x and y, assuming that x and y are normalized to within the +// bounds guaranteed by s2.Point.Normalize. The error is defined with respect to +// the true distance after the points are projected to lie exactly on the sphere. +func (c ChordAngle) MaxPointError() float64 { + // There is a relative error of (2.5*dblEpsilon) when computing the squared + // distance, plus a relative error of 2 * dblEpsilon, plus an absolute error + // of (16 * dblEpsilon**2) because the lengths of the input points may differ + // from 1 by up to (2*dblEpsilon) each. (This is the maximum error in Normalize). + return 4.5*dblEpsilon*float64(c) + 16*dblEpsilon*dblEpsilon +} + +// MaxAngleError returns the maximum error for a ChordAngle constructed +// as an Angle distance. +func (c ChordAngle) MaxAngleError() float64 { + return dblEpsilon * float64(c) +} + +// Add adds the other ChordAngle to this one and returns the resulting value. +// This method assumes the ChordAngles are not special. +func (c ChordAngle) Add(other ChordAngle) ChordAngle { + // Note that this method (and Sub) is much more efficient than converting + // the ChordAngle to an Angle and adding those and converting back. It + // requires only one square root plus a few additions and multiplications. + + // Optimization for the common case where b is an error tolerance + // parameter that happens to be set to zero. + if other == 0 { + return c + } + + // Clamp the angle sum to at most 180 degrees. + if c+other >= maxLength2 { + return StraightChordAngle + } + + // Let a and b be the (non-squared) chord lengths, and let c = a+b. + // Let A, B, and C be the corresponding half-angles (a = 2*sin(A), etc). + // Then the formula below can be derived from c = 2 * sin(A+B) and the + // relationships sin(A+B) = sin(A)*cos(B) + sin(B)*cos(A) + // cos(X) = sqrt(1 - sin^2(X)) + x := float64(c * (1 - 0.25*other)) + y := float64(other * (1 - 0.25*c)) + return ChordAngle(math.Min(maxLength2, x+y+2*math.Sqrt(x*y))) +} + +// Sub subtracts the other ChordAngle from this one and returns the resulting +// value. This method assumes the ChordAngles are not special. +func (c ChordAngle) Sub(other ChordAngle) ChordAngle { + if other == 0 { + return c + } + if c <= other { + return 0 + } + x := float64(c * (1 - 0.25*other)) + y := float64(other * (1 - 0.25*c)) + return ChordAngle(math.Max(0.0, x+y-2*math.Sqrt(x*y))) +} + +// Sin returns the sine of this chord angle. This method is more efficient +// than converting to Angle and performing the computation. +func (c ChordAngle) Sin() float64 { + return math.Sqrt(c.Sin2()) +} + +// Sin2 returns the square of the sine of this chord angle. +// It is more efficient than Sin. +func (c ChordAngle) Sin2() float64 { + // Let a be the (non-squared) chord length, and let A be the corresponding + // half-angle (a = 2*sin(A)). The formula below can be derived from: + // sin(2*A) = 2 * sin(A) * cos(A) + // cos^2(A) = 1 - sin^2(A) + // This is much faster than converting to an angle and computing its sine. + return float64(c * (1 - 0.25*c)) +} + +// Cos returns the cosine of this chord angle. This method is more efficient +// than converting to Angle and performing the computation. +func (c ChordAngle) Cos() float64 { + // cos(2*A) = cos^2(A) - sin^2(A) = 1 - 2*sin^2(A) + return float64(1 - 0.5*c) +} + +// Tan returns the tangent of this chord angle. +func (c ChordAngle) Tan() float64 { + return c.Sin() / c.Cos() +} + +// TODO(roberts): Differences from C++: +// Helpers to/from E5/E6/E7 +// Helpers to/from degrees and radians directly. +// FastUpperBoundFrom(angle Angle) diff --git a/vendor/github.com/golang/geo/s1/doc.go b/vendor/github.com/golang/geo/s1/doc.go new file mode 100644 index 00000000..52a2c526 --- /dev/null +++ b/vendor/github.com/golang/geo/s1/doc.go @@ -0,0 +1,20 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* +Package s1 implements types and functions for working with geometry in S¹ (circular geometry). + +See ../s2 for a more detailed overview. +*/ +package s1 diff --git a/vendor/github.com/golang/geo/s1/interval.go b/vendor/github.com/golang/geo/s1/interval.go new file mode 100644 index 00000000..6fea5221 --- /dev/null +++ b/vendor/github.com/golang/geo/s1/interval.go @@ -0,0 +1,462 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s1 + +import ( + "math" + "strconv" +) + +// An Interval represents a closed interval on a unit circle (also known +// as a 1-dimensional sphere). It is capable of representing the empty +// interval (containing no points), the full interval (containing all +// points), and zero-length intervals (containing a single point). +// +// Points are represented by the angle they make with the positive x-axis in +// the range [-π, π]. An interval is represented by its lower and upper +// bounds (both inclusive, since the interval is closed). The lower bound may +// be greater than the upper bound, in which case the interval is "inverted" +// (i.e. it passes through the point (-1, 0)). +// +// The point (-1, 0) has two valid representations, π and -π. The +// normalized representation of this point is π, so that endpoints +// of normal intervals are in the range (-π, π]. We normalize the latter to +// the former in IntervalFromEndpoints. However, we take advantage of the point +// -π to construct two special intervals: +// The full interval is [-π, π] +// The empty interval is [π, -π]. +// +// Treat the exported fields as read-only. +type Interval struct { + Lo, Hi float64 +} + +// IntervalFromEndpoints constructs a new interval from endpoints. +// Both arguments must be in the range [-π,π]. This function allows inverted intervals +// to be created. +func IntervalFromEndpoints(lo, hi float64) Interval { + i := Interval{lo, hi} + if lo == -math.Pi && hi != math.Pi { + i.Lo = math.Pi + } + if hi == -math.Pi && lo != math.Pi { + i.Hi = math.Pi + } + return i +} + +// IntervalFromPointPair returns the minimal interval containing the two given points. +// Both arguments must be in [-π,π]. +func IntervalFromPointPair(a, b float64) Interval { + if a == -math.Pi { + a = math.Pi + } + if b == -math.Pi { + b = math.Pi + } + if positiveDistance(a, b) <= math.Pi { + return Interval{a, b} + } + return Interval{b, a} +} + +// EmptyInterval returns an empty interval. +func EmptyInterval() Interval { return Interval{math.Pi, -math.Pi} } + +// FullInterval returns a full interval. +func FullInterval() Interval { return Interval{-math.Pi, math.Pi} } + +// IsValid reports whether the interval is valid. +func (i Interval) IsValid() bool { + return (math.Abs(i.Lo) <= math.Pi && math.Abs(i.Hi) <= math.Pi && + !(i.Lo == -math.Pi && i.Hi != math.Pi) && + !(i.Hi == -math.Pi && i.Lo != math.Pi)) +} + +// IsFull reports whether the interval is full. +func (i Interval) IsFull() bool { return i.Lo == -math.Pi && i.Hi == math.Pi } + +// IsEmpty reports whether the interval is empty. +func (i Interval) IsEmpty() bool { return i.Lo == math.Pi && i.Hi == -math.Pi } + +// IsInverted reports whether the interval is inverted; that is, whether Lo > Hi. +func (i Interval) IsInverted() bool { return i.Lo > i.Hi } + +// Invert returns the interval with endpoints swapped. +func (i Interval) Invert() Interval { + return Interval{i.Hi, i.Lo} +} + +// Center returns the midpoint of the interval. +// It is undefined for full and empty intervals. +func (i Interval) Center() float64 { + c := 0.5 * (i.Lo + i.Hi) + if !i.IsInverted() { + return c + } + if c <= 0 { + return c + math.Pi + } + return c - math.Pi +} + +// Length returns the length of the interval. +// The length of an empty interval is negative. +func (i Interval) Length() float64 { + l := i.Hi - i.Lo + if l >= 0 { + return l + } + l += 2 * math.Pi + if l > 0 { + return l + } + return -1 +} + +// Assumes p ∈ (-π,π]. +func (i Interval) fastContains(p float64) bool { + if i.IsInverted() { + return (p >= i.Lo || p <= i.Hi) && !i.IsEmpty() + } + return p >= i.Lo && p <= i.Hi +} + +// Contains returns true iff the interval contains p. +// Assumes p ∈ [-π,π]. +func (i Interval) Contains(p float64) bool { + if p == -math.Pi { + p = math.Pi + } + return i.fastContains(p) +} + +// ContainsInterval returns true iff the interval contains oi. +func (i Interval) ContainsInterval(oi Interval) bool { + if i.IsInverted() { + if oi.IsInverted() { + return oi.Lo >= i.Lo && oi.Hi <= i.Hi + } + return (oi.Lo >= i.Lo || oi.Hi <= i.Hi) && !i.IsEmpty() + } + if oi.IsInverted() { + return i.IsFull() || oi.IsEmpty() + } + return oi.Lo >= i.Lo && oi.Hi <= i.Hi +} + +// InteriorContains returns true iff the interior of the interval contains p. +// Assumes p ∈ [-π,π]. +func (i Interval) InteriorContains(p float64) bool { + if p == -math.Pi { + p = math.Pi + } + if i.IsInverted() { + return p > i.Lo || p < i.Hi + } + return (p > i.Lo && p < i.Hi) || i.IsFull() +} + +// InteriorContainsInterval returns true iff the interior of the interval contains oi. +func (i Interval) InteriorContainsInterval(oi Interval) bool { + if i.IsInverted() { + if oi.IsInverted() { + return (oi.Lo > i.Lo && oi.Hi < i.Hi) || oi.IsEmpty() + } + return oi.Lo > i.Lo || oi.Hi < i.Hi + } + if oi.IsInverted() { + return i.IsFull() || oi.IsEmpty() + } + return (oi.Lo > i.Lo && oi.Hi < i.Hi) || i.IsFull() +} + +// Intersects returns true iff the interval contains any points in common with oi. +func (i Interval) Intersects(oi Interval) bool { + if i.IsEmpty() || oi.IsEmpty() { + return false + } + if i.IsInverted() { + return oi.IsInverted() || oi.Lo <= i.Hi || oi.Hi >= i.Lo + } + if oi.IsInverted() { + return oi.Lo <= i.Hi || oi.Hi >= i.Lo + } + return oi.Lo <= i.Hi && oi.Hi >= i.Lo +} + +// InteriorIntersects returns true iff the interior of the interval contains any points in common with oi, including the latter's boundary. +func (i Interval) InteriorIntersects(oi Interval) bool { + if i.IsEmpty() || oi.IsEmpty() || i.Lo == i.Hi { + return false + } + if i.IsInverted() { + return oi.IsInverted() || oi.Lo < i.Hi || oi.Hi > i.Lo + } + if oi.IsInverted() { + return oi.Lo < i.Hi || oi.Hi > i.Lo + } + return (oi.Lo < i.Hi && oi.Hi > i.Lo) || i.IsFull() +} + +// Compute distance from a to b in [0,2π], in a numerically stable way. +func positiveDistance(a, b float64) float64 { + d := b - a + if d >= 0 { + return d + } + return (b + math.Pi) - (a - math.Pi) +} + +// Union returns the smallest interval that contains both the interval and oi. +func (i Interval) Union(oi Interval) Interval { + if oi.IsEmpty() { + return i + } + if i.fastContains(oi.Lo) { + if i.fastContains(oi.Hi) { + // Either oi ⊂ i, or i ∪ oi is the full interval. + if i.ContainsInterval(oi) { + return i + } + return FullInterval() + } + return Interval{i.Lo, oi.Hi} + } + if i.fastContains(oi.Hi) { + return Interval{oi.Lo, i.Hi} + } + + // Neither endpoint of oi is in i. Either i ⊂ oi, or i and oi are disjoint. + if i.IsEmpty() || oi.fastContains(i.Lo) { + return oi + } + + // This is the only hard case where we need to find the closest pair of endpoints. + if positiveDistance(oi.Hi, i.Lo) < positiveDistance(i.Hi, oi.Lo) { + return Interval{oi.Lo, i.Hi} + } + return Interval{i.Lo, oi.Hi} +} + +// Intersection returns the smallest interval that contains the intersection of the interval and oi. +func (i Interval) Intersection(oi Interval) Interval { + if oi.IsEmpty() { + return EmptyInterval() + } + if i.fastContains(oi.Lo) { + if i.fastContains(oi.Hi) { + // Either oi ⊂ i, or i and oi intersect twice. Neither are empty. + // In the first case we want to return i (which is shorter than oi). + // In the second case one of them is inverted, and the smallest interval + // that covers the two disjoint pieces is the shorter of i and oi. + // We thus want to pick the shorter of i and oi in both cases. + if oi.Length() < i.Length() { + return oi + } + return i + } + return Interval{oi.Lo, i.Hi} + } + if i.fastContains(oi.Hi) { + return Interval{i.Lo, oi.Hi} + } + + // Neither endpoint of oi is in i. Either i ⊂ oi, or i and oi are disjoint. + if oi.fastContains(i.Lo) { + return i + } + return EmptyInterval() +} + +// AddPoint returns the interval expanded by the minimum amount necessary such +// that it contains the given point "p" (an angle in the range [-π, π]). +func (i Interval) AddPoint(p float64) Interval { + if math.Abs(p) > math.Pi { + return i + } + if p == -math.Pi { + p = math.Pi + } + if i.fastContains(p) { + return i + } + if i.IsEmpty() { + return Interval{p, p} + } + if positiveDistance(p, i.Lo) < positiveDistance(i.Hi, p) { + return Interval{p, i.Hi} + } + return Interval{i.Lo, p} +} + +// Define the maximum rounding error for arithmetic operations. Depending on the +// platform the mantissa precision may be different than others, so we choose to +// use specific values to be consistent across all. +// The values come from the C++ implementation. +var ( + // epsilon is a small number that represents a reasonable level of noise between two + // values that can be considered to be equal. + epsilon = 1e-15 + // dblEpsilon is a smaller number for values that require more precision. + dblEpsilon = 2.220446049e-16 +) + +// Expanded returns an interval that has been expanded on each side by margin. +// If margin is negative, then the function shrinks the interval on +// each side by margin instead. The resulting interval may be empty or +// full. Any expansion (positive or negative) of a full interval remains +// full, and any expansion of an empty interval remains empty. +func (i Interval) Expanded(margin float64) Interval { + if margin >= 0 { + if i.IsEmpty() { + return i + } + // Check whether this interval will be full after expansion, allowing + // for a rounding error when computing each endpoint. + if i.Length()+2*margin+2*dblEpsilon >= 2*math.Pi { + return FullInterval() + } + } else { + if i.IsFull() { + return i + } + // Check whether this interval will be empty after expansion, allowing + // for a rounding error when computing each endpoint. + if i.Length()+2*margin-2*dblEpsilon <= 0 { + return EmptyInterval() + } + } + result := IntervalFromEndpoints( + math.Remainder(i.Lo-margin, 2*math.Pi), + math.Remainder(i.Hi+margin, 2*math.Pi), + ) + if result.Lo <= -math.Pi { + result.Lo = math.Pi + } + return result +} + +// ApproxEqual reports whether this interval can be transformed into the given +// interval by moving each endpoint by at most ε, without the +// endpoints crossing (which would invert the interval). Empty and full +// intervals are considered to start at an arbitrary point on the unit circle, +// so any interval with (length <= 2*ε) matches the empty interval, and +// any interval with (length >= 2*π - 2*ε) matches the full interval. +func (i Interval) ApproxEqual(other Interval) bool { + // Full and empty intervals require special cases because the endpoints + // are considered to be positioned arbitrarily. + if i.IsEmpty() { + return other.Length() <= 2*epsilon + } + if other.IsEmpty() { + return i.Length() <= 2*epsilon + } + if i.IsFull() { + return other.Length() >= 2*(math.Pi-epsilon) + } + if other.IsFull() { + return i.Length() >= 2*(math.Pi-epsilon) + } + + // The purpose of the last test below is to verify that moving the endpoints + // does not invert the interval, e.g. [-1e20, 1e20] vs. [1e20, -1e20]. + return (math.Abs(math.Remainder(other.Lo-i.Lo, 2*math.Pi)) <= epsilon && + math.Abs(math.Remainder(other.Hi-i.Hi, 2*math.Pi)) <= epsilon && + math.Abs(i.Length()-other.Length()) <= 2*epsilon) + +} + +func (i Interval) String() string { + // like "[%.7f, %.7f]" + return "[" + strconv.FormatFloat(i.Lo, 'f', 7, 64) + ", " + strconv.FormatFloat(i.Hi, 'f', 7, 64) + "]" +} + +// Complement returns the complement of the interior of the interval. An interval and +// its complement have the same boundary but do not share any interior +// values. The complement operator is not a bijection, since the complement +// of a singleton interval (containing a single value) is the same as the +// complement of an empty interval. +func (i Interval) Complement() Interval { + if i.Lo == i.Hi { + // Singleton. The interval just contains a single point. + return FullInterval() + } + // Handles empty and full. + return Interval{i.Hi, i.Lo} +} + +// ComplementCenter returns the midpoint of the complement of the interval. For full and empty +// intervals, the result is arbitrary. For a singleton interval (containing a +// single point), the result is its antipodal point on S1. +func (i Interval) ComplementCenter() float64 { + if i.Lo != i.Hi { + return i.Complement().Center() + } + // Singleton. The interval just contains a single point. + if i.Hi <= 0 { + return i.Hi + math.Pi + } + return i.Hi - math.Pi +} + +// DirectedHausdorffDistance returns the Hausdorff distance to the given interval. +// For two intervals i and y, this distance is defined by +// h(i, y) = max_{p in i} min_{q in y} d(p, q), +// where d(.,.) is measured along S1. +func (i Interval) DirectedHausdorffDistance(y Interval) Angle { + if y.ContainsInterval(i) { + return 0 // This includes the case i is empty. + } + if y.IsEmpty() { + return Angle(math.Pi) // maximum possible distance on s1. + } + yComplementCenter := y.ComplementCenter() + if i.Contains(yComplementCenter) { + return Angle(positiveDistance(y.Hi, yComplementCenter)) + } + + // The Hausdorff distance is realized by either two i.Hi endpoints or two + // i.Lo endpoints, whichever is farther apart. + hiHi := 0.0 + if IntervalFromEndpoints(y.Hi, yComplementCenter).Contains(i.Hi) { + hiHi = positiveDistance(y.Hi, i.Hi) + } + + loLo := 0.0 + if IntervalFromEndpoints(yComplementCenter, y.Lo).Contains(i.Lo) { + loLo = positiveDistance(i.Lo, y.Lo) + } + + return Angle(math.Max(hiHi, loLo)) +} + +// Project returns the closest point in the interval to the given point p. +// The interval must be non-empty. +func (i Interval) Project(p float64) float64 { + if p == -math.Pi { + p = math.Pi + } + if i.fastContains(p) { + return p + } + // Compute distance from p to each endpoint. + dlo := positiveDistance(p, i.Lo) + dhi := positiveDistance(i.Hi, p) + if dlo < dhi { + return i.Lo + } + return i.Hi +} diff --git a/vendor/github.com/hashicorp/go-immutable-radix/.travis.yml b/vendor/github.com/hashicorp/go-immutable-radix/.travis.yml deleted file mode 100644 index 1a0bbea6..00000000 --- a/vendor/github.com/hashicorp/go-immutable-radix/.travis.yml +++ /dev/null @@ -1,3 +0,0 @@ -language: go -go: - - tip diff --git a/vendor/github.com/hashicorp/go-immutable-radix/CHANGELOG.md b/vendor/github.com/hashicorp/go-immutable-radix/CHANGELOG.md index dd7c0efd..86c6d03f 100644 --- a/vendor/github.com/hashicorp/go-immutable-radix/CHANGELOG.md +++ b/vendor/github.com/hashicorp/go-immutable-radix/CHANGELOG.md @@ -1,3 +1,17 @@ +# UNRELEASED + +# 1.3.0 (September 17th, 2020) + +FEATURES + +* Add reverse tree traversal [[GH-30](https://github.com/hashicorp/go-immutable-radix/pull/30)] + +# 1.2.0 (March 18th, 2020) + +FEATURES + +* Adds a `Clone` method to `Txn` allowing transactions to be split either into two independently mutable trees. [[GH-26](https://github.com/hashicorp/go-immutable-radix/pull/26)] + # 1.1.0 (May 22nd, 2019) FEATURES diff --git a/vendor/github.com/hashicorp/go-immutable-radix/README.md b/vendor/github.com/hashicorp/go-immutable-radix/README.md index 4b6338b5..aca15a64 100644 --- a/vendor/github.com/hashicorp/go-immutable-radix/README.md +++ b/vendor/github.com/hashicorp/go-immutable-radix/README.md @@ -1,4 +1,4 @@ -go-immutable-radix [![Build Status](https://travis-ci.org/hashicorp/go-immutable-radix.png)](https://travis-ci.org/hashicorp/go-immutable-radix) +go-immutable-radix [![CircleCI](https://circleci.com/gh/hashicorp/go-immutable-radix/tree/master.svg?style=svg)](https://circleci.com/gh/hashicorp/go-immutable-radix/tree/master) ========= Provides the `iradix` package that implements an immutable [radix tree](http://en.wikipedia.org/wiki/Radix_tree). diff --git a/vendor/github.com/hashicorp/go-immutable-radix/iradix.go b/vendor/github.com/hashicorp/go-immutable-radix/iradix.go index e5e6e57f..168bda76 100644 --- a/vendor/github.com/hashicorp/go-immutable-radix/iradix.go +++ b/vendor/github.com/hashicorp/go-immutable-radix/iradix.go @@ -86,6 +86,20 @@ func (t *Tree) Txn() *Txn { return txn } +// Clone makes an independent copy of the transaction. The new transaction +// does not track any nodes and has TrackMutate turned off. The cloned transaction will contain any uncommitted writes in the original transaction but further mutations to either will be independent and result in different radix trees on Commit. A cloned transaction may be passed to another goroutine and mutated there independently however each transaction may only be mutated in a single thread. +func (t *Txn) Clone() *Txn { + // reset the writable node cache to avoid leaking future writes into the clone + t.writable = nil + + txn := &Txn{ + root: t.root, + snap: t.snap, + size: t.size, + } + return txn +} + // TrackMutate can be used to toggle if mutations are tracked. If this is enabled // then notifications will be issued for affected internal nodes and leaves when // the transaction is committed. diff --git a/vendor/github.com/hashicorp/go-immutable-radix/iter.go b/vendor/github.com/hashicorp/go-immutable-radix/iter.go index 1ecaf831..f17d0a64 100644 --- a/vendor/github.com/hashicorp/go-immutable-radix/iter.go +++ b/vendor/github.com/hashicorp/go-immutable-radix/iter.go @@ -20,7 +20,7 @@ func (i *Iterator) SeekPrefixWatch(prefix []byte) (watch <-chan struct{}) { watch = n.mutateCh search := prefix for { - // Check for key exhaution + // Check for key exhaustion if len(search) == 0 { i.node = n return @@ -60,10 +60,13 @@ func (i *Iterator) recurseMin(n *Node) *Node { if n.leaf != nil { return n } - if len(n.edges) > 0 { + nEdges := len(n.edges) + if nEdges > 1 { // Add all the other edges to the stack (the min node will be added as // we recurse) i.stack = append(i.stack, n.edges[1:]) + } + if nEdges > 0 { return i.recurseMin(n.edges[0].node) } // Shouldn't be possible @@ -77,16 +80,32 @@ func (i *Iterator) recurseMin(n *Node) *Node { func (i *Iterator) SeekLowerBound(key []byte) { // Wipe the stack. Unlike Prefix iteration, we need to build the stack as we // go because we need only a subset of edges of many nodes in the path to the - // leaf with the lower bound. + // leaf with the lower bound. Note that the iterator will still recurse into + // children that we don't traverse on the way to the reverse lower bound as it + // walks the stack. i.stack = []edges{} + // i.node starts off in the common case as pointing to the root node of the + // tree. By the time we return we have either found a lower bound and setup + // the stack to traverse all larger keys, or we have not and the stack and + // node should both be nil to prevent the iterator from assuming it is just + // iterating the whole tree from the root node. Either way this needs to end + // up as nil so just set it here. n := i.node + i.node = nil search := key found := func(n *Node) { - i.node = n i.stack = append(i.stack, edges{edge{node: n}}) } + findMin := func(n *Node) { + n = i.recurseMin(n) + if n != nil { + found(n) + return + } + } + for { // Compare current prefix with the search key's same-length prefix. var prefixCmp int @@ -100,10 +119,7 @@ func (i *Iterator) SeekLowerBound(key []byte) { // Prefix is larger, that means the lower bound is greater than the search // and from now on we need to follow the minimum path to the smallest // leaf under this subtree. - n = i.recurseMin(n) - if n != nil { - found(n) - } + findMin(n) return } @@ -115,27 +131,29 @@ func (i *Iterator) SeekLowerBound(key []byte) { } // Prefix is equal, we are still heading for an exact match. If this is a - // leaf we're done. - if n.leaf != nil { - if bytes.Compare(n.leaf.key, key) < 0 { - i.node = nil - return - } + // leaf and an exact match we're done. + if n.leaf != nil && bytes.Equal(n.leaf.key, key) { found(n) return } - // Consume the search prefix - if len(n.prefix) > len(search) { - search = []byte{} - } else { - search = search[len(n.prefix):] + // Consume the search prefix if the current node has one. Note that this is + // safe because if n.prefix is longer than the search slice prefixCmp would + // have been > 0 above and the method would have already returned. + search = search[len(n.prefix):] + + if len(search) == 0 { + // We've exhausted the search key, but the current node is not an exact + // match or not a leaf. That means that the leaf value if it exists, and + // all child nodes must be strictly greater, the smallest key in this + // subtree must be the lower bound. + findMin(n) + return } // Otherwise, take the lower bound next edge. idx, lbNode := n.getLowerBoundEdge(search[0]) if lbNode == nil { - i.node = nil return } @@ -144,7 +162,6 @@ func (i *Iterator) SeekLowerBound(key []byte) { i.stack = append(i.stack, n.edges[idx+1:]) } - i.node = lbNode // Recurse n = lbNode } @@ -155,7 +172,7 @@ func (i *Iterator) Next() ([]byte, interface{}, bool) { // Initialize our stack if needed if i.stack == nil && i.node != nil { i.stack = []edges{ - edges{ + { edge{node: i.node}, }, } diff --git a/vendor/github.com/hashicorp/go-immutable-radix/node.go b/vendor/github.com/hashicorp/go-immutable-radix/node.go index 3ab904ed..35985480 100644 --- a/vendor/github.com/hashicorp/go-immutable-radix/node.go +++ b/vendor/github.com/hashicorp/go-immutable-radix/node.go @@ -211,6 +211,12 @@ func (n *Node) Iterator() *Iterator { return &Iterator{node: n} } +// ReverseIterator is used to return an iterator at +// the given node to walk the tree backwards +func (n *Node) ReverseIterator() *ReverseIterator { + return NewReverseIterator(n) +} + // rawIterator is used to return a raw iterator at the given node to walk the // tree. func (n *Node) rawIterator() *rawIterator { @@ -224,6 +230,11 @@ func (n *Node) Walk(fn WalkFn) { recursiveWalk(n, fn) } +// WalkBackwards is used to walk the tree in reverse order +func (n *Node) WalkBackwards(fn WalkFn) { + reverseRecursiveWalk(n, fn) +} + // WalkPrefix is used to walk the tree under a prefix func (n *Node) WalkPrefix(prefix []byte, fn WalkFn) { search := prefix @@ -302,3 +313,22 @@ func recursiveWalk(n *Node, fn WalkFn) bool { } return false } + +// reverseRecursiveWalk is used to do a reverse pre-order +// walk of a node recursively. Returns true if the walk +// should be aborted +func reverseRecursiveWalk(n *Node, fn WalkFn) bool { + // Visit the leaf values if any + if n.leaf != nil && fn(n.leaf.key, n.leaf.val) { + return true + } + + // Recurse on the children in reverse order + for i := len(n.edges) - 1; i >= 0; i-- { + e := n.edges[i] + if reverseRecursiveWalk(e.node, fn) { + return true + } + } + return false +} diff --git a/vendor/github.com/hashicorp/go-immutable-radix/raw_iter.go b/vendor/github.com/hashicorp/go-immutable-radix/raw_iter.go index 04814c13..3c6a2252 100644 --- a/vendor/github.com/hashicorp/go-immutable-radix/raw_iter.go +++ b/vendor/github.com/hashicorp/go-immutable-radix/raw_iter.go @@ -41,7 +41,7 @@ func (i *rawIterator) Next() { // Initialize our stack if needed. if i.stack == nil && i.node != nil { i.stack = []rawStackEntry{ - rawStackEntry{ + { edges: edges{ edge{node: i.node}, }, diff --git a/vendor/github.com/hashicorp/go-immutable-radix/reverse_iter.go b/vendor/github.com/hashicorp/go-immutable-radix/reverse_iter.go new file mode 100644 index 00000000..554fa712 --- /dev/null +++ b/vendor/github.com/hashicorp/go-immutable-radix/reverse_iter.go @@ -0,0 +1,239 @@ +package iradix + +import ( + "bytes" +) + +// ReverseIterator is used to iterate over a set of nodes +// in reverse in-order +type ReverseIterator struct { + i *Iterator + + // expandedParents stores the set of parent nodes whose relevant children have + // already been pushed into the stack. This can happen during seek or during + // iteration. + // + // Unlike forward iteration we need to recurse into children before we can + // output the value stored in an internal leaf since all children are greater. + // We use this to track whether we have already ensured all the children are + // in the stack. + expandedParents map[*Node]struct{} +} + +// NewReverseIterator returns a new ReverseIterator at a node +func NewReverseIterator(n *Node) *ReverseIterator { + return &ReverseIterator{ + i: &Iterator{node: n}, + } +} + +// SeekPrefixWatch is used to seek the iterator to a given prefix +// and returns the watch channel of the finest granularity +func (ri *ReverseIterator) SeekPrefixWatch(prefix []byte) (watch <-chan struct{}) { + return ri.i.SeekPrefixWatch(prefix) +} + +// SeekPrefix is used to seek the iterator to a given prefix +func (ri *ReverseIterator) SeekPrefix(prefix []byte) { + ri.i.SeekPrefixWatch(prefix) +} + +// SeekReverseLowerBound is used to seek the iterator to the largest key that is +// lower or equal to the given key. There is no watch variant as it's hard to +// predict based on the radix structure which node(s) changes might affect the +// result. +func (ri *ReverseIterator) SeekReverseLowerBound(key []byte) { + // Wipe the stack. Unlike Prefix iteration, we need to build the stack as we + // go because we need only a subset of edges of many nodes in the path to the + // leaf with the lower bound. Note that the iterator will still recurse into + // children that we don't traverse on the way to the reverse lower bound as it + // walks the stack. + ri.i.stack = []edges{} + // ri.i.node starts off in the common case as pointing to the root node of the + // tree. By the time we return we have either found a lower bound and setup + // the stack to traverse all larger keys, or we have not and the stack and + // node should both be nil to prevent the iterator from assuming it is just + // iterating the whole tree from the root node. Either way this needs to end + // up as nil so just set it here. + n := ri.i.node + ri.i.node = nil + search := key + + if ri.expandedParents == nil { + ri.expandedParents = make(map[*Node]struct{}) + } + + found := func(n *Node) { + ri.i.stack = append(ri.i.stack, edges{edge{node: n}}) + // We need to mark this node as expanded in advance too otherwise the + // iterator will attempt to walk all of its children even though they are + // greater than the lower bound we have found. We've expanded it in the + // sense that all of its children that we want to walk are already in the + // stack (i.e. none of them). + ri.expandedParents[n] = struct{}{} + } + + for { + // Compare current prefix with the search key's same-length prefix. + var prefixCmp int + if len(n.prefix) < len(search) { + prefixCmp = bytes.Compare(n.prefix, search[0:len(n.prefix)]) + } else { + prefixCmp = bytes.Compare(n.prefix, search) + } + + if prefixCmp < 0 { + // Prefix is smaller than search prefix, that means there is no exact + // match for the search key. But we are looking in reverse, so the reverse + // lower bound will be the largest leaf under this subtree, since it is + // the value that would come right before the current search key if it + // were in the tree. So we need to follow the maximum path in this subtree + // to find it. Note that this is exactly what the iterator will already do + // if it finds a node in the stack that has _not_ been marked as expanded + // so in this one case we don't call `found` and instead let the iterator + // do the expansion and recursion through all the children. + ri.i.stack = append(ri.i.stack, edges{edge{node: n}}) + return + } + + if prefixCmp > 0 { + // Prefix is larger than search prefix, or there is no prefix but we've + // also exhausted the search key. Either way, that means there is no + // reverse lower bound since nothing comes before our current search + // prefix. + return + } + + // If this is a leaf, something needs to happen! Note that if it's a leaf + // and prefixCmp was zero (which it must be to get here) then the leaf value + // is either an exact match for the search, or it's lower. It can't be + // greater. + if n.isLeaf() { + + // Firstly, if it's an exact match, we're done! + if bytes.Equal(n.leaf.key, key) { + found(n) + return + } + + // It's not so this node's leaf value must be lower and could still be a + // valid contender for reverse lower bound. + + // If it has no children then we are also done. + if len(n.edges) == 0 { + // This leaf is the lower bound. + found(n) + return + } + + // Finally, this leaf is internal (has children) so we'll keep searching, + // but we need to add it to the iterator's stack since it has a leaf value + // that needs to be iterated over. It needs to be added to the stack + // before its children below as it comes first. + ri.i.stack = append(ri.i.stack, edges{edge{node: n}}) + // We also need to mark it as expanded since we'll be adding any of its + // relevant children below and so don't want the iterator to re-add them + // on its way back up the stack. + ri.expandedParents[n] = struct{}{} + } + + // Consume the search prefix. Note that this is safe because if n.prefix is + // longer than the search slice prefixCmp would have been > 0 above and the + // method would have already returned. + search = search[len(n.prefix):] + + if len(search) == 0 { + // We've exhausted the search key but we are not at a leaf. That means all + // children are greater than the search key so a reverse lower bound + // doesn't exist in this subtree. Note that there might still be one in + // the whole radix tree by following a different path somewhere further + // up. If that's the case then the iterator's stack will contain all the + // smaller nodes already and Previous will walk through them correctly. + return + } + + // Otherwise, take the lower bound next edge. + idx, lbNode := n.getLowerBoundEdge(search[0]) + + // From here, we need to update the stack with all values lower than + // the lower bound edge. Since getLowerBoundEdge() returns -1 when the + // search prefix is larger than all edges, we need to place idx at the + // last edge index so they can all be place in the stack, since they + // come before our search prefix. + if idx == -1 { + idx = len(n.edges) + } + + // Create stack edges for the all strictly lower edges in this node. + if len(n.edges[:idx]) > 0 { + ri.i.stack = append(ri.i.stack, n.edges[:idx]) + } + + // Exit if there's no lower bound edge. The stack will have the previous + // nodes already. + if lbNode == nil { + return + } + + // Recurse + n = lbNode + } +} + +// Previous returns the previous node in reverse order +func (ri *ReverseIterator) Previous() ([]byte, interface{}, bool) { + // Initialize our stack if needed + if ri.i.stack == nil && ri.i.node != nil { + ri.i.stack = []edges{ + { + edge{node: ri.i.node}, + }, + } + } + + if ri.expandedParents == nil { + ri.expandedParents = make(map[*Node]struct{}) + } + + for len(ri.i.stack) > 0 { + // Inspect the last element of the stack + n := len(ri.i.stack) + last := ri.i.stack[n-1] + m := len(last) + elem := last[m-1].node + + _, alreadyExpanded := ri.expandedParents[elem] + + // If this is an internal node and we've not seen it already, we need to + // leave it in the stack so we can return its possible leaf value _after_ + // we've recursed through all its children. + if len(elem.edges) > 0 && !alreadyExpanded { + // record that we've seen this node! + ri.expandedParents[elem] = struct{}{} + // push child edges onto stack and skip the rest of the loop to recurse + // into the largest one. + ri.i.stack = append(ri.i.stack, elem.edges) + continue + } + + // Remove the node from the stack + if m > 1 { + ri.i.stack[n-1] = last[:m-1] + } else { + ri.i.stack = ri.i.stack[:n-1] + } + // We don't need this state any more as it's no longer in the stack so we + // won't visit it again + if alreadyExpanded { + delete(ri.expandedParents, elem) + } + + // If this is a leaf, return it + if elem.leaf != nil { + return elem.leaf.key, elem.leaf.val, true + } + + // it's not a leaf so keep walking the stack to find the previous leaf + } + return nil, nil, false +} diff --git a/vendor/github.com/inconshreveable/mousetrap/LICENSE b/vendor/github.com/inconshreveable/mousetrap/LICENSE index 5f0d1fb6..5f920e97 100644 --- a/vendor/github.com/inconshreveable/mousetrap/LICENSE +++ b/vendor/github.com/inconshreveable/mousetrap/LICENSE @@ -1,13 +1,201 @@ -Copyright 2014 Alan Shreve + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - http://www.apache.org/licenses/LICENSE-2.0 + 1. Definitions. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2022 Alan Shreve (@inconshreveable) + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/inconshreveable/mousetrap/trap_others.go b/vendor/github.com/inconshreveable/mousetrap/trap_others.go index 9d2d8a4b..06a91f08 100644 --- a/vendor/github.com/inconshreveable/mousetrap/trap_others.go +++ b/vendor/github.com/inconshreveable/mousetrap/trap_others.go @@ -1,3 +1,4 @@ +//go:build !windows // +build !windows package mousetrap diff --git a/vendor/github.com/inconshreveable/mousetrap/trap_windows.go b/vendor/github.com/inconshreveable/mousetrap/trap_windows.go index 336142a5..0c568802 100644 --- a/vendor/github.com/inconshreveable/mousetrap/trap_windows.go +++ b/vendor/github.com/inconshreveable/mousetrap/trap_windows.go @@ -1,81 +1,32 @@ -// +build windows -// +build !go1.4 - package mousetrap import ( - "fmt" - "os" "syscall" "unsafe" ) -const ( - // defined by the Win32 API - th32cs_snapprocess uintptr = 0x2 -) - -var ( - kernel = syscall.MustLoadDLL("kernel32.dll") - CreateToolhelp32Snapshot = kernel.MustFindProc("CreateToolhelp32Snapshot") - Process32First = kernel.MustFindProc("Process32FirstW") - Process32Next = kernel.MustFindProc("Process32NextW") -) - -// ProcessEntry32 structure defined by the Win32 API -type processEntry32 struct { - dwSize uint32 - cntUsage uint32 - th32ProcessID uint32 - th32DefaultHeapID int - th32ModuleID uint32 - cntThreads uint32 - th32ParentProcessID uint32 - pcPriClassBase int32 - dwFlags uint32 - szExeFile [syscall.MAX_PATH]uint16 -} - -func getProcessEntry(pid int) (pe *processEntry32, err error) { - snapshot, _, e1 := CreateToolhelp32Snapshot.Call(th32cs_snapprocess, uintptr(0)) - if snapshot == uintptr(syscall.InvalidHandle) { - err = fmt.Errorf("CreateToolhelp32Snapshot: %v", e1) - return +func getProcessEntry(pid int) (*syscall.ProcessEntry32, error) { + snapshot, err := syscall.CreateToolhelp32Snapshot(syscall.TH32CS_SNAPPROCESS, 0) + if err != nil { + return nil, err } - defer syscall.CloseHandle(syscall.Handle(snapshot)) - - var processEntry processEntry32 - processEntry.dwSize = uint32(unsafe.Sizeof(processEntry)) - ok, _, e1 := Process32First.Call(snapshot, uintptr(unsafe.Pointer(&processEntry))) - if ok == 0 { - err = fmt.Errorf("Process32First: %v", e1) - return + defer syscall.CloseHandle(snapshot) + var procEntry syscall.ProcessEntry32 + procEntry.Size = uint32(unsafe.Sizeof(procEntry)) + if err = syscall.Process32First(snapshot, &procEntry); err != nil { + return nil, err } - for { - if processEntry.th32ProcessID == uint32(pid) { - pe = &processEntry - return + if procEntry.ProcessID == uint32(pid) { + return &procEntry, nil } - - ok, _, e1 = Process32Next.Call(snapshot, uintptr(unsafe.Pointer(&processEntry))) - if ok == 0 { - err = fmt.Errorf("Process32Next: %v", e1) - return + err = syscall.Process32Next(snapshot, &procEntry) + if err != nil { + return nil, err } } } -func getppid() (pid int, err error) { - pe, err := getProcessEntry(os.Getpid()) - if err != nil { - return - } - - pid = int(pe.th32ParentProcessID) - return -} - // StartedByExplorer returns true if the program was invoked by the user double-clicking // on the executable from explorer.exe // @@ -83,16 +34,9 @@ func getppid() (pid int, err error) { // It does not guarantee that the program was run from a terminal. It only can tell you // whether it was launched from explorer.exe func StartedByExplorer() bool { - ppid, err := getppid() + pe, err := getProcessEntry(syscall.Getppid()) if err != nil { return false } - - pe, err := getProcessEntry(ppid) - if err != nil { - return false - } - - name := syscall.UTF16ToString(pe.szExeFile[:]) - return name == "explorer.exe" + return "explorer.exe" == syscall.UTF16ToString(pe.ExeFile[:]) } diff --git a/vendor/github.com/inconshreveable/mousetrap/trap_windows_1.4.go b/vendor/github.com/inconshreveable/mousetrap/trap_windows_1.4.go deleted file mode 100644 index 9a28e57c..00000000 --- a/vendor/github.com/inconshreveable/mousetrap/trap_windows_1.4.go +++ /dev/null @@ -1,46 +0,0 @@ -// +build windows -// +build go1.4 - -package mousetrap - -import ( - "os" - "syscall" - "unsafe" -) - -func getProcessEntry(pid int) (*syscall.ProcessEntry32, error) { - snapshot, err := syscall.CreateToolhelp32Snapshot(syscall.TH32CS_SNAPPROCESS, 0) - if err != nil { - return nil, err - } - defer syscall.CloseHandle(snapshot) - var procEntry syscall.ProcessEntry32 - procEntry.Size = uint32(unsafe.Sizeof(procEntry)) - if err = syscall.Process32First(snapshot, &procEntry); err != nil { - return nil, err - } - for { - if procEntry.ProcessID == uint32(pid) { - return &procEntry, nil - } - err = syscall.Process32Next(snapshot, &procEntry) - if err != nil { - return nil, err - } - } -} - -// StartedByExplorer returns true if the program was invoked by the user double-clicking -// on the executable from explorer.exe -// -// It is conservative and returns false if any of the internal calls fail. -// It does not guarantee that the program was run from a terminal. It only can tell you -// whether it was launched from explorer.exe -func StartedByExplorer() bool { - pe, err := getProcessEntry(os.Getppid()) - if err != nil { - return false - } - return "explorer.exe" == syscall.UTF16ToString(pe.ExeFile[:]) -} diff --git a/vendor/github.com/json-iterator/go/.codecov.yml b/vendor/github.com/json-iterator/go/.codecov.yml new file mode 100644 index 00000000..955dc0be --- /dev/null +++ b/vendor/github.com/json-iterator/go/.codecov.yml @@ -0,0 +1,3 @@ +ignore: + - "output_tests/.*" + diff --git a/vendor/github.com/json-iterator/go/.gitignore b/vendor/github.com/json-iterator/go/.gitignore new file mode 100644 index 00000000..15556530 --- /dev/null +++ b/vendor/github.com/json-iterator/go/.gitignore @@ -0,0 +1,4 @@ +/vendor +/bug_test.go +/coverage.txt +/.idea diff --git a/vendor/github.com/json-iterator/go/.travis.yml b/vendor/github.com/json-iterator/go/.travis.yml new file mode 100644 index 00000000..449e67cd --- /dev/null +++ b/vendor/github.com/json-iterator/go/.travis.yml @@ -0,0 +1,14 @@ +language: go + +go: + - 1.8.x + - 1.x + +before_install: + - go get -t -v ./... + +script: + - ./test.sh + +after_success: + - bash <(curl -s https://codecov.io/bash) diff --git a/vendor/github.com/json-iterator/go/Gopkg.lock b/vendor/github.com/json-iterator/go/Gopkg.lock new file mode 100644 index 00000000..c8a9fbb3 --- /dev/null +++ b/vendor/github.com/json-iterator/go/Gopkg.lock @@ -0,0 +1,21 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[[projects]] + name = "github.com/modern-go/concurrent" + packages = ["."] + revision = "e0a39a4cb4216ea8db28e22a69f4ec25610d513a" + version = "1.0.0" + +[[projects]] + name = "github.com/modern-go/reflect2" + packages = ["."] + revision = "4b7aa43c6742a2c18fdef89dd197aaae7dac7ccd" + version = "1.0.1" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + inputs-digest = "ea54a775e5a354cb015502d2e7aa4b74230fc77e894f34a838b268c25ec8eeb8" + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/vendor/github.com/json-iterator/go/Gopkg.toml b/vendor/github.com/json-iterator/go/Gopkg.toml new file mode 100644 index 00000000..313a0f88 --- /dev/null +++ b/vendor/github.com/json-iterator/go/Gopkg.toml @@ -0,0 +1,26 @@ +# Gopkg.toml example +# +# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md +# for detailed Gopkg.toml documentation. +# +# required = ["github.com/user/thing/cmd/thing"] +# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] +# +# [[constraint]] +# name = "github.com/user/project" +# version = "1.0.0" +# +# [[constraint]] +# name = "github.com/user/project2" +# branch = "dev" +# source = "github.com/myfork/project2" +# +# [[override]] +# name = "github.com/x/y" +# version = "2.4.0" + +ignored = ["github.com/davecgh/go-spew*","github.com/google/gofuzz*","github.com/stretchr/testify*"] + +[[constraint]] + name = "github.com/modern-go/reflect2" + version = "1.0.1" diff --git a/vendor/github.com/json-iterator/go/LICENSE b/vendor/github.com/json-iterator/go/LICENSE new file mode 100644 index 00000000..2cf4f5ab --- /dev/null +++ b/vendor/github.com/json-iterator/go/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2016 json-iterator + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/json-iterator/go/README.md b/vendor/github.com/json-iterator/go/README.md new file mode 100644 index 00000000..c589addf --- /dev/null +++ b/vendor/github.com/json-iterator/go/README.md @@ -0,0 +1,85 @@ +[![Sourcegraph](https://sourcegraph.com/github.com/json-iterator/go/-/badge.svg)](https://sourcegraph.com/github.com/json-iterator/go?badge) +[![GoDoc](http://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)](https://pkg.go.dev/github.com/json-iterator/go) +[![Build Status](https://travis-ci.org/json-iterator/go.svg?branch=master)](https://travis-ci.org/json-iterator/go) +[![codecov](https://codecov.io/gh/json-iterator/go/branch/master/graph/badge.svg)](https://codecov.io/gh/json-iterator/go) +[![rcard](https://goreportcard.com/badge/github.com/json-iterator/go)](https://goreportcard.com/report/github.com/json-iterator/go) +[![License](http://img.shields.io/badge/license-mit-blue.svg?style=flat-square)](https://raw.githubusercontent.com/json-iterator/go/master/LICENSE) +[![Gitter chat](https://badges.gitter.im/gitterHQ/gitter.png)](https://gitter.im/json-iterator/Lobby) + +A high-performance 100% compatible drop-in replacement of "encoding/json" + +# Benchmark + +![benchmark](http://jsoniter.com/benchmarks/go-benchmark.png) + +Source code: https://github.com/json-iterator/go-benchmark/blob/master/src/github.com/json-iterator/go-benchmark/benchmark_medium_payload_test.go + +Raw Result (easyjson requires static code generation) + +| | ns/op | allocation bytes | allocation times | +| --------------- | ----------- | ---------------- | ---------------- | +| std decode | 35510 ns/op | 1960 B/op | 99 allocs/op | +| easyjson decode | 8499 ns/op | 160 B/op | 4 allocs/op | +| jsoniter decode | 5623 ns/op | 160 B/op | 3 allocs/op | +| std encode | 2213 ns/op | 712 B/op | 5 allocs/op | +| easyjson encode | 883 ns/op | 576 B/op | 3 allocs/op | +| jsoniter encode | 837 ns/op | 384 B/op | 4 allocs/op | + +Always benchmark with your own workload. +The result depends heavily on the data input. + +# Usage + +100% compatibility with standard lib + +Replace + +```go +import "encoding/json" +json.Marshal(&data) +``` + +with + +```go +import jsoniter "github.com/json-iterator/go" + +var json = jsoniter.ConfigCompatibleWithStandardLibrary +json.Marshal(&data) +``` + +Replace + +```go +import "encoding/json" +json.Unmarshal(input, &data) +``` + +with + +```go +import jsoniter "github.com/json-iterator/go" + +var json = jsoniter.ConfigCompatibleWithStandardLibrary +json.Unmarshal(input, &data) +``` + +[More documentation](http://jsoniter.com/migrate-from-go-std.html) + +# How to get + +``` +go get github.com/json-iterator/go +``` + +# Contribution Welcomed ! + +Contributors + +- [thockin](https://github.com/thockin) +- [mattn](https://github.com/mattn) +- [cch123](https://github.com/cch123) +- [Oleg Shaldybin](https://github.com/olegshaldybin) +- [Jason Toffaletti](https://github.com/toffaletti) + +Report issue or pull request, or email taowen@gmail.com, or [![Gitter chat](https://badges.gitter.im/gitterHQ/gitter.png)](https://gitter.im/json-iterator/Lobby) diff --git a/vendor/github.com/json-iterator/go/adapter.go b/vendor/github.com/json-iterator/go/adapter.go new file mode 100644 index 00000000..92d2cc4a --- /dev/null +++ b/vendor/github.com/json-iterator/go/adapter.go @@ -0,0 +1,150 @@ +package jsoniter + +import ( + "bytes" + "io" +) + +// RawMessage to make replace json with jsoniter +type RawMessage []byte + +// Unmarshal adapts to json/encoding Unmarshal API +// +// Unmarshal parses the JSON-encoded data and stores the result in the value pointed to by v. +// Refer to https://godoc.org/encoding/json#Unmarshal for more information +func Unmarshal(data []byte, v interface{}) error { + return ConfigDefault.Unmarshal(data, v) +} + +// UnmarshalFromString is a convenient method to read from string instead of []byte +func UnmarshalFromString(str string, v interface{}) error { + return ConfigDefault.UnmarshalFromString(str, v) +} + +// Get quick method to get value from deeply nested JSON structure +func Get(data []byte, path ...interface{}) Any { + return ConfigDefault.Get(data, path...) +} + +// Marshal adapts to json/encoding Marshal API +// +// Marshal returns the JSON encoding of v, adapts to json/encoding Marshal API +// Refer to https://godoc.org/encoding/json#Marshal for more information +func Marshal(v interface{}) ([]byte, error) { + return ConfigDefault.Marshal(v) +} + +// MarshalIndent same as json.MarshalIndent. Prefix is not supported. +func MarshalIndent(v interface{}, prefix, indent string) ([]byte, error) { + return ConfigDefault.MarshalIndent(v, prefix, indent) +} + +// MarshalToString convenient method to write as string instead of []byte +func MarshalToString(v interface{}) (string, error) { + return ConfigDefault.MarshalToString(v) +} + +// NewDecoder adapts to json/stream NewDecoder API. +// +// NewDecoder returns a new decoder that reads from r. +// +// Instead of a json/encoding Decoder, an Decoder is returned +// Refer to https://godoc.org/encoding/json#NewDecoder for more information +func NewDecoder(reader io.Reader) *Decoder { + return ConfigDefault.NewDecoder(reader) +} + +// Decoder reads and decodes JSON values from an input stream. +// Decoder provides identical APIs with json/stream Decoder (Token() and UseNumber() are in progress) +type Decoder struct { + iter *Iterator +} + +// Decode decode JSON into interface{} +func (adapter *Decoder) Decode(obj interface{}) error { + if adapter.iter.head == adapter.iter.tail && adapter.iter.reader != nil { + if !adapter.iter.loadMore() { + return io.EOF + } + } + adapter.iter.ReadVal(obj) + err := adapter.iter.Error + if err == io.EOF { + return nil + } + return adapter.iter.Error +} + +// More is there more? +func (adapter *Decoder) More() bool { + iter := adapter.iter + if iter.Error != nil { + return false + } + c := iter.nextToken() + if c == 0 { + return false + } + iter.unreadByte() + return c != ']' && c != '}' +} + +// Buffered remaining buffer +func (adapter *Decoder) Buffered() io.Reader { + remaining := adapter.iter.buf[adapter.iter.head:adapter.iter.tail] + return bytes.NewReader(remaining) +} + +// UseNumber causes the Decoder to unmarshal a number into an interface{} as a +// Number instead of as a float64. +func (adapter *Decoder) UseNumber() { + cfg := adapter.iter.cfg.configBeforeFrozen + cfg.UseNumber = true + adapter.iter.cfg = cfg.frozeWithCacheReuse(adapter.iter.cfg.extraExtensions) +} + +// DisallowUnknownFields causes the Decoder to return an error when the destination +// is a struct and the input contains object keys which do not match any +// non-ignored, exported fields in the destination. +func (adapter *Decoder) DisallowUnknownFields() { + cfg := adapter.iter.cfg.configBeforeFrozen + cfg.DisallowUnknownFields = true + adapter.iter.cfg = cfg.frozeWithCacheReuse(adapter.iter.cfg.extraExtensions) +} + +// NewEncoder same as json.NewEncoder +func NewEncoder(writer io.Writer) *Encoder { + return ConfigDefault.NewEncoder(writer) +} + +// Encoder same as json.Encoder +type Encoder struct { + stream *Stream +} + +// Encode encode interface{} as JSON to io.Writer +func (adapter *Encoder) Encode(val interface{}) error { + adapter.stream.WriteVal(val) + adapter.stream.WriteRaw("\n") + adapter.stream.Flush() + return adapter.stream.Error +} + +// SetIndent set the indention. Prefix is not supported +func (adapter *Encoder) SetIndent(prefix, indent string) { + config := adapter.stream.cfg.configBeforeFrozen + config.IndentionStep = len(indent) + adapter.stream.cfg = config.frozeWithCacheReuse(adapter.stream.cfg.extraExtensions) +} + +// SetEscapeHTML escape html by default, set to false to disable +func (adapter *Encoder) SetEscapeHTML(escapeHTML bool) { + config := adapter.stream.cfg.configBeforeFrozen + config.EscapeHTML = escapeHTML + adapter.stream.cfg = config.frozeWithCacheReuse(adapter.stream.cfg.extraExtensions) +} + +// Valid reports whether data is a valid JSON encoding. +func Valid(data []byte) bool { + return ConfigDefault.Valid(data) +} diff --git a/vendor/github.com/json-iterator/go/any.go b/vendor/github.com/json-iterator/go/any.go new file mode 100644 index 00000000..f6b8aeab --- /dev/null +++ b/vendor/github.com/json-iterator/go/any.go @@ -0,0 +1,325 @@ +package jsoniter + +import ( + "errors" + "fmt" + "github.com/modern-go/reflect2" + "io" + "reflect" + "strconv" + "unsafe" +) + +// Any generic object representation. +// The lazy json implementation holds []byte and parse lazily. +type Any interface { + LastError() error + ValueType() ValueType + MustBeValid() Any + ToBool() bool + ToInt() int + ToInt32() int32 + ToInt64() int64 + ToUint() uint + ToUint32() uint32 + ToUint64() uint64 + ToFloat32() float32 + ToFloat64() float64 + ToString() string + ToVal(val interface{}) + Get(path ...interface{}) Any + Size() int + Keys() []string + GetInterface() interface{} + WriteTo(stream *Stream) +} + +type baseAny struct{} + +func (any *baseAny) Get(path ...interface{}) Any { + return &invalidAny{baseAny{}, fmt.Errorf("GetIndex %v from simple value", path)} +} + +func (any *baseAny) Size() int { + return 0 +} + +func (any *baseAny) Keys() []string { + return []string{} +} + +func (any *baseAny) ToVal(obj interface{}) { + panic("not implemented") +} + +// WrapInt32 turn int32 into Any interface +func WrapInt32(val int32) Any { + return &int32Any{baseAny{}, val} +} + +// WrapInt64 turn int64 into Any interface +func WrapInt64(val int64) Any { + return &int64Any{baseAny{}, val} +} + +// WrapUint32 turn uint32 into Any interface +func WrapUint32(val uint32) Any { + return &uint32Any{baseAny{}, val} +} + +// WrapUint64 turn uint64 into Any interface +func WrapUint64(val uint64) Any { + return &uint64Any{baseAny{}, val} +} + +// WrapFloat64 turn float64 into Any interface +func WrapFloat64(val float64) Any { + return &floatAny{baseAny{}, val} +} + +// WrapString turn string into Any interface +func WrapString(val string) Any { + return &stringAny{baseAny{}, val} +} + +// Wrap turn a go object into Any interface +func Wrap(val interface{}) Any { + if val == nil { + return &nilAny{} + } + asAny, isAny := val.(Any) + if isAny { + return asAny + } + typ := reflect2.TypeOf(val) + switch typ.Kind() { + case reflect.Slice: + return wrapArray(val) + case reflect.Struct: + return wrapStruct(val) + case reflect.Map: + return wrapMap(val) + case reflect.String: + return WrapString(val.(string)) + case reflect.Int: + if strconv.IntSize == 32 { + return WrapInt32(int32(val.(int))) + } + return WrapInt64(int64(val.(int))) + case reflect.Int8: + return WrapInt32(int32(val.(int8))) + case reflect.Int16: + return WrapInt32(int32(val.(int16))) + case reflect.Int32: + return WrapInt32(val.(int32)) + case reflect.Int64: + return WrapInt64(val.(int64)) + case reflect.Uint: + if strconv.IntSize == 32 { + return WrapUint32(uint32(val.(uint))) + } + return WrapUint64(uint64(val.(uint))) + case reflect.Uintptr: + if ptrSize == 32 { + return WrapUint32(uint32(val.(uintptr))) + } + return WrapUint64(uint64(val.(uintptr))) + case reflect.Uint8: + return WrapUint32(uint32(val.(uint8))) + case reflect.Uint16: + return WrapUint32(uint32(val.(uint16))) + case reflect.Uint32: + return WrapUint32(uint32(val.(uint32))) + case reflect.Uint64: + return WrapUint64(val.(uint64)) + case reflect.Float32: + return WrapFloat64(float64(val.(float32))) + case reflect.Float64: + return WrapFloat64(val.(float64)) + case reflect.Bool: + if val.(bool) == true { + return &trueAny{} + } + return &falseAny{} + } + return &invalidAny{baseAny{}, fmt.Errorf("unsupported type: %v", typ)} +} + +// ReadAny read next JSON element as an Any object. It is a better json.RawMessage. +func (iter *Iterator) ReadAny() Any { + return iter.readAny() +} + +func (iter *Iterator) readAny() Any { + c := iter.nextToken() + switch c { + case '"': + iter.unreadByte() + return &stringAny{baseAny{}, iter.ReadString()} + case 'n': + iter.skipThreeBytes('u', 'l', 'l') // null + return &nilAny{} + case 't': + iter.skipThreeBytes('r', 'u', 'e') // true + return &trueAny{} + case 'f': + iter.skipFourBytes('a', 'l', 's', 'e') // false + return &falseAny{} + case '{': + return iter.readObjectAny() + case '[': + return iter.readArrayAny() + case '-': + return iter.readNumberAny(false) + case 0: + return &invalidAny{baseAny{}, errors.New("input is empty")} + default: + return iter.readNumberAny(true) + } +} + +func (iter *Iterator) readNumberAny(positive bool) Any { + iter.startCapture(iter.head - 1) + iter.skipNumber() + lazyBuf := iter.stopCapture() + return &numberLazyAny{baseAny{}, iter.cfg, lazyBuf, nil} +} + +func (iter *Iterator) readObjectAny() Any { + iter.startCapture(iter.head - 1) + iter.skipObject() + lazyBuf := iter.stopCapture() + return &objectLazyAny{baseAny{}, iter.cfg, lazyBuf, nil} +} + +func (iter *Iterator) readArrayAny() Any { + iter.startCapture(iter.head - 1) + iter.skipArray() + lazyBuf := iter.stopCapture() + return &arrayLazyAny{baseAny{}, iter.cfg, lazyBuf, nil} +} + +func locateObjectField(iter *Iterator, target string) []byte { + var found []byte + iter.ReadObjectCB(func(iter *Iterator, field string) bool { + if field == target { + found = iter.SkipAndReturnBytes() + return false + } + iter.Skip() + return true + }) + return found +} + +func locateArrayElement(iter *Iterator, target int) []byte { + var found []byte + n := 0 + iter.ReadArrayCB(func(iter *Iterator) bool { + if n == target { + found = iter.SkipAndReturnBytes() + return false + } + iter.Skip() + n++ + return true + }) + return found +} + +func locatePath(iter *Iterator, path []interface{}) Any { + for i, pathKeyObj := range path { + switch pathKey := pathKeyObj.(type) { + case string: + valueBytes := locateObjectField(iter, pathKey) + if valueBytes == nil { + return newInvalidAny(path[i:]) + } + iter.ResetBytes(valueBytes) + case int: + valueBytes := locateArrayElement(iter, pathKey) + if valueBytes == nil { + return newInvalidAny(path[i:]) + } + iter.ResetBytes(valueBytes) + case int32: + if '*' == pathKey { + return iter.readAny().Get(path[i:]...) + } + return newInvalidAny(path[i:]) + default: + return newInvalidAny(path[i:]) + } + } + if iter.Error != nil && iter.Error != io.EOF { + return &invalidAny{baseAny{}, iter.Error} + } + return iter.readAny() +} + +var anyType = reflect2.TypeOfPtr((*Any)(nil)).Elem() + +func createDecoderOfAny(ctx *ctx, typ reflect2.Type) ValDecoder { + if typ == anyType { + return &directAnyCodec{} + } + if typ.Implements(anyType) { + return &anyCodec{ + valType: typ, + } + } + return nil +} + +func createEncoderOfAny(ctx *ctx, typ reflect2.Type) ValEncoder { + if typ == anyType { + return &directAnyCodec{} + } + if typ.Implements(anyType) { + return &anyCodec{ + valType: typ, + } + } + return nil +} + +type anyCodec struct { + valType reflect2.Type +} + +func (codec *anyCodec) Decode(ptr unsafe.Pointer, iter *Iterator) { + panic("not implemented") +} + +func (codec *anyCodec) Encode(ptr unsafe.Pointer, stream *Stream) { + obj := codec.valType.UnsafeIndirect(ptr) + any := obj.(Any) + any.WriteTo(stream) +} + +func (codec *anyCodec) IsEmpty(ptr unsafe.Pointer) bool { + obj := codec.valType.UnsafeIndirect(ptr) + any := obj.(Any) + return any.Size() == 0 +} + +type directAnyCodec struct { +} + +func (codec *directAnyCodec) Decode(ptr unsafe.Pointer, iter *Iterator) { + *(*Any)(ptr) = iter.readAny() +} + +func (codec *directAnyCodec) Encode(ptr unsafe.Pointer, stream *Stream) { + any := *(*Any)(ptr) + if any == nil { + stream.WriteNil() + return + } + any.WriteTo(stream) +} + +func (codec *directAnyCodec) IsEmpty(ptr unsafe.Pointer) bool { + any := *(*Any)(ptr) + return any.Size() == 0 +} diff --git a/vendor/github.com/json-iterator/go/any_array.go b/vendor/github.com/json-iterator/go/any_array.go new file mode 100644 index 00000000..0449e9aa --- /dev/null +++ b/vendor/github.com/json-iterator/go/any_array.go @@ -0,0 +1,278 @@ +package jsoniter + +import ( + "reflect" + "unsafe" +) + +type arrayLazyAny struct { + baseAny + cfg *frozenConfig + buf []byte + err error +} + +func (any *arrayLazyAny) ValueType() ValueType { + return ArrayValue +} + +func (any *arrayLazyAny) MustBeValid() Any { + return any +} + +func (any *arrayLazyAny) LastError() error { + return any.err +} + +func (any *arrayLazyAny) ToBool() bool { + iter := any.cfg.BorrowIterator(any.buf) + defer any.cfg.ReturnIterator(iter) + return iter.ReadArray() +} + +func (any *arrayLazyAny) ToInt() int { + if any.ToBool() { + return 1 + } + return 0 +} + +func (any *arrayLazyAny) ToInt32() int32 { + if any.ToBool() { + return 1 + } + return 0 +} + +func (any *arrayLazyAny) ToInt64() int64 { + if any.ToBool() { + return 1 + } + return 0 +} + +func (any *arrayLazyAny) ToUint() uint { + if any.ToBool() { + return 1 + } + return 0 +} + +func (any *arrayLazyAny) ToUint32() uint32 { + if any.ToBool() { + return 1 + } + return 0 +} + +func (any *arrayLazyAny) ToUint64() uint64 { + if any.ToBool() { + return 1 + } + return 0 +} + +func (any *arrayLazyAny) ToFloat32() float32 { + if any.ToBool() { + return 1 + } + return 0 +} + +func (any *arrayLazyAny) ToFloat64() float64 { + if any.ToBool() { + return 1 + } + return 0 +} + +func (any *arrayLazyAny) ToString() string { + return *(*string)(unsafe.Pointer(&any.buf)) +} + +func (any *arrayLazyAny) ToVal(val interface{}) { + iter := any.cfg.BorrowIterator(any.buf) + defer any.cfg.ReturnIterator(iter) + iter.ReadVal(val) +} + +func (any *arrayLazyAny) Get(path ...interface{}) Any { + if len(path) == 0 { + return any + } + switch firstPath := path[0].(type) { + case int: + iter := any.cfg.BorrowIterator(any.buf) + defer any.cfg.ReturnIterator(iter) + valueBytes := locateArrayElement(iter, firstPath) + if valueBytes == nil { + return newInvalidAny(path) + } + iter.ResetBytes(valueBytes) + return locatePath(iter, path[1:]) + case int32: + if '*' == firstPath { + iter := any.cfg.BorrowIterator(any.buf) + defer any.cfg.ReturnIterator(iter) + arr := make([]Any, 0) + iter.ReadArrayCB(func(iter *Iterator) bool { + found := iter.readAny().Get(path[1:]...) + if found.ValueType() != InvalidValue { + arr = append(arr, found) + } + return true + }) + return wrapArray(arr) + } + return newInvalidAny(path) + default: + return newInvalidAny(path) + } +} + +func (any *arrayLazyAny) Size() int { + size := 0 + iter := any.cfg.BorrowIterator(any.buf) + defer any.cfg.ReturnIterator(iter) + iter.ReadArrayCB(func(iter *Iterator) bool { + size++ + iter.Skip() + return true + }) + return size +} + +func (any *arrayLazyAny) WriteTo(stream *Stream) { + stream.Write(any.buf) +} + +func (any *arrayLazyAny) GetInterface() interface{} { + iter := any.cfg.BorrowIterator(any.buf) + defer any.cfg.ReturnIterator(iter) + return iter.Read() +} + +type arrayAny struct { + baseAny + val reflect.Value +} + +func wrapArray(val interface{}) *arrayAny { + return &arrayAny{baseAny{}, reflect.ValueOf(val)} +} + +func (any *arrayAny) ValueType() ValueType { + return ArrayValue +} + +func (any *arrayAny) MustBeValid() Any { + return any +} + +func (any *arrayAny) LastError() error { + return nil +} + +func (any *arrayAny) ToBool() bool { + return any.val.Len() != 0 +} + +func (any *arrayAny) ToInt() int { + if any.val.Len() == 0 { + return 0 + } + return 1 +} + +func (any *arrayAny) ToInt32() int32 { + if any.val.Len() == 0 { + return 0 + } + return 1 +} + +func (any *arrayAny) ToInt64() int64 { + if any.val.Len() == 0 { + return 0 + } + return 1 +} + +func (any *arrayAny) ToUint() uint { + if any.val.Len() == 0 { + return 0 + } + return 1 +} + +func (any *arrayAny) ToUint32() uint32 { + if any.val.Len() == 0 { + return 0 + } + return 1 +} + +func (any *arrayAny) ToUint64() uint64 { + if any.val.Len() == 0 { + return 0 + } + return 1 +} + +func (any *arrayAny) ToFloat32() float32 { + if any.val.Len() == 0 { + return 0 + } + return 1 +} + +func (any *arrayAny) ToFloat64() float64 { + if any.val.Len() == 0 { + return 0 + } + return 1 +} + +func (any *arrayAny) ToString() string { + str, _ := MarshalToString(any.val.Interface()) + return str +} + +func (any *arrayAny) Get(path ...interface{}) Any { + if len(path) == 0 { + return any + } + switch firstPath := path[0].(type) { + case int: + if firstPath < 0 || firstPath >= any.val.Len() { + return newInvalidAny(path) + } + return Wrap(any.val.Index(firstPath).Interface()) + case int32: + if '*' == firstPath { + mappedAll := make([]Any, 0) + for i := 0; i < any.val.Len(); i++ { + mapped := Wrap(any.val.Index(i).Interface()).Get(path[1:]...) + if mapped.ValueType() != InvalidValue { + mappedAll = append(mappedAll, mapped) + } + } + return wrapArray(mappedAll) + } + return newInvalidAny(path) + default: + return newInvalidAny(path) + } +} + +func (any *arrayAny) Size() int { + return any.val.Len() +} + +func (any *arrayAny) WriteTo(stream *Stream) { + stream.WriteVal(any.val) +} + +func (any *arrayAny) GetInterface() interface{} { + return any.val.Interface() +} diff --git a/vendor/github.com/json-iterator/go/any_bool.go b/vendor/github.com/json-iterator/go/any_bool.go new file mode 100644 index 00000000..9452324a --- /dev/null +++ b/vendor/github.com/json-iterator/go/any_bool.go @@ -0,0 +1,137 @@ +package jsoniter + +type trueAny struct { + baseAny +} + +func (any *trueAny) LastError() error { + return nil +} + +func (any *trueAny) ToBool() bool { + return true +} + +func (any *trueAny) ToInt() int { + return 1 +} + +func (any *trueAny) ToInt32() int32 { + return 1 +} + +func (any *trueAny) ToInt64() int64 { + return 1 +} + +func (any *trueAny) ToUint() uint { + return 1 +} + +func (any *trueAny) ToUint32() uint32 { + return 1 +} + +func (any *trueAny) ToUint64() uint64 { + return 1 +} + +func (any *trueAny) ToFloat32() float32 { + return 1 +} + +func (any *trueAny) ToFloat64() float64 { + return 1 +} + +func (any *trueAny) ToString() string { + return "true" +} + +func (any *trueAny) WriteTo(stream *Stream) { + stream.WriteTrue() +} + +func (any *trueAny) Parse() *Iterator { + return nil +} + +func (any *trueAny) GetInterface() interface{} { + return true +} + +func (any *trueAny) ValueType() ValueType { + return BoolValue +} + +func (any *trueAny) MustBeValid() Any { + return any +} + +type falseAny struct { + baseAny +} + +func (any *falseAny) LastError() error { + return nil +} + +func (any *falseAny) ToBool() bool { + return false +} + +func (any *falseAny) ToInt() int { + return 0 +} + +func (any *falseAny) ToInt32() int32 { + return 0 +} + +func (any *falseAny) ToInt64() int64 { + return 0 +} + +func (any *falseAny) ToUint() uint { + return 0 +} + +func (any *falseAny) ToUint32() uint32 { + return 0 +} + +func (any *falseAny) ToUint64() uint64 { + return 0 +} + +func (any *falseAny) ToFloat32() float32 { + return 0 +} + +func (any *falseAny) ToFloat64() float64 { + return 0 +} + +func (any *falseAny) ToString() string { + return "false" +} + +func (any *falseAny) WriteTo(stream *Stream) { + stream.WriteFalse() +} + +func (any *falseAny) Parse() *Iterator { + return nil +} + +func (any *falseAny) GetInterface() interface{} { + return false +} + +func (any *falseAny) ValueType() ValueType { + return BoolValue +} + +func (any *falseAny) MustBeValid() Any { + return any +} diff --git a/vendor/github.com/json-iterator/go/any_float.go b/vendor/github.com/json-iterator/go/any_float.go new file mode 100644 index 00000000..35fdb094 --- /dev/null +++ b/vendor/github.com/json-iterator/go/any_float.go @@ -0,0 +1,83 @@ +package jsoniter + +import ( + "strconv" +) + +type floatAny struct { + baseAny + val float64 +} + +func (any *floatAny) Parse() *Iterator { + return nil +} + +func (any *floatAny) ValueType() ValueType { + return NumberValue +} + +func (any *floatAny) MustBeValid() Any { + return any +} + +func (any *floatAny) LastError() error { + return nil +} + +func (any *floatAny) ToBool() bool { + return any.ToFloat64() != 0 +} + +func (any *floatAny) ToInt() int { + return int(any.val) +} + +func (any *floatAny) ToInt32() int32 { + return int32(any.val) +} + +func (any *floatAny) ToInt64() int64 { + return int64(any.val) +} + +func (any *floatAny) ToUint() uint { + if any.val > 0 { + return uint(any.val) + } + return 0 +} + +func (any *floatAny) ToUint32() uint32 { + if any.val > 0 { + return uint32(any.val) + } + return 0 +} + +func (any *floatAny) ToUint64() uint64 { + if any.val > 0 { + return uint64(any.val) + } + return 0 +} + +func (any *floatAny) ToFloat32() float32 { + return float32(any.val) +} + +func (any *floatAny) ToFloat64() float64 { + return any.val +} + +func (any *floatAny) ToString() string { + return strconv.FormatFloat(any.val, 'E', -1, 64) +} + +func (any *floatAny) WriteTo(stream *Stream) { + stream.WriteFloat64(any.val) +} + +func (any *floatAny) GetInterface() interface{} { + return any.val +} diff --git a/vendor/github.com/json-iterator/go/any_int32.go b/vendor/github.com/json-iterator/go/any_int32.go new file mode 100644 index 00000000..1b56f399 --- /dev/null +++ b/vendor/github.com/json-iterator/go/any_int32.go @@ -0,0 +1,74 @@ +package jsoniter + +import ( + "strconv" +) + +type int32Any struct { + baseAny + val int32 +} + +func (any *int32Any) LastError() error { + return nil +} + +func (any *int32Any) ValueType() ValueType { + return NumberValue +} + +func (any *int32Any) MustBeValid() Any { + return any +} + +func (any *int32Any) ToBool() bool { + return any.val != 0 +} + +func (any *int32Any) ToInt() int { + return int(any.val) +} + +func (any *int32Any) ToInt32() int32 { + return any.val +} + +func (any *int32Any) ToInt64() int64 { + return int64(any.val) +} + +func (any *int32Any) ToUint() uint { + return uint(any.val) +} + +func (any *int32Any) ToUint32() uint32 { + return uint32(any.val) +} + +func (any *int32Any) ToUint64() uint64 { + return uint64(any.val) +} + +func (any *int32Any) ToFloat32() float32 { + return float32(any.val) +} + +func (any *int32Any) ToFloat64() float64 { + return float64(any.val) +} + +func (any *int32Any) ToString() string { + return strconv.FormatInt(int64(any.val), 10) +} + +func (any *int32Any) WriteTo(stream *Stream) { + stream.WriteInt32(any.val) +} + +func (any *int32Any) Parse() *Iterator { + return nil +} + +func (any *int32Any) GetInterface() interface{} { + return any.val +} diff --git a/vendor/github.com/json-iterator/go/any_int64.go b/vendor/github.com/json-iterator/go/any_int64.go new file mode 100644 index 00000000..c440d72b --- /dev/null +++ b/vendor/github.com/json-iterator/go/any_int64.go @@ -0,0 +1,74 @@ +package jsoniter + +import ( + "strconv" +) + +type int64Any struct { + baseAny + val int64 +} + +func (any *int64Any) LastError() error { + return nil +} + +func (any *int64Any) ValueType() ValueType { + return NumberValue +} + +func (any *int64Any) MustBeValid() Any { + return any +} + +func (any *int64Any) ToBool() bool { + return any.val != 0 +} + +func (any *int64Any) ToInt() int { + return int(any.val) +} + +func (any *int64Any) ToInt32() int32 { + return int32(any.val) +} + +func (any *int64Any) ToInt64() int64 { + return any.val +} + +func (any *int64Any) ToUint() uint { + return uint(any.val) +} + +func (any *int64Any) ToUint32() uint32 { + return uint32(any.val) +} + +func (any *int64Any) ToUint64() uint64 { + return uint64(any.val) +} + +func (any *int64Any) ToFloat32() float32 { + return float32(any.val) +} + +func (any *int64Any) ToFloat64() float64 { + return float64(any.val) +} + +func (any *int64Any) ToString() string { + return strconv.FormatInt(any.val, 10) +} + +func (any *int64Any) WriteTo(stream *Stream) { + stream.WriteInt64(any.val) +} + +func (any *int64Any) Parse() *Iterator { + return nil +} + +func (any *int64Any) GetInterface() interface{} { + return any.val +} diff --git a/vendor/github.com/json-iterator/go/any_invalid.go b/vendor/github.com/json-iterator/go/any_invalid.go new file mode 100644 index 00000000..1d859eac --- /dev/null +++ b/vendor/github.com/json-iterator/go/any_invalid.go @@ -0,0 +1,82 @@ +package jsoniter + +import "fmt" + +type invalidAny struct { + baseAny + err error +} + +func newInvalidAny(path []interface{}) *invalidAny { + return &invalidAny{baseAny{}, fmt.Errorf("%v not found", path)} +} + +func (any *invalidAny) LastError() error { + return any.err +} + +func (any *invalidAny) ValueType() ValueType { + return InvalidValue +} + +func (any *invalidAny) MustBeValid() Any { + panic(any.err) +} + +func (any *invalidAny) ToBool() bool { + return false +} + +func (any *invalidAny) ToInt() int { + return 0 +} + +func (any *invalidAny) ToInt32() int32 { + return 0 +} + +func (any *invalidAny) ToInt64() int64 { + return 0 +} + +func (any *invalidAny) ToUint() uint { + return 0 +} + +func (any *invalidAny) ToUint32() uint32 { + return 0 +} + +func (any *invalidAny) ToUint64() uint64 { + return 0 +} + +func (any *invalidAny) ToFloat32() float32 { + return 0 +} + +func (any *invalidAny) ToFloat64() float64 { + return 0 +} + +func (any *invalidAny) ToString() string { + return "" +} + +func (any *invalidAny) WriteTo(stream *Stream) { +} + +func (any *invalidAny) Get(path ...interface{}) Any { + if any.err == nil { + return &invalidAny{baseAny{}, fmt.Errorf("get %v from invalid", path)} + } + return &invalidAny{baseAny{}, fmt.Errorf("%v, get %v from invalid", any.err, path)} +} + +func (any *invalidAny) Parse() *Iterator { + return nil +} + +func (any *invalidAny) GetInterface() interface{} { + return nil +} diff --git a/vendor/github.com/json-iterator/go/any_nil.go b/vendor/github.com/json-iterator/go/any_nil.go new file mode 100644 index 00000000..d04cb54c --- /dev/null +++ b/vendor/github.com/json-iterator/go/any_nil.go @@ -0,0 +1,69 @@ +package jsoniter + +type nilAny struct { + baseAny +} + +func (any *nilAny) LastError() error { + return nil +} + +func (any *nilAny) ValueType() ValueType { + return NilValue +} + +func (any *nilAny) MustBeValid() Any { + return any +} + +func (any *nilAny) ToBool() bool { + return false +} + +func (any *nilAny) ToInt() int { + return 0 +} + +func (any *nilAny) ToInt32() int32 { + return 0 +} + +func (any *nilAny) ToInt64() int64 { + return 0 +} + +func (any *nilAny) ToUint() uint { + return 0 +} + +func (any *nilAny) ToUint32() uint32 { + return 0 +} + +func (any *nilAny) ToUint64() uint64 { + return 0 +} + +func (any *nilAny) ToFloat32() float32 { + return 0 +} + +func (any *nilAny) ToFloat64() float64 { + return 0 +} + +func (any *nilAny) ToString() string { + return "" +} + +func (any *nilAny) WriteTo(stream *Stream) { + stream.WriteNil() +} + +func (any *nilAny) Parse() *Iterator { + return nil +} + +func (any *nilAny) GetInterface() interface{} { + return nil +} diff --git a/vendor/github.com/json-iterator/go/any_number.go b/vendor/github.com/json-iterator/go/any_number.go new file mode 100644 index 00000000..9d1e901a --- /dev/null +++ b/vendor/github.com/json-iterator/go/any_number.go @@ -0,0 +1,123 @@ +package jsoniter + +import ( + "io" + "unsafe" +) + +type numberLazyAny struct { + baseAny + cfg *frozenConfig + buf []byte + err error +} + +func (any *numberLazyAny) ValueType() ValueType { + return NumberValue +} + +func (any *numberLazyAny) MustBeValid() Any { + return any +} + +func (any *numberLazyAny) LastError() error { + return any.err +} + +func (any *numberLazyAny) ToBool() bool { + return any.ToFloat64() != 0 +} + +func (any *numberLazyAny) ToInt() int { + iter := any.cfg.BorrowIterator(any.buf) + defer any.cfg.ReturnIterator(iter) + val := iter.ReadInt() + if iter.Error != nil && iter.Error != io.EOF { + any.err = iter.Error + } + return val +} + +func (any *numberLazyAny) ToInt32() int32 { + iter := any.cfg.BorrowIterator(any.buf) + defer any.cfg.ReturnIterator(iter) + val := iter.ReadInt32() + if iter.Error != nil && iter.Error != io.EOF { + any.err = iter.Error + } + return val +} + +func (any *numberLazyAny) ToInt64() int64 { + iter := any.cfg.BorrowIterator(any.buf) + defer any.cfg.ReturnIterator(iter) + val := iter.ReadInt64() + if iter.Error != nil && iter.Error != io.EOF { + any.err = iter.Error + } + return val +} + +func (any *numberLazyAny) ToUint() uint { + iter := any.cfg.BorrowIterator(any.buf) + defer any.cfg.ReturnIterator(iter) + val := iter.ReadUint() + if iter.Error != nil && iter.Error != io.EOF { + any.err = iter.Error + } + return val +} + +func (any *numberLazyAny) ToUint32() uint32 { + iter := any.cfg.BorrowIterator(any.buf) + defer any.cfg.ReturnIterator(iter) + val := iter.ReadUint32() + if iter.Error != nil && iter.Error != io.EOF { + any.err = iter.Error + } + return val +} + +func (any *numberLazyAny) ToUint64() uint64 { + iter := any.cfg.BorrowIterator(any.buf) + defer any.cfg.ReturnIterator(iter) + val := iter.ReadUint64() + if iter.Error != nil && iter.Error != io.EOF { + any.err = iter.Error + } + return val +} + +func (any *numberLazyAny) ToFloat32() float32 { + iter := any.cfg.BorrowIterator(any.buf) + defer any.cfg.ReturnIterator(iter) + val := iter.ReadFloat32() + if iter.Error != nil && iter.Error != io.EOF { + any.err = iter.Error + } + return val +} + +func (any *numberLazyAny) ToFloat64() float64 { + iter := any.cfg.BorrowIterator(any.buf) + defer any.cfg.ReturnIterator(iter) + val := iter.ReadFloat64() + if iter.Error != nil && iter.Error != io.EOF { + any.err = iter.Error + } + return val +} + +func (any *numberLazyAny) ToString() string { + return *(*string)(unsafe.Pointer(&any.buf)) +} + +func (any *numberLazyAny) WriteTo(stream *Stream) { + stream.Write(any.buf) +} + +func (any *numberLazyAny) GetInterface() interface{} { + iter := any.cfg.BorrowIterator(any.buf) + defer any.cfg.ReturnIterator(iter) + return iter.Read() +} diff --git a/vendor/github.com/json-iterator/go/any_object.go b/vendor/github.com/json-iterator/go/any_object.go new file mode 100644 index 00000000..c44ef5c9 --- /dev/null +++ b/vendor/github.com/json-iterator/go/any_object.go @@ -0,0 +1,374 @@ +package jsoniter + +import ( + "reflect" + "unsafe" +) + +type objectLazyAny struct { + baseAny + cfg *frozenConfig + buf []byte + err error +} + +func (any *objectLazyAny) ValueType() ValueType { + return ObjectValue +} + +func (any *objectLazyAny) MustBeValid() Any { + return any +} + +func (any *objectLazyAny) LastError() error { + return any.err +} + +func (any *objectLazyAny) ToBool() bool { + return true +} + +func (any *objectLazyAny) ToInt() int { + return 0 +} + +func (any *objectLazyAny) ToInt32() int32 { + return 0 +} + +func (any *objectLazyAny) ToInt64() int64 { + return 0 +} + +func (any *objectLazyAny) ToUint() uint { + return 0 +} + +func (any *objectLazyAny) ToUint32() uint32 { + return 0 +} + +func (any *objectLazyAny) ToUint64() uint64 { + return 0 +} + +func (any *objectLazyAny) ToFloat32() float32 { + return 0 +} + +func (any *objectLazyAny) ToFloat64() float64 { + return 0 +} + +func (any *objectLazyAny) ToString() string { + return *(*string)(unsafe.Pointer(&any.buf)) +} + +func (any *objectLazyAny) ToVal(obj interface{}) { + iter := any.cfg.BorrowIterator(any.buf) + defer any.cfg.ReturnIterator(iter) + iter.ReadVal(obj) +} + +func (any *objectLazyAny) Get(path ...interface{}) Any { + if len(path) == 0 { + return any + } + switch firstPath := path[0].(type) { + case string: + iter := any.cfg.BorrowIterator(any.buf) + defer any.cfg.ReturnIterator(iter) + valueBytes := locateObjectField(iter, firstPath) + if valueBytes == nil { + return newInvalidAny(path) + } + iter.ResetBytes(valueBytes) + return locatePath(iter, path[1:]) + case int32: + if '*' == firstPath { + mappedAll := map[string]Any{} + iter := any.cfg.BorrowIterator(any.buf) + defer any.cfg.ReturnIterator(iter) + iter.ReadMapCB(func(iter *Iterator, field string) bool { + mapped := locatePath(iter, path[1:]) + if mapped.ValueType() != InvalidValue { + mappedAll[field] = mapped + } + return true + }) + return wrapMap(mappedAll) + } + return newInvalidAny(path) + default: + return newInvalidAny(path) + } +} + +func (any *objectLazyAny) Keys() []string { + keys := []string{} + iter := any.cfg.BorrowIterator(any.buf) + defer any.cfg.ReturnIterator(iter) + iter.ReadMapCB(func(iter *Iterator, field string) bool { + iter.Skip() + keys = append(keys, field) + return true + }) + return keys +} + +func (any *objectLazyAny) Size() int { + size := 0 + iter := any.cfg.BorrowIterator(any.buf) + defer any.cfg.ReturnIterator(iter) + iter.ReadObjectCB(func(iter *Iterator, field string) bool { + iter.Skip() + size++ + return true + }) + return size +} + +func (any *objectLazyAny) WriteTo(stream *Stream) { + stream.Write(any.buf) +} + +func (any *objectLazyAny) GetInterface() interface{} { + iter := any.cfg.BorrowIterator(any.buf) + defer any.cfg.ReturnIterator(iter) + return iter.Read() +} + +type objectAny struct { + baseAny + err error + val reflect.Value +} + +func wrapStruct(val interface{}) *objectAny { + return &objectAny{baseAny{}, nil, reflect.ValueOf(val)} +} + +func (any *objectAny) ValueType() ValueType { + return ObjectValue +} + +func (any *objectAny) MustBeValid() Any { + return any +} + +func (any *objectAny) Parse() *Iterator { + return nil +} + +func (any *objectAny) LastError() error { + return any.err +} + +func (any *objectAny) ToBool() bool { + return any.val.NumField() != 0 +} + +func (any *objectAny) ToInt() int { + return 0 +} + +func (any *objectAny) ToInt32() int32 { + return 0 +} + +func (any *objectAny) ToInt64() int64 { + return 0 +} + +func (any *objectAny) ToUint() uint { + return 0 +} + +func (any *objectAny) ToUint32() uint32 { + return 0 +} + +func (any *objectAny) ToUint64() uint64 { + return 0 +} + +func (any *objectAny) ToFloat32() float32 { + return 0 +} + +func (any *objectAny) ToFloat64() float64 { + return 0 +} + +func (any *objectAny) ToString() string { + str, err := MarshalToString(any.val.Interface()) + any.err = err + return str +} + +func (any *objectAny) Get(path ...interface{}) Any { + if len(path) == 0 { + return any + } + switch firstPath := path[0].(type) { + case string: + field := any.val.FieldByName(firstPath) + if !field.IsValid() { + return newInvalidAny(path) + } + return Wrap(field.Interface()) + case int32: + if '*' == firstPath { + mappedAll := map[string]Any{} + for i := 0; i < any.val.NumField(); i++ { + field := any.val.Field(i) + if field.CanInterface() { + mapped := Wrap(field.Interface()).Get(path[1:]...) + if mapped.ValueType() != InvalidValue { + mappedAll[any.val.Type().Field(i).Name] = mapped + } + } + } + return wrapMap(mappedAll) + } + return newInvalidAny(path) + default: + return newInvalidAny(path) + } +} + +func (any *objectAny) Keys() []string { + keys := make([]string, 0, any.val.NumField()) + for i := 0; i < any.val.NumField(); i++ { + keys = append(keys, any.val.Type().Field(i).Name) + } + return keys +} + +func (any *objectAny) Size() int { + return any.val.NumField() +} + +func (any *objectAny) WriteTo(stream *Stream) { + stream.WriteVal(any.val) +} + +func (any *objectAny) GetInterface() interface{} { + return any.val.Interface() +} + +type mapAny struct { + baseAny + err error + val reflect.Value +} + +func wrapMap(val interface{}) *mapAny { + return &mapAny{baseAny{}, nil, reflect.ValueOf(val)} +} + +func (any *mapAny) ValueType() ValueType { + return ObjectValue +} + +func (any *mapAny) MustBeValid() Any { + return any +} + +func (any *mapAny) Parse() *Iterator { + return nil +} + +func (any *mapAny) LastError() error { + return any.err +} + +func (any *mapAny) ToBool() bool { + return true +} + +func (any *mapAny) ToInt() int { + return 0 +} + +func (any *mapAny) ToInt32() int32 { + return 0 +} + +func (any *mapAny) ToInt64() int64 { + return 0 +} + +func (any *mapAny) ToUint() uint { + return 0 +} + +func (any *mapAny) ToUint32() uint32 { + return 0 +} + +func (any *mapAny) ToUint64() uint64 { + return 0 +} + +func (any *mapAny) ToFloat32() float32 { + return 0 +} + +func (any *mapAny) ToFloat64() float64 { + return 0 +} + +func (any *mapAny) ToString() string { + str, err := MarshalToString(any.val.Interface()) + any.err = err + return str +} + +func (any *mapAny) Get(path ...interface{}) Any { + if len(path) == 0 { + return any + } + switch firstPath := path[0].(type) { + case int32: + if '*' == firstPath { + mappedAll := map[string]Any{} + for _, key := range any.val.MapKeys() { + keyAsStr := key.String() + element := Wrap(any.val.MapIndex(key).Interface()) + mapped := element.Get(path[1:]...) + if mapped.ValueType() != InvalidValue { + mappedAll[keyAsStr] = mapped + } + } + return wrapMap(mappedAll) + } + return newInvalidAny(path) + default: + value := any.val.MapIndex(reflect.ValueOf(firstPath)) + if !value.IsValid() { + return newInvalidAny(path) + } + return Wrap(value.Interface()) + } +} + +func (any *mapAny) Keys() []string { + keys := make([]string, 0, any.val.Len()) + for _, key := range any.val.MapKeys() { + keys = append(keys, key.String()) + } + return keys +} + +func (any *mapAny) Size() int { + return any.val.Len() +} + +func (any *mapAny) WriteTo(stream *Stream) { + stream.WriteVal(any.val) +} + +func (any *mapAny) GetInterface() interface{} { + return any.val.Interface() +} diff --git a/vendor/github.com/json-iterator/go/any_str.go b/vendor/github.com/json-iterator/go/any_str.go new file mode 100644 index 00000000..1f12f661 --- /dev/null +++ b/vendor/github.com/json-iterator/go/any_str.go @@ -0,0 +1,166 @@ +package jsoniter + +import ( + "fmt" + "strconv" +) + +type stringAny struct { + baseAny + val string +} + +func (any *stringAny) Get(path ...interface{}) Any { + if len(path) == 0 { + return any + } + return &invalidAny{baseAny{}, fmt.Errorf("GetIndex %v from simple value", path)} +} + +func (any *stringAny) Parse() *Iterator { + return nil +} + +func (any *stringAny) ValueType() ValueType { + return StringValue +} + +func (any *stringAny) MustBeValid() Any { + return any +} + +func (any *stringAny) LastError() error { + return nil +} + +func (any *stringAny) ToBool() bool { + str := any.ToString() + if str == "0" { + return false + } + for _, c := range str { + switch c { + case ' ', '\n', '\r', '\t': + default: + return true + } + } + return false +} + +func (any *stringAny) ToInt() int { + return int(any.ToInt64()) + +} + +func (any *stringAny) ToInt32() int32 { + return int32(any.ToInt64()) +} + +func (any *stringAny) ToInt64() int64 { + if any.val == "" { + return 0 + } + + flag := 1 + startPos := 0 + if any.val[0] == '+' || any.val[0] == '-' { + startPos = 1 + } + + if any.val[0] == '-' { + flag = -1 + } + + endPos := startPos + for i := startPos; i < len(any.val); i++ { + if any.val[i] >= '0' && any.val[i] <= '9' { + endPos = i + 1 + } else { + break + } + } + parsed, _ := strconv.ParseInt(any.val[startPos:endPos], 10, 64) + return int64(flag) * parsed +} + +func (any *stringAny) ToUint() uint { + return uint(any.ToUint64()) +} + +func (any *stringAny) ToUint32() uint32 { + return uint32(any.ToUint64()) +} + +func (any *stringAny) ToUint64() uint64 { + if any.val == "" { + return 0 + } + + startPos := 0 + + if any.val[0] == '-' { + return 0 + } + if any.val[0] == '+' { + startPos = 1 + } + + endPos := startPos + for i := startPos; i < len(any.val); i++ { + if any.val[i] >= '0' && any.val[i] <= '9' { + endPos = i + 1 + } else { + break + } + } + parsed, _ := strconv.ParseUint(any.val[startPos:endPos], 10, 64) + return parsed +} + +func (any *stringAny) ToFloat32() float32 { + return float32(any.ToFloat64()) +} + +func (any *stringAny) ToFloat64() float64 { + if len(any.val) == 0 { + return 0 + } + + // first char invalid + if any.val[0] != '+' && any.val[0] != '-' && (any.val[0] > '9' || any.val[0] < '0') { + return 0 + } + + // extract valid num expression from string + // eg 123true => 123, -12.12xxa => -12.12 + endPos := 1 + for i := 1; i < len(any.val); i++ { + if any.val[i] == '.' || any.val[i] == 'e' || any.val[i] == 'E' || any.val[i] == '+' || any.val[i] == '-' { + endPos = i + 1 + continue + } + + // end position is the first char which is not digit + if any.val[i] >= '0' && any.val[i] <= '9' { + endPos = i + 1 + } else { + endPos = i + break + } + } + parsed, _ := strconv.ParseFloat(any.val[:endPos], 64) + return parsed +} + +func (any *stringAny) ToString() string { + return any.val +} + +func (any *stringAny) WriteTo(stream *Stream) { + stream.WriteString(any.val) +} + +func (any *stringAny) GetInterface() interface{} { + return any.val +} diff --git a/vendor/github.com/json-iterator/go/any_uint32.go b/vendor/github.com/json-iterator/go/any_uint32.go new file mode 100644 index 00000000..656bbd33 --- /dev/null +++ b/vendor/github.com/json-iterator/go/any_uint32.go @@ -0,0 +1,74 @@ +package jsoniter + +import ( + "strconv" +) + +type uint32Any struct { + baseAny + val uint32 +} + +func (any *uint32Any) LastError() error { + return nil +} + +func (any *uint32Any) ValueType() ValueType { + return NumberValue +} + +func (any *uint32Any) MustBeValid() Any { + return any +} + +func (any *uint32Any) ToBool() bool { + return any.val != 0 +} + +func (any *uint32Any) ToInt() int { + return int(any.val) +} + +func (any *uint32Any) ToInt32() int32 { + return int32(any.val) +} + +func (any *uint32Any) ToInt64() int64 { + return int64(any.val) +} + +func (any *uint32Any) ToUint() uint { + return uint(any.val) +} + +func (any *uint32Any) ToUint32() uint32 { + return any.val +} + +func (any *uint32Any) ToUint64() uint64 { + return uint64(any.val) +} + +func (any *uint32Any) ToFloat32() float32 { + return float32(any.val) +} + +func (any *uint32Any) ToFloat64() float64 { + return float64(any.val) +} + +func (any *uint32Any) ToString() string { + return strconv.FormatInt(int64(any.val), 10) +} + +func (any *uint32Any) WriteTo(stream *Stream) { + stream.WriteUint32(any.val) +} + +func (any *uint32Any) Parse() *Iterator { + return nil +} + +func (any *uint32Any) GetInterface() interface{} { + return any.val +} diff --git a/vendor/github.com/json-iterator/go/any_uint64.go b/vendor/github.com/json-iterator/go/any_uint64.go new file mode 100644 index 00000000..7df2fce3 --- /dev/null +++ b/vendor/github.com/json-iterator/go/any_uint64.go @@ -0,0 +1,74 @@ +package jsoniter + +import ( + "strconv" +) + +type uint64Any struct { + baseAny + val uint64 +} + +func (any *uint64Any) LastError() error { + return nil +} + +func (any *uint64Any) ValueType() ValueType { + return NumberValue +} + +func (any *uint64Any) MustBeValid() Any { + return any +} + +func (any *uint64Any) ToBool() bool { + return any.val != 0 +} + +func (any *uint64Any) ToInt() int { + return int(any.val) +} + +func (any *uint64Any) ToInt32() int32 { + return int32(any.val) +} + +func (any *uint64Any) ToInt64() int64 { + return int64(any.val) +} + +func (any *uint64Any) ToUint() uint { + return uint(any.val) +} + +func (any *uint64Any) ToUint32() uint32 { + return uint32(any.val) +} + +func (any *uint64Any) ToUint64() uint64 { + return any.val +} + +func (any *uint64Any) ToFloat32() float32 { + return float32(any.val) +} + +func (any *uint64Any) ToFloat64() float64 { + return float64(any.val) +} + +func (any *uint64Any) ToString() string { + return strconv.FormatUint(any.val, 10) +} + +func (any *uint64Any) WriteTo(stream *Stream) { + stream.WriteUint64(any.val) +} + +func (any *uint64Any) Parse() *Iterator { + return nil +} + +func (any *uint64Any) GetInterface() interface{} { + return any.val +} diff --git a/vendor/github.com/json-iterator/go/build.sh b/vendor/github.com/json-iterator/go/build.sh new file mode 100644 index 00000000..b45ef688 --- /dev/null +++ b/vendor/github.com/json-iterator/go/build.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -e +set -x + +if [ ! -d /tmp/build-golang/src/github.com/json-iterator ]; then + mkdir -p /tmp/build-golang/src/github.com/json-iterator + ln -s $PWD /tmp/build-golang/src/github.com/json-iterator/go +fi +export GOPATH=/tmp/build-golang +go get -u github.com/golang/dep/cmd/dep +cd /tmp/build-golang/src/github.com/json-iterator/go +exec $GOPATH/bin/dep ensure -update diff --git a/vendor/github.com/json-iterator/go/config.go b/vendor/github.com/json-iterator/go/config.go new file mode 100644 index 00000000..2adcdc3b --- /dev/null +++ b/vendor/github.com/json-iterator/go/config.go @@ -0,0 +1,375 @@ +package jsoniter + +import ( + "encoding/json" + "io" + "reflect" + "sync" + "unsafe" + + "github.com/modern-go/concurrent" + "github.com/modern-go/reflect2" +) + +// Config customize how the API should behave. +// The API is created from Config by Froze. +type Config struct { + IndentionStep int + MarshalFloatWith6Digits bool + EscapeHTML bool + SortMapKeys bool + UseNumber bool + DisallowUnknownFields bool + TagKey string + OnlyTaggedField bool + ValidateJsonRawMessage bool + ObjectFieldMustBeSimpleString bool + CaseSensitive bool +} + +// API the public interface of this package. +// Primary Marshal and Unmarshal. +type API interface { + IteratorPool + StreamPool + MarshalToString(v interface{}) (string, error) + Marshal(v interface{}) ([]byte, error) + MarshalIndent(v interface{}, prefix, indent string) ([]byte, error) + UnmarshalFromString(str string, v interface{}) error + Unmarshal(data []byte, v interface{}) error + Get(data []byte, path ...interface{}) Any + NewEncoder(writer io.Writer) *Encoder + NewDecoder(reader io.Reader) *Decoder + Valid(data []byte) bool + RegisterExtension(extension Extension) + DecoderOf(typ reflect2.Type) ValDecoder + EncoderOf(typ reflect2.Type) ValEncoder +} + +// ConfigDefault the default API +var ConfigDefault = Config{ + EscapeHTML: true, +}.Froze() + +// ConfigCompatibleWithStandardLibrary tries to be 100% compatible with standard library behavior +var ConfigCompatibleWithStandardLibrary = Config{ + EscapeHTML: true, + SortMapKeys: true, + ValidateJsonRawMessage: true, +}.Froze() + +// ConfigFastest marshals float with only 6 digits precision +var ConfigFastest = Config{ + EscapeHTML: false, + MarshalFloatWith6Digits: true, // will lose precession + ObjectFieldMustBeSimpleString: true, // do not unescape object field +}.Froze() + +type frozenConfig struct { + configBeforeFrozen Config + sortMapKeys bool + indentionStep int + objectFieldMustBeSimpleString bool + onlyTaggedField bool + disallowUnknownFields bool + decoderCache *concurrent.Map + encoderCache *concurrent.Map + encoderExtension Extension + decoderExtension Extension + extraExtensions []Extension + streamPool *sync.Pool + iteratorPool *sync.Pool + caseSensitive bool +} + +func (cfg *frozenConfig) initCache() { + cfg.decoderCache = concurrent.NewMap() + cfg.encoderCache = concurrent.NewMap() +} + +func (cfg *frozenConfig) addDecoderToCache(cacheKey uintptr, decoder ValDecoder) { + cfg.decoderCache.Store(cacheKey, decoder) +} + +func (cfg *frozenConfig) addEncoderToCache(cacheKey uintptr, encoder ValEncoder) { + cfg.encoderCache.Store(cacheKey, encoder) +} + +func (cfg *frozenConfig) getDecoderFromCache(cacheKey uintptr) ValDecoder { + decoder, found := cfg.decoderCache.Load(cacheKey) + if found { + return decoder.(ValDecoder) + } + return nil +} + +func (cfg *frozenConfig) getEncoderFromCache(cacheKey uintptr) ValEncoder { + encoder, found := cfg.encoderCache.Load(cacheKey) + if found { + return encoder.(ValEncoder) + } + return nil +} + +var cfgCache = concurrent.NewMap() + +func getFrozenConfigFromCache(cfg Config) *frozenConfig { + obj, found := cfgCache.Load(cfg) + if found { + return obj.(*frozenConfig) + } + return nil +} + +func addFrozenConfigToCache(cfg Config, frozenConfig *frozenConfig) { + cfgCache.Store(cfg, frozenConfig) +} + +// Froze forge API from config +func (cfg Config) Froze() API { + api := &frozenConfig{ + sortMapKeys: cfg.SortMapKeys, + indentionStep: cfg.IndentionStep, + objectFieldMustBeSimpleString: cfg.ObjectFieldMustBeSimpleString, + onlyTaggedField: cfg.OnlyTaggedField, + disallowUnknownFields: cfg.DisallowUnknownFields, + caseSensitive: cfg.CaseSensitive, + } + api.streamPool = &sync.Pool{ + New: func() interface{} { + return NewStream(api, nil, 512) + }, + } + api.iteratorPool = &sync.Pool{ + New: func() interface{} { + return NewIterator(api) + }, + } + api.initCache() + encoderExtension := EncoderExtension{} + decoderExtension := DecoderExtension{} + if cfg.MarshalFloatWith6Digits { + api.marshalFloatWith6Digits(encoderExtension) + } + if cfg.EscapeHTML { + api.escapeHTML(encoderExtension) + } + if cfg.UseNumber { + api.useNumber(decoderExtension) + } + if cfg.ValidateJsonRawMessage { + api.validateJsonRawMessage(encoderExtension) + } + api.encoderExtension = encoderExtension + api.decoderExtension = decoderExtension + api.configBeforeFrozen = cfg + return api +} + +func (cfg Config) frozeWithCacheReuse(extraExtensions []Extension) *frozenConfig { + api := getFrozenConfigFromCache(cfg) + if api != nil { + return api + } + api = cfg.Froze().(*frozenConfig) + for _, extension := range extraExtensions { + api.RegisterExtension(extension) + } + addFrozenConfigToCache(cfg, api) + return api +} + +func (cfg *frozenConfig) validateJsonRawMessage(extension EncoderExtension) { + encoder := &funcEncoder{func(ptr unsafe.Pointer, stream *Stream) { + rawMessage := *(*json.RawMessage)(ptr) + iter := cfg.BorrowIterator([]byte(rawMessage)) + defer cfg.ReturnIterator(iter) + iter.Read() + if iter.Error != nil && iter.Error != io.EOF { + stream.WriteRaw("null") + } else { + stream.WriteRaw(string(rawMessage)) + } + }, func(ptr unsafe.Pointer) bool { + return len(*((*json.RawMessage)(ptr))) == 0 + }} + extension[reflect2.TypeOfPtr((*json.RawMessage)(nil)).Elem()] = encoder + extension[reflect2.TypeOfPtr((*RawMessage)(nil)).Elem()] = encoder +} + +func (cfg *frozenConfig) useNumber(extension DecoderExtension) { + extension[reflect2.TypeOfPtr((*interface{})(nil)).Elem()] = &funcDecoder{func(ptr unsafe.Pointer, iter *Iterator) { + exitingValue := *((*interface{})(ptr)) + if exitingValue != nil && reflect.TypeOf(exitingValue).Kind() == reflect.Ptr { + iter.ReadVal(exitingValue) + return + } + if iter.WhatIsNext() == NumberValue { + *((*interface{})(ptr)) = json.Number(iter.readNumberAsString()) + } else { + *((*interface{})(ptr)) = iter.Read() + } + }} +} +func (cfg *frozenConfig) getTagKey() string { + tagKey := cfg.configBeforeFrozen.TagKey + if tagKey == "" { + return "json" + } + return tagKey +} + +func (cfg *frozenConfig) RegisterExtension(extension Extension) { + cfg.extraExtensions = append(cfg.extraExtensions, extension) + copied := cfg.configBeforeFrozen + cfg.configBeforeFrozen = copied +} + +type lossyFloat32Encoder struct { +} + +func (encoder *lossyFloat32Encoder) Encode(ptr unsafe.Pointer, stream *Stream) { + stream.WriteFloat32Lossy(*((*float32)(ptr))) +} + +func (encoder *lossyFloat32Encoder) IsEmpty(ptr unsafe.Pointer) bool { + return *((*float32)(ptr)) == 0 +} + +type lossyFloat64Encoder struct { +} + +func (encoder *lossyFloat64Encoder) Encode(ptr unsafe.Pointer, stream *Stream) { + stream.WriteFloat64Lossy(*((*float64)(ptr))) +} + +func (encoder *lossyFloat64Encoder) IsEmpty(ptr unsafe.Pointer) bool { + return *((*float64)(ptr)) == 0 +} + +// EnableLossyFloatMarshalling keeps 10**(-6) precision +// for float variables for better performance. +func (cfg *frozenConfig) marshalFloatWith6Digits(extension EncoderExtension) { + // for better performance + extension[reflect2.TypeOfPtr((*float32)(nil)).Elem()] = &lossyFloat32Encoder{} + extension[reflect2.TypeOfPtr((*float64)(nil)).Elem()] = &lossyFloat64Encoder{} +} + +type htmlEscapedStringEncoder struct { +} + +func (encoder *htmlEscapedStringEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { + str := *((*string)(ptr)) + stream.WriteStringWithHTMLEscaped(str) +} + +func (encoder *htmlEscapedStringEncoder) IsEmpty(ptr unsafe.Pointer) bool { + return *((*string)(ptr)) == "" +} + +func (cfg *frozenConfig) escapeHTML(encoderExtension EncoderExtension) { + encoderExtension[reflect2.TypeOfPtr((*string)(nil)).Elem()] = &htmlEscapedStringEncoder{} +} + +func (cfg *frozenConfig) cleanDecoders() { + typeDecoders = map[string]ValDecoder{} + fieldDecoders = map[string]ValDecoder{} + *cfg = *(cfg.configBeforeFrozen.Froze().(*frozenConfig)) +} + +func (cfg *frozenConfig) cleanEncoders() { + typeEncoders = map[string]ValEncoder{} + fieldEncoders = map[string]ValEncoder{} + *cfg = *(cfg.configBeforeFrozen.Froze().(*frozenConfig)) +} + +func (cfg *frozenConfig) MarshalToString(v interface{}) (string, error) { + stream := cfg.BorrowStream(nil) + defer cfg.ReturnStream(stream) + stream.WriteVal(v) + if stream.Error != nil { + return "", stream.Error + } + return string(stream.Buffer()), nil +} + +func (cfg *frozenConfig) Marshal(v interface{}) ([]byte, error) { + stream := cfg.BorrowStream(nil) + defer cfg.ReturnStream(stream) + stream.WriteVal(v) + if stream.Error != nil { + return nil, stream.Error + } + result := stream.Buffer() + copied := make([]byte, len(result)) + copy(copied, result) + return copied, nil +} + +func (cfg *frozenConfig) MarshalIndent(v interface{}, prefix, indent string) ([]byte, error) { + if prefix != "" { + panic("prefix is not supported") + } + for _, r := range indent { + if r != ' ' { + panic("indent can only be space") + } + } + newCfg := cfg.configBeforeFrozen + newCfg.IndentionStep = len(indent) + return newCfg.frozeWithCacheReuse(cfg.extraExtensions).Marshal(v) +} + +func (cfg *frozenConfig) UnmarshalFromString(str string, v interface{}) error { + data := []byte(str) + iter := cfg.BorrowIterator(data) + defer cfg.ReturnIterator(iter) + iter.ReadVal(v) + c := iter.nextToken() + if c == 0 { + if iter.Error == io.EOF { + return nil + } + return iter.Error + } + iter.ReportError("Unmarshal", "there are bytes left after unmarshal") + return iter.Error +} + +func (cfg *frozenConfig) Get(data []byte, path ...interface{}) Any { + iter := cfg.BorrowIterator(data) + defer cfg.ReturnIterator(iter) + return locatePath(iter, path) +} + +func (cfg *frozenConfig) Unmarshal(data []byte, v interface{}) error { + iter := cfg.BorrowIterator(data) + defer cfg.ReturnIterator(iter) + iter.ReadVal(v) + c := iter.nextToken() + if c == 0 { + if iter.Error == io.EOF { + return nil + } + return iter.Error + } + iter.ReportError("Unmarshal", "there are bytes left after unmarshal") + return iter.Error +} + +func (cfg *frozenConfig) NewEncoder(writer io.Writer) *Encoder { + stream := NewStream(cfg, writer, 512) + return &Encoder{stream} +} + +func (cfg *frozenConfig) NewDecoder(reader io.Reader) *Decoder { + iter := Parse(cfg, reader, 512) + return &Decoder{iter} +} + +func (cfg *frozenConfig) Valid(data []byte) bool { + iter := cfg.BorrowIterator(data) + defer cfg.ReturnIterator(iter) + iter.Skip() + return iter.Error == nil +} diff --git a/vendor/github.com/json-iterator/go/fuzzy_mode_convert_table.md b/vendor/github.com/json-iterator/go/fuzzy_mode_convert_table.md new file mode 100644 index 00000000..3095662b --- /dev/null +++ b/vendor/github.com/json-iterator/go/fuzzy_mode_convert_table.md @@ -0,0 +1,7 @@ +| json type \ dest type | bool | int | uint | float |string| +| --- | --- | --- | --- |--|--| +| number | positive => true
negative => true
zero => false| 23.2 => 23
-32.1 => -32| 12.1 => 12
-12.1 => 0|as normal|same as origin| +| string | empty string => false
string "0" => false
other strings => true | "123.32" => 123
"-123.4" => -123
"123.23xxxw" => 123
"abcde12" => 0
"-32.1" => -32| 13.2 => 13
-1.1 => 0 |12.1 => 12.1
-12.3 => -12.3
12.4xxa => 12.4
+1.1e2 =>110 |same as origin| +| bool | true => true
false => false| true => 1
false => 0 | true => 1
false => 0 |true => 1
false => 0|true => "true"
false => "false"| +| object | true | 0 | 0 |0|originnal json| +| array | empty array => false
nonempty array => true| [] => 0
[1,2] => 1 | [] => 0
[1,2] => 1 |[] => 0
[1,2] => 1|original json| \ No newline at end of file diff --git a/vendor/github.com/json-iterator/go/iter.go b/vendor/github.com/json-iterator/go/iter.go new file mode 100644 index 00000000..29b31cf7 --- /dev/null +++ b/vendor/github.com/json-iterator/go/iter.go @@ -0,0 +1,349 @@ +package jsoniter + +import ( + "encoding/json" + "fmt" + "io" +) + +// ValueType the type for JSON element +type ValueType int + +const ( + // InvalidValue invalid JSON element + InvalidValue ValueType = iota + // StringValue JSON element "string" + StringValue + // NumberValue JSON element 100 or 0.10 + NumberValue + // NilValue JSON element null + NilValue + // BoolValue JSON element true or false + BoolValue + // ArrayValue JSON element [] + ArrayValue + // ObjectValue JSON element {} + ObjectValue +) + +var hexDigits []byte +var valueTypes []ValueType + +func init() { + hexDigits = make([]byte, 256) + for i := 0; i < len(hexDigits); i++ { + hexDigits[i] = 255 + } + for i := '0'; i <= '9'; i++ { + hexDigits[i] = byte(i - '0') + } + for i := 'a'; i <= 'f'; i++ { + hexDigits[i] = byte((i - 'a') + 10) + } + for i := 'A'; i <= 'F'; i++ { + hexDigits[i] = byte((i - 'A') + 10) + } + valueTypes = make([]ValueType, 256) + for i := 0; i < len(valueTypes); i++ { + valueTypes[i] = InvalidValue + } + valueTypes['"'] = StringValue + valueTypes['-'] = NumberValue + valueTypes['0'] = NumberValue + valueTypes['1'] = NumberValue + valueTypes['2'] = NumberValue + valueTypes['3'] = NumberValue + valueTypes['4'] = NumberValue + valueTypes['5'] = NumberValue + valueTypes['6'] = NumberValue + valueTypes['7'] = NumberValue + valueTypes['8'] = NumberValue + valueTypes['9'] = NumberValue + valueTypes['t'] = BoolValue + valueTypes['f'] = BoolValue + valueTypes['n'] = NilValue + valueTypes['['] = ArrayValue + valueTypes['{'] = ObjectValue +} + +// Iterator is a io.Reader like object, with JSON specific read functions. +// Error is not returned as return value, but stored as Error member on this iterator instance. +type Iterator struct { + cfg *frozenConfig + reader io.Reader + buf []byte + head int + tail int + depth int + captureStartedAt int + captured []byte + Error error + Attachment interface{} // open for customized decoder +} + +// NewIterator creates an empty Iterator instance +func NewIterator(cfg API) *Iterator { + return &Iterator{ + cfg: cfg.(*frozenConfig), + reader: nil, + buf: nil, + head: 0, + tail: 0, + depth: 0, + } +} + +// Parse creates an Iterator instance from io.Reader +func Parse(cfg API, reader io.Reader, bufSize int) *Iterator { + return &Iterator{ + cfg: cfg.(*frozenConfig), + reader: reader, + buf: make([]byte, bufSize), + head: 0, + tail: 0, + depth: 0, + } +} + +// ParseBytes creates an Iterator instance from byte array +func ParseBytes(cfg API, input []byte) *Iterator { + return &Iterator{ + cfg: cfg.(*frozenConfig), + reader: nil, + buf: input, + head: 0, + tail: len(input), + depth: 0, + } +} + +// ParseString creates an Iterator instance from string +func ParseString(cfg API, input string) *Iterator { + return ParseBytes(cfg, []byte(input)) +} + +// Pool returns a pool can provide more iterator with same configuration +func (iter *Iterator) Pool() IteratorPool { + return iter.cfg +} + +// Reset reuse iterator instance by specifying another reader +func (iter *Iterator) Reset(reader io.Reader) *Iterator { + iter.reader = reader + iter.head = 0 + iter.tail = 0 + iter.depth = 0 + return iter +} + +// ResetBytes reuse iterator instance by specifying another byte array as input +func (iter *Iterator) ResetBytes(input []byte) *Iterator { + iter.reader = nil + iter.buf = input + iter.head = 0 + iter.tail = len(input) + iter.depth = 0 + return iter +} + +// WhatIsNext gets ValueType of relatively next json element +func (iter *Iterator) WhatIsNext() ValueType { + valueType := valueTypes[iter.nextToken()] + iter.unreadByte() + return valueType +} + +func (iter *Iterator) skipWhitespacesWithoutLoadMore() bool { + for i := iter.head; i < iter.tail; i++ { + c := iter.buf[i] + switch c { + case ' ', '\n', '\t', '\r': + continue + } + iter.head = i + return false + } + return true +} + +func (iter *Iterator) isObjectEnd() bool { + c := iter.nextToken() + if c == ',' { + return false + } + if c == '}' { + return true + } + iter.ReportError("isObjectEnd", "object ended prematurely, unexpected char "+string([]byte{c})) + return true +} + +func (iter *Iterator) nextToken() byte { + // a variation of skip whitespaces, returning the next non-whitespace token + for { + for i := iter.head; i < iter.tail; i++ { + c := iter.buf[i] + switch c { + case ' ', '\n', '\t', '\r': + continue + } + iter.head = i + 1 + return c + } + if !iter.loadMore() { + return 0 + } + } +} + +// ReportError record a error in iterator instance with current position. +func (iter *Iterator) ReportError(operation string, msg string) { + if iter.Error != nil { + if iter.Error != io.EOF { + return + } + } + peekStart := iter.head - 10 + if peekStart < 0 { + peekStart = 0 + } + peekEnd := iter.head + 10 + if peekEnd > iter.tail { + peekEnd = iter.tail + } + parsing := string(iter.buf[peekStart:peekEnd]) + contextStart := iter.head - 50 + if contextStart < 0 { + contextStart = 0 + } + contextEnd := iter.head + 50 + if contextEnd > iter.tail { + contextEnd = iter.tail + } + context := string(iter.buf[contextStart:contextEnd]) + iter.Error = fmt.Errorf("%s: %s, error found in #%v byte of ...|%s|..., bigger context ...|%s|...", + operation, msg, iter.head-peekStart, parsing, context) +} + +// CurrentBuffer gets current buffer as string for debugging purpose +func (iter *Iterator) CurrentBuffer() string { + peekStart := iter.head - 10 + if peekStart < 0 { + peekStart = 0 + } + return fmt.Sprintf("parsing #%v byte, around ...|%s|..., whole buffer ...|%s|...", iter.head, + string(iter.buf[peekStart:iter.head]), string(iter.buf[0:iter.tail])) +} + +func (iter *Iterator) readByte() (ret byte) { + if iter.head == iter.tail { + if iter.loadMore() { + ret = iter.buf[iter.head] + iter.head++ + return ret + } + return 0 + } + ret = iter.buf[iter.head] + iter.head++ + return ret +} + +func (iter *Iterator) loadMore() bool { + if iter.reader == nil { + if iter.Error == nil { + iter.head = iter.tail + iter.Error = io.EOF + } + return false + } + if iter.captured != nil { + iter.captured = append(iter.captured, + iter.buf[iter.captureStartedAt:iter.tail]...) + iter.captureStartedAt = 0 + } + for { + n, err := iter.reader.Read(iter.buf) + if n == 0 { + if err != nil { + if iter.Error == nil { + iter.Error = err + } + return false + } + } else { + iter.head = 0 + iter.tail = n + return true + } + } +} + +func (iter *Iterator) unreadByte() { + if iter.Error != nil { + return + } + iter.head-- + return +} + +// Read read the next JSON element as generic interface{}. +func (iter *Iterator) Read() interface{} { + valueType := iter.WhatIsNext() + switch valueType { + case StringValue: + return iter.ReadString() + case NumberValue: + if iter.cfg.configBeforeFrozen.UseNumber { + return json.Number(iter.readNumberAsString()) + } + return iter.ReadFloat64() + case NilValue: + iter.skipFourBytes('n', 'u', 'l', 'l') + return nil + case BoolValue: + return iter.ReadBool() + case ArrayValue: + arr := []interface{}{} + iter.ReadArrayCB(func(iter *Iterator) bool { + var elem interface{} + iter.ReadVal(&elem) + arr = append(arr, elem) + return true + }) + return arr + case ObjectValue: + obj := map[string]interface{}{} + iter.ReadMapCB(func(Iter *Iterator, field string) bool { + var elem interface{} + iter.ReadVal(&elem) + obj[field] = elem + return true + }) + return obj + default: + iter.ReportError("Read", fmt.Sprintf("unexpected value type: %v", valueType)) + return nil + } +} + +// limit maximum depth of nesting, as allowed by https://tools.ietf.org/html/rfc7159#section-9 +const maxDepth = 10000 + +func (iter *Iterator) incrementDepth() (success bool) { + iter.depth++ + if iter.depth <= maxDepth { + return true + } + iter.ReportError("incrementDepth", "exceeded max depth") + return false +} + +func (iter *Iterator) decrementDepth() (success bool) { + iter.depth-- + if iter.depth >= 0 { + return true + } + iter.ReportError("decrementDepth", "unexpected negative nesting") + return false +} diff --git a/vendor/github.com/json-iterator/go/iter_array.go b/vendor/github.com/json-iterator/go/iter_array.go new file mode 100644 index 00000000..204fe0e0 --- /dev/null +++ b/vendor/github.com/json-iterator/go/iter_array.go @@ -0,0 +1,64 @@ +package jsoniter + +// ReadArray read array element, tells if the array has more element to read. +func (iter *Iterator) ReadArray() (ret bool) { + c := iter.nextToken() + switch c { + case 'n': + iter.skipThreeBytes('u', 'l', 'l') + return false // null + case '[': + c = iter.nextToken() + if c != ']' { + iter.unreadByte() + return true + } + return false + case ']': + return false + case ',': + return true + default: + iter.ReportError("ReadArray", "expect [ or , or ] or n, but found "+string([]byte{c})) + return + } +} + +// ReadArrayCB read array with callback +func (iter *Iterator) ReadArrayCB(callback func(*Iterator) bool) (ret bool) { + c := iter.nextToken() + if c == '[' { + if !iter.incrementDepth() { + return false + } + c = iter.nextToken() + if c != ']' { + iter.unreadByte() + if !callback(iter) { + iter.decrementDepth() + return false + } + c = iter.nextToken() + for c == ',' { + if !callback(iter) { + iter.decrementDepth() + return false + } + c = iter.nextToken() + } + if c != ']' { + iter.ReportError("ReadArrayCB", "expect ] in the end, but found "+string([]byte{c})) + iter.decrementDepth() + return false + } + return iter.decrementDepth() + } + return iter.decrementDepth() + } + if c == 'n' { + iter.skipThreeBytes('u', 'l', 'l') + return true // null + } + iter.ReportError("ReadArrayCB", "expect [ or n, but found "+string([]byte{c})) + return false +} diff --git a/vendor/github.com/json-iterator/go/iter_float.go b/vendor/github.com/json-iterator/go/iter_float.go new file mode 100644 index 00000000..8a3d8b6f --- /dev/null +++ b/vendor/github.com/json-iterator/go/iter_float.go @@ -0,0 +1,342 @@ +package jsoniter + +import ( + "encoding/json" + "io" + "math/big" + "strconv" + "strings" + "unsafe" +) + +var floatDigits []int8 + +const invalidCharForNumber = int8(-1) +const endOfNumber = int8(-2) +const dotInNumber = int8(-3) + +func init() { + floatDigits = make([]int8, 256) + for i := 0; i < len(floatDigits); i++ { + floatDigits[i] = invalidCharForNumber + } + for i := int8('0'); i <= int8('9'); i++ { + floatDigits[i] = i - int8('0') + } + floatDigits[','] = endOfNumber + floatDigits[']'] = endOfNumber + floatDigits['}'] = endOfNumber + floatDigits[' '] = endOfNumber + floatDigits['\t'] = endOfNumber + floatDigits['\n'] = endOfNumber + floatDigits['.'] = dotInNumber +} + +// ReadBigFloat read big.Float +func (iter *Iterator) ReadBigFloat() (ret *big.Float) { + str := iter.readNumberAsString() + if iter.Error != nil && iter.Error != io.EOF { + return nil + } + prec := 64 + if len(str) > prec { + prec = len(str) + } + val, _, err := big.ParseFloat(str, 10, uint(prec), big.ToZero) + if err != nil { + iter.Error = err + return nil + } + return val +} + +// ReadBigInt read big.Int +func (iter *Iterator) ReadBigInt() (ret *big.Int) { + str := iter.readNumberAsString() + if iter.Error != nil && iter.Error != io.EOF { + return nil + } + ret = big.NewInt(0) + var success bool + ret, success = ret.SetString(str, 10) + if !success { + iter.ReportError("ReadBigInt", "invalid big int") + return nil + } + return ret +} + +//ReadFloat32 read float32 +func (iter *Iterator) ReadFloat32() (ret float32) { + c := iter.nextToken() + if c == '-' { + return -iter.readPositiveFloat32() + } + iter.unreadByte() + return iter.readPositiveFloat32() +} + +func (iter *Iterator) readPositiveFloat32() (ret float32) { + i := iter.head + // first char + if i == iter.tail { + return iter.readFloat32SlowPath() + } + c := iter.buf[i] + i++ + ind := floatDigits[c] + switch ind { + case invalidCharForNumber: + return iter.readFloat32SlowPath() + case endOfNumber: + iter.ReportError("readFloat32", "empty number") + return + case dotInNumber: + iter.ReportError("readFloat32", "leading dot is invalid") + return + case 0: + if i == iter.tail { + return iter.readFloat32SlowPath() + } + c = iter.buf[i] + switch c { + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + iter.ReportError("readFloat32", "leading zero is invalid") + return + } + } + value := uint64(ind) + // chars before dot +non_decimal_loop: + for ; i < iter.tail; i++ { + c = iter.buf[i] + ind := floatDigits[c] + switch ind { + case invalidCharForNumber: + return iter.readFloat32SlowPath() + case endOfNumber: + iter.head = i + return float32(value) + case dotInNumber: + break non_decimal_loop + } + if value > uint64SafeToMultiple10 { + return iter.readFloat32SlowPath() + } + value = (value << 3) + (value << 1) + uint64(ind) // value = value * 10 + ind; + } + // chars after dot + if c == '.' { + i++ + decimalPlaces := 0 + if i == iter.tail { + return iter.readFloat32SlowPath() + } + for ; i < iter.tail; i++ { + c = iter.buf[i] + ind := floatDigits[c] + switch ind { + case endOfNumber: + if decimalPlaces > 0 && decimalPlaces < len(pow10) { + iter.head = i + return float32(float64(value) / float64(pow10[decimalPlaces])) + } + // too many decimal places + return iter.readFloat32SlowPath() + case invalidCharForNumber, dotInNumber: + return iter.readFloat32SlowPath() + } + decimalPlaces++ + if value > uint64SafeToMultiple10 { + return iter.readFloat32SlowPath() + } + value = (value << 3) + (value << 1) + uint64(ind) + } + } + return iter.readFloat32SlowPath() +} + +func (iter *Iterator) readNumberAsString() (ret string) { + strBuf := [16]byte{} + str := strBuf[0:0] +load_loop: + for { + for i := iter.head; i < iter.tail; i++ { + c := iter.buf[i] + switch c { + case '+', '-', '.', 'e', 'E', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + str = append(str, c) + continue + default: + iter.head = i + break load_loop + } + } + if !iter.loadMore() { + break + } + } + if iter.Error != nil && iter.Error != io.EOF { + return + } + if len(str) == 0 { + iter.ReportError("readNumberAsString", "invalid number") + } + return *(*string)(unsafe.Pointer(&str)) +} + +func (iter *Iterator) readFloat32SlowPath() (ret float32) { + str := iter.readNumberAsString() + if iter.Error != nil && iter.Error != io.EOF { + return + } + errMsg := validateFloat(str) + if errMsg != "" { + iter.ReportError("readFloat32SlowPath", errMsg) + return + } + val, err := strconv.ParseFloat(str, 32) + if err != nil { + iter.Error = err + return + } + return float32(val) +} + +// ReadFloat64 read float64 +func (iter *Iterator) ReadFloat64() (ret float64) { + c := iter.nextToken() + if c == '-' { + return -iter.readPositiveFloat64() + } + iter.unreadByte() + return iter.readPositiveFloat64() +} + +func (iter *Iterator) readPositiveFloat64() (ret float64) { + i := iter.head + // first char + if i == iter.tail { + return iter.readFloat64SlowPath() + } + c := iter.buf[i] + i++ + ind := floatDigits[c] + switch ind { + case invalidCharForNumber: + return iter.readFloat64SlowPath() + case endOfNumber: + iter.ReportError("readFloat64", "empty number") + return + case dotInNumber: + iter.ReportError("readFloat64", "leading dot is invalid") + return + case 0: + if i == iter.tail { + return iter.readFloat64SlowPath() + } + c = iter.buf[i] + switch c { + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + iter.ReportError("readFloat64", "leading zero is invalid") + return + } + } + value := uint64(ind) + // chars before dot +non_decimal_loop: + for ; i < iter.tail; i++ { + c = iter.buf[i] + ind := floatDigits[c] + switch ind { + case invalidCharForNumber: + return iter.readFloat64SlowPath() + case endOfNumber: + iter.head = i + return float64(value) + case dotInNumber: + break non_decimal_loop + } + if value > uint64SafeToMultiple10 { + return iter.readFloat64SlowPath() + } + value = (value << 3) + (value << 1) + uint64(ind) // value = value * 10 + ind; + } + // chars after dot + if c == '.' { + i++ + decimalPlaces := 0 + if i == iter.tail { + return iter.readFloat64SlowPath() + } + for ; i < iter.tail; i++ { + c = iter.buf[i] + ind := floatDigits[c] + switch ind { + case endOfNumber: + if decimalPlaces > 0 && decimalPlaces < len(pow10) { + iter.head = i + return float64(value) / float64(pow10[decimalPlaces]) + } + // too many decimal places + return iter.readFloat64SlowPath() + case invalidCharForNumber, dotInNumber: + return iter.readFloat64SlowPath() + } + decimalPlaces++ + if value > uint64SafeToMultiple10 { + return iter.readFloat64SlowPath() + } + value = (value << 3) + (value << 1) + uint64(ind) + if value > maxFloat64 { + return iter.readFloat64SlowPath() + } + } + } + return iter.readFloat64SlowPath() +} + +func (iter *Iterator) readFloat64SlowPath() (ret float64) { + str := iter.readNumberAsString() + if iter.Error != nil && iter.Error != io.EOF { + return + } + errMsg := validateFloat(str) + if errMsg != "" { + iter.ReportError("readFloat64SlowPath", errMsg) + return + } + val, err := strconv.ParseFloat(str, 64) + if err != nil { + iter.Error = err + return + } + return val +} + +func validateFloat(str string) string { + // strconv.ParseFloat is not validating `1.` or `1.e1` + if len(str) == 0 { + return "empty number" + } + if str[0] == '-' { + return "-- is not valid" + } + dotPos := strings.IndexByte(str, '.') + if dotPos != -1 { + if dotPos == len(str)-1 { + return "dot can not be last character" + } + switch str[dotPos+1] { + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + default: + return "missing digit after dot" + } + } + return "" +} + +// ReadNumber read json.Number +func (iter *Iterator) ReadNumber() (ret json.Number) { + return json.Number(iter.readNumberAsString()) +} diff --git a/vendor/github.com/json-iterator/go/iter_int.go b/vendor/github.com/json-iterator/go/iter_int.go new file mode 100644 index 00000000..d786a89f --- /dev/null +++ b/vendor/github.com/json-iterator/go/iter_int.go @@ -0,0 +1,346 @@ +package jsoniter + +import ( + "math" + "strconv" +) + +var intDigits []int8 + +const uint32SafeToMultiply10 = uint32(0xffffffff)/10 - 1 +const uint64SafeToMultiple10 = uint64(0xffffffffffffffff)/10 - 1 +const maxFloat64 = 1<<53 - 1 + +func init() { + intDigits = make([]int8, 256) + for i := 0; i < len(intDigits); i++ { + intDigits[i] = invalidCharForNumber + } + for i := int8('0'); i <= int8('9'); i++ { + intDigits[i] = i - int8('0') + } +} + +// ReadUint read uint +func (iter *Iterator) ReadUint() uint { + if strconv.IntSize == 32 { + return uint(iter.ReadUint32()) + } + return uint(iter.ReadUint64()) +} + +// ReadInt read int +func (iter *Iterator) ReadInt() int { + if strconv.IntSize == 32 { + return int(iter.ReadInt32()) + } + return int(iter.ReadInt64()) +} + +// ReadInt8 read int8 +func (iter *Iterator) ReadInt8() (ret int8) { + c := iter.nextToken() + if c == '-' { + val := iter.readUint32(iter.readByte()) + if val > math.MaxInt8+1 { + iter.ReportError("ReadInt8", "overflow: "+strconv.FormatInt(int64(val), 10)) + return + } + return -int8(val) + } + val := iter.readUint32(c) + if val > math.MaxInt8 { + iter.ReportError("ReadInt8", "overflow: "+strconv.FormatInt(int64(val), 10)) + return + } + return int8(val) +} + +// ReadUint8 read uint8 +func (iter *Iterator) ReadUint8() (ret uint8) { + val := iter.readUint32(iter.nextToken()) + if val > math.MaxUint8 { + iter.ReportError("ReadUint8", "overflow: "+strconv.FormatInt(int64(val), 10)) + return + } + return uint8(val) +} + +// ReadInt16 read int16 +func (iter *Iterator) ReadInt16() (ret int16) { + c := iter.nextToken() + if c == '-' { + val := iter.readUint32(iter.readByte()) + if val > math.MaxInt16+1 { + iter.ReportError("ReadInt16", "overflow: "+strconv.FormatInt(int64(val), 10)) + return + } + return -int16(val) + } + val := iter.readUint32(c) + if val > math.MaxInt16 { + iter.ReportError("ReadInt16", "overflow: "+strconv.FormatInt(int64(val), 10)) + return + } + return int16(val) +} + +// ReadUint16 read uint16 +func (iter *Iterator) ReadUint16() (ret uint16) { + val := iter.readUint32(iter.nextToken()) + if val > math.MaxUint16 { + iter.ReportError("ReadUint16", "overflow: "+strconv.FormatInt(int64(val), 10)) + return + } + return uint16(val) +} + +// ReadInt32 read int32 +func (iter *Iterator) ReadInt32() (ret int32) { + c := iter.nextToken() + if c == '-' { + val := iter.readUint32(iter.readByte()) + if val > math.MaxInt32+1 { + iter.ReportError("ReadInt32", "overflow: "+strconv.FormatInt(int64(val), 10)) + return + } + return -int32(val) + } + val := iter.readUint32(c) + if val > math.MaxInt32 { + iter.ReportError("ReadInt32", "overflow: "+strconv.FormatInt(int64(val), 10)) + return + } + return int32(val) +} + +// ReadUint32 read uint32 +func (iter *Iterator) ReadUint32() (ret uint32) { + return iter.readUint32(iter.nextToken()) +} + +func (iter *Iterator) readUint32(c byte) (ret uint32) { + ind := intDigits[c] + if ind == 0 { + iter.assertInteger() + return 0 // single zero + } + if ind == invalidCharForNumber { + iter.ReportError("readUint32", "unexpected character: "+string([]byte{byte(ind)})) + return + } + value := uint32(ind) + if iter.tail-iter.head > 10 { + i := iter.head + ind2 := intDigits[iter.buf[i]] + if ind2 == invalidCharForNumber { + iter.head = i + iter.assertInteger() + return value + } + i++ + ind3 := intDigits[iter.buf[i]] + if ind3 == invalidCharForNumber { + iter.head = i + iter.assertInteger() + return value*10 + uint32(ind2) + } + //iter.head = i + 1 + //value = value * 100 + uint32(ind2) * 10 + uint32(ind3) + i++ + ind4 := intDigits[iter.buf[i]] + if ind4 == invalidCharForNumber { + iter.head = i + iter.assertInteger() + return value*100 + uint32(ind2)*10 + uint32(ind3) + } + i++ + ind5 := intDigits[iter.buf[i]] + if ind5 == invalidCharForNumber { + iter.head = i + iter.assertInteger() + return value*1000 + uint32(ind2)*100 + uint32(ind3)*10 + uint32(ind4) + } + i++ + ind6 := intDigits[iter.buf[i]] + if ind6 == invalidCharForNumber { + iter.head = i + iter.assertInteger() + return value*10000 + uint32(ind2)*1000 + uint32(ind3)*100 + uint32(ind4)*10 + uint32(ind5) + } + i++ + ind7 := intDigits[iter.buf[i]] + if ind7 == invalidCharForNumber { + iter.head = i + iter.assertInteger() + return value*100000 + uint32(ind2)*10000 + uint32(ind3)*1000 + uint32(ind4)*100 + uint32(ind5)*10 + uint32(ind6) + } + i++ + ind8 := intDigits[iter.buf[i]] + if ind8 == invalidCharForNumber { + iter.head = i + iter.assertInteger() + return value*1000000 + uint32(ind2)*100000 + uint32(ind3)*10000 + uint32(ind4)*1000 + uint32(ind5)*100 + uint32(ind6)*10 + uint32(ind7) + } + i++ + ind9 := intDigits[iter.buf[i]] + value = value*10000000 + uint32(ind2)*1000000 + uint32(ind3)*100000 + uint32(ind4)*10000 + uint32(ind5)*1000 + uint32(ind6)*100 + uint32(ind7)*10 + uint32(ind8) + iter.head = i + if ind9 == invalidCharForNumber { + iter.assertInteger() + return value + } + } + for { + for i := iter.head; i < iter.tail; i++ { + ind = intDigits[iter.buf[i]] + if ind == invalidCharForNumber { + iter.head = i + iter.assertInteger() + return value + } + if value > uint32SafeToMultiply10 { + value2 := (value << 3) + (value << 1) + uint32(ind) + if value2 < value { + iter.ReportError("readUint32", "overflow") + return + } + value = value2 + continue + } + value = (value << 3) + (value << 1) + uint32(ind) + } + if !iter.loadMore() { + iter.assertInteger() + return value + } + } +} + +// ReadInt64 read int64 +func (iter *Iterator) ReadInt64() (ret int64) { + c := iter.nextToken() + if c == '-' { + val := iter.readUint64(iter.readByte()) + if val > math.MaxInt64+1 { + iter.ReportError("ReadInt64", "overflow: "+strconv.FormatUint(uint64(val), 10)) + return + } + return -int64(val) + } + val := iter.readUint64(c) + if val > math.MaxInt64 { + iter.ReportError("ReadInt64", "overflow: "+strconv.FormatUint(uint64(val), 10)) + return + } + return int64(val) +} + +// ReadUint64 read uint64 +func (iter *Iterator) ReadUint64() uint64 { + return iter.readUint64(iter.nextToken()) +} + +func (iter *Iterator) readUint64(c byte) (ret uint64) { + ind := intDigits[c] + if ind == 0 { + iter.assertInteger() + return 0 // single zero + } + if ind == invalidCharForNumber { + iter.ReportError("readUint64", "unexpected character: "+string([]byte{byte(ind)})) + return + } + value := uint64(ind) + if iter.tail-iter.head > 10 { + i := iter.head + ind2 := intDigits[iter.buf[i]] + if ind2 == invalidCharForNumber { + iter.head = i + iter.assertInteger() + return value + } + i++ + ind3 := intDigits[iter.buf[i]] + if ind3 == invalidCharForNumber { + iter.head = i + iter.assertInteger() + return value*10 + uint64(ind2) + } + //iter.head = i + 1 + //value = value * 100 + uint32(ind2) * 10 + uint32(ind3) + i++ + ind4 := intDigits[iter.buf[i]] + if ind4 == invalidCharForNumber { + iter.head = i + iter.assertInteger() + return value*100 + uint64(ind2)*10 + uint64(ind3) + } + i++ + ind5 := intDigits[iter.buf[i]] + if ind5 == invalidCharForNumber { + iter.head = i + iter.assertInteger() + return value*1000 + uint64(ind2)*100 + uint64(ind3)*10 + uint64(ind4) + } + i++ + ind6 := intDigits[iter.buf[i]] + if ind6 == invalidCharForNumber { + iter.head = i + iter.assertInteger() + return value*10000 + uint64(ind2)*1000 + uint64(ind3)*100 + uint64(ind4)*10 + uint64(ind5) + } + i++ + ind7 := intDigits[iter.buf[i]] + if ind7 == invalidCharForNumber { + iter.head = i + iter.assertInteger() + return value*100000 + uint64(ind2)*10000 + uint64(ind3)*1000 + uint64(ind4)*100 + uint64(ind5)*10 + uint64(ind6) + } + i++ + ind8 := intDigits[iter.buf[i]] + if ind8 == invalidCharForNumber { + iter.head = i + iter.assertInteger() + return value*1000000 + uint64(ind2)*100000 + uint64(ind3)*10000 + uint64(ind4)*1000 + uint64(ind5)*100 + uint64(ind6)*10 + uint64(ind7) + } + i++ + ind9 := intDigits[iter.buf[i]] + value = value*10000000 + uint64(ind2)*1000000 + uint64(ind3)*100000 + uint64(ind4)*10000 + uint64(ind5)*1000 + uint64(ind6)*100 + uint64(ind7)*10 + uint64(ind8) + iter.head = i + if ind9 == invalidCharForNumber { + iter.assertInteger() + return value + } + } + for { + for i := iter.head; i < iter.tail; i++ { + ind = intDigits[iter.buf[i]] + if ind == invalidCharForNumber { + iter.head = i + iter.assertInteger() + return value + } + if value > uint64SafeToMultiple10 { + value2 := (value << 3) + (value << 1) + uint64(ind) + if value2 < value { + iter.ReportError("readUint64", "overflow") + return + } + value = value2 + continue + } + value = (value << 3) + (value << 1) + uint64(ind) + } + if !iter.loadMore() { + iter.assertInteger() + return value + } + } +} + +func (iter *Iterator) assertInteger() { + if iter.head < iter.tail && iter.buf[iter.head] == '.' { + iter.ReportError("assertInteger", "can not decode float as int") + } +} diff --git a/vendor/github.com/json-iterator/go/iter_object.go b/vendor/github.com/json-iterator/go/iter_object.go new file mode 100644 index 00000000..58ee89c8 --- /dev/null +++ b/vendor/github.com/json-iterator/go/iter_object.go @@ -0,0 +1,267 @@ +package jsoniter + +import ( + "fmt" + "strings" +) + +// ReadObject read one field from object. +// If object ended, returns empty string. +// Otherwise, returns the field name. +func (iter *Iterator) ReadObject() (ret string) { + c := iter.nextToken() + switch c { + case 'n': + iter.skipThreeBytes('u', 'l', 'l') + return "" // null + case '{': + c = iter.nextToken() + if c == '"' { + iter.unreadByte() + field := iter.ReadString() + c = iter.nextToken() + if c != ':' { + iter.ReportError("ReadObject", "expect : after object field, but found "+string([]byte{c})) + } + return field + } + if c == '}' { + return "" // end of object + } + iter.ReportError("ReadObject", `expect " after {, but found `+string([]byte{c})) + return + case ',': + field := iter.ReadString() + c = iter.nextToken() + if c != ':' { + iter.ReportError("ReadObject", "expect : after object field, but found "+string([]byte{c})) + } + return field + case '}': + return "" // end of object + default: + iter.ReportError("ReadObject", fmt.Sprintf(`expect { or , or } or n, but found %s`, string([]byte{c}))) + return + } +} + +// CaseInsensitive +func (iter *Iterator) readFieldHash() int64 { + hash := int64(0x811c9dc5) + c := iter.nextToken() + if c != '"' { + iter.ReportError("readFieldHash", `expect ", but found `+string([]byte{c})) + return 0 + } + for { + for i := iter.head; i < iter.tail; i++ { + // require ascii string and no escape + b := iter.buf[i] + if b == '\\' { + iter.head = i + for _, b := range iter.readStringSlowPath() { + if 'A' <= b && b <= 'Z' && !iter.cfg.caseSensitive { + b += 'a' - 'A' + } + hash ^= int64(b) + hash *= 0x1000193 + } + c = iter.nextToken() + if c != ':' { + iter.ReportError("readFieldHash", `expect :, but found `+string([]byte{c})) + return 0 + } + return hash + } + if b == '"' { + iter.head = i + 1 + c = iter.nextToken() + if c != ':' { + iter.ReportError("readFieldHash", `expect :, but found `+string([]byte{c})) + return 0 + } + return hash + } + if 'A' <= b && b <= 'Z' && !iter.cfg.caseSensitive { + b += 'a' - 'A' + } + hash ^= int64(b) + hash *= 0x1000193 + } + if !iter.loadMore() { + iter.ReportError("readFieldHash", `incomplete field name`) + return 0 + } + } +} + +func calcHash(str string, caseSensitive bool) int64 { + if !caseSensitive { + str = strings.ToLower(str) + } + hash := int64(0x811c9dc5) + for _, b := range []byte(str) { + hash ^= int64(b) + hash *= 0x1000193 + } + return int64(hash) +} + +// ReadObjectCB read object with callback, the key is ascii only and field name not copied +func (iter *Iterator) ReadObjectCB(callback func(*Iterator, string) bool) bool { + c := iter.nextToken() + var field string + if c == '{' { + if !iter.incrementDepth() { + return false + } + c = iter.nextToken() + if c == '"' { + iter.unreadByte() + field = iter.ReadString() + c = iter.nextToken() + if c != ':' { + iter.ReportError("ReadObject", "expect : after object field, but found "+string([]byte{c})) + } + if !callback(iter, field) { + iter.decrementDepth() + return false + } + c = iter.nextToken() + for c == ',' { + field = iter.ReadString() + c = iter.nextToken() + if c != ':' { + iter.ReportError("ReadObject", "expect : after object field, but found "+string([]byte{c})) + } + if !callback(iter, field) { + iter.decrementDepth() + return false + } + c = iter.nextToken() + } + if c != '}' { + iter.ReportError("ReadObjectCB", `object not ended with }`) + iter.decrementDepth() + return false + } + return iter.decrementDepth() + } + if c == '}' { + return iter.decrementDepth() + } + iter.ReportError("ReadObjectCB", `expect " after {, but found `+string([]byte{c})) + iter.decrementDepth() + return false + } + if c == 'n' { + iter.skipThreeBytes('u', 'l', 'l') + return true // null + } + iter.ReportError("ReadObjectCB", `expect { or n, but found `+string([]byte{c})) + return false +} + +// ReadMapCB read map with callback, the key can be any string +func (iter *Iterator) ReadMapCB(callback func(*Iterator, string) bool) bool { + c := iter.nextToken() + if c == '{' { + if !iter.incrementDepth() { + return false + } + c = iter.nextToken() + if c == '"' { + iter.unreadByte() + field := iter.ReadString() + if iter.nextToken() != ':' { + iter.ReportError("ReadMapCB", "expect : after object field, but found "+string([]byte{c})) + iter.decrementDepth() + return false + } + if !callback(iter, field) { + iter.decrementDepth() + return false + } + c = iter.nextToken() + for c == ',' { + field = iter.ReadString() + if iter.nextToken() != ':' { + iter.ReportError("ReadMapCB", "expect : after object field, but found "+string([]byte{c})) + iter.decrementDepth() + return false + } + if !callback(iter, field) { + iter.decrementDepth() + return false + } + c = iter.nextToken() + } + if c != '}' { + iter.ReportError("ReadMapCB", `object not ended with }`) + iter.decrementDepth() + return false + } + return iter.decrementDepth() + } + if c == '}' { + return iter.decrementDepth() + } + iter.ReportError("ReadMapCB", `expect " after {, but found `+string([]byte{c})) + iter.decrementDepth() + return false + } + if c == 'n' { + iter.skipThreeBytes('u', 'l', 'l') + return true // null + } + iter.ReportError("ReadMapCB", `expect { or n, but found `+string([]byte{c})) + return false +} + +func (iter *Iterator) readObjectStart() bool { + c := iter.nextToken() + if c == '{' { + c = iter.nextToken() + if c == '}' { + return false + } + iter.unreadByte() + return true + } else if c == 'n' { + iter.skipThreeBytes('u', 'l', 'l') + return false + } + iter.ReportError("readObjectStart", "expect { or n, but found "+string([]byte{c})) + return false +} + +func (iter *Iterator) readObjectFieldAsBytes() (ret []byte) { + str := iter.ReadStringAsSlice() + if iter.skipWhitespacesWithoutLoadMore() { + if ret == nil { + ret = make([]byte, len(str)) + copy(ret, str) + } + if !iter.loadMore() { + return + } + } + if iter.buf[iter.head] != ':' { + iter.ReportError("readObjectFieldAsBytes", "expect : after object field, but found "+string([]byte{iter.buf[iter.head]})) + return + } + iter.head++ + if iter.skipWhitespacesWithoutLoadMore() { + if ret == nil { + ret = make([]byte, len(str)) + copy(ret, str) + } + if !iter.loadMore() { + return + } + } + if ret == nil { + return str + } + return ret +} diff --git a/vendor/github.com/json-iterator/go/iter_skip.go b/vendor/github.com/json-iterator/go/iter_skip.go new file mode 100644 index 00000000..e91eefb1 --- /dev/null +++ b/vendor/github.com/json-iterator/go/iter_skip.go @@ -0,0 +1,130 @@ +package jsoniter + +import "fmt" + +// ReadNil reads a json object as nil and +// returns whether it's a nil or not +func (iter *Iterator) ReadNil() (ret bool) { + c := iter.nextToken() + if c == 'n' { + iter.skipThreeBytes('u', 'l', 'l') // null + return true + } + iter.unreadByte() + return false +} + +// ReadBool reads a json object as BoolValue +func (iter *Iterator) ReadBool() (ret bool) { + c := iter.nextToken() + if c == 't' { + iter.skipThreeBytes('r', 'u', 'e') + return true + } + if c == 'f' { + iter.skipFourBytes('a', 'l', 's', 'e') + return false + } + iter.ReportError("ReadBool", "expect t or f, but found "+string([]byte{c})) + return +} + +// SkipAndReturnBytes skip next JSON element, and return its content as []byte. +// The []byte can be kept, it is a copy of data. +func (iter *Iterator) SkipAndReturnBytes() []byte { + iter.startCapture(iter.head) + iter.Skip() + return iter.stopCapture() +} + +// SkipAndAppendBytes skips next JSON element and appends its content to +// buffer, returning the result. +func (iter *Iterator) SkipAndAppendBytes(buf []byte) []byte { + iter.startCaptureTo(buf, iter.head) + iter.Skip() + return iter.stopCapture() +} + +func (iter *Iterator) startCaptureTo(buf []byte, captureStartedAt int) { + if iter.captured != nil { + panic("already in capture mode") + } + iter.captureStartedAt = captureStartedAt + iter.captured = buf +} + +func (iter *Iterator) startCapture(captureStartedAt int) { + iter.startCaptureTo(make([]byte, 0, 32), captureStartedAt) +} + +func (iter *Iterator) stopCapture() []byte { + if iter.captured == nil { + panic("not in capture mode") + } + captured := iter.captured + remaining := iter.buf[iter.captureStartedAt:iter.head] + iter.captureStartedAt = -1 + iter.captured = nil + return append(captured, remaining...) +} + +// Skip skips a json object and positions to relatively the next json object +func (iter *Iterator) Skip() { + c := iter.nextToken() + switch c { + case '"': + iter.skipString() + case 'n': + iter.skipThreeBytes('u', 'l', 'l') // null + case 't': + iter.skipThreeBytes('r', 'u', 'e') // true + case 'f': + iter.skipFourBytes('a', 'l', 's', 'e') // false + case '0': + iter.unreadByte() + iter.ReadFloat32() + case '-', '1', '2', '3', '4', '5', '6', '7', '8', '9': + iter.skipNumber() + case '[': + iter.skipArray() + case '{': + iter.skipObject() + default: + iter.ReportError("Skip", fmt.Sprintf("do not know how to skip: %v", c)) + return + } +} + +func (iter *Iterator) skipFourBytes(b1, b2, b3, b4 byte) { + if iter.readByte() != b1 { + iter.ReportError("skipFourBytes", fmt.Sprintf("expect %s", string([]byte{b1, b2, b3, b4}))) + return + } + if iter.readByte() != b2 { + iter.ReportError("skipFourBytes", fmt.Sprintf("expect %s", string([]byte{b1, b2, b3, b4}))) + return + } + if iter.readByte() != b3 { + iter.ReportError("skipFourBytes", fmt.Sprintf("expect %s", string([]byte{b1, b2, b3, b4}))) + return + } + if iter.readByte() != b4 { + iter.ReportError("skipFourBytes", fmt.Sprintf("expect %s", string([]byte{b1, b2, b3, b4}))) + return + } +} + +func (iter *Iterator) skipThreeBytes(b1, b2, b3 byte) { + if iter.readByte() != b1 { + iter.ReportError("skipThreeBytes", fmt.Sprintf("expect %s", string([]byte{b1, b2, b3}))) + return + } + if iter.readByte() != b2 { + iter.ReportError("skipThreeBytes", fmt.Sprintf("expect %s", string([]byte{b1, b2, b3}))) + return + } + if iter.readByte() != b3 { + iter.ReportError("skipThreeBytes", fmt.Sprintf("expect %s", string([]byte{b1, b2, b3}))) + return + } +} diff --git a/vendor/github.com/json-iterator/go/iter_skip_sloppy.go b/vendor/github.com/json-iterator/go/iter_skip_sloppy.go new file mode 100644 index 00000000..9303de41 --- /dev/null +++ b/vendor/github.com/json-iterator/go/iter_skip_sloppy.go @@ -0,0 +1,163 @@ +//+build jsoniter_sloppy + +package jsoniter + +// sloppy but faster implementation, do not validate the input json + +func (iter *Iterator) skipNumber() { + for { + for i := iter.head; i < iter.tail; i++ { + c := iter.buf[i] + switch c { + case ' ', '\n', '\r', '\t', ',', '}', ']': + iter.head = i + return + } + } + if !iter.loadMore() { + return + } + } +} + +func (iter *Iterator) skipArray() { + level := 1 + if !iter.incrementDepth() { + return + } + for { + for i := iter.head; i < iter.tail; i++ { + switch iter.buf[i] { + case '"': // If inside string, skip it + iter.head = i + 1 + iter.skipString() + i = iter.head - 1 // it will be i++ soon + case '[': // If open symbol, increase level + level++ + if !iter.incrementDepth() { + return + } + case ']': // If close symbol, increase level + level-- + if !iter.decrementDepth() { + return + } + + // If we have returned to the original level, we're done + if level == 0 { + iter.head = i + 1 + return + } + } + } + if !iter.loadMore() { + iter.ReportError("skipObject", "incomplete array") + return + } + } +} + +func (iter *Iterator) skipObject() { + level := 1 + if !iter.incrementDepth() { + return + } + + for { + for i := iter.head; i < iter.tail; i++ { + switch iter.buf[i] { + case '"': // If inside string, skip it + iter.head = i + 1 + iter.skipString() + i = iter.head - 1 // it will be i++ soon + case '{': // If open symbol, increase level + level++ + if !iter.incrementDepth() { + return + } + case '}': // If close symbol, increase level + level-- + if !iter.decrementDepth() { + return + } + + // If we have returned to the original level, we're done + if level == 0 { + iter.head = i + 1 + return + } + } + } + if !iter.loadMore() { + iter.ReportError("skipObject", "incomplete object") + return + } + } +} + +func (iter *Iterator) skipString() { + for { + end, escaped := iter.findStringEnd() + if end == -1 { + if !iter.loadMore() { + iter.ReportError("skipString", "incomplete string") + return + } + if escaped { + iter.head = 1 // skip the first char as last char read is \ + } + } else { + iter.head = end + return + } + } +} + +// adapted from: https://github.com/buger/jsonparser/blob/master/parser.go +// Tries to find the end of string +// Support if string contains escaped quote symbols. +func (iter *Iterator) findStringEnd() (int, bool) { + escaped := false + for i := iter.head; i < iter.tail; i++ { + c := iter.buf[i] + if c == '"' { + if !escaped { + return i + 1, false + } + j := i - 1 + for { + if j < iter.head || iter.buf[j] != '\\' { + // even number of backslashes + // either end of buffer, or " found + return i + 1, true + } + j-- + if j < iter.head || iter.buf[j] != '\\' { + // odd number of backslashes + // it is \" or \\\" + break + } + j-- + } + } else if c == '\\' { + escaped = true + } + } + j := iter.tail - 1 + for { + if j < iter.head || iter.buf[j] != '\\' { + // even number of backslashes + // either end of buffer, or " found + return -1, false // do not end with \ + } + j-- + if j < iter.head || iter.buf[j] != '\\' { + // odd number of backslashes + // it is \" or \\\" + break + } + j-- + + } + return -1, true // end with \ +} diff --git a/vendor/github.com/json-iterator/go/iter_skip_strict.go b/vendor/github.com/json-iterator/go/iter_skip_strict.go new file mode 100644 index 00000000..6cf66d04 --- /dev/null +++ b/vendor/github.com/json-iterator/go/iter_skip_strict.go @@ -0,0 +1,99 @@ +//+build !jsoniter_sloppy + +package jsoniter + +import ( + "fmt" + "io" +) + +func (iter *Iterator) skipNumber() { + if !iter.trySkipNumber() { + iter.unreadByte() + if iter.Error != nil && iter.Error != io.EOF { + return + } + iter.ReadFloat64() + if iter.Error != nil && iter.Error != io.EOF { + iter.Error = nil + iter.ReadBigFloat() + } + } +} + +func (iter *Iterator) trySkipNumber() bool { + dotFound := false + for i := iter.head; i < iter.tail; i++ { + c := iter.buf[i] + switch c { + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + case '.': + if dotFound { + iter.ReportError("validateNumber", `more than one dot found in number`) + return true // already failed + } + if i+1 == iter.tail { + return false + } + c = iter.buf[i+1] + switch c { + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + default: + iter.ReportError("validateNumber", `missing digit after dot`) + return true // already failed + } + dotFound = true + default: + switch c { + case ',', ']', '}', ' ', '\t', '\n', '\r': + if iter.head == i { + return false // if - without following digits + } + iter.head = i + return true // must be valid + } + return false // may be invalid + } + } + return false +} + +func (iter *Iterator) skipString() { + if !iter.trySkipString() { + iter.unreadByte() + iter.ReadString() + } +} + +func (iter *Iterator) trySkipString() bool { + for i := iter.head; i < iter.tail; i++ { + c := iter.buf[i] + if c == '"' { + iter.head = i + 1 + return true // valid + } else if c == '\\' { + return false + } else if c < ' ' { + iter.ReportError("trySkipString", + fmt.Sprintf(`invalid control character found: %d`, c)) + return true // already failed + } + } + return false +} + +func (iter *Iterator) skipObject() { + iter.unreadByte() + iter.ReadObjectCB(func(iter *Iterator, field string) bool { + iter.Skip() + return true + }) +} + +func (iter *Iterator) skipArray() { + iter.unreadByte() + iter.ReadArrayCB(func(iter *Iterator) bool { + iter.Skip() + return true + }) +} diff --git a/vendor/github.com/json-iterator/go/iter_str.go b/vendor/github.com/json-iterator/go/iter_str.go new file mode 100644 index 00000000..adc487ea --- /dev/null +++ b/vendor/github.com/json-iterator/go/iter_str.go @@ -0,0 +1,215 @@ +package jsoniter + +import ( + "fmt" + "unicode/utf16" +) + +// ReadString read string from iterator +func (iter *Iterator) ReadString() (ret string) { + c := iter.nextToken() + if c == '"' { + for i := iter.head; i < iter.tail; i++ { + c := iter.buf[i] + if c == '"' { + ret = string(iter.buf[iter.head:i]) + iter.head = i + 1 + return ret + } else if c == '\\' { + break + } else if c < ' ' { + iter.ReportError("ReadString", + fmt.Sprintf(`invalid control character found: %d`, c)) + return + } + } + return iter.readStringSlowPath() + } else if c == 'n' { + iter.skipThreeBytes('u', 'l', 'l') + return "" + } + iter.ReportError("ReadString", `expects " or n, but found `+string([]byte{c})) + return +} + +func (iter *Iterator) readStringSlowPath() (ret string) { + var str []byte + var c byte + for iter.Error == nil { + c = iter.readByte() + if c == '"' { + return string(str) + } + if c == '\\' { + c = iter.readByte() + str = iter.readEscapedChar(c, str) + } else { + str = append(str, c) + } + } + iter.ReportError("readStringSlowPath", "unexpected end of input") + return +} + +func (iter *Iterator) readEscapedChar(c byte, str []byte) []byte { + switch c { + case 'u': + r := iter.readU4() + if utf16.IsSurrogate(r) { + c = iter.readByte() + if iter.Error != nil { + return nil + } + if c != '\\' { + iter.unreadByte() + str = appendRune(str, r) + return str + } + c = iter.readByte() + if iter.Error != nil { + return nil + } + if c != 'u' { + str = appendRune(str, r) + return iter.readEscapedChar(c, str) + } + r2 := iter.readU4() + if iter.Error != nil { + return nil + } + combined := utf16.DecodeRune(r, r2) + if combined == '\uFFFD' { + str = appendRune(str, r) + str = appendRune(str, r2) + } else { + str = appendRune(str, combined) + } + } else { + str = appendRune(str, r) + } + case '"': + str = append(str, '"') + case '\\': + str = append(str, '\\') + case '/': + str = append(str, '/') + case 'b': + str = append(str, '\b') + case 'f': + str = append(str, '\f') + case 'n': + str = append(str, '\n') + case 'r': + str = append(str, '\r') + case 't': + str = append(str, '\t') + default: + iter.ReportError("readEscapedChar", + `invalid escape char after \`) + return nil + } + return str +} + +// ReadStringAsSlice read string from iterator without copying into string form. +// The []byte can not be kept, as it will change after next iterator call. +func (iter *Iterator) ReadStringAsSlice() (ret []byte) { + c := iter.nextToken() + if c == '"' { + for i := iter.head; i < iter.tail; i++ { + // require ascii string and no escape + // for: field name, base64, number + if iter.buf[i] == '"' { + // fast path: reuse the underlying buffer + ret = iter.buf[iter.head:i] + iter.head = i + 1 + return ret + } + } + readLen := iter.tail - iter.head + copied := make([]byte, readLen, readLen*2) + copy(copied, iter.buf[iter.head:iter.tail]) + iter.head = iter.tail + for iter.Error == nil { + c := iter.readByte() + if c == '"' { + return copied + } + copied = append(copied, c) + } + return copied + } + iter.ReportError("ReadStringAsSlice", `expects " or n, but found `+string([]byte{c})) + return +} + +func (iter *Iterator) readU4() (ret rune) { + for i := 0; i < 4; i++ { + c := iter.readByte() + if iter.Error != nil { + return + } + if c >= '0' && c <= '9' { + ret = ret*16 + rune(c-'0') + } else if c >= 'a' && c <= 'f' { + ret = ret*16 + rune(c-'a'+10) + } else if c >= 'A' && c <= 'F' { + ret = ret*16 + rune(c-'A'+10) + } else { + iter.ReportError("readU4", "expects 0~9 or a~f, but found "+string([]byte{c})) + return + } + } + return ret +} + +const ( + t1 = 0x00 // 0000 0000 + tx = 0x80 // 1000 0000 + t2 = 0xC0 // 1100 0000 + t3 = 0xE0 // 1110 0000 + t4 = 0xF0 // 1111 0000 + t5 = 0xF8 // 1111 1000 + + maskx = 0x3F // 0011 1111 + mask2 = 0x1F // 0001 1111 + mask3 = 0x0F // 0000 1111 + mask4 = 0x07 // 0000 0111 + + rune1Max = 1<<7 - 1 + rune2Max = 1<<11 - 1 + rune3Max = 1<<16 - 1 + + surrogateMin = 0xD800 + surrogateMax = 0xDFFF + + maxRune = '\U0010FFFF' // Maximum valid Unicode code point. + runeError = '\uFFFD' // the "error" Rune or "Unicode replacement character" +) + +func appendRune(p []byte, r rune) []byte { + // Negative values are erroneous. Making it unsigned addresses the problem. + switch i := uint32(r); { + case i <= rune1Max: + p = append(p, byte(r)) + return p + case i <= rune2Max: + p = append(p, t2|byte(r>>6)) + p = append(p, tx|byte(r)&maskx) + return p + case i > maxRune, surrogateMin <= i && i <= surrogateMax: + r = runeError + fallthrough + case i <= rune3Max: + p = append(p, t3|byte(r>>12)) + p = append(p, tx|byte(r>>6)&maskx) + p = append(p, tx|byte(r)&maskx) + return p + default: + p = append(p, t4|byte(r>>18)) + p = append(p, tx|byte(r>>12)&maskx) + p = append(p, tx|byte(r>>6)&maskx) + p = append(p, tx|byte(r)&maskx) + return p + } +} diff --git a/vendor/github.com/json-iterator/go/jsoniter.go b/vendor/github.com/json-iterator/go/jsoniter.go new file mode 100644 index 00000000..c2934f91 --- /dev/null +++ b/vendor/github.com/json-iterator/go/jsoniter.go @@ -0,0 +1,18 @@ +// Package jsoniter implements encoding and decoding of JSON as defined in +// RFC 4627 and provides interfaces with identical syntax of standard lib encoding/json. +// Converting from encoding/json to jsoniter is no more than replacing the package with jsoniter +// and variable type declarations (if any). +// jsoniter interfaces gives 100% compatibility with code using standard lib. +// +// "JSON and Go" +// (https://golang.org/doc/articles/json_and_go.html) +// gives a description of how Marshal/Unmarshal operate +// between arbitrary or predefined json objects and bytes, +// and it applies to jsoniter.Marshal/Unmarshal as well. +// +// Besides, jsoniter.Iterator provides a different set of interfaces +// iterating given bytes/string/reader +// and yielding parsed elements one by one. +// This set of interfaces reads input as required and gives +// better performance. +package jsoniter diff --git a/vendor/github.com/json-iterator/go/pool.go b/vendor/github.com/json-iterator/go/pool.go new file mode 100644 index 00000000..e2389b56 --- /dev/null +++ b/vendor/github.com/json-iterator/go/pool.go @@ -0,0 +1,42 @@ +package jsoniter + +import ( + "io" +) + +// IteratorPool a thread safe pool of iterators with same configuration +type IteratorPool interface { + BorrowIterator(data []byte) *Iterator + ReturnIterator(iter *Iterator) +} + +// StreamPool a thread safe pool of streams with same configuration +type StreamPool interface { + BorrowStream(writer io.Writer) *Stream + ReturnStream(stream *Stream) +} + +func (cfg *frozenConfig) BorrowStream(writer io.Writer) *Stream { + stream := cfg.streamPool.Get().(*Stream) + stream.Reset(writer) + return stream +} + +func (cfg *frozenConfig) ReturnStream(stream *Stream) { + stream.out = nil + stream.Error = nil + stream.Attachment = nil + cfg.streamPool.Put(stream) +} + +func (cfg *frozenConfig) BorrowIterator(data []byte) *Iterator { + iter := cfg.iteratorPool.Get().(*Iterator) + iter.ResetBytes(data) + return iter +} + +func (cfg *frozenConfig) ReturnIterator(iter *Iterator) { + iter.Error = nil + iter.Attachment = nil + cfg.iteratorPool.Put(iter) +} diff --git a/vendor/github.com/json-iterator/go/reflect.go b/vendor/github.com/json-iterator/go/reflect.go new file mode 100644 index 00000000..39acb320 --- /dev/null +++ b/vendor/github.com/json-iterator/go/reflect.go @@ -0,0 +1,337 @@ +package jsoniter + +import ( + "fmt" + "reflect" + "unsafe" + + "github.com/modern-go/reflect2" +) + +// ValDecoder is an internal type registered to cache as needed. +// Don't confuse jsoniter.ValDecoder with json.Decoder. +// For json.Decoder's adapter, refer to jsoniter.AdapterDecoder(todo link). +// +// Reflection on type to create decoders, which is then cached +// Reflection on value is avoided as we can, as the reflect.Value itself will allocate, with following exceptions +// 1. create instance of new value, for example *int will need a int to be allocated +// 2. append to slice, if the existing cap is not enough, allocate will be done using Reflect.New +// 3. assignment to map, both key and value will be reflect.Value +// For a simple struct binding, it will be reflect.Value free and allocation free +type ValDecoder interface { + Decode(ptr unsafe.Pointer, iter *Iterator) +} + +// ValEncoder is an internal type registered to cache as needed. +// Don't confuse jsoniter.ValEncoder with json.Encoder. +// For json.Encoder's adapter, refer to jsoniter.AdapterEncoder(todo godoc link). +type ValEncoder interface { + IsEmpty(ptr unsafe.Pointer) bool + Encode(ptr unsafe.Pointer, stream *Stream) +} + +type checkIsEmpty interface { + IsEmpty(ptr unsafe.Pointer) bool +} + +type ctx struct { + *frozenConfig + prefix string + encoders map[reflect2.Type]ValEncoder + decoders map[reflect2.Type]ValDecoder +} + +func (b *ctx) caseSensitive() bool { + if b.frozenConfig == nil { + // default is case-insensitive + return false + } + return b.frozenConfig.caseSensitive +} + +func (b *ctx) append(prefix string) *ctx { + return &ctx{ + frozenConfig: b.frozenConfig, + prefix: b.prefix + " " + prefix, + encoders: b.encoders, + decoders: b.decoders, + } +} + +// ReadVal copy the underlying JSON into go interface, same as json.Unmarshal +func (iter *Iterator) ReadVal(obj interface{}) { + depth := iter.depth + cacheKey := reflect2.RTypeOf(obj) + decoder := iter.cfg.getDecoderFromCache(cacheKey) + if decoder == nil { + typ := reflect2.TypeOf(obj) + if typ == nil || typ.Kind() != reflect.Ptr { + iter.ReportError("ReadVal", "can only unmarshal into pointer") + return + } + decoder = iter.cfg.DecoderOf(typ) + } + ptr := reflect2.PtrOf(obj) + if ptr == nil { + iter.ReportError("ReadVal", "can not read into nil pointer") + return + } + decoder.Decode(ptr, iter) + if iter.depth != depth { + iter.ReportError("ReadVal", "unexpected mismatched nesting") + return + } +} + +// WriteVal copy the go interface into underlying JSON, same as json.Marshal +func (stream *Stream) WriteVal(val interface{}) { + if nil == val { + stream.WriteNil() + return + } + cacheKey := reflect2.RTypeOf(val) + encoder := stream.cfg.getEncoderFromCache(cacheKey) + if encoder == nil { + typ := reflect2.TypeOf(val) + encoder = stream.cfg.EncoderOf(typ) + } + encoder.Encode(reflect2.PtrOf(val), stream) +} + +func (cfg *frozenConfig) DecoderOf(typ reflect2.Type) ValDecoder { + cacheKey := typ.RType() + decoder := cfg.getDecoderFromCache(cacheKey) + if decoder != nil { + return decoder + } + ctx := &ctx{ + frozenConfig: cfg, + prefix: "", + decoders: map[reflect2.Type]ValDecoder{}, + encoders: map[reflect2.Type]ValEncoder{}, + } + ptrType := typ.(*reflect2.UnsafePtrType) + decoder = decoderOfType(ctx, ptrType.Elem()) + cfg.addDecoderToCache(cacheKey, decoder) + return decoder +} + +func decoderOfType(ctx *ctx, typ reflect2.Type) ValDecoder { + decoder := getTypeDecoderFromExtension(ctx, typ) + if decoder != nil { + return decoder + } + decoder = createDecoderOfType(ctx, typ) + for _, extension := range extensions { + decoder = extension.DecorateDecoder(typ, decoder) + } + decoder = ctx.decoderExtension.DecorateDecoder(typ, decoder) + for _, extension := range ctx.extraExtensions { + decoder = extension.DecorateDecoder(typ, decoder) + } + return decoder +} + +func createDecoderOfType(ctx *ctx, typ reflect2.Type) ValDecoder { + decoder := ctx.decoders[typ] + if decoder != nil { + return decoder + } + placeholder := &placeholderDecoder{} + ctx.decoders[typ] = placeholder + decoder = _createDecoderOfType(ctx, typ) + placeholder.decoder = decoder + return decoder +} + +func _createDecoderOfType(ctx *ctx, typ reflect2.Type) ValDecoder { + decoder := createDecoderOfJsonRawMessage(ctx, typ) + if decoder != nil { + return decoder + } + decoder = createDecoderOfJsonNumber(ctx, typ) + if decoder != nil { + return decoder + } + decoder = createDecoderOfMarshaler(ctx, typ) + if decoder != nil { + return decoder + } + decoder = createDecoderOfAny(ctx, typ) + if decoder != nil { + return decoder + } + decoder = createDecoderOfNative(ctx, typ) + if decoder != nil { + return decoder + } + switch typ.Kind() { + case reflect.Interface: + ifaceType, isIFace := typ.(*reflect2.UnsafeIFaceType) + if isIFace { + return &ifaceDecoder{valType: ifaceType} + } + return &efaceDecoder{} + case reflect.Struct: + return decoderOfStruct(ctx, typ) + case reflect.Array: + return decoderOfArray(ctx, typ) + case reflect.Slice: + return decoderOfSlice(ctx, typ) + case reflect.Map: + return decoderOfMap(ctx, typ) + case reflect.Ptr: + return decoderOfOptional(ctx, typ) + default: + return &lazyErrorDecoder{err: fmt.Errorf("%s%s is unsupported type", ctx.prefix, typ.String())} + } +} + +func (cfg *frozenConfig) EncoderOf(typ reflect2.Type) ValEncoder { + cacheKey := typ.RType() + encoder := cfg.getEncoderFromCache(cacheKey) + if encoder != nil { + return encoder + } + ctx := &ctx{ + frozenConfig: cfg, + prefix: "", + decoders: map[reflect2.Type]ValDecoder{}, + encoders: map[reflect2.Type]ValEncoder{}, + } + encoder = encoderOfType(ctx, typ) + if typ.LikePtr() { + encoder = &onePtrEncoder{encoder} + } + cfg.addEncoderToCache(cacheKey, encoder) + return encoder +} + +type onePtrEncoder struct { + encoder ValEncoder +} + +func (encoder *onePtrEncoder) IsEmpty(ptr unsafe.Pointer) bool { + return encoder.encoder.IsEmpty(unsafe.Pointer(&ptr)) +} + +func (encoder *onePtrEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { + encoder.encoder.Encode(unsafe.Pointer(&ptr), stream) +} + +func encoderOfType(ctx *ctx, typ reflect2.Type) ValEncoder { + encoder := getTypeEncoderFromExtension(ctx, typ) + if encoder != nil { + return encoder + } + encoder = createEncoderOfType(ctx, typ) + for _, extension := range extensions { + encoder = extension.DecorateEncoder(typ, encoder) + } + encoder = ctx.encoderExtension.DecorateEncoder(typ, encoder) + for _, extension := range ctx.extraExtensions { + encoder = extension.DecorateEncoder(typ, encoder) + } + return encoder +} + +func createEncoderOfType(ctx *ctx, typ reflect2.Type) ValEncoder { + encoder := ctx.encoders[typ] + if encoder != nil { + return encoder + } + placeholder := &placeholderEncoder{} + ctx.encoders[typ] = placeholder + encoder = _createEncoderOfType(ctx, typ) + placeholder.encoder = encoder + return encoder +} +func _createEncoderOfType(ctx *ctx, typ reflect2.Type) ValEncoder { + encoder := createEncoderOfJsonRawMessage(ctx, typ) + if encoder != nil { + return encoder + } + encoder = createEncoderOfJsonNumber(ctx, typ) + if encoder != nil { + return encoder + } + encoder = createEncoderOfMarshaler(ctx, typ) + if encoder != nil { + return encoder + } + encoder = createEncoderOfAny(ctx, typ) + if encoder != nil { + return encoder + } + encoder = createEncoderOfNative(ctx, typ) + if encoder != nil { + return encoder + } + kind := typ.Kind() + switch kind { + case reflect.Interface: + return &dynamicEncoder{typ} + case reflect.Struct: + return encoderOfStruct(ctx, typ) + case reflect.Array: + return encoderOfArray(ctx, typ) + case reflect.Slice: + return encoderOfSlice(ctx, typ) + case reflect.Map: + return encoderOfMap(ctx, typ) + case reflect.Ptr: + return encoderOfOptional(ctx, typ) + default: + return &lazyErrorEncoder{err: fmt.Errorf("%s%s is unsupported type", ctx.prefix, typ.String())} + } +} + +type lazyErrorDecoder struct { + err error +} + +func (decoder *lazyErrorDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { + if iter.WhatIsNext() != NilValue { + if iter.Error == nil { + iter.Error = decoder.err + } + } else { + iter.Skip() + } +} + +type lazyErrorEncoder struct { + err error +} + +func (encoder *lazyErrorEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { + if ptr == nil { + stream.WriteNil() + } else if stream.Error == nil { + stream.Error = encoder.err + } +} + +func (encoder *lazyErrorEncoder) IsEmpty(ptr unsafe.Pointer) bool { + return false +} + +type placeholderDecoder struct { + decoder ValDecoder +} + +func (decoder *placeholderDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { + decoder.decoder.Decode(ptr, iter) +} + +type placeholderEncoder struct { + encoder ValEncoder +} + +func (encoder *placeholderEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { + encoder.encoder.Encode(ptr, stream) +} + +func (encoder *placeholderEncoder) IsEmpty(ptr unsafe.Pointer) bool { + return encoder.encoder.IsEmpty(ptr) +} diff --git a/vendor/github.com/json-iterator/go/reflect_array.go b/vendor/github.com/json-iterator/go/reflect_array.go new file mode 100644 index 00000000..13a0b7b0 --- /dev/null +++ b/vendor/github.com/json-iterator/go/reflect_array.go @@ -0,0 +1,104 @@ +package jsoniter + +import ( + "fmt" + "github.com/modern-go/reflect2" + "io" + "unsafe" +) + +func decoderOfArray(ctx *ctx, typ reflect2.Type) ValDecoder { + arrayType := typ.(*reflect2.UnsafeArrayType) + decoder := decoderOfType(ctx.append("[arrayElem]"), arrayType.Elem()) + return &arrayDecoder{arrayType, decoder} +} + +func encoderOfArray(ctx *ctx, typ reflect2.Type) ValEncoder { + arrayType := typ.(*reflect2.UnsafeArrayType) + if arrayType.Len() == 0 { + return emptyArrayEncoder{} + } + encoder := encoderOfType(ctx.append("[arrayElem]"), arrayType.Elem()) + return &arrayEncoder{arrayType, encoder} +} + +type emptyArrayEncoder struct{} + +func (encoder emptyArrayEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { + stream.WriteEmptyArray() +} + +func (encoder emptyArrayEncoder) IsEmpty(ptr unsafe.Pointer) bool { + return true +} + +type arrayEncoder struct { + arrayType *reflect2.UnsafeArrayType + elemEncoder ValEncoder +} + +func (encoder *arrayEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { + stream.WriteArrayStart() + elemPtr := unsafe.Pointer(ptr) + encoder.elemEncoder.Encode(elemPtr, stream) + for i := 1; i < encoder.arrayType.Len(); i++ { + stream.WriteMore() + elemPtr = encoder.arrayType.UnsafeGetIndex(ptr, i) + encoder.elemEncoder.Encode(elemPtr, stream) + } + stream.WriteArrayEnd() + if stream.Error != nil && stream.Error != io.EOF { + stream.Error = fmt.Errorf("%v: %s", encoder.arrayType, stream.Error.Error()) + } +} + +func (encoder *arrayEncoder) IsEmpty(ptr unsafe.Pointer) bool { + return false +} + +type arrayDecoder struct { + arrayType *reflect2.UnsafeArrayType + elemDecoder ValDecoder +} + +func (decoder *arrayDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { + decoder.doDecode(ptr, iter) + if iter.Error != nil && iter.Error != io.EOF { + iter.Error = fmt.Errorf("%v: %s", decoder.arrayType, iter.Error.Error()) + } +} + +func (decoder *arrayDecoder) doDecode(ptr unsafe.Pointer, iter *Iterator) { + c := iter.nextToken() + arrayType := decoder.arrayType + if c == 'n' { + iter.skipThreeBytes('u', 'l', 'l') + return + } + if c != '[' { + iter.ReportError("decode array", "expect [ or n, but found "+string([]byte{c})) + return + } + c = iter.nextToken() + if c == ']' { + return + } + iter.unreadByte() + elemPtr := arrayType.UnsafeGetIndex(ptr, 0) + decoder.elemDecoder.Decode(elemPtr, iter) + length := 1 + for c = iter.nextToken(); c == ','; c = iter.nextToken() { + if length >= arrayType.Len() { + iter.Skip() + continue + } + idx := length + length += 1 + elemPtr = arrayType.UnsafeGetIndex(ptr, idx) + decoder.elemDecoder.Decode(elemPtr, iter) + } + if c != ']' { + iter.ReportError("decode array", "expect ], but found "+string([]byte{c})) + return + } +} diff --git a/vendor/github.com/json-iterator/go/reflect_dynamic.go b/vendor/github.com/json-iterator/go/reflect_dynamic.go new file mode 100644 index 00000000..8b6bc8b4 --- /dev/null +++ b/vendor/github.com/json-iterator/go/reflect_dynamic.go @@ -0,0 +1,70 @@ +package jsoniter + +import ( + "github.com/modern-go/reflect2" + "reflect" + "unsafe" +) + +type dynamicEncoder struct { + valType reflect2.Type +} + +func (encoder *dynamicEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { + obj := encoder.valType.UnsafeIndirect(ptr) + stream.WriteVal(obj) +} + +func (encoder *dynamicEncoder) IsEmpty(ptr unsafe.Pointer) bool { + return encoder.valType.UnsafeIndirect(ptr) == nil +} + +type efaceDecoder struct { +} + +func (decoder *efaceDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { + pObj := (*interface{})(ptr) + obj := *pObj + if obj == nil { + *pObj = iter.Read() + return + } + typ := reflect2.TypeOf(obj) + if typ.Kind() != reflect.Ptr { + *pObj = iter.Read() + return + } + ptrType := typ.(*reflect2.UnsafePtrType) + ptrElemType := ptrType.Elem() + if iter.WhatIsNext() == NilValue { + if ptrElemType.Kind() != reflect.Ptr { + iter.skipFourBytes('n', 'u', 'l', 'l') + *pObj = nil + return + } + } + if reflect2.IsNil(obj) { + obj := ptrElemType.New() + iter.ReadVal(obj) + *pObj = obj + return + } + iter.ReadVal(obj) +} + +type ifaceDecoder struct { + valType *reflect2.UnsafeIFaceType +} + +func (decoder *ifaceDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { + if iter.ReadNil() { + decoder.valType.UnsafeSet(ptr, decoder.valType.UnsafeNew()) + return + } + obj := decoder.valType.UnsafeIndirect(ptr) + if reflect2.IsNil(obj) { + iter.ReportError("decode non empty interface", "can not unmarshal into nil") + return + } + iter.ReadVal(obj) +} diff --git a/vendor/github.com/json-iterator/go/reflect_extension.go b/vendor/github.com/json-iterator/go/reflect_extension.go new file mode 100644 index 00000000..74a97bfe --- /dev/null +++ b/vendor/github.com/json-iterator/go/reflect_extension.go @@ -0,0 +1,483 @@ +package jsoniter + +import ( + "fmt" + "github.com/modern-go/reflect2" + "reflect" + "sort" + "strings" + "unicode" + "unsafe" +) + +var typeDecoders = map[string]ValDecoder{} +var fieldDecoders = map[string]ValDecoder{} +var typeEncoders = map[string]ValEncoder{} +var fieldEncoders = map[string]ValEncoder{} +var extensions = []Extension{} + +// StructDescriptor describe how should we encode/decode the struct +type StructDescriptor struct { + Type reflect2.Type + Fields []*Binding +} + +// GetField get one field from the descriptor by its name. +// Can not use map here to keep field orders. +func (structDescriptor *StructDescriptor) GetField(fieldName string) *Binding { + for _, binding := range structDescriptor.Fields { + if binding.Field.Name() == fieldName { + return binding + } + } + return nil +} + +// Binding describe how should we encode/decode the struct field +type Binding struct { + levels []int + Field reflect2.StructField + FromNames []string + ToNames []string + Encoder ValEncoder + Decoder ValDecoder +} + +// Extension the one for all SPI. Customize encoding/decoding by specifying alternate encoder/decoder. +// Can also rename fields by UpdateStructDescriptor. +type Extension interface { + UpdateStructDescriptor(structDescriptor *StructDescriptor) + CreateMapKeyDecoder(typ reflect2.Type) ValDecoder + CreateMapKeyEncoder(typ reflect2.Type) ValEncoder + CreateDecoder(typ reflect2.Type) ValDecoder + CreateEncoder(typ reflect2.Type) ValEncoder + DecorateDecoder(typ reflect2.Type, decoder ValDecoder) ValDecoder + DecorateEncoder(typ reflect2.Type, encoder ValEncoder) ValEncoder +} + +// DummyExtension embed this type get dummy implementation for all methods of Extension +type DummyExtension struct { +} + +// UpdateStructDescriptor No-op +func (extension *DummyExtension) UpdateStructDescriptor(structDescriptor *StructDescriptor) { +} + +// CreateMapKeyDecoder No-op +func (extension *DummyExtension) CreateMapKeyDecoder(typ reflect2.Type) ValDecoder { + return nil +} + +// CreateMapKeyEncoder No-op +func (extension *DummyExtension) CreateMapKeyEncoder(typ reflect2.Type) ValEncoder { + return nil +} + +// CreateDecoder No-op +func (extension *DummyExtension) CreateDecoder(typ reflect2.Type) ValDecoder { + return nil +} + +// CreateEncoder No-op +func (extension *DummyExtension) CreateEncoder(typ reflect2.Type) ValEncoder { + return nil +} + +// DecorateDecoder No-op +func (extension *DummyExtension) DecorateDecoder(typ reflect2.Type, decoder ValDecoder) ValDecoder { + return decoder +} + +// DecorateEncoder No-op +func (extension *DummyExtension) DecorateEncoder(typ reflect2.Type, encoder ValEncoder) ValEncoder { + return encoder +} + +type EncoderExtension map[reflect2.Type]ValEncoder + +// UpdateStructDescriptor No-op +func (extension EncoderExtension) UpdateStructDescriptor(structDescriptor *StructDescriptor) { +} + +// CreateDecoder No-op +func (extension EncoderExtension) CreateDecoder(typ reflect2.Type) ValDecoder { + return nil +} + +// CreateEncoder get encoder from map +func (extension EncoderExtension) CreateEncoder(typ reflect2.Type) ValEncoder { + return extension[typ] +} + +// CreateMapKeyDecoder No-op +func (extension EncoderExtension) CreateMapKeyDecoder(typ reflect2.Type) ValDecoder { + return nil +} + +// CreateMapKeyEncoder No-op +func (extension EncoderExtension) CreateMapKeyEncoder(typ reflect2.Type) ValEncoder { + return nil +} + +// DecorateDecoder No-op +func (extension EncoderExtension) DecorateDecoder(typ reflect2.Type, decoder ValDecoder) ValDecoder { + return decoder +} + +// DecorateEncoder No-op +func (extension EncoderExtension) DecorateEncoder(typ reflect2.Type, encoder ValEncoder) ValEncoder { + return encoder +} + +type DecoderExtension map[reflect2.Type]ValDecoder + +// UpdateStructDescriptor No-op +func (extension DecoderExtension) UpdateStructDescriptor(structDescriptor *StructDescriptor) { +} + +// CreateMapKeyDecoder No-op +func (extension DecoderExtension) CreateMapKeyDecoder(typ reflect2.Type) ValDecoder { + return nil +} + +// CreateMapKeyEncoder No-op +func (extension DecoderExtension) CreateMapKeyEncoder(typ reflect2.Type) ValEncoder { + return nil +} + +// CreateDecoder get decoder from map +func (extension DecoderExtension) CreateDecoder(typ reflect2.Type) ValDecoder { + return extension[typ] +} + +// CreateEncoder No-op +func (extension DecoderExtension) CreateEncoder(typ reflect2.Type) ValEncoder { + return nil +} + +// DecorateDecoder No-op +func (extension DecoderExtension) DecorateDecoder(typ reflect2.Type, decoder ValDecoder) ValDecoder { + return decoder +} + +// DecorateEncoder No-op +func (extension DecoderExtension) DecorateEncoder(typ reflect2.Type, encoder ValEncoder) ValEncoder { + return encoder +} + +type funcDecoder struct { + fun DecoderFunc +} + +func (decoder *funcDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { + decoder.fun(ptr, iter) +} + +type funcEncoder struct { + fun EncoderFunc + isEmptyFunc func(ptr unsafe.Pointer) bool +} + +func (encoder *funcEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { + encoder.fun(ptr, stream) +} + +func (encoder *funcEncoder) IsEmpty(ptr unsafe.Pointer) bool { + if encoder.isEmptyFunc == nil { + return false + } + return encoder.isEmptyFunc(ptr) +} + +// DecoderFunc the function form of TypeDecoder +type DecoderFunc func(ptr unsafe.Pointer, iter *Iterator) + +// EncoderFunc the function form of TypeEncoder +type EncoderFunc func(ptr unsafe.Pointer, stream *Stream) + +// RegisterTypeDecoderFunc register TypeDecoder for a type with function +func RegisterTypeDecoderFunc(typ string, fun DecoderFunc) { + typeDecoders[typ] = &funcDecoder{fun} +} + +// RegisterTypeDecoder register TypeDecoder for a typ +func RegisterTypeDecoder(typ string, decoder ValDecoder) { + typeDecoders[typ] = decoder +} + +// RegisterFieldDecoderFunc register TypeDecoder for a struct field with function +func RegisterFieldDecoderFunc(typ string, field string, fun DecoderFunc) { + RegisterFieldDecoder(typ, field, &funcDecoder{fun}) +} + +// RegisterFieldDecoder register TypeDecoder for a struct field +func RegisterFieldDecoder(typ string, field string, decoder ValDecoder) { + fieldDecoders[fmt.Sprintf("%s/%s", typ, field)] = decoder +} + +// RegisterTypeEncoderFunc register TypeEncoder for a type with encode/isEmpty function +func RegisterTypeEncoderFunc(typ string, fun EncoderFunc, isEmptyFunc func(unsafe.Pointer) bool) { + typeEncoders[typ] = &funcEncoder{fun, isEmptyFunc} +} + +// RegisterTypeEncoder register TypeEncoder for a type +func RegisterTypeEncoder(typ string, encoder ValEncoder) { + typeEncoders[typ] = encoder +} + +// RegisterFieldEncoderFunc register TypeEncoder for a struct field with encode/isEmpty function +func RegisterFieldEncoderFunc(typ string, field string, fun EncoderFunc, isEmptyFunc func(unsafe.Pointer) bool) { + RegisterFieldEncoder(typ, field, &funcEncoder{fun, isEmptyFunc}) +} + +// RegisterFieldEncoder register TypeEncoder for a struct field +func RegisterFieldEncoder(typ string, field string, encoder ValEncoder) { + fieldEncoders[fmt.Sprintf("%s/%s", typ, field)] = encoder +} + +// RegisterExtension register extension +func RegisterExtension(extension Extension) { + extensions = append(extensions, extension) +} + +func getTypeDecoderFromExtension(ctx *ctx, typ reflect2.Type) ValDecoder { + decoder := _getTypeDecoderFromExtension(ctx, typ) + if decoder != nil { + for _, extension := range extensions { + decoder = extension.DecorateDecoder(typ, decoder) + } + decoder = ctx.decoderExtension.DecorateDecoder(typ, decoder) + for _, extension := range ctx.extraExtensions { + decoder = extension.DecorateDecoder(typ, decoder) + } + } + return decoder +} +func _getTypeDecoderFromExtension(ctx *ctx, typ reflect2.Type) ValDecoder { + for _, extension := range extensions { + decoder := extension.CreateDecoder(typ) + if decoder != nil { + return decoder + } + } + decoder := ctx.decoderExtension.CreateDecoder(typ) + if decoder != nil { + return decoder + } + for _, extension := range ctx.extraExtensions { + decoder := extension.CreateDecoder(typ) + if decoder != nil { + return decoder + } + } + typeName := typ.String() + decoder = typeDecoders[typeName] + if decoder != nil { + return decoder + } + if typ.Kind() == reflect.Ptr { + ptrType := typ.(*reflect2.UnsafePtrType) + decoder := typeDecoders[ptrType.Elem().String()] + if decoder != nil { + return &OptionalDecoder{ptrType.Elem(), decoder} + } + } + return nil +} + +func getTypeEncoderFromExtension(ctx *ctx, typ reflect2.Type) ValEncoder { + encoder := _getTypeEncoderFromExtension(ctx, typ) + if encoder != nil { + for _, extension := range extensions { + encoder = extension.DecorateEncoder(typ, encoder) + } + encoder = ctx.encoderExtension.DecorateEncoder(typ, encoder) + for _, extension := range ctx.extraExtensions { + encoder = extension.DecorateEncoder(typ, encoder) + } + } + return encoder +} + +func _getTypeEncoderFromExtension(ctx *ctx, typ reflect2.Type) ValEncoder { + for _, extension := range extensions { + encoder := extension.CreateEncoder(typ) + if encoder != nil { + return encoder + } + } + encoder := ctx.encoderExtension.CreateEncoder(typ) + if encoder != nil { + return encoder + } + for _, extension := range ctx.extraExtensions { + encoder := extension.CreateEncoder(typ) + if encoder != nil { + return encoder + } + } + typeName := typ.String() + encoder = typeEncoders[typeName] + if encoder != nil { + return encoder + } + if typ.Kind() == reflect.Ptr { + typePtr := typ.(*reflect2.UnsafePtrType) + encoder := typeEncoders[typePtr.Elem().String()] + if encoder != nil { + return &OptionalEncoder{encoder} + } + } + return nil +} + +func describeStruct(ctx *ctx, typ reflect2.Type) *StructDescriptor { + structType := typ.(*reflect2.UnsafeStructType) + embeddedBindings := []*Binding{} + bindings := []*Binding{} + for i := 0; i < structType.NumField(); i++ { + field := structType.Field(i) + tag, hastag := field.Tag().Lookup(ctx.getTagKey()) + if ctx.onlyTaggedField && !hastag && !field.Anonymous() { + continue + } + if tag == "-" || field.Name() == "_" { + continue + } + tagParts := strings.Split(tag, ",") + if field.Anonymous() && (tag == "" || tagParts[0] == "") { + if field.Type().Kind() == reflect.Struct { + structDescriptor := describeStruct(ctx, field.Type()) + for _, binding := range structDescriptor.Fields { + binding.levels = append([]int{i}, binding.levels...) + omitempty := binding.Encoder.(*structFieldEncoder).omitempty + binding.Encoder = &structFieldEncoder{field, binding.Encoder, omitempty} + binding.Decoder = &structFieldDecoder{field, binding.Decoder} + embeddedBindings = append(embeddedBindings, binding) + } + continue + } else if field.Type().Kind() == reflect.Ptr { + ptrType := field.Type().(*reflect2.UnsafePtrType) + if ptrType.Elem().Kind() == reflect.Struct { + structDescriptor := describeStruct(ctx, ptrType.Elem()) + for _, binding := range structDescriptor.Fields { + binding.levels = append([]int{i}, binding.levels...) + omitempty := binding.Encoder.(*structFieldEncoder).omitempty + binding.Encoder = &dereferenceEncoder{binding.Encoder} + binding.Encoder = &structFieldEncoder{field, binding.Encoder, omitempty} + binding.Decoder = &dereferenceDecoder{ptrType.Elem(), binding.Decoder} + binding.Decoder = &structFieldDecoder{field, binding.Decoder} + embeddedBindings = append(embeddedBindings, binding) + } + continue + } + } + } + fieldNames := calcFieldNames(field.Name(), tagParts[0], tag) + fieldCacheKey := fmt.Sprintf("%s/%s", typ.String(), field.Name()) + decoder := fieldDecoders[fieldCacheKey] + if decoder == nil { + decoder = decoderOfType(ctx.append(field.Name()), field.Type()) + } + encoder := fieldEncoders[fieldCacheKey] + if encoder == nil { + encoder = encoderOfType(ctx.append(field.Name()), field.Type()) + } + binding := &Binding{ + Field: field, + FromNames: fieldNames, + ToNames: fieldNames, + Decoder: decoder, + Encoder: encoder, + } + binding.levels = []int{i} + bindings = append(bindings, binding) + } + return createStructDescriptor(ctx, typ, bindings, embeddedBindings) +} +func createStructDescriptor(ctx *ctx, typ reflect2.Type, bindings []*Binding, embeddedBindings []*Binding) *StructDescriptor { + structDescriptor := &StructDescriptor{ + Type: typ, + Fields: bindings, + } + for _, extension := range extensions { + extension.UpdateStructDescriptor(structDescriptor) + } + ctx.encoderExtension.UpdateStructDescriptor(structDescriptor) + ctx.decoderExtension.UpdateStructDescriptor(structDescriptor) + for _, extension := range ctx.extraExtensions { + extension.UpdateStructDescriptor(structDescriptor) + } + processTags(structDescriptor, ctx.frozenConfig) + // merge normal & embedded bindings & sort with original order + allBindings := sortableBindings(append(embeddedBindings, structDescriptor.Fields...)) + sort.Sort(allBindings) + structDescriptor.Fields = allBindings + return structDescriptor +} + +type sortableBindings []*Binding + +func (bindings sortableBindings) Len() int { + return len(bindings) +} + +func (bindings sortableBindings) Less(i, j int) bool { + left := bindings[i].levels + right := bindings[j].levels + k := 0 + for { + if left[k] < right[k] { + return true + } else if left[k] > right[k] { + return false + } + k++ + } +} + +func (bindings sortableBindings) Swap(i, j int) { + bindings[i], bindings[j] = bindings[j], bindings[i] +} + +func processTags(structDescriptor *StructDescriptor, cfg *frozenConfig) { + for _, binding := range structDescriptor.Fields { + shouldOmitEmpty := false + tagParts := strings.Split(binding.Field.Tag().Get(cfg.getTagKey()), ",") + for _, tagPart := range tagParts[1:] { + if tagPart == "omitempty" { + shouldOmitEmpty = true + } else if tagPart == "string" { + if binding.Field.Type().Kind() == reflect.String { + binding.Decoder = &stringModeStringDecoder{binding.Decoder, cfg} + binding.Encoder = &stringModeStringEncoder{binding.Encoder, cfg} + } else { + binding.Decoder = &stringModeNumberDecoder{binding.Decoder} + binding.Encoder = &stringModeNumberEncoder{binding.Encoder} + } + } + } + binding.Decoder = &structFieldDecoder{binding.Field, binding.Decoder} + binding.Encoder = &structFieldEncoder{binding.Field, binding.Encoder, shouldOmitEmpty} + } +} + +func calcFieldNames(originalFieldName string, tagProvidedFieldName string, wholeTag string) []string { + // ignore? + if wholeTag == "-" { + return []string{} + } + // rename? + var fieldNames []string + if tagProvidedFieldName == "" { + fieldNames = []string{originalFieldName} + } else { + fieldNames = []string{tagProvidedFieldName} + } + // private? + isNotExported := unicode.IsLower(rune(originalFieldName[0])) || originalFieldName[0] == '_' + if isNotExported { + fieldNames = []string{} + } + return fieldNames +} diff --git a/vendor/github.com/json-iterator/go/reflect_json_number.go b/vendor/github.com/json-iterator/go/reflect_json_number.go new file mode 100644 index 00000000..98d45c1e --- /dev/null +++ b/vendor/github.com/json-iterator/go/reflect_json_number.go @@ -0,0 +1,112 @@ +package jsoniter + +import ( + "encoding/json" + "github.com/modern-go/reflect2" + "strconv" + "unsafe" +) + +type Number string + +// String returns the literal text of the number. +func (n Number) String() string { return string(n) } + +// Float64 returns the number as a float64. +func (n Number) Float64() (float64, error) { + return strconv.ParseFloat(string(n), 64) +} + +// Int64 returns the number as an int64. +func (n Number) Int64() (int64, error) { + return strconv.ParseInt(string(n), 10, 64) +} + +func CastJsonNumber(val interface{}) (string, bool) { + switch typedVal := val.(type) { + case json.Number: + return string(typedVal), true + case Number: + return string(typedVal), true + } + return "", false +} + +var jsonNumberType = reflect2.TypeOfPtr((*json.Number)(nil)).Elem() +var jsoniterNumberType = reflect2.TypeOfPtr((*Number)(nil)).Elem() + +func createDecoderOfJsonNumber(ctx *ctx, typ reflect2.Type) ValDecoder { + if typ.AssignableTo(jsonNumberType) { + return &jsonNumberCodec{} + } + if typ.AssignableTo(jsoniterNumberType) { + return &jsoniterNumberCodec{} + } + return nil +} + +func createEncoderOfJsonNumber(ctx *ctx, typ reflect2.Type) ValEncoder { + if typ.AssignableTo(jsonNumberType) { + return &jsonNumberCodec{} + } + if typ.AssignableTo(jsoniterNumberType) { + return &jsoniterNumberCodec{} + } + return nil +} + +type jsonNumberCodec struct { +} + +func (codec *jsonNumberCodec) Decode(ptr unsafe.Pointer, iter *Iterator) { + switch iter.WhatIsNext() { + case StringValue: + *((*json.Number)(ptr)) = json.Number(iter.ReadString()) + case NilValue: + iter.skipFourBytes('n', 'u', 'l', 'l') + *((*json.Number)(ptr)) = "" + default: + *((*json.Number)(ptr)) = json.Number([]byte(iter.readNumberAsString())) + } +} + +func (codec *jsonNumberCodec) Encode(ptr unsafe.Pointer, stream *Stream) { + number := *((*json.Number)(ptr)) + if len(number) == 0 { + stream.writeByte('0') + } else { + stream.WriteRaw(string(number)) + } +} + +func (codec *jsonNumberCodec) IsEmpty(ptr unsafe.Pointer) bool { + return len(*((*json.Number)(ptr))) == 0 +} + +type jsoniterNumberCodec struct { +} + +func (codec *jsoniterNumberCodec) Decode(ptr unsafe.Pointer, iter *Iterator) { + switch iter.WhatIsNext() { + case StringValue: + *((*Number)(ptr)) = Number(iter.ReadString()) + case NilValue: + iter.skipFourBytes('n', 'u', 'l', 'l') + *((*Number)(ptr)) = "" + default: + *((*Number)(ptr)) = Number([]byte(iter.readNumberAsString())) + } +} + +func (codec *jsoniterNumberCodec) Encode(ptr unsafe.Pointer, stream *Stream) { + number := *((*Number)(ptr)) + if len(number) == 0 { + stream.writeByte('0') + } else { + stream.WriteRaw(string(number)) + } +} + +func (codec *jsoniterNumberCodec) IsEmpty(ptr unsafe.Pointer) bool { + return len(*((*Number)(ptr))) == 0 +} diff --git a/vendor/github.com/json-iterator/go/reflect_json_raw_message.go b/vendor/github.com/json-iterator/go/reflect_json_raw_message.go new file mode 100644 index 00000000..eba434f2 --- /dev/null +++ b/vendor/github.com/json-iterator/go/reflect_json_raw_message.go @@ -0,0 +1,76 @@ +package jsoniter + +import ( + "encoding/json" + "github.com/modern-go/reflect2" + "unsafe" +) + +var jsonRawMessageType = reflect2.TypeOfPtr((*json.RawMessage)(nil)).Elem() +var jsoniterRawMessageType = reflect2.TypeOfPtr((*RawMessage)(nil)).Elem() + +func createEncoderOfJsonRawMessage(ctx *ctx, typ reflect2.Type) ValEncoder { + if typ == jsonRawMessageType { + return &jsonRawMessageCodec{} + } + if typ == jsoniterRawMessageType { + return &jsoniterRawMessageCodec{} + } + return nil +} + +func createDecoderOfJsonRawMessage(ctx *ctx, typ reflect2.Type) ValDecoder { + if typ == jsonRawMessageType { + return &jsonRawMessageCodec{} + } + if typ == jsoniterRawMessageType { + return &jsoniterRawMessageCodec{} + } + return nil +} + +type jsonRawMessageCodec struct { +} + +func (codec *jsonRawMessageCodec) Decode(ptr unsafe.Pointer, iter *Iterator) { + if iter.ReadNil() { + *((*json.RawMessage)(ptr)) = nil + } else { + *((*json.RawMessage)(ptr)) = iter.SkipAndReturnBytes() + } +} + +func (codec *jsonRawMessageCodec) Encode(ptr unsafe.Pointer, stream *Stream) { + if *((*json.RawMessage)(ptr)) == nil { + stream.WriteNil() + } else { + stream.WriteRaw(string(*((*json.RawMessage)(ptr)))) + } +} + +func (codec *jsonRawMessageCodec) IsEmpty(ptr unsafe.Pointer) bool { + return len(*((*json.RawMessage)(ptr))) == 0 +} + +type jsoniterRawMessageCodec struct { +} + +func (codec *jsoniterRawMessageCodec) Decode(ptr unsafe.Pointer, iter *Iterator) { + if iter.ReadNil() { + *((*RawMessage)(ptr)) = nil + } else { + *((*RawMessage)(ptr)) = iter.SkipAndReturnBytes() + } +} + +func (codec *jsoniterRawMessageCodec) Encode(ptr unsafe.Pointer, stream *Stream) { + if *((*RawMessage)(ptr)) == nil { + stream.WriteNil() + } else { + stream.WriteRaw(string(*((*RawMessage)(ptr)))) + } +} + +func (codec *jsoniterRawMessageCodec) IsEmpty(ptr unsafe.Pointer) bool { + return len(*((*RawMessage)(ptr))) == 0 +} diff --git a/vendor/github.com/json-iterator/go/reflect_map.go b/vendor/github.com/json-iterator/go/reflect_map.go new file mode 100644 index 00000000..58296713 --- /dev/null +++ b/vendor/github.com/json-iterator/go/reflect_map.go @@ -0,0 +1,346 @@ +package jsoniter + +import ( + "fmt" + "github.com/modern-go/reflect2" + "io" + "reflect" + "sort" + "unsafe" +) + +func decoderOfMap(ctx *ctx, typ reflect2.Type) ValDecoder { + mapType := typ.(*reflect2.UnsafeMapType) + keyDecoder := decoderOfMapKey(ctx.append("[mapKey]"), mapType.Key()) + elemDecoder := decoderOfType(ctx.append("[mapElem]"), mapType.Elem()) + return &mapDecoder{ + mapType: mapType, + keyType: mapType.Key(), + elemType: mapType.Elem(), + keyDecoder: keyDecoder, + elemDecoder: elemDecoder, + } +} + +func encoderOfMap(ctx *ctx, typ reflect2.Type) ValEncoder { + mapType := typ.(*reflect2.UnsafeMapType) + if ctx.sortMapKeys { + return &sortKeysMapEncoder{ + mapType: mapType, + keyEncoder: encoderOfMapKey(ctx.append("[mapKey]"), mapType.Key()), + elemEncoder: encoderOfType(ctx.append("[mapElem]"), mapType.Elem()), + } + } + return &mapEncoder{ + mapType: mapType, + keyEncoder: encoderOfMapKey(ctx.append("[mapKey]"), mapType.Key()), + elemEncoder: encoderOfType(ctx.append("[mapElem]"), mapType.Elem()), + } +} + +func decoderOfMapKey(ctx *ctx, typ reflect2.Type) ValDecoder { + decoder := ctx.decoderExtension.CreateMapKeyDecoder(typ) + if decoder != nil { + return decoder + } + for _, extension := range ctx.extraExtensions { + decoder := extension.CreateMapKeyDecoder(typ) + if decoder != nil { + return decoder + } + } + + ptrType := reflect2.PtrTo(typ) + if ptrType.Implements(unmarshalerType) { + return &referenceDecoder{ + &unmarshalerDecoder{ + valType: ptrType, + }, + } + } + if typ.Implements(unmarshalerType) { + return &unmarshalerDecoder{ + valType: typ, + } + } + if ptrType.Implements(textUnmarshalerType) { + return &referenceDecoder{ + &textUnmarshalerDecoder{ + valType: ptrType, + }, + } + } + if typ.Implements(textUnmarshalerType) { + return &textUnmarshalerDecoder{ + valType: typ, + } + } + + switch typ.Kind() { + case reflect.String: + return decoderOfType(ctx, reflect2.DefaultTypeOfKind(reflect.String)) + case reflect.Bool, + reflect.Uint8, reflect.Int8, + reflect.Uint16, reflect.Int16, + reflect.Uint32, reflect.Int32, + reflect.Uint64, reflect.Int64, + reflect.Uint, reflect.Int, + reflect.Float32, reflect.Float64, + reflect.Uintptr: + typ = reflect2.DefaultTypeOfKind(typ.Kind()) + return &numericMapKeyDecoder{decoderOfType(ctx, typ)} + default: + return &lazyErrorDecoder{err: fmt.Errorf("unsupported map key type: %v", typ)} + } +} + +func encoderOfMapKey(ctx *ctx, typ reflect2.Type) ValEncoder { + encoder := ctx.encoderExtension.CreateMapKeyEncoder(typ) + if encoder != nil { + return encoder + } + for _, extension := range ctx.extraExtensions { + encoder := extension.CreateMapKeyEncoder(typ) + if encoder != nil { + return encoder + } + } + + if typ == textMarshalerType { + return &directTextMarshalerEncoder{ + stringEncoder: ctx.EncoderOf(reflect2.TypeOf("")), + } + } + if typ.Implements(textMarshalerType) { + return &textMarshalerEncoder{ + valType: typ, + stringEncoder: ctx.EncoderOf(reflect2.TypeOf("")), + } + } + + switch typ.Kind() { + case reflect.String: + return encoderOfType(ctx, reflect2.DefaultTypeOfKind(reflect.String)) + case reflect.Bool, + reflect.Uint8, reflect.Int8, + reflect.Uint16, reflect.Int16, + reflect.Uint32, reflect.Int32, + reflect.Uint64, reflect.Int64, + reflect.Uint, reflect.Int, + reflect.Float32, reflect.Float64, + reflect.Uintptr: + typ = reflect2.DefaultTypeOfKind(typ.Kind()) + return &numericMapKeyEncoder{encoderOfType(ctx, typ)} + default: + if typ.Kind() == reflect.Interface { + return &dynamicMapKeyEncoder{ctx, typ} + } + return &lazyErrorEncoder{err: fmt.Errorf("unsupported map key type: %v", typ)} + } +} + +type mapDecoder struct { + mapType *reflect2.UnsafeMapType + keyType reflect2.Type + elemType reflect2.Type + keyDecoder ValDecoder + elemDecoder ValDecoder +} + +func (decoder *mapDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { + mapType := decoder.mapType + c := iter.nextToken() + if c == 'n' { + iter.skipThreeBytes('u', 'l', 'l') + *(*unsafe.Pointer)(ptr) = nil + mapType.UnsafeSet(ptr, mapType.UnsafeNew()) + return + } + if mapType.UnsafeIsNil(ptr) { + mapType.UnsafeSet(ptr, mapType.UnsafeMakeMap(0)) + } + if c != '{' { + iter.ReportError("ReadMapCB", `expect { or n, but found `+string([]byte{c})) + return + } + c = iter.nextToken() + if c == '}' { + return + } + iter.unreadByte() + key := decoder.keyType.UnsafeNew() + decoder.keyDecoder.Decode(key, iter) + c = iter.nextToken() + if c != ':' { + iter.ReportError("ReadMapCB", "expect : after object field, but found "+string([]byte{c})) + return + } + elem := decoder.elemType.UnsafeNew() + decoder.elemDecoder.Decode(elem, iter) + decoder.mapType.UnsafeSetIndex(ptr, key, elem) + for c = iter.nextToken(); c == ','; c = iter.nextToken() { + key := decoder.keyType.UnsafeNew() + decoder.keyDecoder.Decode(key, iter) + c = iter.nextToken() + if c != ':' { + iter.ReportError("ReadMapCB", "expect : after object field, but found "+string([]byte{c})) + return + } + elem := decoder.elemType.UnsafeNew() + decoder.elemDecoder.Decode(elem, iter) + decoder.mapType.UnsafeSetIndex(ptr, key, elem) + } + if c != '}' { + iter.ReportError("ReadMapCB", `expect }, but found `+string([]byte{c})) + } +} + +type numericMapKeyDecoder struct { + decoder ValDecoder +} + +func (decoder *numericMapKeyDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { + c := iter.nextToken() + if c != '"' { + iter.ReportError("ReadMapCB", `expect ", but found `+string([]byte{c})) + return + } + decoder.decoder.Decode(ptr, iter) + c = iter.nextToken() + if c != '"' { + iter.ReportError("ReadMapCB", `expect ", but found `+string([]byte{c})) + return + } +} + +type numericMapKeyEncoder struct { + encoder ValEncoder +} + +func (encoder *numericMapKeyEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { + stream.writeByte('"') + encoder.encoder.Encode(ptr, stream) + stream.writeByte('"') +} + +func (encoder *numericMapKeyEncoder) IsEmpty(ptr unsafe.Pointer) bool { + return false +} + +type dynamicMapKeyEncoder struct { + ctx *ctx + valType reflect2.Type +} + +func (encoder *dynamicMapKeyEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { + obj := encoder.valType.UnsafeIndirect(ptr) + encoderOfMapKey(encoder.ctx, reflect2.TypeOf(obj)).Encode(reflect2.PtrOf(obj), stream) +} + +func (encoder *dynamicMapKeyEncoder) IsEmpty(ptr unsafe.Pointer) bool { + obj := encoder.valType.UnsafeIndirect(ptr) + return encoderOfMapKey(encoder.ctx, reflect2.TypeOf(obj)).IsEmpty(reflect2.PtrOf(obj)) +} + +type mapEncoder struct { + mapType *reflect2.UnsafeMapType + keyEncoder ValEncoder + elemEncoder ValEncoder +} + +func (encoder *mapEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { + if *(*unsafe.Pointer)(ptr) == nil { + stream.WriteNil() + return + } + stream.WriteObjectStart() + iter := encoder.mapType.UnsafeIterate(ptr) + for i := 0; iter.HasNext(); i++ { + if i != 0 { + stream.WriteMore() + } + key, elem := iter.UnsafeNext() + encoder.keyEncoder.Encode(key, stream) + if stream.indention > 0 { + stream.writeTwoBytes(byte(':'), byte(' ')) + } else { + stream.writeByte(':') + } + encoder.elemEncoder.Encode(elem, stream) + } + stream.WriteObjectEnd() +} + +func (encoder *mapEncoder) IsEmpty(ptr unsafe.Pointer) bool { + iter := encoder.mapType.UnsafeIterate(ptr) + return !iter.HasNext() +} + +type sortKeysMapEncoder struct { + mapType *reflect2.UnsafeMapType + keyEncoder ValEncoder + elemEncoder ValEncoder +} + +func (encoder *sortKeysMapEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { + if *(*unsafe.Pointer)(ptr) == nil { + stream.WriteNil() + return + } + stream.WriteObjectStart() + mapIter := encoder.mapType.UnsafeIterate(ptr) + subStream := stream.cfg.BorrowStream(nil) + subStream.Attachment = stream.Attachment + subIter := stream.cfg.BorrowIterator(nil) + keyValues := encodedKeyValues{} + for mapIter.HasNext() { + key, elem := mapIter.UnsafeNext() + subStreamIndex := subStream.Buffered() + encoder.keyEncoder.Encode(key, subStream) + if subStream.Error != nil && subStream.Error != io.EOF && stream.Error == nil { + stream.Error = subStream.Error + } + encodedKey := subStream.Buffer()[subStreamIndex:] + subIter.ResetBytes(encodedKey) + decodedKey := subIter.ReadString() + if stream.indention > 0 { + subStream.writeTwoBytes(byte(':'), byte(' ')) + } else { + subStream.writeByte(':') + } + encoder.elemEncoder.Encode(elem, subStream) + keyValues = append(keyValues, encodedKV{ + key: decodedKey, + keyValue: subStream.Buffer()[subStreamIndex:], + }) + } + sort.Sort(keyValues) + for i, keyValue := range keyValues { + if i != 0 { + stream.WriteMore() + } + stream.Write(keyValue.keyValue) + } + if subStream.Error != nil && stream.Error == nil { + stream.Error = subStream.Error + } + stream.WriteObjectEnd() + stream.cfg.ReturnStream(subStream) + stream.cfg.ReturnIterator(subIter) +} + +func (encoder *sortKeysMapEncoder) IsEmpty(ptr unsafe.Pointer) bool { + iter := encoder.mapType.UnsafeIterate(ptr) + return !iter.HasNext() +} + +type encodedKeyValues []encodedKV + +type encodedKV struct { + key string + keyValue []byte +} + +func (sv encodedKeyValues) Len() int { return len(sv) } +func (sv encodedKeyValues) Swap(i, j int) { sv[i], sv[j] = sv[j], sv[i] } +func (sv encodedKeyValues) Less(i, j int) bool { return sv[i].key < sv[j].key } diff --git a/vendor/github.com/json-iterator/go/reflect_marshaler.go b/vendor/github.com/json-iterator/go/reflect_marshaler.go new file mode 100644 index 00000000..3e21f375 --- /dev/null +++ b/vendor/github.com/json-iterator/go/reflect_marshaler.go @@ -0,0 +1,225 @@ +package jsoniter + +import ( + "encoding" + "encoding/json" + "unsafe" + + "github.com/modern-go/reflect2" +) + +var marshalerType = reflect2.TypeOfPtr((*json.Marshaler)(nil)).Elem() +var unmarshalerType = reflect2.TypeOfPtr((*json.Unmarshaler)(nil)).Elem() +var textMarshalerType = reflect2.TypeOfPtr((*encoding.TextMarshaler)(nil)).Elem() +var textUnmarshalerType = reflect2.TypeOfPtr((*encoding.TextUnmarshaler)(nil)).Elem() + +func createDecoderOfMarshaler(ctx *ctx, typ reflect2.Type) ValDecoder { + ptrType := reflect2.PtrTo(typ) + if ptrType.Implements(unmarshalerType) { + return &referenceDecoder{ + &unmarshalerDecoder{ptrType}, + } + } + if ptrType.Implements(textUnmarshalerType) { + return &referenceDecoder{ + &textUnmarshalerDecoder{ptrType}, + } + } + return nil +} + +func createEncoderOfMarshaler(ctx *ctx, typ reflect2.Type) ValEncoder { + if typ == marshalerType { + checkIsEmpty := createCheckIsEmpty(ctx, typ) + var encoder ValEncoder = &directMarshalerEncoder{ + checkIsEmpty: checkIsEmpty, + } + return encoder + } + if typ.Implements(marshalerType) { + checkIsEmpty := createCheckIsEmpty(ctx, typ) + var encoder ValEncoder = &marshalerEncoder{ + valType: typ, + checkIsEmpty: checkIsEmpty, + } + return encoder + } + ptrType := reflect2.PtrTo(typ) + if ctx.prefix != "" && ptrType.Implements(marshalerType) { + checkIsEmpty := createCheckIsEmpty(ctx, ptrType) + var encoder ValEncoder = &marshalerEncoder{ + valType: ptrType, + checkIsEmpty: checkIsEmpty, + } + return &referenceEncoder{encoder} + } + if typ == textMarshalerType { + checkIsEmpty := createCheckIsEmpty(ctx, typ) + var encoder ValEncoder = &directTextMarshalerEncoder{ + checkIsEmpty: checkIsEmpty, + stringEncoder: ctx.EncoderOf(reflect2.TypeOf("")), + } + return encoder + } + if typ.Implements(textMarshalerType) { + checkIsEmpty := createCheckIsEmpty(ctx, typ) + var encoder ValEncoder = &textMarshalerEncoder{ + valType: typ, + stringEncoder: ctx.EncoderOf(reflect2.TypeOf("")), + checkIsEmpty: checkIsEmpty, + } + return encoder + } + // if prefix is empty, the type is the root type + if ctx.prefix != "" && ptrType.Implements(textMarshalerType) { + checkIsEmpty := createCheckIsEmpty(ctx, ptrType) + var encoder ValEncoder = &textMarshalerEncoder{ + valType: ptrType, + stringEncoder: ctx.EncoderOf(reflect2.TypeOf("")), + checkIsEmpty: checkIsEmpty, + } + return &referenceEncoder{encoder} + } + return nil +} + +type marshalerEncoder struct { + checkIsEmpty checkIsEmpty + valType reflect2.Type +} + +func (encoder *marshalerEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { + obj := encoder.valType.UnsafeIndirect(ptr) + if encoder.valType.IsNullable() && reflect2.IsNil(obj) { + stream.WriteNil() + return + } + marshaler := obj.(json.Marshaler) + bytes, err := marshaler.MarshalJSON() + if err != nil { + stream.Error = err + } else { + // html escape was already done by jsoniter + // but the extra '\n' should be trimed + l := len(bytes) + if l > 0 && bytes[l-1] == '\n' { + bytes = bytes[:l-1] + } + stream.Write(bytes) + } +} + +func (encoder *marshalerEncoder) IsEmpty(ptr unsafe.Pointer) bool { + return encoder.checkIsEmpty.IsEmpty(ptr) +} + +type directMarshalerEncoder struct { + checkIsEmpty checkIsEmpty +} + +func (encoder *directMarshalerEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { + marshaler := *(*json.Marshaler)(ptr) + if marshaler == nil { + stream.WriteNil() + return + } + bytes, err := marshaler.MarshalJSON() + if err != nil { + stream.Error = err + } else { + stream.Write(bytes) + } +} + +func (encoder *directMarshalerEncoder) IsEmpty(ptr unsafe.Pointer) bool { + return encoder.checkIsEmpty.IsEmpty(ptr) +} + +type textMarshalerEncoder struct { + valType reflect2.Type + stringEncoder ValEncoder + checkIsEmpty checkIsEmpty +} + +func (encoder *textMarshalerEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { + obj := encoder.valType.UnsafeIndirect(ptr) + if encoder.valType.IsNullable() && reflect2.IsNil(obj) { + stream.WriteNil() + return + } + marshaler := (obj).(encoding.TextMarshaler) + bytes, err := marshaler.MarshalText() + if err != nil { + stream.Error = err + } else { + str := string(bytes) + encoder.stringEncoder.Encode(unsafe.Pointer(&str), stream) + } +} + +func (encoder *textMarshalerEncoder) IsEmpty(ptr unsafe.Pointer) bool { + return encoder.checkIsEmpty.IsEmpty(ptr) +} + +type directTextMarshalerEncoder struct { + stringEncoder ValEncoder + checkIsEmpty checkIsEmpty +} + +func (encoder *directTextMarshalerEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { + marshaler := *(*encoding.TextMarshaler)(ptr) + if marshaler == nil { + stream.WriteNil() + return + } + bytes, err := marshaler.MarshalText() + if err != nil { + stream.Error = err + } else { + str := string(bytes) + encoder.stringEncoder.Encode(unsafe.Pointer(&str), stream) + } +} + +func (encoder *directTextMarshalerEncoder) IsEmpty(ptr unsafe.Pointer) bool { + return encoder.checkIsEmpty.IsEmpty(ptr) +} + +type unmarshalerDecoder struct { + valType reflect2.Type +} + +func (decoder *unmarshalerDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { + valType := decoder.valType + obj := valType.UnsafeIndirect(ptr) + unmarshaler := obj.(json.Unmarshaler) + iter.nextToken() + iter.unreadByte() // skip spaces + bytes := iter.SkipAndReturnBytes() + err := unmarshaler.UnmarshalJSON(bytes) + if err != nil { + iter.ReportError("unmarshalerDecoder", err.Error()) + } +} + +type textUnmarshalerDecoder struct { + valType reflect2.Type +} + +func (decoder *textUnmarshalerDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { + valType := decoder.valType + obj := valType.UnsafeIndirect(ptr) + if reflect2.IsNil(obj) { + ptrType := valType.(*reflect2.UnsafePtrType) + elemType := ptrType.Elem() + elem := elemType.UnsafeNew() + ptrType.UnsafeSet(ptr, unsafe.Pointer(&elem)) + obj = valType.UnsafeIndirect(ptr) + } + unmarshaler := (obj).(encoding.TextUnmarshaler) + str := iter.ReadString() + err := unmarshaler.UnmarshalText([]byte(str)) + if err != nil { + iter.ReportError("textUnmarshalerDecoder", err.Error()) + } +} diff --git a/vendor/github.com/json-iterator/go/reflect_native.go b/vendor/github.com/json-iterator/go/reflect_native.go new file mode 100644 index 00000000..f88722d1 --- /dev/null +++ b/vendor/github.com/json-iterator/go/reflect_native.go @@ -0,0 +1,453 @@ +package jsoniter + +import ( + "encoding/base64" + "reflect" + "strconv" + "unsafe" + + "github.com/modern-go/reflect2" +) + +const ptrSize = 32 << uintptr(^uintptr(0)>>63) + +func createEncoderOfNative(ctx *ctx, typ reflect2.Type) ValEncoder { + if typ.Kind() == reflect.Slice && typ.(reflect2.SliceType).Elem().Kind() == reflect.Uint8 { + sliceDecoder := decoderOfSlice(ctx, typ) + return &base64Codec{sliceDecoder: sliceDecoder} + } + typeName := typ.String() + kind := typ.Kind() + switch kind { + case reflect.String: + if typeName != "string" { + return encoderOfType(ctx, reflect2.TypeOfPtr((*string)(nil)).Elem()) + } + return &stringCodec{} + case reflect.Int: + if typeName != "int" { + return encoderOfType(ctx, reflect2.TypeOfPtr((*int)(nil)).Elem()) + } + if strconv.IntSize == 32 { + return &int32Codec{} + } + return &int64Codec{} + case reflect.Int8: + if typeName != "int8" { + return encoderOfType(ctx, reflect2.TypeOfPtr((*int8)(nil)).Elem()) + } + return &int8Codec{} + case reflect.Int16: + if typeName != "int16" { + return encoderOfType(ctx, reflect2.TypeOfPtr((*int16)(nil)).Elem()) + } + return &int16Codec{} + case reflect.Int32: + if typeName != "int32" { + return encoderOfType(ctx, reflect2.TypeOfPtr((*int32)(nil)).Elem()) + } + return &int32Codec{} + case reflect.Int64: + if typeName != "int64" { + return encoderOfType(ctx, reflect2.TypeOfPtr((*int64)(nil)).Elem()) + } + return &int64Codec{} + case reflect.Uint: + if typeName != "uint" { + return encoderOfType(ctx, reflect2.TypeOfPtr((*uint)(nil)).Elem()) + } + if strconv.IntSize == 32 { + return &uint32Codec{} + } + return &uint64Codec{} + case reflect.Uint8: + if typeName != "uint8" { + return encoderOfType(ctx, reflect2.TypeOfPtr((*uint8)(nil)).Elem()) + } + return &uint8Codec{} + case reflect.Uint16: + if typeName != "uint16" { + return encoderOfType(ctx, reflect2.TypeOfPtr((*uint16)(nil)).Elem()) + } + return &uint16Codec{} + case reflect.Uint32: + if typeName != "uint32" { + return encoderOfType(ctx, reflect2.TypeOfPtr((*uint32)(nil)).Elem()) + } + return &uint32Codec{} + case reflect.Uintptr: + if typeName != "uintptr" { + return encoderOfType(ctx, reflect2.TypeOfPtr((*uintptr)(nil)).Elem()) + } + if ptrSize == 32 { + return &uint32Codec{} + } + return &uint64Codec{} + case reflect.Uint64: + if typeName != "uint64" { + return encoderOfType(ctx, reflect2.TypeOfPtr((*uint64)(nil)).Elem()) + } + return &uint64Codec{} + case reflect.Float32: + if typeName != "float32" { + return encoderOfType(ctx, reflect2.TypeOfPtr((*float32)(nil)).Elem()) + } + return &float32Codec{} + case reflect.Float64: + if typeName != "float64" { + return encoderOfType(ctx, reflect2.TypeOfPtr((*float64)(nil)).Elem()) + } + return &float64Codec{} + case reflect.Bool: + if typeName != "bool" { + return encoderOfType(ctx, reflect2.TypeOfPtr((*bool)(nil)).Elem()) + } + return &boolCodec{} + } + return nil +} + +func createDecoderOfNative(ctx *ctx, typ reflect2.Type) ValDecoder { + if typ.Kind() == reflect.Slice && typ.(reflect2.SliceType).Elem().Kind() == reflect.Uint8 { + sliceDecoder := decoderOfSlice(ctx, typ) + return &base64Codec{sliceDecoder: sliceDecoder} + } + typeName := typ.String() + switch typ.Kind() { + case reflect.String: + if typeName != "string" { + return decoderOfType(ctx, reflect2.TypeOfPtr((*string)(nil)).Elem()) + } + return &stringCodec{} + case reflect.Int: + if typeName != "int" { + return decoderOfType(ctx, reflect2.TypeOfPtr((*int)(nil)).Elem()) + } + if strconv.IntSize == 32 { + return &int32Codec{} + } + return &int64Codec{} + case reflect.Int8: + if typeName != "int8" { + return decoderOfType(ctx, reflect2.TypeOfPtr((*int8)(nil)).Elem()) + } + return &int8Codec{} + case reflect.Int16: + if typeName != "int16" { + return decoderOfType(ctx, reflect2.TypeOfPtr((*int16)(nil)).Elem()) + } + return &int16Codec{} + case reflect.Int32: + if typeName != "int32" { + return decoderOfType(ctx, reflect2.TypeOfPtr((*int32)(nil)).Elem()) + } + return &int32Codec{} + case reflect.Int64: + if typeName != "int64" { + return decoderOfType(ctx, reflect2.TypeOfPtr((*int64)(nil)).Elem()) + } + return &int64Codec{} + case reflect.Uint: + if typeName != "uint" { + return decoderOfType(ctx, reflect2.TypeOfPtr((*uint)(nil)).Elem()) + } + if strconv.IntSize == 32 { + return &uint32Codec{} + } + return &uint64Codec{} + case reflect.Uint8: + if typeName != "uint8" { + return decoderOfType(ctx, reflect2.TypeOfPtr((*uint8)(nil)).Elem()) + } + return &uint8Codec{} + case reflect.Uint16: + if typeName != "uint16" { + return decoderOfType(ctx, reflect2.TypeOfPtr((*uint16)(nil)).Elem()) + } + return &uint16Codec{} + case reflect.Uint32: + if typeName != "uint32" { + return decoderOfType(ctx, reflect2.TypeOfPtr((*uint32)(nil)).Elem()) + } + return &uint32Codec{} + case reflect.Uintptr: + if typeName != "uintptr" { + return decoderOfType(ctx, reflect2.TypeOfPtr((*uintptr)(nil)).Elem()) + } + if ptrSize == 32 { + return &uint32Codec{} + } + return &uint64Codec{} + case reflect.Uint64: + if typeName != "uint64" { + return decoderOfType(ctx, reflect2.TypeOfPtr((*uint64)(nil)).Elem()) + } + return &uint64Codec{} + case reflect.Float32: + if typeName != "float32" { + return decoderOfType(ctx, reflect2.TypeOfPtr((*float32)(nil)).Elem()) + } + return &float32Codec{} + case reflect.Float64: + if typeName != "float64" { + return decoderOfType(ctx, reflect2.TypeOfPtr((*float64)(nil)).Elem()) + } + return &float64Codec{} + case reflect.Bool: + if typeName != "bool" { + return decoderOfType(ctx, reflect2.TypeOfPtr((*bool)(nil)).Elem()) + } + return &boolCodec{} + } + return nil +} + +type stringCodec struct { +} + +func (codec *stringCodec) Decode(ptr unsafe.Pointer, iter *Iterator) { + *((*string)(ptr)) = iter.ReadString() +} + +func (codec *stringCodec) Encode(ptr unsafe.Pointer, stream *Stream) { + str := *((*string)(ptr)) + stream.WriteString(str) +} + +func (codec *stringCodec) IsEmpty(ptr unsafe.Pointer) bool { + return *((*string)(ptr)) == "" +} + +type int8Codec struct { +} + +func (codec *int8Codec) Decode(ptr unsafe.Pointer, iter *Iterator) { + if !iter.ReadNil() { + *((*int8)(ptr)) = iter.ReadInt8() + } +} + +func (codec *int8Codec) Encode(ptr unsafe.Pointer, stream *Stream) { + stream.WriteInt8(*((*int8)(ptr))) +} + +func (codec *int8Codec) IsEmpty(ptr unsafe.Pointer) bool { + return *((*int8)(ptr)) == 0 +} + +type int16Codec struct { +} + +func (codec *int16Codec) Decode(ptr unsafe.Pointer, iter *Iterator) { + if !iter.ReadNil() { + *((*int16)(ptr)) = iter.ReadInt16() + } +} + +func (codec *int16Codec) Encode(ptr unsafe.Pointer, stream *Stream) { + stream.WriteInt16(*((*int16)(ptr))) +} + +func (codec *int16Codec) IsEmpty(ptr unsafe.Pointer) bool { + return *((*int16)(ptr)) == 0 +} + +type int32Codec struct { +} + +func (codec *int32Codec) Decode(ptr unsafe.Pointer, iter *Iterator) { + if !iter.ReadNil() { + *((*int32)(ptr)) = iter.ReadInt32() + } +} + +func (codec *int32Codec) Encode(ptr unsafe.Pointer, stream *Stream) { + stream.WriteInt32(*((*int32)(ptr))) +} + +func (codec *int32Codec) IsEmpty(ptr unsafe.Pointer) bool { + return *((*int32)(ptr)) == 0 +} + +type int64Codec struct { +} + +func (codec *int64Codec) Decode(ptr unsafe.Pointer, iter *Iterator) { + if !iter.ReadNil() { + *((*int64)(ptr)) = iter.ReadInt64() + } +} + +func (codec *int64Codec) Encode(ptr unsafe.Pointer, stream *Stream) { + stream.WriteInt64(*((*int64)(ptr))) +} + +func (codec *int64Codec) IsEmpty(ptr unsafe.Pointer) bool { + return *((*int64)(ptr)) == 0 +} + +type uint8Codec struct { +} + +func (codec *uint8Codec) Decode(ptr unsafe.Pointer, iter *Iterator) { + if !iter.ReadNil() { + *((*uint8)(ptr)) = iter.ReadUint8() + } +} + +func (codec *uint8Codec) Encode(ptr unsafe.Pointer, stream *Stream) { + stream.WriteUint8(*((*uint8)(ptr))) +} + +func (codec *uint8Codec) IsEmpty(ptr unsafe.Pointer) bool { + return *((*uint8)(ptr)) == 0 +} + +type uint16Codec struct { +} + +func (codec *uint16Codec) Decode(ptr unsafe.Pointer, iter *Iterator) { + if !iter.ReadNil() { + *((*uint16)(ptr)) = iter.ReadUint16() + } +} + +func (codec *uint16Codec) Encode(ptr unsafe.Pointer, stream *Stream) { + stream.WriteUint16(*((*uint16)(ptr))) +} + +func (codec *uint16Codec) IsEmpty(ptr unsafe.Pointer) bool { + return *((*uint16)(ptr)) == 0 +} + +type uint32Codec struct { +} + +func (codec *uint32Codec) Decode(ptr unsafe.Pointer, iter *Iterator) { + if !iter.ReadNil() { + *((*uint32)(ptr)) = iter.ReadUint32() + } +} + +func (codec *uint32Codec) Encode(ptr unsafe.Pointer, stream *Stream) { + stream.WriteUint32(*((*uint32)(ptr))) +} + +func (codec *uint32Codec) IsEmpty(ptr unsafe.Pointer) bool { + return *((*uint32)(ptr)) == 0 +} + +type uint64Codec struct { +} + +func (codec *uint64Codec) Decode(ptr unsafe.Pointer, iter *Iterator) { + if !iter.ReadNil() { + *((*uint64)(ptr)) = iter.ReadUint64() + } +} + +func (codec *uint64Codec) Encode(ptr unsafe.Pointer, stream *Stream) { + stream.WriteUint64(*((*uint64)(ptr))) +} + +func (codec *uint64Codec) IsEmpty(ptr unsafe.Pointer) bool { + return *((*uint64)(ptr)) == 0 +} + +type float32Codec struct { +} + +func (codec *float32Codec) Decode(ptr unsafe.Pointer, iter *Iterator) { + if !iter.ReadNil() { + *((*float32)(ptr)) = iter.ReadFloat32() + } +} + +func (codec *float32Codec) Encode(ptr unsafe.Pointer, stream *Stream) { + stream.WriteFloat32(*((*float32)(ptr))) +} + +func (codec *float32Codec) IsEmpty(ptr unsafe.Pointer) bool { + return *((*float32)(ptr)) == 0 +} + +type float64Codec struct { +} + +func (codec *float64Codec) Decode(ptr unsafe.Pointer, iter *Iterator) { + if !iter.ReadNil() { + *((*float64)(ptr)) = iter.ReadFloat64() + } +} + +func (codec *float64Codec) Encode(ptr unsafe.Pointer, stream *Stream) { + stream.WriteFloat64(*((*float64)(ptr))) +} + +func (codec *float64Codec) IsEmpty(ptr unsafe.Pointer) bool { + return *((*float64)(ptr)) == 0 +} + +type boolCodec struct { +} + +func (codec *boolCodec) Decode(ptr unsafe.Pointer, iter *Iterator) { + if !iter.ReadNil() { + *((*bool)(ptr)) = iter.ReadBool() + } +} + +func (codec *boolCodec) Encode(ptr unsafe.Pointer, stream *Stream) { + stream.WriteBool(*((*bool)(ptr))) +} + +func (codec *boolCodec) IsEmpty(ptr unsafe.Pointer) bool { + return !(*((*bool)(ptr))) +} + +type base64Codec struct { + sliceType *reflect2.UnsafeSliceType + sliceDecoder ValDecoder +} + +func (codec *base64Codec) Decode(ptr unsafe.Pointer, iter *Iterator) { + if iter.ReadNil() { + codec.sliceType.UnsafeSetNil(ptr) + return + } + switch iter.WhatIsNext() { + case StringValue: + src := iter.ReadString() + dst, err := base64.StdEncoding.DecodeString(src) + if err != nil { + iter.ReportError("decode base64", err.Error()) + } else { + codec.sliceType.UnsafeSet(ptr, unsafe.Pointer(&dst)) + } + case ArrayValue: + codec.sliceDecoder.Decode(ptr, iter) + default: + iter.ReportError("base64Codec", "invalid input") + } +} + +func (codec *base64Codec) Encode(ptr unsafe.Pointer, stream *Stream) { + if codec.sliceType.UnsafeIsNil(ptr) { + stream.WriteNil() + return + } + src := *((*[]byte)(ptr)) + encoding := base64.StdEncoding + stream.writeByte('"') + if len(src) != 0 { + size := encoding.EncodedLen(len(src)) + buf := make([]byte, size) + encoding.Encode(buf, src) + stream.buf = append(stream.buf, buf...) + } + stream.writeByte('"') +} + +func (codec *base64Codec) IsEmpty(ptr unsafe.Pointer) bool { + return len(*((*[]byte)(ptr))) == 0 +} diff --git a/vendor/github.com/json-iterator/go/reflect_optional.go b/vendor/github.com/json-iterator/go/reflect_optional.go new file mode 100644 index 00000000..fa71f474 --- /dev/null +++ b/vendor/github.com/json-iterator/go/reflect_optional.go @@ -0,0 +1,129 @@ +package jsoniter + +import ( + "github.com/modern-go/reflect2" + "unsafe" +) + +func decoderOfOptional(ctx *ctx, typ reflect2.Type) ValDecoder { + ptrType := typ.(*reflect2.UnsafePtrType) + elemType := ptrType.Elem() + decoder := decoderOfType(ctx, elemType) + return &OptionalDecoder{elemType, decoder} +} + +func encoderOfOptional(ctx *ctx, typ reflect2.Type) ValEncoder { + ptrType := typ.(*reflect2.UnsafePtrType) + elemType := ptrType.Elem() + elemEncoder := encoderOfType(ctx, elemType) + encoder := &OptionalEncoder{elemEncoder} + return encoder +} + +type OptionalDecoder struct { + ValueType reflect2.Type + ValueDecoder ValDecoder +} + +func (decoder *OptionalDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { + if iter.ReadNil() { + *((*unsafe.Pointer)(ptr)) = nil + } else { + if *((*unsafe.Pointer)(ptr)) == nil { + //pointer to null, we have to allocate memory to hold the value + newPtr := decoder.ValueType.UnsafeNew() + decoder.ValueDecoder.Decode(newPtr, iter) + *((*unsafe.Pointer)(ptr)) = newPtr + } else { + //reuse existing instance + decoder.ValueDecoder.Decode(*((*unsafe.Pointer)(ptr)), iter) + } + } +} + +type dereferenceDecoder struct { + // only to deference a pointer + valueType reflect2.Type + valueDecoder ValDecoder +} + +func (decoder *dereferenceDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { + if *((*unsafe.Pointer)(ptr)) == nil { + //pointer to null, we have to allocate memory to hold the value + newPtr := decoder.valueType.UnsafeNew() + decoder.valueDecoder.Decode(newPtr, iter) + *((*unsafe.Pointer)(ptr)) = newPtr + } else { + //reuse existing instance + decoder.valueDecoder.Decode(*((*unsafe.Pointer)(ptr)), iter) + } +} + +type OptionalEncoder struct { + ValueEncoder ValEncoder +} + +func (encoder *OptionalEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { + if *((*unsafe.Pointer)(ptr)) == nil { + stream.WriteNil() + } else { + encoder.ValueEncoder.Encode(*((*unsafe.Pointer)(ptr)), stream) + } +} + +func (encoder *OptionalEncoder) IsEmpty(ptr unsafe.Pointer) bool { + return *((*unsafe.Pointer)(ptr)) == nil +} + +type dereferenceEncoder struct { + ValueEncoder ValEncoder +} + +func (encoder *dereferenceEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { + if *((*unsafe.Pointer)(ptr)) == nil { + stream.WriteNil() + } else { + encoder.ValueEncoder.Encode(*((*unsafe.Pointer)(ptr)), stream) + } +} + +func (encoder *dereferenceEncoder) IsEmpty(ptr unsafe.Pointer) bool { + dePtr := *((*unsafe.Pointer)(ptr)) + if dePtr == nil { + return true + } + return encoder.ValueEncoder.IsEmpty(dePtr) +} + +func (encoder *dereferenceEncoder) IsEmbeddedPtrNil(ptr unsafe.Pointer) bool { + deReferenced := *((*unsafe.Pointer)(ptr)) + if deReferenced == nil { + return true + } + isEmbeddedPtrNil, converted := encoder.ValueEncoder.(IsEmbeddedPtrNil) + if !converted { + return false + } + fieldPtr := unsafe.Pointer(deReferenced) + return isEmbeddedPtrNil.IsEmbeddedPtrNil(fieldPtr) +} + +type referenceEncoder struct { + encoder ValEncoder +} + +func (encoder *referenceEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { + encoder.encoder.Encode(unsafe.Pointer(&ptr), stream) +} + +func (encoder *referenceEncoder) IsEmpty(ptr unsafe.Pointer) bool { + return encoder.encoder.IsEmpty(unsafe.Pointer(&ptr)) +} + +type referenceDecoder struct { + decoder ValDecoder +} + +func (decoder *referenceDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { + decoder.decoder.Decode(unsafe.Pointer(&ptr), iter) +} diff --git a/vendor/github.com/json-iterator/go/reflect_slice.go b/vendor/github.com/json-iterator/go/reflect_slice.go new file mode 100644 index 00000000..9441d79d --- /dev/null +++ b/vendor/github.com/json-iterator/go/reflect_slice.go @@ -0,0 +1,99 @@ +package jsoniter + +import ( + "fmt" + "github.com/modern-go/reflect2" + "io" + "unsafe" +) + +func decoderOfSlice(ctx *ctx, typ reflect2.Type) ValDecoder { + sliceType := typ.(*reflect2.UnsafeSliceType) + decoder := decoderOfType(ctx.append("[sliceElem]"), sliceType.Elem()) + return &sliceDecoder{sliceType, decoder} +} + +func encoderOfSlice(ctx *ctx, typ reflect2.Type) ValEncoder { + sliceType := typ.(*reflect2.UnsafeSliceType) + encoder := encoderOfType(ctx.append("[sliceElem]"), sliceType.Elem()) + return &sliceEncoder{sliceType, encoder} +} + +type sliceEncoder struct { + sliceType *reflect2.UnsafeSliceType + elemEncoder ValEncoder +} + +func (encoder *sliceEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { + if encoder.sliceType.UnsafeIsNil(ptr) { + stream.WriteNil() + return + } + length := encoder.sliceType.UnsafeLengthOf(ptr) + if length == 0 { + stream.WriteEmptyArray() + return + } + stream.WriteArrayStart() + encoder.elemEncoder.Encode(encoder.sliceType.UnsafeGetIndex(ptr, 0), stream) + for i := 1; i < length; i++ { + stream.WriteMore() + elemPtr := encoder.sliceType.UnsafeGetIndex(ptr, i) + encoder.elemEncoder.Encode(elemPtr, stream) + } + stream.WriteArrayEnd() + if stream.Error != nil && stream.Error != io.EOF { + stream.Error = fmt.Errorf("%v: %s", encoder.sliceType, stream.Error.Error()) + } +} + +func (encoder *sliceEncoder) IsEmpty(ptr unsafe.Pointer) bool { + return encoder.sliceType.UnsafeLengthOf(ptr) == 0 +} + +type sliceDecoder struct { + sliceType *reflect2.UnsafeSliceType + elemDecoder ValDecoder +} + +func (decoder *sliceDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { + decoder.doDecode(ptr, iter) + if iter.Error != nil && iter.Error != io.EOF { + iter.Error = fmt.Errorf("%v: %s", decoder.sliceType, iter.Error.Error()) + } +} + +func (decoder *sliceDecoder) doDecode(ptr unsafe.Pointer, iter *Iterator) { + c := iter.nextToken() + sliceType := decoder.sliceType + if c == 'n' { + iter.skipThreeBytes('u', 'l', 'l') + sliceType.UnsafeSetNil(ptr) + return + } + if c != '[' { + iter.ReportError("decode slice", "expect [ or n, but found "+string([]byte{c})) + return + } + c = iter.nextToken() + if c == ']' { + sliceType.UnsafeSet(ptr, sliceType.UnsafeMakeSlice(0, 0)) + return + } + iter.unreadByte() + sliceType.UnsafeGrow(ptr, 1) + elemPtr := sliceType.UnsafeGetIndex(ptr, 0) + decoder.elemDecoder.Decode(elemPtr, iter) + length := 1 + for c = iter.nextToken(); c == ','; c = iter.nextToken() { + idx := length + length += 1 + sliceType.UnsafeGrow(ptr, length) + elemPtr = sliceType.UnsafeGetIndex(ptr, idx) + decoder.elemDecoder.Decode(elemPtr, iter) + } + if c != ']' { + iter.ReportError("decode slice", "expect ], but found "+string([]byte{c})) + return + } +} diff --git a/vendor/github.com/json-iterator/go/reflect_struct_decoder.go b/vendor/github.com/json-iterator/go/reflect_struct_decoder.go new file mode 100644 index 00000000..92ae912d --- /dev/null +++ b/vendor/github.com/json-iterator/go/reflect_struct_decoder.go @@ -0,0 +1,1097 @@ +package jsoniter + +import ( + "fmt" + "io" + "strings" + "unsafe" + + "github.com/modern-go/reflect2" +) + +func decoderOfStruct(ctx *ctx, typ reflect2.Type) ValDecoder { + bindings := map[string]*Binding{} + structDescriptor := describeStruct(ctx, typ) + for _, binding := range structDescriptor.Fields { + for _, fromName := range binding.FromNames { + old := bindings[fromName] + if old == nil { + bindings[fromName] = binding + continue + } + ignoreOld, ignoreNew := resolveConflictBinding(ctx.frozenConfig, old, binding) + if ignoreOld { + delete(bindings, fromName) + } + if !ignoreNew { + bindings[fromName] = binding + } + } + } + fields := map[string]*structFieldDecoder{} + for k, binding := range bindings { + fields[k] = binding.Decoder.(*structFieldDecoder) + } + + if !ctx.caseSensitive() { + for k, binding := range bindings { + if _, found := fields[strings.ToLower(k)]; !found { + fields[strings.ToLower(k)] = binding.Decoder.(*structFieldDecoder) + } + } + } + + return createStructDecoder(ctx, typ, fields) +} + +func createStructDecoder(ctx *ctx, typ reflect2.Type, fields map[string]*structFieldDecoder) ValDecoder { + if ctx.disallowUnknownFields { + return &generalStructDecoder{typ: typ, fields: fields, disallowUnknownFields: true} + } + knownHash := map[int64]struct{}{ + 0: {}, + } + + switch len(fields) { + case 0: + return &skipObjectDecoder{typ} + case 1: + for fieldName, fieldDecoder := range fields { + fieldHash := calcHash(fieldName, ctx.caseSensitive()) + _, known := knownHash[fieldHash] + if known { + return &generalStructDecoder{typ, fields, false} + } + knownHash[fieldHash] = struct{}{} + return &oneFieldStructDecoder{typ, fieldHash, fieldDecoder} + } + case 2: + var fieldHash1 int64 + var fieldHash2 int64 + var fieldDecoder1 *structFieldDecoder + var fieldDecoder2 *structFieldDecoder + for fieldName, fieldDecoder := range fields { + fieldHash := calcHash(fieldName, ctx.caseSensitive()) + _, known := knownHash[fieldHash] + if known { + return &generalStructDecoder{typ, fields, false} + } + knownHash[fieldHash] = struct{}{} + if fieldHash1 == 0 { + fieldHash1 = fieldHash + fieldDecoder1 = fieldDecoder + } else { + fieldHash2 = fieldHash + fieldDecoder2 = fieldDecoder + } + } + return &twoFieldsStructDecoder{typ, fieldHash1, fieldDecoder1, fieldHash2, fieldDecoder2} + case 3: + var fieldName1 int64 + var fieldName2 int64 + var fieldName3 int64 + var fieldDecoder1 *structFieldDecoder + var fieldDecoder2 *structFieldDecoder + var fieldDecoder3 *structFieldDecoder + for fieldName, fieldDecoder := range fields { + fieldHash := calcHash(fieldName, ctx.caseSensitive()) + _, known := knownHash[fieldHash] + if known { + return &generalStructDecoder{typ, fields, false} + } + knownHash[fieldHash] = struct{}{} + if fieldName1 == 0 { + fieldName1 = fieldHash + fieldDecoder1 = fieldDecoder + } else if fieldName2 == 0 { + fieldName2 = fieldHash + fieldDecoder2 = fieldDecoder + } else { + fieldName3 = fieldHash + fieldDecoder3 = fieldDecoder + } + } + return &threeFieldsStructDecoder{typ, + fieldName1, fieldDecoder1, + fieldName2, fieldDecoder2, + fieldName3, fieldDecoder3} + case 4: + var fieldName1 int64 + var fieldName2 int64 + var fieldName3 int64 + var fieldName4 int64 + var fieldDecoder1 *structFieldDecoder + var fieldDecoder2 *structFieldDecoder + var fieldDecoder3 *structFieldDecoder + var fieldDecoder4 *structFieldDecoder + for fieldName, fieldDecoder := range fields { + fieldHash := calcHash(fieldName, ctx.caseSensitive()) + _, known := knownHash[fieldHash] + if known { + return &generalStructDecoder{typ, fields, false} + } + knownHash[fieldHash] = struct{}{} + if fieldName1 == 0 { + fieldName1 = fieldHash + fieldDecoder1 = fieldDecoder + } else if fieldName2 == 0 { + fieldName2 = fieldHash + fieldDecoder2 = fieldDecoder + } else if fieldName3 == 0 { + fieldName3 = fieldHash + fieldDecoder3 = fieldDecoder + } else { + fieldName4 = fieldHash + fieldDecoder4 = fieldDecoder + } + } + return &fourFieldsStructDecoder{typ, + fieldName1, fieldDecoder1, + fieldName2, fieldDecoder2, + fieldName3, fieldDecoder3, + fieldName4, fieldDecoder4} + case 5: + var fieldName1 int64 + var fieldName2 int64 + var fieldName3 int64 + var fieldName4 int64 + var fieldName5 int64 + var fieldDecoder1 *structFieldDecoder + var fieldDecoder2 *structFieldDecoder + var fieldDecoder3 *structFieldDecoder + var fieldDecoder4 *structFieldDecoder + var fieldDecoder5 *structFieldDecoder + for fieldName, fieldDecoder := range fields { + fieldHash := calcHash(fieldName, ctx.caseSensitive()) + _, known := knownHash[fieldHash] + if known { + return &generalStructDecoder{typ, fields, false} + } + knownHash[fieldHash] = struct{}{} + if fieldName1 == 0 { + fieldName1 = fieldHash + fieldDecoder1 = fieldDecoder + } else if fieldName2 == 0 { + fieldName2 = fieldHash + fieldDecoder2 = fieldDecoder + } else if fieldName3 == 0 { + fieldName3 = fieldHash + fieldDecoder3 = fieldDecoder + } else if fieldName4 == 0 { + fieldName4 = fieldHash + fieldDecoder4 = fieldDecoder + } else { + fieldName5 = fieldHash + fieldDecoder5 = fieldDecoder + } + } + return &fiveFieldsStructDecoder{typ, + fieldName1, fieldDecoder1, + fieldName2, fieldDecoder2, + fieldName3, fieldDecoder3, + fieldName4, fieldDecoder4, + fieldName5, fieldDecoder5} + case 6: + var fieldName1 int64 + var fieldName2 int64 + var fieldName3 int64 + var fieldName4 int64 + var fieldName5 int64 + var fieldName6 int64 + var fieldDecoder1 *structFieldDecoder + var fieldDecoder2 *structFieldDecoder + var fieldDecoder3 *structFieldDecoder + var fieldDecoder4 *structFieldDecoder + var fieldDecoder5 *structFieldDecoder + var fieldDecoder6 *structFieldDecoder + for fieldName, fieldDecoder := range fields { + fieldHash := calcHash(fieldName, ctx.caseSensitive()) + _, known := knownHash[fieldHash] + if known { + return &generalStructDecoder{typ, fields, false} + } + knownHash[fieldHash] = struct{}{} + if fieldName1 == 0 { + fieldName1 = fieldHash + fieldDecoder1 = fieldDecoder + } else if fieldName2 == 0 { + fieldName2 = fieldHash + fieldDecoder2 = fieldDecoder + } else if fieldName3 == 0 { + fieldName3 = fieldHash + fieldDecoder3 = fieldDecoder + } else if fieldName4 == 0 { + fieldName4 = fieldHash + fieldDecoder4 = fieldDecoder + } else if fieldName5 == 0 { + fieldName5 = fieldHash + fieldDecoder5 = fieldDecoder + } else { + fieldName6 = fieldHash + fieldDecoder6 = fieldDecoder + } + } + return &sixFieldsStructDecoder{typ, + fieldName1, fieldDecoder1, + fieldName2, fieldDecoder2, + fieldName3, fieldDecoder3, + fieldName4, fieldDecoder4, + fieldName5, fieldDecoder5, + fieldName6, fieldDecoder6} + case 7: + var fieldName1 int64 + var fieldName2 int64 + var fieldName3 int64 + var fieldName4 int64 + var fieldName5 int64 + var fieldName6 int64 + var fieldName7 int64 + var fieldDecoder1 *structFieldDecoder + var fieldDecoder2 *structFieldDecoder + var fieldDecoder3 *structFieldDecoder + var fieldDecoder4 *structFieldDecoder + var fieldDecoder5 *structFieldDecoder + var fieldDecoder6 *structFieldDecoder + var fieldDecoder7 *structFieldDecoder + for fieldName, fieldDecoder := range fields { + fieldHash := calcHash(fieldName, ctx.caseSensitive()) + _, known := knownHash[fieldHash] + if known { + return &generalStructDecoder{typ, fields, false} + } + knownHash[fieldHash] = struct{}{} + if fieldName1 == 0 { + fieldName1 = fieldHash + fieldDecoder1 = fieldDecoder + } else if fieldName2 == 0 { + fieldName2 = fieldHash + fieldDecoder2 = fieldDecoder + } else if fieldName3 == 0 { + fieldName3 = fieldHash + fieldDecoder3 = fieldDecoder + } else if fieldName4 == 0 { + fieldName4 = fieldHash + fieldDecoder4 = fieldDecoder + } else if fieldName5 == 0 { + fieldName5 = fieldHash + fieldDecoder5 = fieldDecoder + } else if fieldName6 == 0 { + fieldName6 = fieldHash + fieldDecoder6 = fieldDecoder + } else { + fieldName7 = fieldHash + fieldDecoder7 = fieldDecoder + } + } + return &sevenFieldsStructDecoder{typ, + fieldName1, fieldDecoder1, + fieldName2, fieldDecoder2, + fieldName3, fieldDecoder3, + fieldName4, fieldDecoder4, + fieldName5, fieldDecoder5, + fieldName6, fieldDecoder6, + fieldName7, fieldDecoder7} + case 8: + var fieldName1 int64 + var fieldName2 int64 + var fieldName3 int64 + var fieldName4 int64 + var fieldName5 int64 + var fieldName6 int64 + var fieldName7 int64 + var fieldName8 int64 + var fieldDecoder1 *structFieldDecoder + var fieldDecoder2 *structFieldDecoder + var fieldDecoder3 *structFieldDecoder + var fieldDecoder4 *structFieldDecoder + var fieldDecoder5 *structFieldDecoder + var fieldDecoder6 *structFieldDecoder + var fieldDecoder7 *structFieldDecoder + var fieldDecoder8 *structFieldDecoder + for fieldName, fieldDecoder := range fields { + fieldHash := calcHash(fieldName, ctx.caseSensitive()) + _, known := knownHash[fieldHash] + if known { + return &generalStructDecoder{typ, fields, false} + } + knownHash[fieldHash] = struct{}{} + if fieldName1 == 0 { + fieldName1 = fieldHash + fieldDecoder1 = fieldDecoder + } else if fieldName2 == 0 { + fieldName2 = fieldHash + fieldDecoder2 = fieldDecoder + } else if fieldName3 == 0 { + fieldName3 = fieldHash + fieldDecoder3 = fieldDecoder + } else if fieldName4 == 0 { + fieldName4 = fieldHash + fieldDecoder4 = fieldDecoder + } else if fieldName5 == 0 { + fieldName5 = fieldHash + fieldDecoder5 = fieldDecoder + } else if fieldName6 == 0 { + fieldName6 = fieldHash + fieldDecoder6 = fieldDecoder + } else if fieldName7 == 0 { + fieldName7 = fieldHash + fieldDecoder7 = fieldDecoder + } else { + fieldName8 = fieldHash + fieldDecoder8 = fieldDecoder + } + } + return &eightFieldsStructDecoder{typ, + fieldName1, fieldDecoder1, + fieldName2, fieldDecoder2, + fieldName3, fieldDecoder3, + fieldName4, fieldDecoder4, + fieldName5, fieldDecoder5, + fieldName6, fieldDecoder6, + fieldName7, fieldDecoder7, + fieldName8, fieldDecoder8} + case 9: + var fieldName1 int64 + var fieldName2 int64 + var fieldName3 int64 + var fieldName4 int64 + var fieldName5 int64 + var fieldName6 int64 + var fieldName7 int64 + var fieldName8 int64 + var fieldName9 int64 + var fieldDecoder1 *structFieldDecoder + var fieldDecoder2 *structFieldDecoder + var fieldDecoder3 *structFieldDecoder + var fieldDecoder4 *structFieldDecoder + var fieldDecoder5 *structFieldDecoder + var fieldDecoder6 *structFieldDecoder + var fieldDecoder7 *structFieldDecoder + var fieldDecoder8 *structFieldDecoder + var fieldDecoder9 *structFieldDecoder + for fieldName, fieldDecoder := range fields { + fieldHash := calcHash(fieldName, ctx.caseSensitive()) + _, known := knownHash[fieldHash] + if known { + return &generalStructDecoder{typ, fields, false} + } + knownHash[fieldHash] = struct{}{} + if fieldName1 == 0 { + fieldName1 = fieldHash + fieldDecoder1 = fieldDecoder + } else if fieldName2 == 0 { + fieldName2 = fieldHash + fieldDecoder2 = fieldDecoder + } else if fieldName3 == 0 { + fieldName3 = fieldHash + fieldDecoder3 = fieldDecoder + } else if fieldName4 == 0 { + fieldName4 = fieldHash + fieldDecoder4 = fieldDecoder + } else if fieldName5 == 0 { + fieldName5 = fieldHash + fieldDecoder5 = fieldDecoder + } else if fieldName6 == 0 { + fieldName6 = fieldHash + fieldDecoder6 = fieldDecoder + } else if fieldName7 == 0 { + fieldName7 = fieldHash + fieldDecoder7 = fieldDecoder + } else if fieldName8 == 0 { + fieldName8 = fieldHash + fieldDecoder8 = fieldDecoder + } else { + fieldName9 = fieldHash + fieldDecoder9 = fieldDecoder + } + } + return &nineFieldsStructDecoder{typ, + fieldName1, fieldDecoder1, + fieldName2, fieldDecoder2, + fieldName3, fieldDecoder3, + fieldName4, fieldDecoder4, + fieldName5, fieldDecoder5, + fieldName6, fieldDecoder6, + fieldName7, fieldDecoder7, + fieldName8, fieldDecoder8, + fieldName9, fieldDecoder9} + case 10: + var fieldName1 int64 + var fieldName2 int64 + var fieldName3 int64 + var fieldName4 int64 + var fieldName5 int64 + var fieldName6 int64 + var fieldName7 int64 + var fieldName8 int64 + var fieldName9 int64 + var fieldName10 int64 + var fieldDecoder1 *structFieldDecoder + var fieldDecoder2 *structFieldDecoder + var fieldDecoder3 *structFieldDecoder + var fieldDecoder4 *structFieldDecoder + var fieldDecoder5 *structFieldDecoder + var fieldDecoder6 *structFieldDecoder + var fieldDecoder7 *structFieldDecoder + var fieldDecoder8 *structFieldDecoder + var fieldDecoder9 *structFieldDecoder + var fieldDecoder10 *structFieldDecoder + for fieldName, fieldDecoder := range fields { + fieldHash := calcHash(fieldName, ctx.caseSensitive()) + _, known := knownHash[fieldHash] + if known { + return &generalStructDecoder{typ, fields, false} + } + knownHash[fieldHash] = struct{}{} + if fieldName1 == 0 { + fieldName1 = fieldHash + fieldDecoder1 = fieldDecoder + } else if fieldName2 == 0 { + fieldName2 = fieldHash + fieldDecoder2 = fieldDecoder + } else if fieldName3 == 0 { + fieldName3 = fieldHash + fieldDecoder3 = fieldDecoder + } else if fieldName4 == 0 { + fieldName4 = fieldHash + fieldDecoder4 = fieldDecoder + } else if fieldName5 == 0 { + fieldName5 = fieldHash + fieldDecoder5 = fieldDecoder + } else if fieldName6 == 0 { + fieldName6 = fieldHash + fieldDecoder6 = fieldDecoder + } else if fieldName7 == 0 { + fieldName7 = fieldHash + fieldDecoder7 = fieldDecoder + } else if fieldName8 == 0 { + fieldName8 = fieldHash + fieldDecoder8 = fieldDecoder + } else if fieldName9 == 0 { + fieldName9 = fieldHash + fieldDecoder9 = fieldDecoder + } else { + fieldName10 = fieldHash + fieldDecoder10 = fieldDecoder + } + } + return &tenFieldsStructDecoder{typ, + fieldName1, fieldDecoder1, + fieldName2, fieldDecoder2, + fieldName3, fieldDecoder3, + fieldName4, fieldDecoder4, + fieldName5, fieldDecoder5, + fieldName6, fieldDecoder6, + fieldName7, fieldDecoder7, + fieldName8, fieldDecoder8, + fieldName9, fieldDecoder9, + fieldName10, fieldDecoder10} + } + return &generalStructDecoder{typ, fields, false} +} + +type generalStructDecoder struct { + typ reflect2.Type + fields map[string]*structFieldDecoder + disallowUnknownFields bool +} + +func (decoder *generalStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { + if !iter.readObjectStart() { + return + } + if !iter.incrementDepth() { + return + } + var c byte + for c = ','; c == ','; c = iter.nextToken() { + decoder.decodeOneField(ptr, iter) + } + if iter.Error != nil && iter.Error != io.EOF && len(decoder.typ.Type1().Name()) != 0 { + iter.Error = fmt.Errorf("%v.%s", decoder.typ, iter.Error.Error()) + } + if c != '}' { + iter.ReportError("struct Decode", `expect }, but found `+string([]byte{c})) + } + iter.decrementDepth() +} + +func (decoder *generalStructDecoder) decodeOneField(ptr unsafe.Pointer, iter *Iterator) { + var field string + var fieldDecoder *structFieldDecoder + if iter.cfg.objectFieldMustBeSimpleString { + fieldBytes := iter.ReadStringAsSlice() + field = *(*string)(unsafe.Pointer(&fieldBytes)) + fieldDecoder = decoder.fields[field] + if fieldDecoder == nil && !iter.cfg.caseSensitive { + fieldDecoder = decoder.fields[strings.ToLower(field)] + } + } else { + field = iter.ReadString() + fieldDecoder = decoder.fields[field] + if fieldDecoder == nil && !iter.cfg.caseSensitive { + fieldDecoder = decoder.fields[strings.ToLower(field)] + } + } + if fieldDecoder == nil { + if decoder.disallowUnknownFields { + msg := "found unknown field: " + field + iter.ReportError("ReadObject", msg) + } + c := iter.nextToken() + if c != ':' { + iter.ReportError("ReadObject", "expect : after object field, but found "+string([]byte{c})) + } + iter.Skip() + return + } + c := iter.nextToken() + if c != ':' { + iter.ReportError("ReadObject", "expect : after object field, but found "+string([]byte{c})) + } + fieldDecoder.Decode(ptr, iter) +} + +type skipObjectDecoder struct { + typ reflect2.Type +} + +func (decoder *skipObjectDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { + valueType := iter.WhatIsNext() + if valueType != ObjectValue && valueType != NilValue { + iter.ReportError("skipObjectDecoder", "expect object or null") + return + } + iter.Skip() +} + +type oneFieldStructDecoder struct { + typ reflect2.Type + fieldHash int64 + fieldDecoder *structFieldDecoder +} + +func (decoder *oneFieldStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { + if !iter.readObjectStart() { + return + } + if !iter.incrementDepth() { + return + } + for { + if iter.readFieldHash() == decoder.fieldHash { + decoder.fieldDecoder.Decode(ptr, iter) + } else { + iter.Skip() + } + if iter.isObjectEnd() { + break + } + } + if iter.Error != nil && iter.Error != io.EOF && len(decoder.typ.Type1().Name()) != 0 { + iter.Error = fmt.Errorf("%v.%s", decoder.typ, iter.Error.Error()) + } + iter.decrementDepth() +} + +type twoFieldsStructDecoder struct { + typ reflect2.Type + fieldHash1 int64 + fieldDecoder1 *structFieldDecoder + fieldHash2 int64 + fieldDecoder2 *structFieldDecoder +} + +func (decoder *twoFieldsStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { + if !iter.readObjectStart() { + return + } + if !iter.incrementDepth() { + return + } + for { + switch iter.readFieldHash() { + case decoder.fieldHash1: + decoder.fieldDecoder1.Decode(ptr, iter) + case decoder.fieldHash2: + decoder.fieldDecoder2.Decode(ptr, iter) + default: + iter.Skip() + } + if iter.isObjectEnd() { + break + } + } + if iter.Error != nil && iter.Error != io.EOF && len(decoder.typ.Type1().Name()) != 0 { + iter.Error = fmt.Errorf("%v.%s", decoder.typ, iter.Error.Error()) + } + iter.decrementDepth() +} + +type threeFieldsStructDecoder struct { + typ reflect2.Type + fieldHash1 int64 + fieldDecoder1 *structFieldDecoder + fieldHash2 int64 + fieldDecoder2 *structFieldDecoder + fieldHash3 int64 + fieldDecoder3 *structFieldDecoder +} + +func (decoder *threeFieldsStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { + if !iter.readObjectStart() { + return + } + if !iter.incrementDepth() { + return + } + for { + switch iter.readFieldHash() { + case decoder.fieldHash1: + decoder.fieldDecoder1.Decode(ptr, iter) + case decoder.fieldHash2: + decoder.fieldDecoder2.Decode(ptr, iter) + case decoder.fieldHash3: + decoder.fieldDecoder3.Decode(ptr, iter) + default: + iter.Skip() + } + if iter.isObjectEnd() { + break + } + } + if iter.Error != nil && iter.Error != io.EOF && len(decoder.typ.Type1().Name()) != 0 { + iter.Error = fmt.Errorf("%v.%s", decoder.typ, iter.Error.Error()) + } + iter.decrementDepth() +} + +type fourFieldsStructDecoder struct { + typ reflect2.Type + fieldHash1 int64 + fieldDecoder1 *structFieldDecoder + fieldHash2 int64 + fieldDecoder2 *structFieldDecoder + fieldHash3 int64 + fieldDecoder3 *structFieldDecoder + fieldHash4 int64 + fieldDecoder4 *structFieldDecoder +} + +func (decoder *fourFieldsStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { + if !iter.readObjectStart() { + return + } + if !iter.incrementDepth() { + return + } + for { + switch iter.readFieldHash() { + case decoder.fieldHash1: + decoder.fieldDecoder1.Decode(ptr, iter) + case decoder.fieldHash2: + decoder.fieldDecoder2.Decode(ptr, iter) + case decoder.fieldHash3: + decoder.fieldDecoder3.Decode(ptr, iter) + case decoder.fieldHash4: + decoder.fieldDecoder4.Decode(ptr, iter) + default: + iter.Skip() + } + if iter.isObjectEnd() { + break + } + } + if iter.Error != nil && iter.Error != io.EOF && len(decoder.typ.Type1().Name()) != 0 { + iter.Error = fmt.Errorf("%v.%s", decoder.typ, iter.Error.Error()) + } + iter.decrementDepth() +} + +type fiveFieldsStructDecoder struct { + typ reflect2.Type + fieldHash1 int64 + fieldDecoder1 *structFieldDecoder + fieldHash2 int64 + fieldDecoder2 *structFieldDecoder + fieldHash3 int64 + fieldDecoder3 *structFieldDecoder + fieldHash4 int64 + fieldDecoder4 *structFieldDecoder + fieldHash5 int64 + fieldDecoder5 *structFieldDecoder +} + +func (decoder *fiveFieldsStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { + if !iter.readObjectStart() { + return + } + if !iter.incrementDepth() { + return + } + for { + switch iter.readFieldHash() { + case decoder.fieldHash1: + decoder.fieldDecoder1.Decode(ptr, iter) + case decoder.fieldHash2: + decoder.fieldDecoder2.Decode(ptr, iter) + case decoder.fieldHash3: + decoder.fieldDecoder3.Decode(ptr, iter) + case decoder.fieldHash4: + decoder.fieldDecoder4.Decode(ptr, iter) + case decoder.fieldHash5: + decoder.fieldDecoder5.Decode(ptr, iter) + default: + iter.Skip() + } + if iter.isObjectEnd() { + break + } + } + if iter.Error != nil && iter.Error != io.EOF && len(decoder.typ.Type1().Name()) != 0 { + iter.Error = fmt.Errorf("%v.%s", decoder.typ, iter.Error.Error()) + } + iter.decrementDepth() +} + +type sixFieldsStructDecoder struct { + typ reflect2.Type + fieldHash1 int64 + fieldDecoder1 *structFieldDecoder + fieldHash2 int64 + fieldDecoder2 *structFieldDecoder + fieldHash3 int64 + fieldDecoder3 *structFieldDecoder + fieldHash4 int64 + fieldDecoder4 *structFieldDecoder + fieldHash5 int64 + fieldDecoder5 *structFieldDecoder + fieldHash6 int64 + fieldDecoder6 *structFieldDecoder +} + +func (decoder *sixFieldsStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { + if !iter.readObjectStart() { + return + } + if !iter.incrementDepth() { + return + } + for { + switch iter.readFieldHash() { + case decoder.fieldHash1: + decoder.fieldDecoder1.Decode(ptr, iter) + case decoder.fieldHash2: + decoder.fieldDecoder2.Decode(ptr, iter) + case decoder.fieldHash3: + decoder.fieldDecoder3.Decode(ptr, iter) + case decoder.fieldHash4: + decoder.fieldDecoder4.Decode(ptr, iter) + case decoder.fieldHash5: + decoder.fieldDecoder5.Decode(ptr, iter) + case decoder.fieldHash6: + decoder.fieldDecoder6.Decode(ptr, iter) + default: + iter.Skip() + } + if iter.isObjectEnd() { + break + } + } + if iter.Error != nil && iter.Error != io.EOF && len(decoder.typ.Type1().Name()) != 0 { + iter.Error = fmt.Errorf("%v.%s", decoder.typ, iter.Error.Error()) + } + iter.decrementDepth() +} + +type sevenFieldsStructDecoder struct { + typ reflect2.Type + fieldHash1 int64 + fieldDecoder1 *structFieldDecoder + fieldHash2 int64 + fieldDecoder2 *structFieldDecoder + fieldHash3 int64 + fieldDecoder3 *structFieldDecoder + fieldHash4 int64 + fieldDecoder4 *structFieldDecoder + fieldHash5 int64 + fieldDecoder5 *structFieldDecoder + fieldHash6 int64 + fieldDecoder6 *structFieldDecoder + fieldHash7 int64 + fieldDecoder7 *structFieldDecoder +} + +func (decoder *sevenFieldsStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { + if !iter.readObjectStart() { + return + } + if !iter.incrementDepth() { + return + } + for { + switch iter.readFieldHash() { + case decoder.fieldHash1: + decoder.fieldDecoder1.Decode(ptr, iter) + case decoder.fieldHash2: + decoder.fieldDecoder2.Decode(ptr, iter) + case decoder.fieldHash3: + decoder.fieldDecoder3.Decode(ptr, iter) + case decoder.fieldHash4: + decoder.fieldDecoder4.Decode(ptr, iter) + case decoder.fieldHash5: + decoder.fieldDecoder5.Decode(ptr, iter) + case decoder.fieldHash6: + decoder.fieldDecoder6.Decode(ptr, iter) + case decoder.fieldHash7: + decoder.fieldDecoder7.Decode(ptr, iter) + default: + iter.Skip() + } + if iter.isObjectEnd() { + break + } + } + if iter.Error != nil && iter.Error != io.EOF && len(decoder.typ.Type1().Name()) != 0 { + iter.Error = fmt.Errorf("%v.%s", decoder.typ, iter.Error.Error()) + } + iter.decrementDepth() +} + +type eightFieldsStructDecoder struct { + typ reflect2.Type + fieldHash1 int64 + fieldDecoder1 *structFieldDecoder + fieldHash2 int64 + fieldDecoder2 *structFieldDecoder + fieldHash3 int64 + fieldDecoder3 *structFieldDecoder + fieldHash4 int64 + fieldDecoder4 *structFieldDecoder + fieldHash5 int64 + fieldDecoder5 *structFieldDecoder + fieldHash6 int64 + fieldDecoder6 *structFieldDecoder + fieldHash7 int64 + fieldDecoder7 *structFieldDecoder + fieldHash8 int64 + fieldDecoder8 *structFieldDecoder +} + +func (decoder *eightFieldsStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { + if !iter.readObjectStart() { + return + } + if !iter.incrementDepth() { + return + } + for { + switch iter.readFieldHash() { + case decoder.fieldHash1: + decoder.fieldDecoder1.Decode(ptr, iter) + case decoder.fieldHash2: + decoder.fieldDecoder2.Decode(ptr, iter) + case decoder.fieldHash3: + decoder.fieldDecoder3.Decode(ptr, iter) + case decoder.fieldHash4: + decoder.fieldDecoder4.Decode(ptr, iter) + case decoder.fieldHash5: + decoder.fieldDecoder5.Decode(ptr, iter) + case decoder.fieldHash6: + decoder.fieldDecoder6.Decode(ptr, iter) + case decoder.fieldHash7: + decoder.fieldDecoder7.Decode(ptr, iter) + case decoder.fieldHash8: + decoder.fieldDecoder8.Decode(ptr, iter) + default: + iter.Skip() + } + if iter.isObjectEnd() { + break + } + } + if iter.Error != nil && iter.Error != io.EOF && len(decoder.typ.Type1().Name()) != 0 { + iter.Error = fmt.Errorf("%v.%s", decoder.typ, iter.Error.Error()) + } + iter.decrementDepth() +} + +type nineFieldsStructDecoder struct { + typ reflect2.Type + fieldHash1 int64 + fieldDecoder1 *structFieldDecoder + fieldHash2 int64 + fieldDecoder2 *structFieldDecoder + fieldHash3 int64 + fieldDecoder3 *structFieldDecoder + fieldHash4 int64 + fieldDecoder4 *structFieldDecoder + fieldHash5 int64 + fieldDecoder5 *structFieldDecoder + fieldHash6 int64 + fieldDecoder6 *structFieldDecoder + fieldHash7 int64 + fieldDecoder7 *structFieldDecoder + fieldHash8 int64 + fieldDecoder8 *structFieldDecoder + fieldHash9 int64 + fieldDecoder9 *structFieldDecoder +} + +func (decoder *nineFieldsStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { + if !iter.readObjectStart() { + return + } + if !iter.incrementDepth() { + return + } + for { + switch iter.readFieldHash() { + case decoder.fieldHash1: + decoder.fieldDecoder1.Decode(ptr, iter) + case decoder.fieldHash2: + decoder.fieldDecoder2.Decode(ptr, iter) + case decoder.fieldHash3: + decoder.fieldDecoder3.Decode(ptr, iter) + case decoder.fieldHash4: + decoder.fieldDecoder4.Decode(ptr, iter) + case decoder.fieldHash5: + decoder.fieldDecoder5.Decode(ptr, iter) + case decoder.fieldHash6: + decoder.fieldDecoder6.Decode(ptr, iter) + case decoder.fieldHash7: + decoder.fieldDecoder7.Decode(ptr, iter) + case decoder.fieldHash8: + decoder.fieldDecoder8.Decode(ptr, iter) + case decoder.fieldHash9: + decoder.fieldDecoder9.Decode(ptr, iter) + default: + iter.Skip() + } + if iter.isObjectEnd() { + break + } + } + if iter.Error != nil && iter.Error != io.EOF && len(decoder.typ.Type1().Name()) != 0 { + iter.Error = fmt.Errorf("%v.%s", decoder.typ, iter.Error.Error()) + } + iter.decrementDepth() +} + +type tenFieldsStructDecoder struct { + typ reflect2.Type + fieldHash1 int64 + fieldDecoder1 *structFieldDecoder + fieldHash2 int64 + fieldDecoder2 *structFieldDecoder + fieldHash3 int64 + fieldDecoder3 *structFieldDecoder + fieldHash4 int64 + fieldDecoder4 *structFieldDecoder + fieldHash5 int64 + fieldDecoder5 *structFieldDecoder + fieldHash6 int64 + fieldDecoder6 *structFieldDecoder + fieldHash7 int64 + fieldDecoder7 *structFieldDecoder + fieldHash8 int64 + fieldDecoder8 *structFieldDecoder + fieldHash9 int64 + fieldDecoder9 *structFieldDecoder + fieldHash10 int64 + fieldDecoder10 *structFieldDecoder +} + +func (decoder *tenFieldsStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { + if !iter.readObjectStart() { + return + } + if !iter.incrementDepth() { + return + } + for { + switch iter.readFieldHash() { + case decoder.fieldHash1: + decoder.fieldDecoder1.Decode(ptr, iter) + case decoder.fieldHash2: + decoder.fieldDecoder2.Decode(ptr, iter) + case decoder.fieldHash3: + decoder.fieldDecoder3.Decode(ptr, iter) + case decoder.fieldHash4: + decoder.fieldDecoder4.Decode(ptr, iter) + case decoder.fieldHash5: + decoder.fieldDecoder5.Decode(ptr, iter) + case decoder.fieldHash6: + decoder.fieldDecoder6.Decode(ptr, iter) + case decoder.fieldHash7: + decoder.fieldDecoder7.Decode(ptr, iter) + case decoder.fieldHash8: + decoder.fieldDecoder8.Decode(ptr, iter) + case decoder.fieldHash9: + decoder.fieldDecoder9.Decode(ptr, iter) + case decoder.fieldHash10: + decoder.fieldDecoder10.Decode(ptr, iter) + default: + iter.Skip() + } + if iter.isObjectEnd() { + break + } + } + if iter.Error != nil && iter.Error != io.EOF && len(decoder.typ.Type1().Name()) != 0 { + iter.Error = fmt.Errorf("%v.%s", decoder.typ, iter.Error.Error()) + } + iter.decrementDepth() +} + +type structFieldDecoder struct { + field reflect2.StructField + fieldDecoder ValDecoder +} + +func (decoder *structFieldDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { + fieldPtr := decoder.field.UnsafeGet(ptr) + decoder.fieldDecoder.Decode(fieldPtr, iter) + if iter.Error != nil && iter.Error != io.EOF { + iter.Error = fmt.Errorf("%s: %s", decoder.field.Name(), iter.Error.Error()) + } +} + +type stringModeStringDecoder struct { + elemDecoder ValDecoder + cfg *frozenConfig +} + +func (decoder *stringModeStringDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { + decoder.elemDecoder.Decode(ptr, iter) + str := *((*string)(ptr)) + tempIter := decoder.cfg.BorrowIterator([]byte(str)) + defer decoder.cfg.ReturnIterator(tempIter) + *((*string)(ptr)) = tempIter.ReadString() +} + +type stringModeNumberDecoder struct { + elemDecoder ValDecoder +} + +func (decoder *stringModeNumberDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { + if iter.WhatIsNext() == NilValue { + decoder.elemDecoder.Decode(ptr, iter) + return + } + + c := iter.nextToken() + if c != '"' { + iter.ReportError("stringModeNumberDecoder", `expect ", but found `+string([]byte{c})) + return + } + decoder.elemDecoder.Decode(ptr, iter) + if iter.Error != nil { + return + } + c = iter.readByte() + if c != '"' { + iter.ReportError("stringModeNumberDecoder", `expect ", but found `+string([]byte{c})) + return + } +} diff --git a/vendor/github.com/json-iterator/go/reflect_struct_encoder.go b/vendor/github.com/json-iterator/go/reflect_struct_encoder.go new file mode 100644 index 00000000..152e3ef5 --- /dev/null +++ b/vendor/github.com/json-iterator/go/reflect_struct_encoder.go @@ -0,0 +1,211 @@ +package jsoniter + +import ( + "fmt" + "github.com/modern-go/reflect2" + "io" + "reflect" + "unsafe" +) + +func encoderOfStruct(ctx *ctx, typ reflect2.Type) ValEncoder { + type bindingTo struct { + binding *Binding + toName string + ignored bool + } + orderedBindings := []*bindingTo{} + structDescriptor := describeStruct(ctx, typ) + for _, binding := range structDescriptor.Fields { + for _, toName := range binding.ToNames { + new := &bindingTo{ + binding: binding, + toName: toName, + } + for _, old := range orderedBindings { + if old.toName != toName { + continue + } + old.ignored, new.ignored = resolveConflictBinding(ctx.frozenConfig, old.binding, new.binding) + } + orderedBindings = append(orderedBindings, new) + } + } + if len(orderedBindings) == 0 { + return &emptyStructEncoder{} + } + finalOrderedFields := []structFieldTo{} + for _, bindingTo := range orderedBindings { + if !bindingTo.ignored { + finalOrderedFields = append(finalOrderedFields, structFieldTo{ + encoder: bindingTo.binding.Encoder.(*structFieldEncoder), + toName: bindingTo.toName, + }) + } + } + return &structEncoder{typ, finalOrderedFields} +} + +func createCheckIsEmpty(ctx *ctx, typ reflect2.Type) checkIsEmpty { + encoder := createEncoderOfNative(ctx, typ) + if encoder != nil { + return encoder + } + kind := typ.Kind() + switch kind { + case reflect.Interface: + return &dynamicEncoder{typ} + case reflect.Struct: + return &structEncoder{typ: typ} + case reflect.Array: + return &arrayEncoder{} + case reflect.Slice: + return &sliceEncoder{} + case reflect.Map: + return encoderOfMap(ctx, typ) + case reflect.Ptr: + return &OptionalEncoder{} + default: + return &lazyErrorEncoder{err: fmt.Errorf("unsupported type: %v", typ)} + } +} + +func resolveConflictBinding(cfg *frozenConfig, old, new *Binding) (ignoreOld, ignoreNew bool) { + newTagged := new.Field.Tag().Get(cfg.getTagKey()) != "" + oldTagged := old.Field.Tag().Get(cfg.getTagKey()) != "" + if newTagged { + if oldTagged { + if len(old.levels) > len(new.levels) { + return true, false + } else if len(new.levels) > len(old.levels) { + return false, true + } else { + return true, true + } + } else { + return true, false + } + } else { + if oldTagged { + return true, false + } + if len(old.levels) > len(new.levels) { + return true, false + } else if len(new.levels) > len(old.levels) { + return false, true + } else { + return true, true + } + } +} + +type structFieldEncoder struct { + field reflect2.StructField + fieldEncoder ValEncoder + omitempty bool +} + +func (encoder *structFieldEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { + fieldPtr := encoder.field.UnsafeGet(ptr) + encoder.fieldEncoder.Encode(fieldPtr, stream) + if stream.Error != nil && stream.Error != io.EOF { + stream.Error = fmt.Errorf("%s: %s", encoder.field.Name(), stream.Error.Error()) + } +} + +func (encoder *structFieldEncoder) IsEmpty(ptr unsafe.Pointer) bool { + fieldPtr := encoder.field.UnsafeGet(ptr) + return encoder.fieldEncoder.IsEmpty(fieldPtr) +} + +func (encoder *structFieldEncoder) IsEmbeddedPtrNil(ptr unsafe.Pointer) bool { + isEmbeddedPtrNil, converted := encoder.fieldEncoder.(IsEmbeddedPtrNil) + if !converted { + return false + } + fieldPtr := encoder.field.UnsafeGet(ptr) + return isEmbeddedPtrNil.IsEmbeddedPtrNil(fieldPtr) +} + +type IsEmbeddedPtrNil interface { + IsEmbeddedPtrNil(ptr unsafe.Pointer) bool +} + +type structEncoder struct { + typ reflect2.Type + fields []structFieldTo +} + +type structFieldTo struct { + encoder *structFieldEncoder + toName string +} + +func (encoder *structEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { + stream.WriteObjectStart() + isNotFirst := false + for _, field := range encoder.fields { + if field.encoder.omitempty && field.encoder.IsEmpty(ptr) { + continue + } + if field.encoder.IsEmbeddedPtrNil(ptr) { + continue + } + if isNotFirst { + stream.WriteMore() + } + stream.WriteObjectField(field.toName) + field.encoder.Encode(ptr, stream) + isNotFirst = true + } + stream.WriteObjectEnd() + if stream.Error != nil && stream.Error != io.EOF { + stream.Error = fmt.Errorf("%v.%s", encoder.typ, stream.Error.Error()) + } +} + +func (encoder *structEncoder) IsEmpty(ptr unsafe.Pointer) bool { + return false +} + +type emptyStructEncoder struct { +} + +func (encoder *emptyStructEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { + stream.WriteEmptyObject() +} + +func (encoder *emptyStructEncoder) IsEmpty(ptr unsafe.Pointer) bool { + return false +} + +type stringModeNumberEncoder struct { + elemEncoder ValEncoder +} + +func (encoder *stringModeNumberEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { + stream.writeByte('"') + encoder.elemEncoder.Encode(ptr, stream) + stream.writeByte('"') +} + +func (encoder *stringModeNumberEncoder) IsEmpty(ptr unsafe.Pointer) bool { + return encoder.elemEncoder.IsEmpty(ptr) +} + +type stringModeStringEncoder struct { + elemEncoder ValEncoder + cfg *frozenConfig +} + +func (encoder *stringModeStringEncoder) Encode(ptr unsafe.Pointer, stream *Stream) { + tempStream := encoder.cfg.BorrowStream(nil) + tempStream.Attachment = stream.Attachment + defer encoder.cfg.ReturnStream(tempStream) + encoder.elemEncoder.Encode(ptr, tempStream) + stream.WriteString(string(tempStream.Buffer())) +} + +func (encoder *stringModeStringEncoder) IsEmpty(ptr unsafe.Pointer) bool { + return encoder.elemEncoder.IsEmpty(ptr) +} diff --git a/vendor/github.com/json-iterator/go/stream.go b/vendor/github.com/json-iterator/go/stream.go new file mode 100644 index 00000000..23d8a3ad --- /dev/null +++ b/vendor/github.com/json-iterator/go/stream.go @@ -0,0 +1,210 @@ +package jsoniter + +import ( + "io" +) + +// stream is a io.Writer like object, with JSON specific write functions. +// Error is not returned as return value, but stored as Error member on this stream instance. +type Stream struct { + cfg *frozenConfig + out io.Writer + buf []byte + Error error + indention int + Attachment interface{} // open for customized encoder +} + +// NewStream create new stream instance. +// cfg can be jsoniter.ConfigDefault. +// out can be nil if write to internal buffer. +// bufSize is the initial size for the internal buffer in bytes. +func NewStream(cfg API, out io.Writer, bufSize int) *Stream { + return &Stream{ + cfg: cfg.(*frozenConfig), + out: out, + buf: make([]byte, 0, bufSize), + Error: nil, + indention: 0, + } +} + +// Pool returns a pool can provide more stream with same configuration +func (stream *Stream) Pool() StreamPool { + return stream.cfg +} + +// Reset reuse this stream instance by assign a new writer +func (stream *Stream) Reset(out io.Writer) { + stream.out = out + stream.buf = stream.buf[:0] +} + +// Available returns how many bytes are unused in the buffer. +func (stream *Stream) Available() int { + return cap(stream.buf) - len(stream.buf) +} + +// Buffered returns the number of bytes that have been written into the current buffer. +func (stream *Stream) Buffered() int { + return len(stream.buf) +} + +// Buffer if writer is nil, use this method to take the result +func (stream *Stream) Buffer() []byte { + return stream.buf +} + +// SetBuffer allows to append to the internal buffer directly +func (stream *Stream) SetBuffer(buf []byte) { + stream.buf = buf +} + +// Write writes the contents of p into the buffer. +// It returns the number of bytes written. +// If nn < len(p), it also returns an error explaining +// why the write is short. +func (stream *Stream) Write(p []byte) (nn int, err error) { + stream.buf = append(stream.buf, p...) + if stream.out != nil { + nn, err = stream.out.Write(stream.buf) + stream.buf = stream.buf[nn:] + return + } + return len(p), nil +} + +// WriteByte writes a single byte. +func (stream *Stream) writeByte(c byte) { + stream.buf = append(stream.buf, c) +} + +func (stream *Stream) writeTwoBytes(c1 byte, c2 byte) { + stream.buf = append(stream.buf, c1, c2) +} + +func (stream *Stream) writeThreeBytes(c1 byte, c2 byte, c3 byte) { + stream.buf = append(stream.buf, c1, c2, c3) +} + +func (stream *Stream) writeFourBytes(c1 byte, c2 byte, c3 byte, c4 byte) { + stream.buf = append(stream.buf, c1, c2, c3, c4) +} + +func (stream *Stream) writeFiveBytes(c1 byte, c2 byte, c3 byte, c4 byte, c5 byte) { + stream.buf = append(stream.buf, c1, c2, c3, c4, c5) +} + +// Flush writes any buffered data to the underlying io.Writer. +func (stream *Stream) Flush() error { + if stream.out == nil { + return nil + } + if stream.Error != nil { + return stream.Error + } + _, err := stream.out.Write(stream.buf) + if err != nil { + if stream.Error == nil { + stream.Error = err + } + return err + } + stream.buf = stream.buf[:0] + return nil +} + +// WriteRaw write string out without quotes, just like []byte +func (stream *Stream) WriteRaw(s string) { + stream.buf = append(stream.buf, s...) +} + +// WriteNil write null to stream +func (stream *Stream) WriteNil() { + stream.writeFourBytes('n', 'u', 'l', 'l') +} + +// WriteTrue write true to stream +func (stream *Stream) WriteTrue() { + stream.writeFourBytes('t', 'r', 'u', 'e') +} + +// WriteFalse write false to stream +func (stream *Stream) WriteFalse() { + stream.writeFiveBytes('f', 'a', 'l', 's', 'e') +} + +// WriteBool write true or false into stream +func (stream *Stream) WriteBool(val bool) { + if val { + stream.WriteTrue() + } else { + stream.WriteFalse() + } +} + +// WriteObjectStart write { with possible indention +func (stream *Stream) WriteObjectStart() { + stream.indention += stream.cfg.indentionStep + stream.writeByte('{') + stream.writeIndention(0) +} + +// WriteObjectField write "field": with possible indention +func (stream *Stream) WriteObjectField(field string) { + stream.WriteString(field) + if stream.indention > 0 { + stream.writeTwoBytes(':', ' ') + } else { + stream.writeByte(':') + } +} + +// WriteObjectEnd write } with possible indention +func (stream *Stream) WriteObjectEnd() { + stream.writeIndention(stream.cfg.indentionStep) + stream.indention -= stream.cfg.indentionStep + stream.writeByte('}') +} + +// WriteEmptyObject write {} +func (stream *Stream) WriteEmptyObject() { + stream.writeByte('{') + stream.writeByte('}') +} + +// WriteMore write , with possible indention +func (stream *Stream) WriteMore() { + stream.writeByte(',') + stream.writeIndention(0) +} + +// WriteArrayStart write [ with possible indention +func (stream *Stream) WriteArrayStart() { + stream.indention += stream.cfg.indentionStep + stream.writeByte('[') + stream.writeIndention(0) +} + +// WriteEmptyArray write [] +func (stream *Stream) WriteEmptyArray() { + stream.writeTwoBytes('[', ']') +} + +// WriteArrayEnd write ] with possible indention +func (stream *Stream) WriteArrayEnd() { + stream.writeIndention(stream.cfg.indentionStep) + stream.indention -= stream.cfg.indentionStep + stream.writeByte(']') +} + +func (stream *Stream) writeIndention(delta int) { + if stream.indention == 0 { + return + } + stream.writeByte('\n') + toWrite := stream.indention - delta + for i := 0; i < toWrite; i++ { + stream.buf = append(stream.buf, ' ') + } +} diff --git a/vendor/github.com/json-iterator/go/stream_float.go b/vendor/github.com/json-iterator/go/stream_float.go new file mode 100644 index 00000000..826aa594 --- /dev/null +++ b/vendor/github.com/json-iterator/go/stream_float.go @@ -0,0 +1,111 @@ +package jsoniter + +import ( + "fmt" + "math" + "strconv" +) + +var pow10 []uint64 + +func init() { + pow10 = []uint64{1, 10, 100, 1000, 10000, 100000, 1000000} +} + +// WriteFloat32 write float32 to stream +func (stream *Stream) WriteFloat32(val float32) { + if math.IsInf(float64(val), 0) || math.IsNaN(float64(val)) { + stream.Error = fmt.Errorf("unsupported value: %f", val) + return + } + abs := math.Abs(float64(val)) + fmt := byte('f') + // Note: Must use float32 comparisons for underlying float32 value to get precise cutoffs right. + if abs != 0 { + if float32(abs) < 1e-6 || float32(abs) >= 1e21 { + fmt = 'e' + } + } + stream.buf = strconv.AppendFloat(stream.buf, float64(val), fmt, -1, 32) +} + +// WriteFloat32Lossy write float32 to stream with ONLY 6 digits precision although much much faster +func (stream *Stream) WriteFloat32Lossy(val float32) { + if math.IsInf(float64(val), 0) || math.IsNaN(float64(val)) { + stream.Error = fmt.Errorf("unsupported value: %f", val) + return + } + if val < 0 { + stream.writeByte('-') + val = -val + } + if val > 0x4ffffff { + stream.WriteFloat32(val) + return + } + precision := 6 + exp := uint64(1000000) // 6 + lval := uint64(float64(val)*float64(exp) + 0.5) + stream.WriteUint64(lval / exp) + fval := lval % exp + if fval == 0 { + return + } + stream.writeByte('.') + for p := precision - 1; p > 0 && fval < pow10[p]; p-- { + stream.writeByte('0') + } + stream.WriteUint64(fval) + for stream.buf[len(stream.buf)-1] == '0' { + stream.buf = stream.buf[:len(stream.buf)-1] + } +} + +// WriteFloat64 write float64 to stream +func (stream *Stream) WriteFloat64(val float64) { + if math.IsInf(val, 0) || math.IsNaN(val) { + stream.Error = fmt.Errorf("unsupported value: %f", val) + return + } + abs := math.Abs(val) + fmt := byte('f') + // Note: Must use float32 comparisons for underlying float32 value to get precise cutoffs right. + if abs != 0 { + if abs < 1e-6 || abs >= 1e21 { + fmt = 'e' + } + } + stream.buf = strconv.AppendFloat(stream.buf, float64(val), fmt, -1, 64) +} + +// WriteFloat64Lossy write float64 to stream with ONLY 6 digits precision although much much faster +func (stream *Stream) WriteFloat64Lossy(val float64) { + if math.IsInf(val, 0) || math.IsNaN(val) { + stream.Error = fmt.Errorf("unsupported value: %f", val) + return + } + if val < 0 { + stream.writeByte('-') + val = -val + } + if val > 0x4ffffff { + stream.WriteFloat64(val) + return + } + precision := 6 + exp := uint64(1000000) // 6 + lval := uint64(val*float64(exp) + 0.5) + stream.WriteUint64(lval / exp) + fval := lval % exp + if fval == 0 { + return + } + stream.writeByte('.') + for p := precision - 1; p > 0 && fval < pow10[p]; p-- { + stream.writeByte('0') + } + stream.WriteUint64(fval) + for stream.buf[len(stream.buf)-1] == '0' { + stream.buf = stream.buf[:len(stream.buf)-1] + } +} diff --git a/vendor/github.com/json-iterator/go/stream_int.go b/vendor/github.com/json-iterator/go/stream_int.go new file mode 100644 index 00000000..d1059ee4 --- /dev/null +++ b/vendor/github.com/json-iterator/go/stream_int.go @@ -0,0 +1,190 @@ +package jsoniter + +var digits []uint32 + +func init() { + digits = make([]uint32, 1000) + for i := uint32(0); i < 1000; i++ { + digits[i] = (((i / 100) + '0') << 16) + ((((i / 10) % 10) + '0') << 8) + i%10 + '0' + if i < 10 { + digits[i] += 2 << 24 + } else if i < 100 { + digits[i] += 1 << 24 + } + } +} + +func writeFirstBuf(space []byte, v uint32) []byte { + start := v >> 24 + if start == 0 { + space = append(space, byte(v>>16), byte(v>>8)) + } else if start == 1 { + space = append(space, byte(v>>8)) + } + space = append(space, byte(v)) + return space +} + +func writeBuf(buf []byte, v uint32) []byte { + return append(buf, byte(v>>16), byte(v>>8), byte(v)) +} + +// WriteUint8 write uint8 to stream +func (stream *Stream) WriteUint8(val uint8) { + stream.buf = writeFirstBuf(stream.buf, digits[val]) +} + +// WriteInt8 write int8 to stream +func (stream *Stream) WriteInt8(nval int8) { + var val uint8 + if nval < 0 { + val = uint8(-nval) + stream.buf = append(stream.buf, '-') + } else { + val = uint8(nval) + } + stream.buf = writeFirstBuf(stream.buf, digits[val]) +} + +// WriteUint16 write uint16 to stream +func (stream *Stream) WriteUint16(val uint16) { + q1 := val / 1000 + if q1 == 0 { + stream.buf = writeFirstBuf(stream.buf, digits[val]) + return + } + r1 := val - q1*1000 + stream.buf = writeFirstBuf(stream.buf, digits[q1]) + stream.buf = writeBuf(stream.buf, digits[r1]) + return +} + +// WriteInt16 write int16 to stream +func (stream *Stream) WriteInt16(nval int16) { + var val uint16 + if nval < 0 { + val = uint16(-nval) + stream.buf = append(stream.buf, '-') + } else { + val = uint16(nval) + } + stream.WriteUint16(val) +} + +// WriteUint32 write uint32 to stream +func (stream *Stream) WriteUint32(val uint32) { + q1 := val / 1000 + if q1 == 0 { + stream.buf = writeFirstBuf(stream.buf, digits[val]) + return + } + r1 := val - q1*1000 + q2 := q1 / 1000 + if q2 == 0 { + stream.buf = writeFirstBuf(stream.buf, digits[q1]) + stream.buf = writeBuf(stream.buf, digits[r1]) + return + } + r2 := q1 - q2*1000 + q3 := q2 / 1000 + if q3 == 0 { + stream.buf = writeFirstBuf(stream.buf, digits[q2]) + } else { + r3 := q2 - q3*1000 + stream.buf = append(stream.buf, byte(q3+'0')) + stream.buf = writeBuf(stream.buf, digits[r3]) + } + stream.buf = writeBuf(stream.buf, digits[r2]) + stream.buf = writeBuf(stream.buf, digits[r1]) +} + +// WriteInt32 write int32 to stream +func (stream *Stream) WriteInt32(nval int32) { + var val uint32 + if nval < 0 { + val = uint32(-nval) + stream.buf = append(stream.buf, '-') + } else { + val = uint32(nval) + } + stream.WriteUint32(val) +} + +// WriteUint64 write uint64 to stream +func (stream *Stream) WriteUint64(val uint64) { + q1 := val / 1000 + if q1 == 0 { + stream.buf = writeFirstBuf(stream.buf, digits[val]) + return + } + r1 := val - q1*1000 + q2 := q1 / 1000 + if q2 == 0 { + stream.buf = writeFirstBuf(stream.buf, digits[q1]) + stream.buf = writeBuf(stream.buf, digits[r1]) + return + } + r2 := q1 - q2*1000 + q3 := q2 / 1000 + if q3 == 0 { + stream.buf = writeFirstBuf(stream.buf, digits[q2]) + stream.buf = writeBuf(stream.buf, digits[r2]) + stream.buf = writeBuf(stream.buf, digits[r1]) + return + } + r3 := q2 - q3*1000 + q4 := q3 / 1000 + if q4 == 0 { + stream.buf = writeFirstBuf(stream.buf, digits[q3]) + stream.buf = writeBuf(stream.buf, digits[r3]) + stream.buf = writeBuf(stream.buf, digits[r2]) + stream.buf = writeBuf(stream.buf, digits[r1]) + return + } + r4 := q3 - q4*1000 + q5 := q4 / 1000 + if q5 == 0 { + stream.buf = writeFirstBuf(stream.buf, digits[q4]) + stream.buf = writeBuf(stream.buf, digits[r4]) + stream.buf = writeBuf(stream.buf, digits[r3]) + stream.buf = writeBuf(stream.buf, digits[r2]) + stream.buf = writeBuf(stream.buf, digits[r1]) + return + } + r5 := q4 - q5*1000 + q6 := q5 / 1000 + if q6 == 0 { + stream.buf = writeFirstBuf(stream.buf, digits[q5]) + } else { + stream.buf = writeFirstBuf(stream.buf, digits[q6]) + r6 := q5 - q6*1000 + stream.buf = writeBuf(stream.buf, digits[r6]) + } + stream.buf = writeBuf(stream.buf, digits[r5]) + stream.buf = writeBuf(stream.buf, digits[r4]) + stream.buf = writeBuf(stream.buf, digits[r3]) + stream.buf = writeBuf(stream.buf, digits[r2]) + stream.buf = writeBuf(stream.buf, digits[r1]) +} + +// WriteInt64 write int64 to stream +func (stream *Stream) WriteInt64(nval int64) { + var val uint64 + if nval < 0 { + val = uint64(-nval) + stream.buf = append(stream.buf, '-') + } else { + val = uint64(nval) + } + stream.WriteUint64(val) +} + +// WriteInt write int to stream +func (stream *Stream) WriteInt(val int) { + stream.WriteInt64(int64(val)) +} + +// WriteUint write uint to stream +func (stream *Stream) WriteUint(val uint) { + stream.WriteUint64(uint64(val)) +} diff --git a/vendor/github.com/json-iterator/go/stream_str.go b/vendor/github.com/json-iterator/go/stream_str.go new file mode 100644 index 00000000..54c2ba0b --- /dev/null +++ b/vendor/github.com/json-iterator/go/stream_str.go @@ -0,0 +1,372 @@ +package jsoniter + +import ( + "unicode/utf8" +) + +// htmlSafeSet holds the value true if the ASCII character with the given +// array position can be safely represented inside a JSON string, embedded +// inside of HTML