diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 6be30e4d..98f16bba 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -20,7 +20,7 @@ jobs: - name: Install Go uses: actions/setup-go@v2 with: - go-version: 1.20.x + go-version: 1.21.x - uses: actions/cache@v1 with: diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e271728..94fbb63f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,11 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan We use *breaking :warning:* to mark changes that are not backward compatible (relates only to v0.y.z releases.) ## Unreleased +- [#38](https://github.com/thanos-io/objstore/pull/38) GCS: Upgrade cloud.google.com/go/storage version to `v1.43.0`. +- [#145](https://github.com/thanos-io/objstore/pull/145) Include content length in the response of Get and GetRange. ### Fixed +- [#153](https://github.com/thanos-io/objstore/pull/153) Metrics: Fix `objstore_bucket_operation_duration_seconds_*` for `get` and `get_range` operations. - [#117](https://github.com/thanos-io/objstore/pull/117) Metrics: Fix `objstore_bucket_operation_failures_total` incorrectly incremented if context is cancelled while reading object contents. - [#115](https://github.com/thanos-io/objstore/pull/115) GCS: Fix creation of bucket with GRPC connections. Also update storage client to `v1.40.0`. - [#102](https://github.com/thanos-io/objstore/pull/102) Azure: bump azblob sdk to get concurrency fixes. @@ -50,6 +53,11 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#116](https://github.com/thanos-io/objstore/pull/116) Azure: Add new storage_create_container configuration property - [#128](https://github.com/thanos-io/objstore/pull/128) GCS: Add support for `ChunkSize` for writer. - [#130](https://github.com/thanos-io/objstore/pull/130) feat: Decouple creating bucket metrics from instrumenting the bucket +- [#147](https://github.com/thanos-io/objstore/pull/147) feat: Add MaxRetries config to cos, gcs and obs. +- [#150](https://github.com/thanos-io/objstore/pull/150) Add support for roundtripper wrapper. +- [#63](https://github.com/thanos-io/objstore/pull/63) Implement a `IterWithAttributes` method on the bucket client. +- [#155](https://github.com/thanos-io/objstore/pull/155) Add a `Provider` method on `objstore.Client`. + ### Changed - [#38](https://github.com/thanos-io/objstore/pull/38) *: Upgrade minio-go version to `v7.0.45`. diff --git a/README.md b/README.md index 6d848e79..516d1070 100644 --- a/README.md +++ b/README.md @@ -48,13 +48,15 @@ See [MAINTAINERS.md](https://github.com/thanos-io/thanos/blob/main/MAINTAINERS.m The core this module is the [`Bucket` interface](objstore.go): -```go mdox-exec="sed -n '37,50p' objstore.go" +```go mdox-exec="sed -n '55,73p' objstore.go" // Bucket provides read and write access to an object storage bucket. // NOTE: We assume strong consistency for write-read flow. type Bucket interface { io.Closer BucketReader + Provider() ObjProvider + // Upload the contents of the reader as an object into the bucket. // Upload should be idempotent. Upload(ctx context.Context, name string, r io.Reader) error @@ -63,18 +65,31 @@ type Bucket interface { // If object does not exist in the moment of deletion, Delete should throw error. Delete(ctx context.Context, name string) error + // Name returns the bucket name for the provider. + Name() string +} ``` All [provider implementations](providers) have to implement `Bucket` interface that allows common read and write operations that all supported by all object providers. If you want to limit the code that will do bucket operation to only read access (smart idea, allowing to limit access permissions), you can use the [`BucketReader` interface](objstore.go): -```go mdox-exec="sed -n '68,93p' objstore.go" - +```go mdox-exec="sed -n '89,124p' objstore.go" // BucketReader provides read access to an object storage bucket. type BucketReader interface { // Iter calls f for each entry in the given directory (not recursive.). The argument to f is the full // object name including the prefix of the inspected directory. + // Entries are passed to function in sorted order. - Iter(ctx context.Context, dir string, f func(string) error, options ...IterOption) error + Iter(ctx context.Context, dir string, f func(name string) error, options ...IterOption) error + + // IterWithAttributes calls f for each entry in the given directory similar to Iter. + // In addition to Name, it also includes requested object attributes in the argument to f. + // + // Attributes can be requested using IterOption. + // Not all IterOptions are supported by all providers, requesting for an unsupported option will fail with ErrOptionNotSupported. + IterWithAttributes(ctx context.Context, dir string, f func(attrs IterObjectAttributes) error, options ...IterOption) error + + // SupportedIterOptions returns a list of supported IterOptions by the underlying provider. + SupportedIterOptions() []IterOptionType // Get returns a reader for the given object name. Get(ctx context.Context, name string) (io.ReadCloser, error) @@ -374,6 +389,7 @@ config: server_name: "" insecure_skip_verify: false disable_compression: false + chunk_size_bytes: 0 prefix: "" ``` @@ -447,6 +463,7 @@ config: storage_account: "" storage_account_key: "" storage_connection_string: "" + storage_create_container: false container: "" endpoint: "" user_assigned_id: "" diff --git a/client/factory.go b/client/factory.go index 9e1adec9..089fd843 100644 --- a/client/factory.go +++ b/client/factory.go @@ -6,6 +6,7 @@ package client import ( "context" "fmt" + "net/http" "strings" "github.com/thanos-io/objstore" @@ -26,30 +27,15 @@ import ( "gopkg.in/yaml.v2" ) -type ObjProvider string - -const ( - FILESYSTEM ObjProvider = "FILESYSTEM" - GCS ObjProvider = "GCS" - S3 ObjProvider = "S3" - AZURE ObjProvider = "AZURE" - SWIFT ObjProvider = "SWIFT" - COS ObjProvider = "COS" - ALIYUNOSS ObjProvider = "ALIYUNOSS" - BOS ObjProvider = "BOS" - OCI ObjProvider = "OCI" - OBS ObjProvider = "OBS" -) - type BucketConfig struct { - Type ObjProvider `yaml:"type"` - Config interface{} `yaml:"config"` - Prefix string `yaml:"prefix" default:""` + Type objstore.ObjProvider `yaml:"type"` + Config interface{} `yaml:"config"` + Prefix string `yaml:"prefix" default:""` } // NewBucket initializes and returns new object storage clients. // NOTE: confContentYaml can contain secrets. -func NewBucket(logger log.Logger, confContentYaml []byte, component string) (objstore.Bucket, error) { +func NewBucket(logger log.Logger, confContentYaml []byte, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (objstore.Bucket, error) { level.Info(logger).Log("msg", "loading bucket configuration") bucketConf := &BucketConfig{} if err := yaml.UnmarshalStrict(confContentYaml, bucketConf); err != nil { @@ -63,25 +49,25 @@ func NewBucket(logger log.Logger, confContentYaml []byte, component string) (obj var bucket objstore.Bucket switch strings.ToUpper(string(bucketConf.Type)) { - case string(GCS): - bucket, err = gcs.NewBucket(context.Background(), logger, config, component) - case string(S3): - bucket, err = s3.NewBucket(logger, config, component) - case string(AZURE): - bucket, err = azure.NewBucket(logger, config, component) - case string(SWIFT): - bucket, err = swift.NewContainer(logger, config) - case string(COS): - bucket, err = cos.NewBucket(logger, config, component) - case string(ALIYUNOSS): - bucket, err = oss.NewBucket(logger, config, component) - case string(FILESYSTEM): + case string(objstore.GCS): + bucket, err = gcs.NewBucket(context.Background(), logger, config, component, wrapRoundtripper) + case string(objstore.S3): + bucket, err = s3.NewBucket(logger, config, component, wrapRoundtripper) + case string(objstore.AZURE): + bucket, err = azure.NewBucket(logger, config, component, wrapRoundtripper) + case string(objstore.SWIFT): + bucket, err = swift.NewContainer(logger, config, wrapRoundtripper) + case string(objstore.COS): + bucket, err = cos.NewBucket(logger, config, component, wrapRoundtripper) + case string(objstore.ALIYUNOSS): + bucket, err = oss.NewBucket(logger, config, component, wrapRoundtripper) + case string(objstore.FILESYSTEM): bucket, err = filesystem.NewBucketFromConfig(config) - case string(BOS): + case string(objstore.BOS): bucket, err = bos.NewBucket(logger, config, component) - case string(OCI): - bucket, err = oci.NewBucket(logger, config) - case string(OBS): + case string(objstore.OCI): + bucket, err = oci.NewBucket(logger, config, wrapRoundtripper) + case string(objstore.OBS): bucket, err = obs.NewBucket(logger, config) default: return nil, errors.Errorf("bucket with type %s is not supported", bucketConf.Type) diff --git a/client/factory_test.go b/client/factory_test.go index 4a9cf879..cd008add 100644 --- a/client/factory_test.go +++ b/client/factory_test.go @@ -23,7 +23,7 @@ func ExampleBucket() { } // Create a new bucket. - bucket, err := NewBucket(log.NewNopLogger(), confContentYaml, "example") + bucket, err := NewBucket(log.NewNopLogger(), confContentYaml, "example", nil) if err != nil { panic(err) } @@ -46,7 +46,7 @@ func ExampleTracingBucketUsingOpenTracing() { //nolint:govet } // Create a new bucket. - bucket, err := NewBucket(log.NewNopLogger(), confContentYaml, "example") + bucket, err := NewBucket(log.NewNopLogger(), confContentYaml, "example", nil) if err != nil { panic(err) } @@ -72,7 +72,7 @@ func ExampleTracingBucketUsingOpenTelemetry() { //nolint:govet } // Create a new bucket. - bucket, err := NewBucket(log.NewNopLogger(), confContentYaml, "example") + bucket, err := NewBucket(log.NewNopLogger(), confContentYaml, "example", nil) if err != nil { panic(err) } diff --git a/errutil/rt_error.go b/errutil/rt_error.go new file mode 100644 index 00000000..b6b2e9c9 --- /dev/null +++ b/errutil/rt_error.go @@ -0,0 +1,26 @@ +package errutil + +import ( + "net/http" + + "github.com/pkg/errors" +) + +var rtErr = errors.New("RoundTripper error") + +func IsMockedError(err error) bool { + return errors.Is(err, rtErr) +} + +// ErrorRoundTripper is a custom RoundTripper that always returns an error. +type ErrorRoundTripper struct { + Err error +} + +func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { + return nil, ert.Err +} + +func WrapWithErrRoundtripper(rt http.RoundTripper) http.RoundTripper { + return &ErrorRoundTripper{Err: rtErr} +} diff --git a/go.mod b/go.mod index 8d742017..12d617f2 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,9 @@ module github.com/thanos-io/objstore -go 1.21 +go 1.22 require ( - cloud.google.com/go/storage v1.40.0 + cloud.google.com/go/storage v1.43.0 github.com/aliyun/aliyun-oss-go-sdk v2.2.2+incompatible github.com/aws/aws-sdk-go-v2 v1.30.4 github.com/aws/aws-sdk-go-v2/config v1.27.30 @@ -14,7 +14,7 @@ require ( github.com/fullstorydev/emulators/storage v0.0.0-20240401123056-edc69752f474 github.com/go-kit/log v0.2.1 github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.3+incompatible - github.com/minio/minio-go/v7 v7.0.72 + github.com/minio/minio-go/v7 v7.0.80 github.com/ncw/swift v1.0.53 github.com/opentracing/opentracing-go v1.2.0 github.com/oracle/oci-go-sdk/v65 v65.41.1 @@ -25,19 +25,20 @@ require ( go.opentelemetry.io/otel v1.24.0 go.opentelemetry.io/otel/trace v1.24.0 go.uber.org/atomic v1.9.0 - golang.org/x/oauth2 v0.18.0 - golang.org/x/sync v0.7.0 - google.golang.org/api v0.172.0 - google.golang.org/grpc v1.62.1 + golang.org/x/oauth2 v0.21.0 + golang.org/x/sync v0.8.0 + google.golang.org/api v0.187.0 + google.golang.org/grpc v1.66.0 gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/yaml.v2 v2.4.0 ) require ( - cloud.google.com/go v0.112.1 // indirect - cloud.google.com/go/compute v1.24.0 // indirect - cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v1.1.7 // indirect + cloud.google.com/go v0.115.0 // indirect + cloud.google.com/go/auth v0.6.1 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect + cloud.google.com/go/compute/metadata v0.3.0 // indirect + cloud.google.com/go/iam v1.1.8 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect @@ -56,11 +57,12 @@ require ( github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bluele/gcache v0.0.2 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/clbanning/mxj v1.8.4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-ini/ini v1.67.0 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -74,9 +76,9 @@ require ( github.com/google/s2a-go v0.1.7 // indirect github.com/google/uuid v1.6.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect - github.com/googleapis/gax-go/v2 v2.12.3 // indirect + github.com/googleapis/gax-go/v2 v2.12.5 // indirect github.com/jpillora/backoff v1.0.0 // indirect - github.com/klauspost/compress v1.17.9 // indirect + github.com/klauspost/compress v1.17.11 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect @@ -87,22 +89,21 @@ require ( github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect github.com/prometheus/procfs v0.11.1 // indirect - github.com/rs/xid v1.5.0 // indirect + github.com/rs/xid v1.6.0 // indirect github.com/sony/gobreaker v0.5.0 // indirect + github.com/stretchr/objx v0.5.2 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect go.opentelemetry.io/otel/metric v1.24.0 // indirect - golang.org/x/net v0.26.0 // indirect - golang.org/x/sys v0.21.0 // indirect - golang.org/x/text v0.16.0 // indirect + golang.org/x/net v0.30.0 // indirect + golang.org/x/sys v0.26.0 // indirect + golang.org/x/text v0.19.0 // indirect golang.org/x/time v0.5.0 // indirect - google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect - google.golang.org/protobuf v1.33.0 // indirect - gopkg.in/ini.v1 v1.67.0 // indirect + google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d // indirect + google.golang.org/protobuf v1.34.2 // indirect ) require ( @@ -111,5 +112,5 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.0 github.com/kr/text v0.2.0 // indirect github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b // indirect - golang.org/x/crypto v0.24.0 // indirect + golang.org/x/crypto v0.28.0 // indirect ) diff --git a/go.sum b/go.sum index 37429f78..8ee7b244 100644 --- a/go.sum +++ b/go.sum @@ -1,14 +1,18 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.112.1 h1:uJSeirPke5UNZHIb4SxfZklVSiWWVqW4oXlETwZziwM= -cloud.google.com/go v0.112.1/go.mod h1:+Vbu+Y1UU+I1rjmzeMOb/8RfkKJK2Gyxi1X6jJCZLo4= -cloud.google.com/go/compute v1.24.0 h1:phWcR2eWzRJaL/kOiJwfFsPs4BaKq1j6vnpZrc1YlVg= -cloud.google.com/go/compute v1.24.0/go.mod h1:kw1/T+h/+tK2LJK0wiPPx1intgdAM3j/g3hFDlscY40= -cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= -cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/iam v1.1.7 h1:z4VHOhwKLF/+UYXAJDFwGtNF0b6gjsW1Pk9Ml0U/IoM= -cloud.google.com/go/iam v1.1.7/go.mod h1:J4PMPg8TtyurAUvSmPj8FF3EDgY1SPRZxcUGrn7WXGA= -cloud.google.com/go/storage v1.40.0 h1:VEpDQV5CJxFmJ6ueWNsKxcr1QAYOXEgxDa+sBbJahPw= -cloud.google.com/go/storage v1.40.0/go.mod h1:Rrj7/hKlG87BLqDJYtwR0fbPld8uJPbQ2ucUMY7Ir0g= +cloud.google.com/go v0.115.0 h1:CnFSK6Xo3lDYRoBKEcAtia6VSC837/ZkJuRduSFnr14= +cloud.google.com/go v0.115.0/go.mod h1:8jIM5vVgoAEoiVxQ/O4BFTfHqulPZgs/ufEzMcFMdWU= +cloud.google.com/go/auth v0.6.1 h1:T0Zw1XM5c1GlpN2HYr2s+m3vr1p2wy+8VN+Z1FKxW38= +cloud.google.com/go/auth v0.6.1/go.mod h1:eFHG7zDzbXHKmjJddFG/rBlcGp6t25SwRUiEQSlO4x4= +cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKFt//oWu7HX4= +cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q= +cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= +cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= +cloud.google.com/go/iam v1.1.8 h1:r7umDwhj+BQyz0ScZMp4QrGXjSTI3ZINnpgU2nlB/K0= +cloud.google.com/go/iam v1.1.8/go.mod h1:GvE6lyMmfxXauzNq8NbgJbeVQNspG+tcdL/W8QO1+zE= +cloud.google.com/go/longrunning v0.5.7 h1:WLbHekDbjK1fVFD3ibpFFVoyizlLRl73I7YKuAKilhU= +cloud.google.com/go/longrunning v0.5.7/go.mod h1:8GClkudohy1Fxm3owmBGid8W0pSgodEMwEAztp38Xng= +cloud.google.com/go/storage v1.43.0 h1:CcxnSohZwizt4LCzQHWvBf1/kvtHUn7gk9QERXPyXFs= +cloud.google.com/go/storage v1.43.0/go.mod h1:ajvxEa7WmZS1PxvKRq4bq0tFT3vMd502JwstCcYv0Q0= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2 h1:c4k2FIYIh4xtwqrQwV0Ct1v5+ehlNXj5NI/MWVsiTkQ= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2/go.mod h1:5FDJtLEO/GxwNgUxbwrY3LP0pEoThTQJtk2oysdXHxM= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 h1:sO0/P7g68FrryJzljemN+6GTssUXdANk6aJ7T1ZxnsQ= @@ -64,8 +68,8 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I= github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -92,6 +96,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fullstorydev/emulators/storage v0.0.0-20240401123056-edc69752f474 h1:TufioMBjkJ6/Oqmlye/ReuxHFS35HyLmypj/BNy/8GY= github.com/fullstorydev/emulators/storage v0.0.0-20240401123056-edc69752f474/go.mod h1:PQwxF4UU8wuL+srGxr3BOhIW5zXqgucwVlO/nPZLsxw= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= @@ -121,8 +127,6 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= @@ -135,13 +139,14 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= -github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= -github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -150,14 +155,14 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= -github.com/googleapis/gax-go/v2 v2.12.3 h1:5/zPPDvw8Q1SuXjrqrZslrqT7dL/uJT2CQii/cLCKqA= -github.com/googleapis/gax-go/v2 v2.12.3/go.mod h1:AKloxT6GtNbaLm8QTNSidHUVsHYcBHwWRvkNFJUQcS4= +github.com/googleapis/gax-go/v2 v2.12.5 h1:8gw9KZK8TiVKB6q3zHY3SBzLnrGp6HQjyfYBYGmXdxA= +github.com/googleapis/gax-go/v2 v2.12.5/go.mod h1:BUDKcWo+RaKq5SC9vVYL0wLADa3VcfswbOMMRmB9H3E= github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.3+incompatible h1:tKTaPHNVwikS3I1rdyf1INNvgJXWSf/+TzqsiGbrgnQ= github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.3+incompatible/go.mod h1:l7VUhRbTKCzdOacdT4oWCwATKyvZqUOlOqr0Ous3k4s= github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= -github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= @@ -171,8 +176,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zk github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.72 h1:ZSbxs2BfJensLyHdVOgHv+pfmvxYraaUy07ER04dWnA= -github.com/minio/minio-go/v7 v7.0.72/go.mod h1:4yBA8v80xGA30cfM3fz0DKYMXunWl/AV/6tWEs9ryzo= +github.com/minio/minio-go/v7 v7.0.80 h1:2mdUHXEykRdY/BigLt3Iuu1otL0JTogT0Nmltg0wujk= +github.com/minio/minio-go/v7 v7.0.80/go.mod h1:84gmIilaX4zcvAWWzJ5Z1WI5axN+hAbM5w25xf8xvC0= github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ= @@ -202,29 +207,29 @@ github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwa github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= -github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b h1:gQZ0qzfKHQIybLANtM3mBXNUtOfsCFXeTsnBqCsx1KM= github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sony/gobreaker v0.5.0 h1:dRCvqm0P490vZPmy7ppEk2qCnCieBooFJ+YoXGYB+yg= github.com/sony/gobreaker v0.5.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.194/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.194/go.mod h1:yrBKWhChnDqNz1xuXdSbWXG56XawEq0G5j1lg4VwBD4= github.com/tencentyun/cos-go-sdk-v5 v0.7.40 h1:W6vDGKCHe4wBACI1d2UgE6+50sJFhRWU4O8IB2ozzxM= github.com/tencentyun/cos-go-sdk-v5 v0.7.40/go.mod h1:4dCEtLHGh8QPxHEkgq+nFaky7yZxQuYwgSJM87icDaw= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= @@ -235,64 +240,50 @@ go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= -go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw= -go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc= +go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= +go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= -golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= -golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -300,34 +291,27 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= -google.golang.org/api v0.172.0 h1:/1OcMZGPmW1rX2LCu2CmGUD1KXK1+pfzxotxyRUCCdk= -google.golang.org/api v0.172.0/go.mod h1:+fJZq6QXWfa9pXhnIzsjx4yI22d4aI9ZpLb58gvXjis= +google.golang.org/api v0.187.0 h1:Mxs7VATVC2v7CY+7Xwm4ndkX71hpElcvx0D1Ji/p1eo= +google.golang.org/api v0.187.0/go.mod h1:KIHlTc4x7N7gKKuVsdmfBXN13yEEWXWFURWY6SBp2gk= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= -google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 h1:9+tzLLstTlPTRyJTh+ah5wIMsBW5c4tQwGTN3thOW9Y= -google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:mqHbVIp48Muh7Ywss/AD6I5kNVKZMmAa/QEW58Gxp2s= -google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c h1:kaI7oewGK5YnVwj+Y+EJBO/YN1ht8iTL9XkFHtVZLsc= -google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c/go.mod h1:VQW3tUculP/D4B+xVCo+VgSq8As6wA9ZjHl//pmk+6s= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d h1:PksQg4dV6Sem3/HkBX+Ltq8T0ke0PKIRBNBatoDTVls= +google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d/go.mod h1:s7iA721uChleev562UJO2OYB0PPT9CMFjV+Ce7VJH5M= +google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 h1:MuYw1wJzT+ZkybKfaOXKp5hJiZDn2iHaXRw0mRYdHSc= +google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4/go.mod h1:px9SlOOZBg1wM1zdnr8jEL4CNGUBZ+ZKYtNPApNQc4c= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d h1:k3zyW3BYYR30e8v3x0bTDdE9vpYFjZHK+HcyqkrppWk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= -google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= +google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= +google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -337,17 +321,13 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= -gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/inmem.go b/inmem.go index 3f6f35e9..6a344066 100644 --- a/inmem.go +++ b/inmem.go @@ -34,6 +34,8 @@ func NewInMemBucket() *InMemBucket { } } +func (b *InMemBucket) Provider() ObjProvider { return MEMORY } + // Objects returns a copy of the internally stored objects. // NOTE: For assert purposes. func (b *InMemBucket) Objects() map[string][]byte { @@ -106,6 +108,20 @@ func (b *InMemBucket) Iter(_ context.Context, dir string, f func(string) error, return nil } +func (i *InMemBucket) SupportedIterOptions() []IterOptionType { + return []IterOptionType{Recursive} +} + +func (b *InMemBucket) IterWithAttributes(ctx context.Context, dir string, f func(attrs IterObjectAttributes) error, options ...IterOption) error { + if err := ValidateIterOptions(b.SupportedIterOptions(), options...); err != nil { + return err + } + + return b.Iter(ctx, dir, func(name string) error { + return f(IterObjectAttributes{Name: name}) + }, options...) +} + // Get returns a reader for the given object name. func (b *InMemBucket) Get(_ context.Context, name string) (io.ReadCloser, error) { if name == "" { @@ -119,7 +135,12 @@ func (b *InMemBucket) Get(_ context.Context, name string) (io.ReadCloser, error) return nil, errNotFound } - return io.NopCloser(bytes.NewReader(file)), nil + return ObjectSizerReadCloser{ + ReadCloser: io.NopCloser(bytes.NewReader(file)), + Size: func() (int64, error) { + return int64(len(file)), nil + }, + }, nil } // GetRange returns a new range reader for the given object name and range. @@ -136,15 +157,27 @@ func (b *InMemBucket) GetRange(_ context.Context, name string, off, length int64 } if int64(len(file)) < off { - return io.NopCloser(bytes.NewReader(nil)), nil + return ObjectSizerReadCloser{ + ReadCloser: io.NopCloser(bytes.NewReader(nil)), + Size: func() (int64, error) { return 0, nil }, + }, nil } if length == -1 { - return io.NopCloser(bytes.NewReader(file[off:])), nil + return ObjectSizerReadCloser{ + ReadCloser: io.NopCloser(bytes.NewReader(file[off:])), + Size: func() (int64, error) { + return int64(len(file[off:])), nil + }, + }, nil } if length <= 0 { - return io.NopCloser(bytes.NewReader(nil)), errors.New("length cannot be smaller or equal 0") + // wrap with ObjectSizerReadCloser to return 0 size. + return ObjectSizerReadCloser{ + ReadCloser: io.NopCloser(bytes.NewReader(nil)), + Size: func() (int64, error) { return 0, nil }, + }, errors.New("length cannot be smaller or equal 0") } if int64(len(file)) <= off+length { @@ -152,7 +185,12 @@ func (b *InMemBucket) GetRange(_ context.Context, name string, off, length int64 length = int64(len(file)) - off } - return io.NopCloser(bytes.NewReader(file[off : off+length])), nil + return ObjectSizerReadCloser{ + ReadCloser: io.NopCloser(bytes.NewReader(file[off : off+length])), + Size: func() (int64, error) { + return length, nil + }, + }, nil } // Exists checks if the given directory exists in memory. diff --git a/objstore.go b/objstore.go index 87ec9e98..86ecfa26 100644 --- a/objstore.go +++ b/objstore.go @@ -6,11 +6,13 @@ package objstore import ( "bytes" "context" + "fmt" "io" "io/fs" "os" "path" "path/filepath" + "slices" "strings" "sync" "time" @@ -24,6 +26,22 @@ import ( "golang.org/x/sync/errgroup" ) +type ObjProvider string + +const ( + MEMORY ObjProvider = "MEMORY" + FILESYSTEM ObjProvider = "FILESYSTEM" + GCS ObjProvider = "GCS" + S3 ObjProvider = "S3" + AZURE ObjProvider = "AZURE" + SWIFT ObjProvider = "SWIFT" + COS ObjProvider = "COS" + ALIYUNOSS ObjProvider = "ALIYUNOSS" + BOS ObjProvider = "BOS" + OCI ObjProvider = "OCI" + OBS ObjProvider = "OBS" +) + const ( OpIter = "iter" OpGet = "get" @@ -40,6 +58,8 @@ type Bucket interface { io.Closer BucketReader + Provider() ObjProvider + // Upload the contents of the reader as an object into the bucket. // Upload should be idempotent. Upload(ctx context.Context, name string, r io.Reader) error @@ -70,8 +90,19 @@ type InstrumentedBucket interface { type BucketReader interface { // Iter calls f for each entry in the given directory (not recursive.). The argument to f is the full // object name including the prefix of the inspected directory. + // Entries are passed to function in sorted order. - Iter(ctx context.Context, dir string, f func(string) error, options ...IterOption) error + Iter(ctx context.Context, dir string, f func(name string) error, options ...IterOption) error + + // IterWithAttributes calls f for each entry in the given directory similar to Iter. + // In addition to Name, it also includes requested object attributes in the argument to f. + // + // Attributes can be requested using IterOption. + // Not all IterOptions are supported by all providers, requesting for an unsupported option will fail with ErrOptionNotSupported. + IterWithAttributes(ctx context.Context, dir string, f func(attrs IterObjectAttributes) error, options ...IterOption) error + + // SupportedIterOptions returns a list of supported IterOptions by the underlying provider. + SupportedIterOptions() []IterOptionType // Get returns a reader for the given object name. Get(ctx context.Context, name string) (io.ReadCloser, error) @@ -101,24 +132,66 @@ type InstrumentedBucketReader interface { ReaderWithExpectedErrs(IsOpFailureExpectedFunc) BucketReader } +var ErrOptionNotSupported = errors.New("iter option is not supported") + +// IterOptionType is used for type-safe option support checking. +type IterOptionType int + +const ( + Recursive IterOptionType = iota + UpdatedAt +) + // IterOption configures the provided params. -type IterOption func(params *IterParams) +type IterOption struct { + Type IterOptionType + Apply func(params *IterParams) +} // WithRecursiveIter is an option that can be applied to Iter() to recursively list objects // in the bucket. -func WithRecursiveIter(params *IterParams) { - params.Recursive = true +func WithRecursiveIter() IterOption { + return IterOption{ + Type: Recursive, + Apply: func(params *IterParams) { + params.Recursive = true + }, + } +} + +// WithUpdatedAt is an option that can be applied to Iter() to +// include the last modified time in the attributes. +// NB: Prefixes may not report last modified time. +// This option is currently supported for the azure, s3, bos, gcs and filesystem providers. +func WithUpdatedAt() IterOption { + return IterOption{ + Type: UpdatedAt, + Apply: func(params *IterParams) { + params.LastModified = true + }, + } } // IterParams holds the Iter() parameters and is used by objstore clients implementations. type IterParams struct { - Recursive bool + Recursive bool + LastModified bool +} + +func ValidateIterOptions(supportedOptions []IterOptionType, options ...IterOption) error { + for _, opt := range options { + if !slices.Contains(supportedOptions, opt.Type) { + return fmt.Errorf("%w: %v", ErrOptionNotSupported, opt.Type) + } + } + + return nil } func ApplyIterOptions(options ...IterOption) IterParams { out := IterParams{} for _, opt := range options { - opt(&out) + opt.Apply(&out) } return out } @@ -189,6 +262,20 @@ type ObjectAttributes struct { LastModified time.Time `json:"last_modified"` } +type IterObjectAttributes struct { + Name string + lastModified time.Time +} + +func (i *IterObjectAttributes) SetLastModified(t time.Time) { + i.lastModified = t +} + +// LastModified returns the timestamp the object was last modified. Returns false if the timestamp is not available. +func (i *IterObjectAttributes) LastModified() (time.Time, bool) { + return i.lastModified, !i.lastModified.IsZero() +} + // TryToGetSize tries to get upfront size from reader. // Some implementations may return only size of unread data in the reader, so it's best to call this method before // doing any reading. @@ -211,6 +298,8 @@ func TryToGetSize(r io.Reader) (int64, error) { return f.Size(), nil case ObjectSizer: return f.ObjectSize() + case *io.LimitedReader: + return f.N, nil } return 0, errors.Errorf("unsupported type of io.Reader: %T", r) } @@ -512,6 +601,10 @@ type metricBucket struct { metrics *Metrics } +func (b *metricBucket) Provider() ObjProvider { + return b.bkt.Provider() +} + func (b *metricBucket) WithExpectedErrs(fn IsOpFailureExpectedFunc) Bucket { return &metricBucket{ bkt: b.bkt, @@ -531,21 +624,43 @@ func (b *metricBucket) ReaderWithExpectedErrs(fn IsOpFailureExpectedFunc) Bucket return b.WithExpectedErrs(fn) } -func (b *metricBucket) Iter(ctx context.Context, dir string, f func(name string) error, options ...IterOption) error { +func (b *metricBucket) Iter(ctx context.Context, dir string, f func(string) error, options ...IterOption) error { const op = OpIter b.metrics.ops.WithLabelValues(op).Inc() - start := time.Now() + timer := prometheus.NewTimer(b.metrics.opsDuration.WithLabelValues(op)) + defer timer.ObserveDuration() + err := b.bkt.Iter(ctx, dir, f, options...) if err != nil { if !b.metrics.isOpFailureExpected(err) && ctx.Err() != context.Canceled { b.metrics.opsFailures.WithLabelValues(op).Inc() } } - b.metrics.opsDuration.WithLabelValues(op).Observe(time.Since(start).Seconds()) return err } +func (b *metricBucket) IterWithAttributes(ctx context.Context, dir string, f func(IterObjectAttributes) error, options ...IterOption) error { + const op = OpIter + b.metrics.ops.WithLabelValues(op).Inc() + + timer := prometheus.NewTimer(b.metrics.opsDuration.WithLabelValues(op)) + defer timer.ObserveDuration() + + err := b.bkt.IterWithAttributes(ctx, dir, f, options...) + if err != nil { + if !b.metrics.isOpFailureExpected(err) && ctx.Err() != context.Canceled { + b.metrics.opsFailures.WithLabelValues(op).Inc() + } + } + + return err +} + +func (b *metricBucket) SupportedIterOptions() []IterOptionType { + return b.bkt.SupportedIterOptions() +} + func (b *metricBucket) Attributes(ctx context.Context, name string) (ObjectAttributes, error) { const op = OpAttributes b.metrics.ops.WithLabelValues(op).Inc() @@ -566,14 +681,18 @@ func (b *metricBucket) Get(ctx context.Context, name string) (io.ReadCloser, err const op = OpGet b.metrics.ops.WithLabelValues(op).Inc() + start := time.Now() + rc, err := b.bkt.Get(ctx, name) if err != nil { if !b.metrics.isOpFailureExpected(err) && ctx.Err() != context.Canceled { b.metrics.opsFailures.WithLabelValues(op).Inc() } + b.metrics.opsDuration.WithLabelValues(op).Observe(time.Since(start).Seconds()) return nil, err } return newTimingReader( + start, rc, true, op, @@ -589,14 +708,18 @@ func (b *metricBucket) GetRange(ctx context.Context, name string, off, length in const op = OpGetRange b.metrics.ops.WithLabelValues(op).Inc() + start := time.Now() + rc, err := b.bkt.GetRange(ctx, name, off, length) if err != nil { if !b.metrics.isOpFailureExpected(err) && ctx.Err() != context.Canceled { b.metrics.opsFailures.WithLabelValues(op).Inc() } + b.metrics.opsDuration.WithLabelValues(op).Observe(time.Since(start).Seconds()) return nil, err } return newTimingReader( + start, rc, true, op, @@ -628,7 +751,10 @@ func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) err const op = OpUpload b.metrics.ops.WithLabelValues(op).Inc() + start := time.Now() + trc := newTimingReader( + start, r, false, op, @@ -705,7 +831,7 @@ type timingReader struct { transferredBytes *prometheus.HistogramVec } -func newTimingReader(r io.Reader, closeReader bool, op string, dur *prometheus.HistogramVec, failed *prometheus.CounterVec, isFailureExpected IsOpFailureExpectedFunc, fetchedBytes *prometheus.CounterVec, transferredBytes *prometheus.HistogramVec) io.ReadCloser { +func newTimingReader(start time.Time, r io.Reader, closeReader bool, op string, dur *prometheus.HistogramVec, failed *prometheus.CounterVec, isFailureExpected IsOpFailureExpectedFunc, fetchedBytes *prometheus.CounterVec, transferredBytes *prometheus.HistogramVec) io.ReadCloser { // Initialize the metrics with 0. dur.WithLabelValues(op) failed.WithLabelValues(op) @@ -716,7 +842,7 @@ func newTimingReader(r io.Reader, closeReader bool, op string, dur *prometheus.H closeReader: closeReader, objSize: objSize, objSizeErr: objSizeErr, - start: time.Now(), + start: start, op: op, duration: dur, failed: failed, @@ -728,7 +854,6 @@ func newTimingReader(r io.Reader, closeReader bool, op string, dur *prometheus.H _, isSeeker := r.(io.Seeker) _, isReaderAt := r.(io.ReaderAt) - if isSeeker && isReaderAt { // The assumption is that in most cases when io.ReaderAt() is implemented then // io.Seeker is implemented too (e.g. os.File). @@ -737,6 +862,9 @@ func newTimingReader(r io.Reader, closeReader bool, op string, dur *prometheus.H if isSeeker { return &timingReaderSeeker{timingReader: trc} } + if _, isWriterTo := r.(io.WriterTo); isWriterTo { + return &timingReaderWriterTo{timingReader: trc} + } return &trc } @@ -772,11 +900,16 @@ func (r *timingReader) Close() error { func (r *timingReader) Read(b []byte) (n int, err error) { n, err = r.Reader.Read(b) + r.updateMetrics(n, err) + return n, err +} + +func (r *timingReader) updateMetrics(n int, err error) { if r.fetchedBytes != nil { r.fetchedBytes.WithLabelValues(r.op).Add(float64(n)) } - r.readBytes += int64(n) + // Report metric just once. if !r.alreadyGotErr && err != nil && err != io.EOF { if !r.isFailureExpected(err) && !errors.Is(err, context.Canceled) { @@ -784,7 +917,6 @@ func (r *timingReader) Read(b []byte) (n int, err error) { } r.alreadyGotErr = true } - return n, err } type timingReaderSeeker struct { @@ -802,3 +934,27 @@ type timingReaderSeekerReaderAt struct { func (rsc *timingReaderSeekerReaderAt) ReadAt(p []byte, off int64) (int, error) { return (rsc.Reader).(io.ReaderAt).ReadAt(p, off) } + +type timingReaderWriterTo struct { + timingReader +} + +func (t *timingReaderWriterTo) WriteTo(w io.Writer) (n int64, err error) { + n, err = (t.Reader).(io.WriterTo).WriteTo(w) + t.timingReader.updateMetrics(int(n), err) + return n, err +} + +type ObjectSizerReadCloser struct { + io.ReadCloser + Size func() (int64, error) +} + +// ObjectSize implement ObjectSizer. +func (o ObjectSizerReadCloser) ObjectSize() (int64, error) { + if o.Size == nil { + return 0, errors.New("unknown size") + } + + return o.Size() +} diff --git a/objstore_test.go b/objstore_test.go index b62858fb..b1f82922 100644 --- a/objstore_test.go +++ b/objstore_test.go @@ -11,6 +11,7 @@ import ( "path/filepath" "strings" "testing" + "time" "github.com/efficientgo/core/testutil" "github.com/go-kit/log" @@ -412,7 +413,7 @@ func TestDownloadUploadDirConcurrency(t *testing.T) { func TestTimingReader(t *testing.T) { m := WrapWithMetrics(NewInMemBucket(), nil, "") r := bytes.NewReader([]byte("hello world")) - tr := newTimingReader(r, true, OpGet, m.metrics.opsDuration, m.metrics.opsFailures, func(err error) bool { + tr := newTimingReader(time.Now(), r, true, OpGet, m.metrics.opsDuration, m.metrics.opsFailures, func(err error) bool { return false }, m.metrics.opsFetchedBytes, m.metrics.opsTransferredBytes) @@ -447,7 +448,7 @@ func TestTimingReader_ExpectedError(t *testing.T) { m := WrapWithMetrics(NewInMemBucket(), nil, "") r := dummyReader{readerErr} - tr := newTimingReader(r, true, OpGet, m.metrics.opsDuration, m.metrics.opsFailures, func(err error) bool { return errors.Is(err, readerErr) }, m.metrics.opsFetchedBytes, m.metrics.opsTransferredBytes) + tr := newTimingReader(time.Now(), r, true, OpGet, m.metrics.opsDuration, m.metrics.opsFailures, func(err error) bool { return errors.Is(err, readerErr) }, m.metrics.opsFetchedBytes, m.metrics.opsTransferredBytes) buf := make([]byte, 1) _, err := io.ReadFull(tr, buf) @@ -461,7 +462,7 @@ func TestTimingReader_UnexpectedError(t *testing.T) { m := WrapWithMetrics(NewInMemBucket(), nil, "") r := dummyReader{readerErr} - tr := newTimingReader(r, true, OpGet, m.metrics.opsDuration, m.metrics.opsFailures, func(err error) bool { return false }, m.metrics.opsFetchedBytes, m.metrics.opsTransferredBytes) + tr := newTimingReader(time.Now(), r, true, OpGet, m.metrics.opsDuration, m.metrics.opsFailures, func(err error) bool { return false }, m.metrics.opsFetchedBytes, m.metrics.opsTransferredBytes) buf := make([]byte, 1) _, err := io.ReadFull(tr, buf) @@ -476,7 +477,7 @@ func TestTimingReader_ContextCancellation(t *testing.T) { m := WrapWithMetrics(NewInMemBucket(), nil, "") r := dummyReader{ctx.Err()} - tr := newTimingReader(r, true, OpGet, m.metrics.opsDuration, m.metrics.opsFailures, func(err error) bool { return false }, m.metrics.opsFetchedBytes, m.metrics.opsTransferredBytes) + tr := newTimingReader(time.Now(), r, true, OpGet, m.metrics.opsDuration, m.metrics.opsFailures, func(err error) bool { return false }, m.metrics.opsFetchedBytes, m.metrics.opsTransferredBytes) buf := make([]byte, 1) _, err := io.ReadFull(tr, buf) @@ -506,7 +507,7 @@ func TestTimingReader_ShouldCorrectlyWrapFile(t *testing.T) { }) m := WrapWithMetrics(NewInMemBucket(), nil, "") - r := newTimingReader(file, true, "", m.metrics.opsDuration, m.metrics.opsFailures, func(err error) bool { + r := newTimingReader(time.Now(), file, true, "", m.metrics.opsDuration, m.metrics.opsFailures, func(err error) bool { return false }, m.metrics.opsFetchedBytes, m.metrics.opsTransferredBytes) @@ -594,3 +595,11 @@ func (b *mockBucket) GetRange(ctx context.Context, name string, off, length int6 } return nil, errors.New("GetRange has not been mocked") } + +func Test_TryToGetSizeLimitedReader(t *testing.T) { + b := &bytes.Buffer{} + r := io.LimitReader(b, 1024) + size, err := TryToGetSize(r) + testutil.Ok(t, err) + testutil.Equals(t, int64(1024), size) +} diff --git a/objtesting/foreach.go b/objtesting/foreach.go index 87d9ed1b..29d16c39 100644 --- a/objtesting/foreach.go +++ b/objtesting/foreach.go @@ -9,7 +9,6 @@ import ( "testing" "github.com/thanos-io/objstore" - "github.com/thanos-io/objstore/client" "github.com/thanos-io/objstore/providers/azure" "github.com/thanos-io/objstore/providers/bos" "github.com/thanos-io/objstore/providers/cos" @@ -26,7 +25,7 @@ import ( // IsObjStoreSkipped returns true if given provider ID is found in THANOS_TEST_OBJSTORE_SKIP array delimited by comma e.g: // THANOS_TEST_OBJSTORE_SKIP=GCS,S3,AZURE,SWIFT,COS,ALIYUNOSS,BOS,OCI. -func IsObjStoreSkipped(t *testing.T, provider client.ObjProvider) bool { +func IsObjStoreSkipped(t *testing.T, provider objstore.ObjProvider) bool { if e, ok := os.LookupEnv("THANOS_TEST_OBJSTORE_SKIP"); ok { obstores := strings.Split(e, ",") for _, objstore := range obstores { @@ -69,7 +68,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket)) }) // Optional GCS. - if !IsObjStoreSkipped(t, client.GCS) { + if !IsObjStoreSkipped(t, objstore.GCS) { t.Run("gcs", func(t *testing.T) { bkt, closeFn, err := gcs.NewTestBucket(t, os.Getenv("GCP_PROJECT")) testutil.Ok(t, err) @@ -84,7 +83,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket)) } // Optional S3. - if !IsObjStoreSkipped(t, client.S3) { + if !IsObjStoreSkipped(t, objstore.S3) { t.Run("aws s3", func(t *testing.T) { // TODO(bwplotka): Allow taking location from envvar. bkt, closeFn, err := s3.NewTestBucket(t, "us-west-2") @@ -103,7 +102,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket)) } // Optional Azure. - if !IsObjStoreSkipped(t, client.AZURE) { + if !IsObjStoreSkipped(t, objstore.AZURE) { t.Run("azure", func(t *testing.T) { bkt, closeFn, err := azure.NewTestBucket(t, "e2e-tests") testutil.Ok(t, err) @@ -117,7 +116,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket)) } // Optional SWIFT. - if !IsObjStoreSkipped(t, client.SWIFT) { + if !IsObjStoreSkipped(t, objstore.SWIFT) { t.Run("swift", func(t *testing.T) { container, closeFn, err := swift.NewTestContainer(t) testutil.Ok(t, err) @@ -131,7 +130,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket)) } // Optional COS. - if !IsObjStoreSkipped(t, client.COS) { + if !IsObjStoreSkipped(t, objstore.COS) { t.Run("Tencent cos", func(t *testing.T) { bkt, closeFn, err := cos.NewTestBucket(t) testutil.Ok(t, err) @@ -145,7 +144,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket)) } // Optional OSS. - if !IsObjStoreSkipped(t, client.ALIYUNOSS) { + if !IsObjStoreSkipped(t, objstore.ALIYUNOSS) { t.Run("AliYun oss", func(t *testing.T) { bkt, closeFn, err := oss.NewTestBucket(t) testutil.Ok(t, err) @@ -159,7 +158,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket)) } // Optional BOS. - if !IsObjStoreSkipped(t, client.BOS) { + if !IsObjStoreSkipped(t, objstore.BOS) { t.Run("Baidu BOS", func(t *testing.T) { bkt, closeFn, err := bos.NewTestBucket(t) testutil.Ok(t, err) @@ -173,7 +172,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket)) } // Optional OCI. - if !IsObjStoreSkipped(t, client.OCI) { + if !IsObjStoreSkipped(t, objstore.OCI) { t.Run("oci", func(t *testing.T) { bkt, closeFn, err := oci.NewTestBucket(t) testutil.Ok(t, err) @@ -186,7 +185,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket)) } // Optional OBS. - if !IsObjStoreSkipped(t, client.OBS) { + if !IsObjStoreSkipped(t, objstore.OBS) { t.Run("obs", func(t *testing.T) { bkt, closeFn, err := obs.NewTestBucket(t, "cn-south-1") testutil.Ok(t, err) diff --git a/prefixed_bucket.go b/prefixed_bucket.go index f2b71434..a37450ca 100644 --- a/prefixed_bucket.go +++ b/prefixed_bucket.go @@ -39,6 +39,8 @@ func withPrefix(prefix, name string) string { return prefix + DirDelim + name } +func (p *PrefixedBucket) Provider() ObjProvider { return p.bkt.Provider() } + func (p *PrefixedBucket) Close() error { return p.bkt.Close() } @@ -54,6 +56,19 @@ func (p *PrefixedBucket) Iter(ctx context.Context, dir string, f func(string) er }, options...) } +func (p *PrefixedBucket) IterWithAttributes(ctx context.Context, dir string, f func(IterObjectAttributes) error, options ...IterOption) error { + pdir := withPrefix(p.prefix, dir) + + return p.bkt.IterWithAttributes(ctx, pdir, func(attrs IterObjectAttributes) error { + attrs.Name = strings.TrimPrefix(attrs.Name, p.prefix+DirDelim) + return f(attrs) + }, options...) +} + +func (p *PrefixedBucket) SupportedIterOptions() []IterOptionType { + return p.bkt.SupportedIterOptions() +} + // Get returns a reader for the given object name. func (p *PrefixedBucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { return p.bkt.Get(ctx, conditionalPrefix(p.prefix, name)) @@ -80,7 +95,7 @@ func (p *PrefixedBucket) IsAccessDeniedErr(err error) bool { } // Attributes returns information about the specified object. -func (p PrefixedBucket) Attributes(ctx context.Context, name string) (ObjectAttributes, error) { +func (p *PrefixedBucket) Attributes(ctx context.Context, name string) (ObjectAttributes, error) { return p.bkt.Attributes(ctx, conditionalPrefix(p.prefix, name)) } diff --git a/prefixed_bucket_test.go b/prefixed_bucket_test.go index f93c8580..6252e05d 100644 --- a/prefixed_bucket_test.go +++ b/prefixed_bucket_test.go @@ -74,7 +74,7 @@ func UsesPrefixTest(t *testing.T, bkt Bucket, prefix string) { testutil.Ok(t, pBkt.Iter(context.Background(), "", func(fn string) error { seen = append(seen, fn) return nil - }, WithRecursiveIter)) + }, WithRecursiveIter())) expected := []string{"dir/file1.jpg", "file1.jpg"} sort.Strings(expected) sort.Strings(seen) diff --git a/providers/azure/azure.go b/providers/azure/azure.go index 63bd2c1a..8d055e77 100644 --- a/providers/azure/azure.go +++ b/providers/azure/azure.go @@ -6,6 +6,7 @@ package azure import ( "context" "io" + "net/http" "os" "strings" "testing" @@ -145,7 +146,7 @@ type Bucket struct { } // NewBucket returns a new Bucket using the provided Azure config. -func NewBucket(logger log.Logger, azureConfig []byte, component string) (*Bucket, error) { +func NewBucket(logger log.Logger, azureConfig []byte, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error) { level.Debug(logger).Log("msg", "creating new Azure bucket connection", "component", component) conf, err := parseConfig(azureConfig) if err != nil { @@ -154,16 +155,16 @@ func NewBucket(logger log.Logger, azureConfig []byte, component string) (*Bucket if conf.MSIResource != "" { level.Warn(logger).Log("msg", "The field msi_resource has been deprecated and should no longer be set") } - return NewBucketWithConfig(logger, conf, component) + return NewBucketWithConfig(logger, conf, component, wrapRoundtripper) } // NewBucketWithConfig returns a new Bucket using the provided Azure config struct. -func NewBucketWithConfig(logger log.Logger, conf Config, component string) (*Bucket, error) { +func NewBucketWithConfig(logger log.Logger, conf Config, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error) { if err := conf.validate(); err != nil { return nil, err } - containerClient, err := getContainerClient(conf) + containerClient, err := getContainerClient(conf, wrapRoundtripper) if err != nil { return nil, err } @@ -192,9 +193,17 @@ func NewBucketWithConfig(logger log.Logger, conf Config, component string) (*Buc return bkt, nil } -// Iter calls f for each entry in the given directory. The argument to f is the full -// object name including the prefix of the inspected directory. -func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, options ...objstore.IterOption) error { +func (b *Bucket) Provider() objstore.ObjProvider { return objstore.AZURE } + +func (b *Bucket) SupportedIterOptions() []objstore.IterOptionType { + return []objstore.IterOptionType{objstore.Recursive, objstore.UpdatedAt} +} + +func (b *Bucket) IterWithAttributes(ctx context.Context, dir string, f func(attrs objstore.IterObjectAttributes) error, options ...objstore.IterOption) error { + if err := objstore.ValidateIterOptions(b.SupportedIterOptions(), options...); err != nil { + return err + } + prefix := dir if prefix != "" && !strings.HasSuffix(prefix, DirDelim) { prefix += DirDelim @@ -210,7 +219,13 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt return err } for _, blob := range resp.Segment.BlobItems { - if err := f(*blob.Name); err != nil { + attrs := objstore.IterObjectAttributes{ + Name: *blob.Name, + } + if params.LastModified { + attrs.SetLastModified(*blob.Properties.LastModified) + } + if err := f(attrs); err != nil { return err } } @@ -226,12 +241,18 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt return err } for _, blobItem := range resp.Segment.BlobItems { - if err := f(*blobItem.Name); err != nil { + attrs := objstore.IterObjectAttributes{ + Name: *blobItem.Name, + } + if params.LastModified { + attrs.SetLastModified(*blobItem.Properties.LastModified) + } + if err := f(attrs); err != nil { return err } } for _, blobPrefix := range resp.Segment.BlobPrefixes { - if err := f(*blobPrefix.Name); err != nil { + if err := f(objstore.IterObjectAttributes{Name: *blobPrefix.Name}); err != nil { return err } } @@ -239,6 +260,23 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt return nil } +// Iter calls f for each entry in the given directory. The argument to f is the full +// object name including the prefix of the inspected directory. +func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opts ...objstore.IterOption) error { + // Only include recursive option since attributes are not used in this method. + var filteredOpts []objstore.IterOption + for _, opt := range opts { + if opt.Type == objstore.Recursive { + filteredOpts = append(filteredOpts, opt) + break + } + } + + return b.IterWithAttributes(ctx, dir, func(attrs objstore.IterObjectAttributes) error { + return f(attrs.Name) + }, filteredOpts...) +} + // IsObjNotFoundErr returns true if error means that object is not found. Relevant to Get operations. func (b *Bucket) IsObjNotFoundErr(err error) bool { if err == nil { @@ -269,7 +307,13 @@ func (b *Bucket) getBlobReader(ctx context.Context, name string, httpRange blob. return nil, errors.Wrapf(err, "cannot download blob, address: %s", blobClient.URL()) } retryOpts := azblob.RetryReaderOptions{MaxRetries: int32(b.readerMaxRetries)} - return resp.NewRetryReader(ctx, &retryOpts), nil + + return objstore.ObjectSizerReadCloser{ + ReadCloser: resp.NewRetryReader(ctx, &retryOpts), + Size: func() (int64, error) { + return *resp.ContentLength, nil + }, + }, nil } // Get returns a reader for the given object name. @@ -355,7 +399,7 @@ func NewTestBucket(t testing.TB, component string) (objstore.Bucket, func(), err if err != nil { return nil, nil, err } - bkt, err := NewBucket(log.NewNopLogger(), bc, component) + bkt, err := NewBucket(log.NewNopLogger(), bc, component, nil) if err != nil { t.Errorf("Cannot create Azure storage container:") return nil, nil, err diff --git a/providers/azure/azure_test.go b/providers/azure/azure_test.go index c49695ab..a96dcefb 100644 --- a/providers/azure/azure_test.go +++ b/providers/azure/azure_test.go @@ -8,7 +8,9 @@ import ( "time" "github.com/efficientgo/core/testutil" + "github.com/go-kit/log" + "github.com/thanos-io/objstore/errutil" "github.com/thanos-io/objstore/exthttp" ) @@ -20,7 +22,7 @@ type TestCase struct { } var validConfig = []byte(`storage_account: "myStorageAccount" -storage_account_key: "abc123" +storage_account_key: "bXlTdXBlclNlY3JldEtleTEyMyFAIw==" container: "MyContainer" endpoint: "blob.core.windows.net" reader_config: @@ -222,3 +224,14 @@ http_config: testutil.Ok(t, err) testutil.Equals(t, true, transport.TLSClientConfig.InsecureSkipVerify) } + +func TestNewBucketWithErrorRoundTripper(t *testing.T) { + cfg, err := parseConfig(validConfig) + testutil.Ok(t, err) + + _, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test", errutil.WrapWithErrRoundtripper) + + // We expect an error from the RoundTripper + testutil.NotOk(t, err) + testutil.Assert(t, errutil.IsMockedError(err), "Expected RoundTripper error, got: %v", err) +} diff --git a/providers/azure/helpers.go b/providers/azure/helpers.go index 846394a0..deb86d03 100644 --- a/providers/azure/helpers.go +++ b/providers/azure/helpers.go @@ -19,11 +19,18 @@ import ( // DirDelim is the delimiter used to model a directory structure in an object store bucket. const DirDelim = "/" -func getContainerClient(conf Config) (*container.Client, error) { - dt, err := exthttp.DefaultTransport(conf.HTTPConfig) +func getContainerClient(conf Config, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*container.Client, error) { + var rt http.RoundTripper + rt, err := exthttp.DefaultTransport(conf.HTTPConfig) if err != nil { return nil, err } + if conf.HTTPConfig.Transport != nil { + rt = conf.HTTPConfig.Transport + } + if wrapRoundtripper != nil { + rt = wrapRoundtripper(rt) + } opt := &container.ClientOptions{ ClientOptions: azcore.ClientOptions{ Retry: policy.RetryOptions{ @@ -35,7 +42,7 @@ func getContainerClient(conf Config) (*container.Client, error) { Telemetry: policy.TelemetryOptions{ ApplicationID: "Thanos", }, - Transport: &http.Client{Transport: dt}, + Transport: &http.Client{Transport: rt}, }, } diff --git a/providers/bos/bos.go b/providers/bos/bos.go index 72e1b1e0..0cc4352c 100644 --- a/providers/bos/bos.go +++ b/providers/bos/bos.go @@ -66,6 +66,7 @@ func parseConfig(conf []byte) (Config, error) { // NewBucket new bos bucket. func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error) { + // TODO(https://github.com/thanos-io/objstore/pull/150): Add support for roundtripper wrapper. if logger == nil { logger = log.NewNopLogger() } @@ -99,6 +100,8 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B return bkt, nil } +func (b *Bucket) Provider() objstore.ObjProvider { return objstore.BOS } + // Name returns the bucket name for the provider. func (b *Bucket) Name() string { return b.name @@ -175,16 +178,23 @@ func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) error { return nil } -// Iter calls f for each entry in the given directory (not recursive). The argument to f is the full -// object name including the prefix of the inspected directory. -func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt ...objstore.IterOption) error { +func (b *Bucket) SupportedIterOptions() []objstore.IterOptionType { + return []objstore.IterOptionType{objstore.Recursive, objstore.UpdatedAt} +} + +func (b *Bucket) IterWithAttributes(ctx context.Context, dir string, f func(attrs objstore.IterObjectAttributes) error, options ...objstore.IterOption) error { + if err := objstore.ValidateIterOptions(b.SupportedIterOptions(), options...); err != nil { + return err + } + if dir != "" { dir = strings.TrimSuffix(dir, objstore.DirDelim) + objstore.DirDelim } delimiter := objstore.DirDelim - if objstore.ApplyIterOptions(opt...).Recursive { + params := objstore.ApplyIterOptions(options...) + if params.Recursive { delimiter = "" } @@ -206,13 +216,25 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt marker = objects.NextMarker for _, object := range objects.Contents { - if err := f(object.Key); err != nil { + attrs := objstore.IterObjectAttributes{ + Name: object.Key, + } + + if params.LastModified && object.LastModified != "" { + lastModified, err := time.Parse(time.RFC1123, object.LastModified) + if err != nil { + return fmt.Errorf("iter: get last modified: %w", err) + } + attrs.SetLastModified(lastModified) + } + + if err := f(attrs); err != nil { return err } } for _, object := range objects.CommonPrefixes { - if err := f(object.Prefix); err != nil { + if err := f(objstore.IterObjectAttributes{Name: object.Prefix}); err != nil { return err } } @@ -223,6 +245,23 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt return nil } +// Iter calls f for each entry in the given directory. The argument to f is the full +// object name including the prefix of the inspected directory. +func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opts ...objstore.IterOption) error { + // Only include recursive option since attributes are not used in this method. + var filteredOpts []objstore.IterOption + for _, opt := range opts { + if opt.Type == objstore.Recursive { + filteredOpts = append(filteredOpts, opt) + break + } + } + + return b.IterWithAttributes(ctx, dir, func(attrs objstore.IterObjectAttributes) error { + return f(attrs.Name) + }, filteredOpts...) +} + // Get returns a reader for the given object name. func (b *Bucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { return b.getRange(ctx, b.name, name, 0, -1) @@ -307,7 +346,12 @@ func (b *Bucket) getRange(_ context.Context, bucketName, objectKey string, off, return nil, err } - return obj.Body, nil + return objstore.ObjectSizerReadCloser{ + ReadCloser: obj.Body, + Size: func() (int64, error) { + return obj.ContentLength, nil + }, + }, err } func configFromEnv() Config { diff --git a/providers/cos/cos.go b/providers/cos/cos.go index e518cae2..1851c984 100644 --- a/providers/cos/cos.go +++ b/providers/cos/cos.go @@ -59,6 +59,7 @@ type Config struct { Endpoint string `yaml:"endpoint"` SecretKey string `yaml:"secret_key"` SecretId string `yaml:"secret_id"` + MaxRetries int `yaml:"max_retries"` HTTPConfig exthttp.HTTPConfig `yaml:"http_config"` } @@ -95,7 +96,7 @@ func parseConfig(conf []byte) (Config, error) { } // NewBucket returns a new Bucket using the provided cos configuration. -func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error) { +func NewBucket(logger log.Logger, conf []byte, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error) { if logger == nil { logger = log.NewNopLogger() } @@ -104,12 +105,11 @@ func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error if err != nil { return nil, errors.Wrap(err, "parsing cos configuration") } - - return NewBucketWithConfig(logger, config, component) + return NewBucketWithConfig(logger, config, component, wrapRoundtripper) } // NewBucketWithConfig returns a new Bucket using the provided cos config values. -func NewBucketWithConfig(logger log.Logger, config Config, component string) (*Bucket, error) { +func NewBucketWithConfig(logger log.Logger, config Config, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error) { if err := config.validate(); err != nil { return nil, errors.Wrap(err, "validate cos configuration") } @@ -128,15 +128,29 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B } } b := &cos.BaseURL{BucketURL: bucketURL} - tpt, _ := exthttp.DefaultTransport(config.HTTPConfig) + var rt http.RoundTripper + rt, err = exthttp.DefaultTransport(config.HTTPConfig) + if err != nil { + return nil, err + } + if config.HTTPConfig.Transport != nil { + rt = config.HTTPConfig.Transport + } + if wrapRoundtripper != nil { + rt = wrapRoundtripper(rt) + } client := cos.NewClient(b, &http.Client{ Transport: &cos.AuthorizationTransport{ SecretID: config.SecretId, SecretKey: config.SecretKey, - Transport: tpt, + Transport: rt, }, }) + if config.MaxRetries > 0 { + client.Conf.RetryOpt.Count = config.MaxRetries + } + bkt := &Bucket{ logger: logger, client: client, @@ -145,6 +159,8 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B return bkt, nil } +func (b *Bucket) Provider() objstore.ObjProvider { return objstore.COS } + // Name returns the bucket name for COS. func (b *Bucket) Name() string { return b.name @@ -267,7 +283,11 @@ func (b *Bucket) Delete(ctx context.Context, name string) error { return nil } -// Iter calls f for each entry in the given directory (not recursive.). The argument to f is the full +func (b *Bucket) SupportedIterOptions() []objstore.IterOptionType { + return []objstore.IterOptionType{objstore.Recursive} +} + +// Iter calls f for each entry in the given directory. The argument to f is the full // object name including the prefix of the inspected directory. func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, options ...objstore.IterOption) error { if dir != "" { @@ -289,6 +309,16 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt return nil } +func (b *Bucket) IterWithAttributes(ctx context.Context, dir string, f func(attrs objstore.IterObjectAttributes) error, options ...objstore.IterOption) error { + if err := objstore.ValidateIterOptions(b.SupportedIterOptions(), options...); err != nil { + return err + } + + return b.Iter(ctx, dir, func(name string) error { + return f(objstore.IterObjectAttributes{Name: name}) + }, options...) +} + func (b *Bucket) getRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error) { if name == "" { return nil, errors.New("given object name should not empty") @@ -314,18 +344,12 @@ func (b *Bucket) getRange(ctx context.Context, name string, off, length int64) ( return nil, err } // Add size info into reader to pass it to Upload function. - r := objectSizerReadCloser{ReadCloser: resp.Body, size: resp.ContentLength} - return r, nil -} - -type objectSizerReadCloser struct { - io.ReadCloser - size int64 -} - -// ObjectSize implement objstore.ObjectSizer. -func (o objectSizerReadCloser) ObjectSize() (int64, error) { - return o.size, nil + return objstore.ObjectSizerReadCloser{ + ReadCloser: resp.Body, + Size: func() (int64, error) { + return resp.ContentLength, nil + }, + }, nil } // Get returns a reader for the given object name. @@ -485,12 +509,12 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) { return nil, nil, err } - b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test") + b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test", nil) if err != nil { return nil, nil, err } - if err := b.Iter(context.Background(), "", func(f string) error { + if err := b.Iter(context.Background(), "", func(_ string) error { return errors.Errorf("bucket %s is not empty", c.Bucket) }); err != nil { return nil, nil, errors.Wrapf(err, "cos check bucket %s", c.Bucket) @@ -506,7 +530,7 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) { return nil, nil, err } - b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test") + b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test", nil) if err != nil { return nil, nil, err } diff --git a/providers/cos/cos_test.go b/providers/cos/cos_test.go index 6d4316f2..4f7a56d6 100644 --- a/providers/cos/cos_test.go +++ b/providers/cos/cos_test.go @@ -4,12 +4,15 @@ package cos import ( + "context" "testing" "time" "github.com/efficientgo/core/testutil" + "github.com/go-kit/log" "github.com/prometheus/common/model" + "github.com/thanos-io/objstore/errutil" "github.com/thanos-io/objstore/exthttp" ) @@ -137,3 +140,20 @@ func TestConfig_validate(t *testing.T) { }) } } + +func TestNewBucketWithErrorRoundTripper(t *testing.T) { + config := Config{ + Bucket: "bucket", + AppId: "123", + Region: "test", + SecretId: "sid", + SecretKey: "skey", + } + + bkt, err := NewBucketWithConfig(log.NewNopLogger(), config, "test", errutil.WrapWithErrRoundtripper) + testutil.Ok(t, err) + _, err = bkt.Get(context.Background(), "Test") + // We expect an error from the RoundTripper + testutil.NotOk(t, err) + testutil.Assert(t, errutil.IsMockedError(err), "Expected RoundTripper error, got: %v", err) +} diff --git a/providers/filesystem/filesystem.go b/providers/filesystem/filesystem.go index 21c70485..df602877 100644 --- a/providers/filesystem/filesystem.go +++ b/providers/filesystem/filesystem.go @@ -50,13 +50,21 @@ func NewBucket(rootDir string) (*Bucket, error) { return &Bucket{rootDir: absDir}, nil } -// Iter calls f for each entry in the given directory. The argument to f is the full -// object name including the prefix of the inspected directory. -func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, options ...objstore.IterOption) error { +func (b *Bucket) Provider() objstore.ObjProvider { return objstore.FILESYSTEM } + +func (b *Bucket) SupportedIterOptions() []objstore.IterOptionType { + return []objstore.IterOptionType{objstore.Recursive, objstore.UpdatedAt} +} + +func (b *Bucket) IterWithAttributes(ctx context.Context, dir string, f func(attrs objstore.IterObjectAttributes) error, options ...objstore.IterOption) error { if ctx.Err() != nil { return ctx.Err() } + if err := objstore.ValidateIterOptions(b.SupportedIterOptions(), options...); err != nil { + return err + } + params := objstore.ApplyIterOptions(options...) absDir := filepath.Join(b.rootDir, dir) info, err := os.Stat(absDir) @@ -92,7 +100,7 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt if params.Recursive { // Recursively list files in the subdirectory. - if err := b.Iter(ctx, name, f, options...); err != nil { + if err := b.IterWithAttributes(ctx, name, f, options...); err != nil { return err } @@ -101,13 +109,42 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt continue } } - if err := f(name); err != nil { + + attrs := objstore.IterObjectAttributes{ + Name: name, + } + if params.LastModified { + absPath := filepath.Join(absDir, file.Name()) + stat, err := os.Stat(absPath) + if err != nil { + return errors.Wrapf(err, "stat %s", name) + } + attrs.SetLastModified(stat.ModTime()) + } + if err := f(attrs); err != nil { return err } } return nil } +// Iter calls f for each entry in the given directory. The argument to f is the full +// object name including the prefix of the inspected directory. +func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opts ...objstore.IterOption) error { + // Only include recursive option since attributes are not used in this method. + var filteredOpts []objstore.IterOption + for _, opt := range opts { + if opt.Type == objstore.Recursive { + filteredOpts = append(filteredOpts, opt) + break + } + } + + return b.IterWithAttributes(ctx, dir, func(attrs objstore.IterObjectAttributes) error { + return f(attrs.Name) + }, filteredOpts...) +} + // Get returns a reader for the given object name. func (b *Bucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { return b.GetRange(ctx, name, 0, -1) @@ -150,8 +187,12 @@ func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) ( return nil, errors.New("object name is empty") } - file := filepath.Join(b.rootDir, name) - if _, err := os.Stat(file); err != nil { + var ( + file = filepath.Join(b.rootDir, name) + stat os.FileInfo + err error + ) + if stat, err = os.Stat(file); err != nil { return nil, errors.Wrapf(err, "stat %s", file) } @@ -160,18 +201,33 @@ func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) ( return nil, err } + var newOffset int64 if off > 0 { - _, err := f.Seek(off, 0) + newOffset, err = f.Seek(off, 0) if err != nil { return nil, errors.Wrapf(err, "seek %v", off) } } + size := stat.Size() - newOffset if length == -1 { - return f, nil + return objstore.ObjectSizerReadCloser{ + ReadCloser: f, + Size: func() (int64, error) { + return size, nil + }, + }, nil } - return &rangeReaderCloser{Reader: io.LimitReader(f, length), f: f}, nil + return objstore.ObjectSizerReadCloser{ + ReadCloser: &rangeReaderCloser{ + Reader: io.LimitReader(f, length), + f: f, + }, + Size: func() (int64, error) { + return min(length, size), nil + }, + }, nil } // Exists checks if the given directory exists in memory. diff --git a/providers/filesystem/filesystem_test.go b/providers/filesystem/filesystem_test.go index c3621fe0..105aab8e 100644 --- a/providers/filesystem/filesystem_test.go +++ b/providers/filesystem/filesystem_test.go @@ -6,11 +6,15 @@ package filesystem import ( "bytes" "context" + "os" "strings" "sync" "testing" + "time" "github.com/efficientgo/core/testutil" + + "github.com/thanos-io/objstore" ) func TestDelete_EmptyDirDeletionRaceCondition(t *testing.T) { @@ -61,6 +65,60 @@ func TestIter_CancelledContext(t *testing.T) { testutil.Equals(t, context.Canceled, err) } +func TestIterWithAttributes(t *testing.T) { + dir := t.TempDir() + f, err := os.CreateTemp(dir, "test") + testutil.Ok(t, err) + defer f.Close() + + stat, err := f.Stat() + testutil.Ok(t, err) + + cases := []struct { + name string + opts []objstore.IterOption + expectedUpdatedAt time.Time + }{ + { + name: "no options", + opts: nil, + }, + { + name: "with updated at", + opts: []objstore.IterOption{ + objstore.WithUpdatedAt(), + }, + expectedUpdatedAt: stat.ModTime(), + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + b, err := NewBucket(dir) + testutil.Ok(t, err) + + var attrs objstore.IterObjectAttributes + + ctx := context.Background() + err = b.IterWithAttributes(ctx, "", func(objectAttrs objstore.IterObjectAttributes) error { + attrs = objectAttrs + return nil + }, tc.opts...) + + testutil.Ok(t, err) + + lastModified, ok := attrs.LastModified() + if zero := tc.expectedUpdatedAt.IsZero(); zero { + testutil.Equals(t, false, ok) + } else { + testutil.Equals(t, true, ok) + testutil.Equals(t, tc.expectedUpdatedAt, lastModified) + } + }) + + } +} + func TestGet_CancelledContext(t *testing.T) { b, err := NewBucket(t.TempDir()) testutil.Ok(t, err) diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index a697e597..b89f8735 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -22,9 +22,7 @@ import ( "google.golang.org/api/iterator" "google.golang.org/api/option" htransport "google.golang.org/api/transport/http" - "google.golang.org/grpc" "google.golang.org/grpc/codes" - "google.golang.org/grpc/experimental" "google.golang.org/grpc/status" "gopkg.in/yaml.v2" @@ -54,7 +52,13 @@ type Config struct { // ChunkSizeBytes controls the maximum number of bytes of the object that the // Writer will attempt to send to the server in a single request // Used as storage.Writer.ChunkSize of https://pkg.go.dev/google.golang.org/cloud/storage#Writer - ChunkSizeBytes int `yaml:"chunk_size_bytes"` + ChunkSizeBytes int `yaml:"chunk_size_bytes"` + noAuth bool `yaml:"no_auth"` + + // MaxRetries controls the number of retries for idempotent operations. + // Overrides the default gcs storage client behavior if this value is greater than 0. + // Set this to 1 to disable retries. + MaxRetries int `yaml:"max_retries"` } // Bucket implements the store.Bucket and shipper.Bucket interfaces against GCS. @@ -78,16 +82,16 @@ func parseConfig(conf []byte) (Config, error) { } // NewBucket returns a new Bucket against the given bucket handle. -func NewBucket(ctx context.Context, logger log.Logger, conf []byte, component string) (*Bucket, error) { +func NewBucket(ctx context.Context, logger log.Logger, conf []byte, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error) { config, err := parseConfig(conf) if err != nil { return nil, err } - return NewBucketWithConfig(ctx, logger, config, component) + return NewBucketWithConfig(ctx, logger, config, component, wrapRoundtripper) } // NewBucketWithConfig returns a new Bucket with gcs Config struct. -func NewBucketWithConfig(ctx context.Context, logger log.Logger, gc Config, component string) (*Bucket, error) { +func NewBucketWithConfig(ctx context.Context, logger log.Logger, gc Config, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error) { if gc.Bucket == "" { return nil, errors.New("missing Google Cloud Storage bucket name for stored blocks") } @@ -102,14 +106,16 @@ func NewBucketWithConfig(ctx context.Context, logger log.Logger, gc Config, comp } opts = append(opts, option.WithCredentials(credentials)) } - + if gc.noAuth { + opts = append(opts, option.WithoutAuthentication()) + } opts = append(opts, option.WithUserAgent(fmt.Sprintf("thanos-%s/%s (%s)", component, version.Version, runtime.Version())), ) if !gc.UseGRPC { var err error - opts, err = appendHttpOptions(gc, opts) + opts, err = appendHttpOptions(gc, opts, wrapRoundtripper) if err != nil { return nil, err } @@ -118,18 +124,19 @@ func NewBucketWithConfig(ctx context.Context, logger log.Logger, gc Config, comp return newBucket(ctx, logger, gc, opts) } -func appendHttpOptions(gc Config, opts []option.ClientOption) ([]option.ClientOption, error) { +func appendHttpOptions(gc Config, opts []option.ClientOption, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) ([]option.ClientOption, error) { // Check if a roundtripper has been set in the config // otherwise build the default transport. var rt http.RoundTripper + rt, err := exthttp.DefaultTransport(gc.HTTPConfig) + if err != nil { + return nil, err + } if gc.HTTPConfig.Transport != nil { rt = gc.HTTPConfig.Transport - } else { - var err error - rt, err = exthttp.DefaultTransport(gc.HTTPConfig) - if err != nil { - return nil, err - } + } + if wrapRoundtripper != nil { + rt = wrapRoundtripper(rt) } // GCS uses some defaults when "options.WithHTTPClient" is not used that are important when we call @@ -155,7 +162,6 @@ func newBucket(ctx context.Context, logger log.Logger, gc Config, opts []option. ) if gc.UseGRPC { opts = append(opts, - option.WithGRPCDialOption(experimental.WithRecvBufferPool(grpc.NewSharedBufferPool())), option.WithGRPCConnectionPool(gc.GRPCConnPoolSize), ) gcsClient, err = storage.NewGRPCClient(ctx, opts...) @@ -172,26 +178,41 @@ func newBucket(ctx context.Context, logger log.Logger, gc Config, opts []option. name: gc.Bucket, chunkSize: gc.ChunkSizeBytes, } + + if gc.MaxRetries > 0 { + bkt.bkt = bkt.bkt.Retryer(storage.WithMaxAttempts(gc.MaxRetries)) + } + return bkt, nil } +func (b *Bucket) Provider() objstore.ObjProvider { return objstore.GCS } + // Name returns the bucket name for gcs. func (b *Bucket) Name() string { return b.name } -// Iter calls f for each entry in the given directory. The argument to f is the full -// object name including the prefix of the inspected directory. -func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, options ...objstore.IterOption) error { +func (b *Bucket) SupportedIterOptions() []objstore.IterOptionType { + return []objstore.IterOptionType{objstore.Recursive, objstore.UpdatedAt} +} + +func (b *Bucket) IterWithAttributes(ctx context.Context, dir string, f func(attrs objstore.IterObjectAttributes) error, options ...objstore.IterOption) error { + if err := objstore.ValidateIterOptions(b.SupportedIterOptions(), options...); err != nil { + return err + } + // Ensure the object name actually ends with a dir suffix. Otherwise we'll just iterate the // object itself as one prefix item. if dir != "" { dir = strings.TrimSuffix(dir, DirDelim) + DirDelim } + appliedOpts := objstore.ApplyIterOptions(options...) + // If recursive iteration is enabled we should pass an empty delimiter. delimiter := DirDelim - if objstore.ApplyIterOptions(options...).Recursive { + if appliedOpts.Recursive { delimiter = "" } @@ -199,11 +220,15 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt Prefix: dir, Delimiter: delimiter, } - err := query.SetAttrSelection([]string{"Name"}) - if err != nil { - return err + if appliedOpts.LastModified { + if err := query.SetAttrSelection([]string{"Name", "Updated"}); err != nil { + return err + } + } else { + if err := query.SetAttrSelection([]string{"Name"}); err != nil { + return err + } } - it := b.bkt.Objects(ctx, query) for { select { @@ -218,20 +243,63 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt if err != nil { return err } - if err := f(attrs.Prefix + attrs.Name); err != nil { + + objAttrs := objstore.IterObjectAttributes{Name: attrs.Prefix + attrs.Name} + if appliedOpts.LastModified { + objAttrs.SetLastModified(attrs.Updated) + } + if err := f(objAttrs); err != nil { return err } } } +// Iter calls f for each entry in the given directory. The argument to f is the full +// object name including the prefix of the inspected directory. +func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opts ...objstore.IterOption) error { + // Only include recursive option since attributes are not used in this method. + var filteredOpts []objstore.IterOption + for _, opt := range opts { + if opt.Type == objstore.Recursive { + filteredOpts = append(filteredOpts, opt) + break + } + } + + return b.IterWithAttributes(ctx, dir, func(attrs objstore.IterObjectAttributes) error { + return f(attrs.Name) + }, filteredOpts...) +} + // Get returns a reader for the given object name. func (b *Bucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { - return b.bkt.Object(name).NewReader(ctx) + r, err := b.bkt.Object(name).NewReader(ctx) + if err != nil { + return r, err + } + + return objstore.ObjectSizerReadCloser{ + ReadCloser: r, + Size: func() (int64, error) { + return r.Attrs.Size, nil + }, + }, nil } // GetRange returns a new range reader for the given object name and range. func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error) { - return b.bkt.Object(name).NewRangeReader(ctx, off, length) + r, err := b.bkt.Object(name).NewRangeReader(ctx, off, length) + if err != nil { + return r, err + } + + sz := r.Remain() + return objstore.ObjectSizerReadCloser{ + ReadCloser: r, + Size: func() (int64, error) { + return sz, nil + }, + }, nil } // Attributes returns information about the specified object. @@ -315,7 +383,7 @@ func NewTestBucket(t testing.TB, project string) (objstore.Bucket, func(), error return nil, nil, err } - b, err := NewBucket(ctx, log.NewNopLogger(), bc, "thanos-e2e-test") + b, err := NewBucket(ctx, log.NewNopLogger(), bc, "thanos-e2e-test", nil) if err != nil { return nil, nil, err } diff --git a/providers/gcs/gcs_test.go b/providers/gcs/gcs_test.go index d03c5dd9..80951d7a 100644 --- a/providers/gcs/gcs_test.go +++ b/providers/gcs/gcs_test.go @@ -16,6 +16,7 @@ import ( "github.com/fullstorydev/emulators/storage/gcsemu" "github.com/go-kit/log" "github.com/prometheus/common/model" + "github.com/thanos-io/objstore/errutil" "google.golang.org/api/option" ) @@ -66,7 +67,7 @@ func TestNewBucketWithConfig_ShouldCreateGRPC(t *testing.T) { err = os.Setenv("STORAGE_EMULATOR_HOST_GRPC", svr.Addr) testutil.Ok(t, err) - bkt, err := NewBucketWithConfig(context.Background(), log.NewNopLogger(), cfg, "test-bucket") + bkt, err := NewBucketWithConfig(context.Background(), log.NewNopLogger(), cfg, "test-bucket", nil) testutil.Ok(t, err) // Check if the bucket is created. @@ -157,3 +158,23 @@ http_config: }) } } + +func TestNewBucketWithErrorRoundTripper(t *testing.T) { + cfg := Config{ + Bucket: "test-bucket", + ServiceAccount: "", + UseGRPC: false, + noAuth: true, + } + svr, err := gcsemu.NewServer("127.0.0.1:0", gcsemu.Options{}) + testutil.Ok(t, err) + defer svr.Close() + err = os.Setenv("STORAGE_EMULATOR_HOST", svr.Addr) + testutil.Ok(t, err) + + bkt, err := NewBucketWithConfig(context.Background(), log.NewNopLogger(), cfg, "test-bucket", errutil.WrapWithErrRoundtripper) + testutil.Ok(t, err) + _, err = bkt.Get(context.Background(), "test-bucket") + testutil.NotOk(t, err) + testutil.Assert(t, errutil.IsMockedError(err), "Expected RoundTripper error, got: %v", err) +} diff --git a/providers/obs/obs.go b/providers/obs/obs.go index 7bd9666b..20294cc6 100644 --- a/providers/obs/obs.go +++ b/providers/obs/obs.go @@ -46,6 +46,7 @@ type Config struct { Endpoint string `yaml:"endpoint"` AccessKey string `yaml:"access_key"` SecretKey string `yaml:"secret_key"` + MaxRetries int `yaml:"max_retries"` HTTPConfig exthttp.HTTPConfig `yaml:"http_config"` } @@ -75,11 +76,11 @@ type Bucket struct { } func NewBucket(logger log.Logger, conf []byte) (*Bucket, error) { + // TODO(https://github.com/thanos-io/objstore/pull/150): Add support for roundtripper wrapper. config, err := parseConfig(conf) if err != nil { return nil, errors.Wrap(err, "parsing cos configuration") } - return NewBucketWithConfig(logger, config) } @@ -102,7 +103,13 @@ func NewBucketWithConfig(logger log.Logger, config Config) (*Bucket, error) { return nil, errors.Wrap(err, "get http transport err") } - client, err := obs.New(config.AccessKey, config.SecretKey, config.Endpoint, obs.WithHttpTransport(rt)) + var client *obs.ObsClient + if config.MaxRetries > 0 { + client, err = obs.New(config.AccessKey, config.SecretKey, config.Endpoint, obs.WithHttpTransport(rt), obs.WithMaxRetryCount(config.MaxRetries)) + } else { + client, err = obs.New(config.AccessKey, config.SecretKey, config.Endpoint, obs.WithHttpTransport(rt)) + } + if err != nil { return nil, errors.Wrap(err, "initialize obs client err") } @@ -115,6 +122,8 @@ func NewBucketWithConfig(logger log.Logger, config Config) (*Bucket, error) { return bkt, nil } +func (b *Bucket) Provider() objstore.ObjProvider { return objstore.OBS } + // Name returns the bucket name for the provider. func (b *Bucket) Name() string { return b.name @@ -232,6 +241,10 @@ func (b *Bucket) multipartUpload(size int64, key, uploadId string, body io.Reade func (b *Bucket) Close() error { return nil } +func (b *Bucket) SupportedIterOptions() []objstore.IterOptionType { + return []objstore.IterOptionType{objstore.Recursive} +} + // Iter calls f for each entry in the given directory (not recursive.) func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, options ...objstore.IterOption) error { if dir != "" { @@ -270,6 +283,16 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt return nil } +func (b *Bucket) IterWithAttributes(ctx context.Context, dir string, f func(attrs objstore.IterObjectAttributes) error, options ...objstore.IterOption) error { + if err := objstore.ValidateIterOptions(b.SupportedIterOptions(), options...); err != nil { + return err + } + + return b.Iter(ctx, dir, func(name string) error { + return f(objstore.IterObjectAttributes{Name: name}) + }, options...) +} + // Get returns a reader for the given object name. func (b *Bucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { return b.getRange(ctx, name, 0, -1) @@ -299,7 +322,12 @@ func (b *Bucket) getRange(_ context.Context, name string, off, length int64) (io if err != nil { return nil, errors.Wrap(err, "failed to get object") } - return output.Body, nil + return objstore.ObjectSizerReadCloser{ + ReadCloser: output.Body, + Size: func() (int64, error) { + return output.ContentLength, nil + }, + }, nil } // Exists checks if the given object exists in the bucket. @@ -376,7 +404,7 @@ func NewTestBucketFromConfig(t testing.TB, c Config, reuseBucket bool, location bktToCreate := c.Bucket if c.Bucket != "" && reuseBucket { - if err := b.Iter(ctx, "", func(f string) error { + if err := b.Iter(ctx, "", func(_ string) error { return errors.Errorf("bucket %s is not empty", c.Bucket) }); err != nil { return nil, nil, err diff --git a/providers/oci/oci.go b/providers/oci/oci.go index 2db35461..7b4230ec 100644 --- a/providers/oci/oci.go +++ b/providers/oci/oci.go @@ -21,8 +21,9 @@ import ( "github.com/oracle/oci-go-sdk/v65/objectstorage/transfer" "github.com/pkg/errors" "github.com/prometheus/common/model" - "github.com/thanos-io/objstore" "gopkg.in/yaml.v2" + + "github.com/thanos-io/objstore" ) // DirDelim is the delimiter used to model a directory structure in an object store bucket. @@ -58,13 +59,14 @@ type HTTPConfig struct { ResponseHeaderTimeout model.Duration `yaml:"response_header_timeout"` InsecureSkipVerify bool `yaml:"insecure_skip_verify"` - TLSHandshakeTimeout model.Duration `yaml:"tls_handshake_timeout"` - ExpectContinueTimeout model.Duration `yaml:"expect_continue_timeout"` - MaxIdleConns int `yaml:"max_idle_conns"` - MaxIdleConnsPerHost int `yaml:"max_idle_conns_per_host"` - MaxConnsPerHost int `yaml:"max_conns_per_host"` - DisableCompression bool `yaml:"disable_compression"` - ClientTimeout time.Duration `yaml:"client_timeout"` + TLSHandshakeTimeout model.Duration `yaml:"tls_handshake_timeout"` + ExpectContinueTimeout model.Duration `yaml:"expect_continue_timeout"` + MaxIdleConns int `yaml:"max_idle_conns"` + MaxIdleConnsPerHost int `yaml:"max_idle_conns_per_host"` + MaxConnsPerHost int `yaml:"max_conns_per_host"` + DisableCompression bool `yaml:"disable_compression"` + ClientTimeout time.Duration `yaml:"client_timeout"` + Transport http.RoundTripper `yaml:"-"` } // Config stores the configuration for oci bucket. @@ -94,12 +96,18 @@ type Bucket struct { requestMetadata common.RequestMetadata } +func (b *Bucket) Provider() objstore.ObjProvider { return objstore.OCI } + // Name returns the bucket name for the provider. func (b *Bucket) Name() string { return b.name } -// Iter calls f for each entry in the given directory (not recursive). The argument to f is the full +func (b *Bucket) SupportedIterOptions() []objstore.IterOptionType { + return []objstore.IterOptionType{objstore.Recursive} +} + +// Iter calls f for each entry in the given directory. The argument to f is the full // object name including the prefix of the inspected directory. func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, options ...objstore.IterOption) error { // Ensure the object name actually ends with a dir suffix. Otherwise we'll just iterate the @@ -119,6 +127,7 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt if objectName == "" || objectName == dir { continue } + if err := f(objectName); err != nil { return err } @@ -127,13 +136,28 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt return nil } +func (b *Bucket) IterWithAttributes(ctx context.Context, dir string, f func(attrs objstore.IterObjectAttributes) error, options ...objstore.IterOption) error { + if err := objstore.ValidateIterOptions(b.SupportedIterOptions(), options...); err != nil { + return err + } + + return b.Iter(ctx, dir, func(name string) error { + return f(objstore.IterObjectAttributes{Name: name}) + }, options...) +} + // Get returns a reader for the given object name. func (b *Bucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { response, err := getObject(ctx, *b, name, "") if err != nil { return nil, err } - return response.Content, nil + return objstore.ObjectSizerReadCloser{ + ReadCloser: response.Content, + Size: func() (int64, error) { + return *response.ContentLength, nil + }, + }, nil } // GetRange returns a new range reader for the given object name and range. @@ -163,7 +187,11 @@ func (b *Bucket) GetRange(ctx context.Context, name string, offset, length int64 if err != nil { return nil, err } - return response.Content, nil + return objstore.ObjectSizerReadCloser{ReadCloser: response.Content, + Size: func() (int64, error) { + return *response.ContentLength, nil + }, + }, nil } // Upload the contents of the reader as an object into the bucket. @@ -288,7 +316,7 @@ func (b *Bucket) deleteBucket(ctx context.Context) (err error) { } // NewBucket returns a new Bucket using the provided oci config values. -func NewBucket(logger log.Logger, ociConfig []byte) (*Bucket, error) { +func NewBucket(logger log.Logger, ociConfig []byte, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error) { level.Debug(logger).Log("msg", "creating new oci bucket connection") var config = DefaultConfig var configurationProvider common.ConfigurationProvider @@ -334,9 +362,16 @@ func NewBucket(logger log.Logger, ociConfig []byte) (*Bucket, error) { if err != nil { return nil, errors.Wrapf(err, "unable to create ObjectStorage client with the given oci configurations") } - + var rt http.RoundTripper + rt = CustomTransport(config) + if config.HTTPConfig.Transport != nil { + rt = config.HTTPConfig.Transport + } + if wrapRoundtripper != nil { + rt = wrapRoundtripper(rt) + } httpClient := http.Client{ - Transport: CustomTransport(config), + Transport: rt, Timeout: config.HTTPConfig.ClientTimeout, } client.HTTPClient = &httpClient @@ -375,7 +410,7 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) { return nil, nil, err } - bkt, err := NewBucket(log.NewNopLogger(), ociConfig) + bkt, err := NewBucket(log.NewNopLogger(), ociConfig, nil) if err != nil { return nil, nil, err } diff --git a/providers/oci/oci_test.go b/providers/oci/oci_test.go new file mode 100644 index 00000000..6b44a0e7 --- /dev/null +++ b/providers/oci/oci_test.go @@ -0,0 +1,44 @@ +package oci + +import ( + "testing" + + "github.com/efficientgo/core/testutil" + "github.com/go-kit/log" + "github.com/thanos-io/objstore/errutil" + "gopkg.in/yaml.v2" +) + +func TestNewBucketWithErrorRoundTripper(t *testing.T) { + const mockPrivateKey = `-----BEGIN RSA PRIVATE KEY----- +MIICXgIBAAKBgQDCFENGw33yGihy92pDjZQhl0C36rPJj+CvfSC8+q28hxA161QF +NUd13wuCTUcq0Qd2qsBe/2hFyc2DCJJg0h1L78+6Z4UMR7EOcpfdUE9Hf3m/hs+F +UR45uBJeDK1HSFHD8bHKD6kv8FPGfJTotc+2xjJwoYi+1hqp1fIekaxsyQIDAQAB +AoGBAJR8ZkCUvx5kzv+utdl7T5MnordT1TvoXXJGXK7ZZ+UuvMNUCdN2QPc4sBiA +QWvLw1cSKt5DsKZ8UETpYPy8pPYnnDEz2dDYiaew9+xEpubyeW2oH4Zx71wqBtOK +kqwrXa/pzdpiucRRjk6vE6YY7EBBs/g7uanVpGibOVAEsqH1AkEA7DkjVH28WDUg +f1nqvfn2Kj6CT7nIcE3jGJsZZ7zlZmBmHFDONMLUrXR/Zm3pR5m0tCmBqa5RK95u +412jt1dPIwJBANJT3v8pnkth48bQo/fKel6uEYyboRtA5/uHuHkZ6FQF7OUkGogc +mSJluOdc5t6hI1VsLn0QZEjQZMEOWr+wKSMCQQCC4kXJEsHAve77oP6HtG/IiEn7 +kpyUXRNvFsDE0czpJJBvL/aRFUJxuRK91jhjC68sA7NsKMGg5OXb5I5Jj36xAkEA +gIT7aFOYBFwGgQAQkWNKLvySgKbAZRTeLBacpHMuQdl1DfdntvAyqpAZ0lY0RKmW +G6aFKaqQfOXKCyWoUiVknQJAXrlgySFci/2ueKlIE1QqIiLSZ8V8OlpFLRnb1pzI +7U1yQXnTAEFYM560yJlzUpOb1V4cScGd365tiSMvxLOvTA== +-----END RSA PRIVATE KEY-----` + + config := DefaultConfig + config.Provider = "raw" + config.Tenancy = "test" + config.User = "test" + config.Region = "test" + config.Fingerprint = "123" + config.PrivateKey = mockPrivateKey + config.Passphrase = "123" + ociConfig, err := yaml.Marshal(config) + testutil.Ok(t, err) + + _, err = NewBucket(log.NewNopLogger(), ociConfig, errutil.WrapWithErrRoundtripper) + // We expect an error from the RoundTripper + testutil.NotOk(t, err) + testutil.Assert(t, errutil.IsMockedError(err), "Expected RoundTripper error, got: %v", err) +} diff --git a/providers/oss/oss.go b/providers/oss/oss.go index e7e3a648..2a6cb219 100644 --- a/providers/oss/oss.go +++ b/providers/oss/oss.go @@ -16,14 +16,15 @@ import ( "testing" "time" + "github.com/aliyun/aliyun-oss-go-sdk/oss" alioss "github.com/aliyun/aliyun-oss-go-sdk/oss" "github.com/go-kit/log" "github.com/pkg/errors" "gopkg.in/yaml.v2" - "github.com/thanos-io/objstore/clientutil" - "github.com/thanos-io/objstore" + "github.com/thanos-io/objstore/clientutil" + "github.com/thanos-io/objstore/exthttp" ) // PartSize is a part size for multi part upload. @@ -67,6 +68,8 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) { return NewTestBucketFromConfig(t, c, false) } +func (b *Bucket) Provider() objstore.ObjProvider { return objstore.ALIYUNOSS } + // Upload the contents of the reader as an object into the bucket. func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) error { // TODO(https://github.com/thanos-io/thanos/issues/678): Remove guessing length when minio provider will support multipart upload without this. @@ -158,22 +161,32 @@ func (b *Bucket) Attributes(ctx context.Context, name string) (objstore.ObjectAt } // NewBucket returns a new Bucket using the provided oss config values. -func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error) { +func NewBucket(logger log.Logger, conf []byte, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error) { var config Config if err := yaml.Unmarshal(conf, &config); err != nil { return nil, errors.Wrap(err, "parse aliyun oss config file failed") } - - return NewBucketWithConfig(logger, config, component) + return NewBucketWithConfig(logger, config, component, wrapRoundtripper) } // NewBucketWithConfig returns a new Bucket using the provided oss config struct. -func NewBucketWithConfig(logger log.Logger, config Config, component string) (*Bucket, error) { +func NewBucketWithConfig(logger log.Logger, config Config, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error) { if err := validate(config); err != nil { return nil, err } - - client, err := alioss.New(config.Endpoint, config.AccessKeyID, config.AccessKeySecret) + var clientOptions []alioss.ClientOption + if wrapRoundtripper != nil { + rt, err := exthttp.DefaultTransport(exthttp.DefaultHTTPConfig) + if err != nil { + return nil, err + } + clientOptions = append(clientOptions, func(client *alioss.Client) { + client.HTTPClient = &http.Client{ + Transport: wrapRoundtripper(rt), + } + }) + } + client, err := alioss.New(config.Endpoint, config.AccessKeyID, config.AccessKeySecret, clientOptions...) if err != nil { return nil, errors.Wrap(err, "create aliyun oss client failed") } @@ -204,7 +217,11 @@ func validate(config Config) error { return nil } -// Iter calls f for each entry in the given directory (not recursive). The argument to f is the full +func (b *Bucket) SupportedIterOptions() []objstore.IterOptionType { + return []objstore.IterOptionType{objstore.Recursive} +} + +// Iter calls f for each entry in the given directory. The argument to f is the full // object name including the prefix of the inspected directory. func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, options ...objstore.IterOption) error { if dir != "" { @@ -246,6 +263,16 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt return nil } +func (b *Bucket) IterWithAttributes(ctx context.Context, dir string, f func(attrs objstore.IterObjectAttributes) error, options ...objstore.IterOption) error { + if err := objstore.ValidateIterOptions(b.SupportedIterOptions(), options...); err != nil { + return err + } + + return b.Iter(ctx, dir, func(name string) error { + return f(objstore.IterObjectAttributes{Name: name}) + }, options...) +} + func (b *Bucket) Name() string { return b.name } @@ -274,13 +301,13 @@ func NewTestBucketFromConfig(t testing.TB, c Config, reuseBucket bool) (objstore return nil, nil, err } - b, err := NewBucket(log.NewNopLogger(), bc, "thanos-aliyun-oss-test") + b, err := NewBucket(log.NewNopLogger(), bc, "thanos-aliyun-oss-test", nil) if err != nil { return nil, nil, err } if reuseBucket { - if err := b.Iter(context.Background(), "", func(f string) error { + if err := b.Iter(context.Background(), "", func(_ string) error { return errors.Errorf("bucket %s is not empty", c.Bucket) }); err != nil { return nil, nil, errors.Wrapf(err, "oss check bucket %s", c.Bucket) @@ -338,12 +365,22 @@ func (b *Bucket) getRange(_ context.Context, name string, off, length int64) (io opts = append(opts, opt) } - resp, err := b.bucket.GetObject(name, opts...) + resp, err := b.bucket.DoGetObject(&oss.GetObjectRequest{ObjectKey: name}, opts) if err != nil { return nil, err } - return resp, nil + size, err := clientutil.ParseContentLength(resp.Response.Headers) + if err == nil { + return objstore.ObjectSizerReadCloser{ + ReadCloser: resp.Response, + Size: func() (int64, error) { + return size, nil + }, + }, nil + } + + return resp.Response, nil } // Get returns a reader for the given object name. diff --git a/providers/oss/oss_test.go b/providers/oss/oss_test.go new file mode 100644 index 00000000..b43d3077 --- /dev/null +++ b/providers/oss/oss_test.go @@ -0,0 +1,26 @@ +package oss + +import ( + "context" + "testing" + + "github.com/efficientgo/core/testutil" + "github.com/go-kit/log" + "github.com/thanos-io/objstore/errutil" +) + +func TestNewBucketWithErrorRoundTripper(t *testing.T) { + config := Config{ + Endpoint: "http://test.com/", + AccessKeyID: "123", + AccessKeySecret: "123", + Bucket: "test", + } + + bkt, err := NewBucketWithConfig(log.NewNopLogger(), config, "test", errutil.WrapWithErrRoundtripper) + // We expect an error from the RoundTripper + testutil.Ok(t, err) + _, err = bkt.Get(context.Background(), "test") + testutil.NotOk(t, err) + testutil.Assert(t, errutil.IsMockedError(err), "Expected RoundTripper error, got: %v", err) +} diff --git a/providers/s3/s3.go b/providers/s3/s3.go index dad89e66..58590486 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -136,6 +136,7 @@ type Config struct { PartSize uint64 `yaml:"part_size"` SSEConfig SSEConfig `yaml:"sse_config"` STSEndpoint string `yaml:"sts_endpoint"` + MaxRetries int `yaml:"max_retries"` } // SSEConfig deals with the configuration of SSE for Minio. The following options are valid: @@ -176,13 +177,13 @@ func parseConfig(conf []byte) (Config, error) { } // NewBucket returns a new Bucket using the provided s3 config values. -func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error) { +func NewBucket(logger log.Logger, conf []byte, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error) { config, err := parseConfig(conf) if err != nil { return nil, err } - return NewBucketWithConfig(logger, config, component) + return NewBucketWithConfig(logger, config, component, wrapRoundtripper) } type overrideSignerType struct { @@ -202,7 +203,7 @@ func (s *overrideSignerType) Retrieve() (credentials.Value, error) { } // NewBucketWithConfig returns a new Bucket using the provided s3 config values. -func NewBucketWithConfig(logger log.Logger, config Config, component string) (*Bucket, error) { +func NewBucketWithConfig(logger log.Logger, config Config, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error) { var chain []credentials.Provider // TODO(bwplotka): Don't do flags as they won't scale, use actual params like v2, v4 instead @@ -245,23 +246,25 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B // Check if a roundtripper has been set in the config // otherwise build the default transport. - var rt http.RoundTripper + var tpt http.RoundTripper + tpt, err := exthttp.DefaultTransport(config.HTTPConfig) + if err != nil { + return nil, err + } if config.HTTPConfig.Transport != nil { - rt = config.HTTPConfig.Transport - } else { - var err error - rt, err = exthttp.DefaultTransport(config.HTTPConfig) - if err != nil { - return nil, err - } + tpt = config.HTTPConfig.Transport + } + if wrapRoundtripper != nil { + tpt = wrapRoundtripper(tpt) } client, err := minio.New(config.Endpoint, &minio.Options{ Creds: credentials.NewChainCredentials(chain), Secure: !config.Insecure, Region: config.Region, - Transport: rt, + Transport: tpt, BucketLookup: config.BucketLookupType.MinioType(), + MaxRetries: config.MaxRetries, }) if err != nil { return nil, errors.Wrap(err, "initialize s3 client") @@ -342,6 +345,8 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B return bkt, nil } +func (b *Bucket) Provider() objstore.ObjProvider { return objstore.S3 } + // Name returns the bucket name for s3. func (b *Bucket) Name() string { return b.name @@ -386,18 +391,26 @@ func ValidateForTests(conf Config) error { return nil } -// Iter calls f for each entry in the given directory. The argument to f is the full -// object name including the prefix of the inspected directory. -func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, options ...objstore.IterOption) error { +func (b *Bucket) SupportedIterOptions() []objstore.IterOptionType { + return []objstore.IterOptionType{objstore.Recursive, objstore.UpdatedAt} +} + +func (b *Bucket) IterWithAttributes(ctx context.Context, dir string, f func(attrs objstore.IterObjectAttributes) error, options ...objstore.IterOption) error { + if err := objstore.ValidateIterOptions(b.SupportedIterOptions(), options...); err != nil { + return err + } + // Ensure the object name actually ends with a dir suffix. Otherwise we'll just iterate the // object itself as one prefix item. if dir != "" { dir = strings.TrimSuffix(dir, DirDelim) + DirDelim } + appliedOpts := objstore.ApplyIterOptions(options...) + opts := minio.ListObjectsOptions{ Prefix: dir, - Recursive: objstore.ApplyIterOptions(options...).Recursive, + Recursive: appliedOpts.Recursive, UseV1: b.listObjectsV1, } @@ -414,7 +427,15 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt if object.Key == dir { continue } - if err := f(object.Key); err != nil { + + attr := objstore.IterObjectAttributes{ + Name: object.Key, + } + if appliedOpts.LastModified { + attr.SetLastModified(object.LastModified) + } + + if err := f(attr); err != nil { return err } } @@ -422,6 +443,21 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt return ctx.Err() } +func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opts ...objstore.IterOption) error { + // Only include recursive option since attributes are not used in this method. + var filteredOpts []objstore.IterOption + for _, opt := range opts { + if opt.Type == objstore.Recursive { + filteredOpts = append(filteredOpts, opt) + break + } + } + + return b.IterWithAttributes(ctx, dir, func(attrs objstore.IterObjectAttributes) error { + return f(attrs.Name) + }, filteredOpts...) +} + func (b *Bucket) getRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error) { sse, err := b.getServerSideEncryption(ctx) if err != nil { @@ -452,7 +488,17 @@ func (b *Bucket) getRange(ctx context.Context, name string, off, length int64) ( return nil, err } - return r, nil + return objstore.ObjectSizerReadCloser{ + ReadCloser: r, + Size: func() (int64, error) { + stat, err := r.Stat() + if err != nil { + return 0, err + } + + return stat.Size, nil + }, + }, nil } // Get returns a reader for the given object name. @@ -611,14 +657,14 @@ func NewTestBucketFromConfig(t testing.TB, location string, c Config, reuseBucke if err != nil { return nil, nil, err } - b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test") + b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test", nil) if err != nil { return nil, nil, err } bktToCreate := c.Bucket if c.Bucket != "" && reuseBucket { - if err := b.Iter(ctx, "", func(f string) error { + if err := b.Iter(ctx, "", func(string) error { return errors.Errorf("bucket %s is not empty", c.Bucket) }); err != nil { return nil, nil, errors.Wrapf(err, "s3 check bucket %s", c.Bucket) diff --git a/providers/s3/s3_e2e_test.go b/providers/s3/s3_e2e_test.go index 4b75a014..ac9ec261 100644 --- a/providers/s3/s3_e2e_test.go +++ b/providers/s3/s3_e2e_test.go @@ -37,6 +37,7 @@ func BenchmarkUpload(b *testing.B) { log.NewNopLogger(), e2ethanos.NewS3Config(bucket, m.Endpoint("https"), m.Dir()), "test-feed", + nil, ) testutil.Ok(b, err) diff --git a/providers/s3/s3_test.go b/providers/s3/s3_test.go index cdab39c3..3040cd81 100644 --- a/providers/s3/s3_test.go +++ b/providers/s3/s3_test.go @@ -17,6 +17,7 @@ import ( "github.com/go-kit/log" "github.com/minio/minio-go/v7/pkg/encrypt" + "github.com/thanos-io/objstore/errutil" "github.com/thanos-io/objstore/exthttp" ) @@ -324,7 +325,7 @@ func TestBucket_getServerSideEncryption(t *testing.T) { // Default config should return no SSE config. cfg := DefaultConfig cfg.Endpoint = endpoint - bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test") + bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test", nil) testutil.Ok(t, err) sse, err := bkt.getServerSideEncryption(context.Background()) @@ -335,7 +336,7 @@ func TestBucket_getServerSideEncryption(t *testing.T) { cfg = DefaultConfig cfg.Endpoint = endpoint cfg.SSEConfig = SSEConfig{Type: SSES3} - bkt, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test") + bkt, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test", nil) testutil.Ok(t, err) sse, err = bkt.getServerSideEncryption(context.Background()) @@ -351,7 +352,7 @@ func TestBucket_getServerSideEncryption(t *testing.T) { Type: SSEKMS, KMSKeyID: "key", } - bkt, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test") + bkt, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test", nil) testutil.Ok(t, err) sse, err = bkt.getServerSideEncryption(context.Background()) @@ -375,7 +376,7 @@ func TestBucket_getServerSideEncryption(t *testing.T) { KMSKeyID: "key", KMSEncryptionContext: map[string]string{"foo": "bar"}, } - bkt, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test") + bkt, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test", nil) testutil.Ok(t, err) sse, err = bkt.getServerSideEncryption(context.Background()) @@ -396,7 +397,7 @@ func TestBucket_getServerSideEncryption(t *testing.T) { override, err := encrypt.NewSSEKMS("test", nil) testutil.Ok(t, err) - bkt, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test") + bkt, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test", nil) testutil.Ok(t, err) sse, err = bkt.getServerSideEncryption(context.WithValue(context.Background(), sseConfigKey, override)) @@ -423,7 +424,7 @@ func TestBucket_Get_ShouldReturnErrorIfServerTruncateResponse(t *testing.T) { cfg.AccessKey = "test" cfg.SecretKey = "test" - bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test") + bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test", nil) testutil.Ok(t, err) reader, err := bkt.Get(context.Background(), "test") @@ -448,7 +449,7 @@ func TestParseConfig_CustomStorageClass(t *testing.T) { cfg.Endpoint = endpoint storageClass := "STANDARD_IA" cfg.PutUserMetadata[testCase.storageClassKey] = storageClass - bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test") + bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test", nil) testutil.Ok(t, err) testutil.Equals(t, storageClass, bkt.storageClass) }) @@ -458,7 +459,19 @@ func TestParseConfig_CustomStorageClass(t *testing.T) { func TestParseConfig_DefaultStorageClassIsZero(t *testing.T) { cfg := DefaultConfig cfg.Endpoint = endpoint - bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test") + bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test", nil) testutil.Ok(t, err) testutil.Equals(t, "", bkt.storageClass) } + +func TestNewBucketWithErrorRoundTripper(t *testing.T) { + cfg := DefaultConfig + cfg.Endpoint = endpoint + cfg.Bucket = "test" + bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test", errutil.WrapWithErrRoundtripper) + testutil.Ok(t, err) + _, err = bkt.Get(context.Background(), "test") + // We expect an error from the RoundTripper + testutil.NotOk(t, err) + testutil.Assert(t, errutil.IsMockedError(err), "Expected RoundTripper error, got: %v", err) +} diff --git a/providers/swift/swift.go b/providers/swift/swift.go index a8c56c55..19eb0d45 100644 --- a/providers/swift/swift.go +++ b/providers/swift/swift.go @@ -21,6 +21,7 @@ import ( "github.com/ncw/swift" "github.com/pkg/errors" "github.com/prometheus/common/model" + "github.com/thanos-io/objstore" "github.com/thanos-io/objstore/exthttp" "gopkg.in/yaml.v2" @@ -154,12 +155,12 @@ type Container struct { segmentsContainer string } -func NewContainer(logger log.Logger, conf []byte) (*Container, error) { +func NewContainer(logger log.Logger, conf []byte, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Container, error) { sc, err := parseConfig(conf) if err != nil { return nil, errors.Wrap(err, "parse config") } - return NewContainerFromConfig(logger, sc, false) + return NewContainerFromConfig(logger, sc, false, wrapRoundtripper) } func ensureContainer(connection *swift.Connection, name string, createIfNotExist bool) error { @@ -178,19 +179,19 @@ func ensureContainer(connection *swift.Connection, name string, createIfNotExist return nil } -func NewContainerFromConfig(logger log.Logger, sc *Config, createContainer bool) (*Container, error) { - +func NewContainerFromConfig(logger log.Logger, sc *Config, createContainer bool, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Container, error) { // Check if a roundtripper has been set in the config // otherwise build the default transport. var rt http.RoundTripper + rt, err := exthttp.DefaultTransport(sc.HTTPConfig) + if err != nil { + return nil, err + } if sc.HTTPConfig.Transport != nil { rt = sc.HTTPConfig.Transport - } else { - var err error - rt, err = exthttp.DefaultTransport(sc.HTTPConfig) - if err != nil { - return nil, err - } + } + if wrapRoundtripper != nil { + rt = wrapRoundtripper(rt) } connection := connectionFromConfig(sc, rt) @@ -217,14 +218,20 @@ func NewContainerFromConfig(logger log.Logger, sc *Config, createContainer bool) }, nil } +func (c *Container) Provider() objstore.ObjProvider { return objstore.SWIFT } + // Name returns the container name for swift. func (c *Container) Name() string { return c.name } +func (c *Container) SupportedIterOptions() []objstore.IterOptionType { + return []objstore.IterOptionType{objstore.Recursive} +} + // Iter calls f for each entry in the given directory. The argument to f is the full // object name including the prefix of the inspected directory. -func (c *Container) Iter(_ context.Context, dir string, f func(string) error, options ...objstore.IterOption) error { +func (c *Container) Iter(ctx context.Context, dir string, f func(string) error, options ...objstore.IterOption) error { if dir != "" { dir = strings.TrimSuffix(dir, string(DirDelim)) + string(DirDelim) } @@ -242,6 +249,7 @@ func (c *Container) Iter(_ context.Context, dir string, f func(string) error, op if err != nil { return objects, errors.Wrap(err, "list object names") } + for _, object := range objects { if object == SegmentsDir { continue @@ -254,6 +262,16 @@ func (c *Container) Iter(_ context.Context, dir string, f func(string) error, op }) } +func (c *Container) IterWithAttributes(ctx context.Context, dir string, f func(attrs objstore.IterObjectAttributes) error, options ...objstore.IterOption) error { + if err := objstore.ValidateIterOptions(c.SupportedIterOptions(), options...); err != nil { + return err + } + + return c.Iter(ctx, dir, func(name string) error { + return f(objstore.IterObjectAttributes{Name: name}) + }, options...) +} + func (c *Container) get(name string, headers swift.Headers, checkHash bool) (io.ReadCloser, error) { if name == "" { return nil, errors.New("object name cannot be empty") @@ -262,7 +280,11 @@ func (c *Container) get(name string, headers swift.Headers, checkHash bool) (io. if err != nil { return nil, errors.Wrap(err, "open object") } - return file, err + + return objstore.ObjectSizerReadCloser{ + ReadCloser: file, + Size: file.Length, + }, nil } // Get returns a reader for the given object name. @@ -378,7 +400,7 @@ func NewTestContainer(t testing.TB) (objstore.Bucket, func(), error) { "needs to be manually cleared. This means that it is only useful to run one test in a time. This is due " + "to safety (accidentally pointing prod container for test) as well as swift not being fully strong consistent.") } - c, err := NewContainerFromConfig(log.NewNopLogger(), config, false) + c, err := NewContainerFromConfig(log.NewNopLogger(), config, false, nil) if err != nil { return nil, nil, errors.Wrap(err, "initializing new container") } @@ -392,7 +414,7 @@ func NewTestContainer(t testing.TB) (objstore.Bucket, func(), error) { } config.ContainerName = objstore.CreateTemporaryTestBucketName(t) config.SegmentContainerName = config.ContainerName - c, err := NewContainerFromConfig(log.NewNopLogger(), config, true) + c, err := NewContainerFromConfig(log.NewNopLogger(), config, true, nil) if err != nil { return nil, nil, errors.Wrap(err, "initializing new container") } diff --git a/providers/swift/swift_test.go b/providers/swift/swift_test.go index 656e7756..b17a5e2b 100644 --- a/providers/swift/swift_test.go +++ b/providers/swift/swift_test.go @@ -8,7 +8,9 @@ import ( "time" "github.com/efficientgo/core/testutil" + "github.com/go-kit/log" "github.com/prometheus/common/model" + "github.com/thanos-io/objstore/errutil" ) func TestParseConfig(t *testing.T) { @@ -64,3 +66,13 @@ http_config: testutil.Equals(t, false, cfg.HTTPConfig.InsecureSkipVerify) } + +func TestNewBucketWithErrorRoundTripper(t *testing.T) { + config := DefaultConfig + config.AuthUrl = "http://identity.something.com/v3" + _, err := NewContainerFromConfig(log.NewNopLogger(), &config, false, errutil.WrapWithErrRoundtripper) + + // We expect an error from the RoundTripper + testutil.NotOk(t, err) + testutil.Assert(t, errutil.IsMockedError(err), "Expected RoundTripper error, got: %v", err) +} diff --git a/scripts/cfggen/main.go b/scripts/cfggen/main.go index 424bf9b0..e1bc6410 100644 --- a/scripts/cfggen/main.go +++ b/scripts/cfggen/main.go @@ -5,6 +5,7 @@ package main import ( "fmt" + "github.com/thanos-io/objstore" "io" "os" "path/filepath" @@ -35,17 +36,17 @@ var ( configs map[string]interface{} possibleValues []string - bucketConfigs = map[client.ObjProvider]interface{}{ - client.AZURE: azure.Config{}, - client.GCS: gcs.Config{}, - client.S3: s3.DefaultConfig, - client.SWIFT: swift.DefaultConfig, - client.COS: cos.DefaultConfig, - client.ALIYUNOSS: oss.Config{}, - client.FILESYSTEM: filesystem.Config{}, - client.BOS: bos.Config{}, - client.OCI: oci.Config{}, - client.OBS: obs.DefaultConfig, + bucketConfigs = map[objstore.ObjProvider]interface{}{ + objstore.AZURE: azure.Config{}, + objstore.GCS: gcs.Config{}, + objstore.S3: s3.DefaultConfig, + objstore.SWIFT: swift.DefaultConfig, + objstore.COS: cos.DefaultConfig, + objstore.ALIYUNOSS: oss.Config{}, + objstore.FILESYSTEM: filesystem.Config{}, + objstore.BOS: bos.Config{}, + objstore.OCI: oci.Config{}, + objstore.OBS: obs.DefaultConfig, } ) diff --git a/testing.go b/testing.go index b8e3744c..80f1e198 100644 --- a/testing.go +++ b/testing.go @@ -106,6 +106,11 @@ func AcceptanceTest(t *testing.T, bkt Bucket) { rc1, err := bkt.Get(ctx, "id1/obj_1.some") testutil.Ok(t, err) defer func() { testutil.Ok(t, rc1.Close()) }() + + sz, err := TryToGetSize(rc1) + testutil.Ok(t, err) + testutil.Equals(t, int64(11), sz, "expected size to be equal to 11") + content, err := io.ReadAll(rc1) testutil.Ok(t, err) testutil.Equals(t, "@test-data@", string(content)) @@ -118,6 +123,11 @@ func AcceptanceTest(t *testing.T, bkt Bucket) { rc2, err := bkt.GetRange(ctx, "id1/obj_1.some", 1, 3) testutil.Ok(t, err) defer func() { testutil.Ok(t, rc2.Close()) }() + + sz, err = TryToGetSize(rc2) + testutil.Ok(t, err) + testutil.Equals(t, int64(3), sz, "expected size to be equal to 3") + content, err = io.ReadAll(rc2) testutil.Ok(t, err) testutil.Equals(t, "tes", string(content)) @@ -126,6 +136,11 @@ func AcceptanceTest(t *testing.T, bkt Bucket) { rcUnspecifiedLen, err := bkt.GetRange(ctx, "id1/obj_1.some", 1, -1) testutil.Ok(t, err) defer func() { testutil.Ok(t, rcUnspecifiedLen.Close()) }() + + sz, err = TryToGetSize(rcUnspecifiedLen) + testutil.Ok(t, err) + testutil.Equals(t, int64(10), sz, "expected size to be equal to 10") + content, err = io.ReadAll(rcUnspecifiedLen) testutil.Ok(t, err) testutil.Equals(t, "test-data@", string(content)) @@ -141,6 +156,11 @@ func AcceptanceTest(t *testing.T, bkt Bucket) { rcLength, err := bkt.GetRange(ctx, "id1/obj_1.some", 3, 9999) testutil.Ok(t, err) defer func() { testutil.Ok(t, rcLength.Close()) }() + + sz, err = TryToGetSize(rcLength) + testutil.Ok(t, err) + testutil.Equals(t, int64(8), sz, "expected size to be equal to 8") + content, err = io.ReadAll(rcLength) testutil.Ok(t, err) testutil.Equals(t, "st-data@", string(content)) @@ -175,7 +195,7 @@ func AcceptanceTest(t *testing.T, bkt Bucket) { testutil.Ok(t, bkt.Iter(ctx, "", func(fn string) error { seen = append(seen, fn) return nil - }, WithRecursiveIter)) + }, WithRecursiveIter())) expected = []string{"id1/obj_1.some", "id1/obj_2.some", "id1/obj_3.some", "id1/sub/subobj_1.some", "id1/sub/subobj_2.some", "id2/obj_4.some", "obj_5.some"} sort.Strings(expected) sort.Strings(seen) @@ -194,7 +214,7 @@ func AcceptanceTest(t *testing.T, bkt Bucket) { testutil.Ok(t, bkt.Iter(ctx, "id1/", func(fn string) error { seen = append(seen, fn) return nil - }, WithRecursiveIter)) + }, WithRecursiveIter())) testutil.Equals(t, []string{"id1/obj_1.some", "id1/obj_2.some", "id1/obj_3.some", "id1/sub/subobj_1.some", "id1/sub/subobj_2.some"}, seen) // Can we iter over items from id1 dir? @@ -210,7 +230,7 @@ func AcceptanceTest(t *testing.T, bkt Bucket) { testutil.Ok(t, bkt.Iter(ctx, "id1", func(fn string) error { seen = append(seen, fn) return nil - }, WithRecursiveIter)) + }, WithRecursiveIter())) testutil.Equals(t, []string{"id1/obj_1.some", "id1/obj_2.some", "id1/obj_3.some", "id1/sub/subobj_1.some", "id1/sub/subobj_2.some"}, seen) // Can we iter over items from not existing dir? @@ -260,6 +280,8 @@ func WithDelay(bkt Bucket, delay time.Duration) Bucket { return &delayingBucket{bkt: bkt, delay: delay} } +func (d *delayingBucket) Provider() ObjProvider { return d.bkt.Provider() } + func (d *delayingBucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { time.Sleep(d.delay) return d.bkt.Get(ctx, name) @@ -275,6 +297,15 @@ func (d *delayingBucket) Iter(ctx context.Context, dir string, f func(string) er return d.bkt.Iter(ctx, dir, f, options...) } +func (d *delayingBucket) IterWithAttributes(ctx context.Context, dir string, f func(IterObjectAttributes) error, options ...IterOption) error { + time.Sleep(d.delay) + return d.bkt.IterWithAttributes(ctx, dir, f, options...) +} + +func (d *delayingBucket) SupportedIterOptions() []IterOptionType { + return d.bkt.SupportedIterOptions() +} + func (d *delayingBucket) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error) { time.Sleep(d.delay) return d.bkt.GetRange(ctx, name, off, length) diff --git a/tracing/opentelemetry/opentelemetry.go b/tracing/opentelemetry/opentelemetry.go index f65b0f0e..d5f9bd8c 100644 --- a/tracing/opentelemetry/opentelemetry.go +++ b/tracing/opentelemetry/opentelemetry.go @@ -23,6 +23,8 @@ func WrapWithTraces(bkt objstore.Bucket, tracer trace.Tracer) objstore.Instrumen return TracingBucket{tracer: tracer, bkt: bkt} } +func (t TracingBucket) Provider() objstore.ObjProvider { return t.bkt.Provider() } + func (t TracingBucket) Iter(ctx context.Context, dir string, f func(string) error, options ...objstore.IterOption) (err error) { ctx, span := t.tracer.Start(ctx, "bucket_iter") defer span.End() @@ -36,6 +38,24 @@ func (t TracingBucket) Iter(ctx context.Context, dir string, f func(string) erro return t.bkt.Iter(ctx, dir, f, options...) } +func (t TracingBucket) IterWithAttributes(ctx context.Context, dir string, f func(attrs objstore.IterObjectAttributes) error, options ...objstore.IterOption) (err error) { + ctx, span := t.tracer.Start(ctx, "bucket_iter_with_attrs") + defer span.End() + span.SetAttributes(attribute.String("dir", dir)) + + defer func() { + if err != nil { + span.RecordError(err) + } + }() + return t.bkt.IterWithAttributes(ctx, dir, f, options...) +} + +// SupportedIterOptions returns a list of supported IterOptions by the underlying provider. +func (t TracingBucket) SupportedIterOptions() []objstore.IterOptionType { + return t.bkt.SupportedIterOptions() +} + func (t TracingBucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { ctx, span := t.tracer.Start(ctx, "bucket_get") defer span.End() diff --git a/tracing/opentracing/opentracing.go b/tracing/opentracing/opentracing.go index 0a26ceeb..58bdea07 100644 --- a/tracing/opentracing/opentracing.go +++ b/tracing/opentracing/opentracing.go @@ -44,6 +44,8 @@ func WrapWithTraces(bkt objstore.Bucket) objstore.InstrumentedBucket { return TracingBucket{bkt: bkt} } +func (t TracingBucket) Provider() objstore.ObjProvider { return t.bkt.Provider() } + func (t TracingBucket) Iter(ctx context.Context, dir string, f func(string) error, options ...objstore.IterOption) (err error) { doWithSpan(ctx, "bucket_iter", func(spanCtx context.Context, span opentracing.Span) { span.LogKV("dir", dir) @@ -52,6 +54,18 @@ func (t TracingBucket) Iter(ctx context.Context, dir string, f func(string) erro return } +func (t TracingBucket) IterWithAttributes(ctx context.Context, dir string, f func(attrs objstore.IterObjectAttributes) error, options ...objstore.IterOption) (err error) { + doWithSpan(ctx, "bucket_iter_with_attrs", func(spanCtx context.Context, span opentracing.Span) { + span.LogKV("dir", dir) + err = t.bkt.IterWithAttributes(spanCtx, dir, f, options...) + }) + return +} + +func (t TracingBucket) SupportedIterOptions() []objstore.IterOptionType { + return t.bkt.SupportedIterOptions() +} + func (t TracingBucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { span, spanCtx := startSpan(ctx, "bucket_get") span.LogKV("name", name)