diff --git a/CHANGELOG.md b/CHANGELOG.md index 85ba8e80..ebdbdbe4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,7 +26,6 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#79](https://github.com/thanos-io/objstore/pull/79) Metrics: Fix `objstore_bucket_operation_duration_seconds` for `iter` operations. ### Added -- [#63](https://github.com/thanos-io/objstore/pull/63) Implement a `IterWithAttributes` method on the bucket client. - [#15](https://github.com/thanos-io/objstore/pull/15) Add Oracle Cloud Infrastructure Object Storage Bucket support. - [#25](https://github.com/thanos-io/objstore/pull/25) S3: Support specifying S3 storage class. - [#32](https://github.com/thanos-io/objstore/pull/32) Swift: Support authentication using application credentials. @@ -57,6 +56,9 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#130](https://github.com/thanos-io/objstore/pull/130) feat: Decouple creating bucket metrics from instrumenting the bucket - [#147](https://github.com/thanos-io/objstore/pull/147) feat: Add MaxRetries config to cos, gcs and obs. - [#150](https://github.com/thanos-io/objstore/pull/150) Add support for roundtripper wrapper. +- [#63](https://github.com/thanos-io/objstore/pull/63) Implement a `IterWithAttributes` method on the bucket client. +- [#155](https://github.com/thanos-io/objstore/pull/155) Add a `Provider` method on `objstore.Client`. + ### Changed - [#38](https://github.com/thanos-io/objstore/pull/38) *: Upgrade minio-go version to `v7.0.45`. @@ -67,4 +69,6 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#71](https://github.com/thanos-io/objstore/pull/71) Replace method `IsCustomerManagedKeyError` for a more generic `IsAccessDeniedErr` on the bucket interface. - [#89](https://github.com/thanos-io/objstore/pull/89) GCS: Upgrade cloud.google.com/go/storage version to `v1.35.1`. - [#123](https://github.com/thanos-io/objstore/pull/123) *: Upgrade minio-go version to `v7.0.71`. +- [#132](https://github.com/thanos-io/objstore/pull/132) s3: Upgrade aws-sdk-go-v2/config version to `v1.27.30` + ### Removed diff --git a/README.md b/README.md index d8f58023..516d1070 100644 --- a/README.md +++ b/README.md @@ -48,13 +48,15 @@ See [MAINTAINERS.md](https://github.com/thanos-io/thanos/blob/main/MAINTAINERS.m The core this module is the [`Bucket` interface](objstore.go): -```go mdox-exec="sed -n '39,55p' objstore.go" +```go mdox-exec="sed -n '55,73p' objstore.go" // Bucket provides read and write access to an object storage bucket. // NOTE: We assume strong consistency for write-read flow. type Bucket interface { io.Closer BucketReader + Provider() ObjProvider + // Upload the contents of the reader as an object into the bucket. // Upload should be idempotent. Upload(ctx context.Context, name string, r io.Reader) error @@ -70,7 +72,7 @@ type Bucket interface { All [provider implementations](providers) have to implement `Bucket` interface that allows common read and write operations that all supported by all object providers. If you want to limit the code that will do bucket operation to only read access (smart idea, allowing to limit access permissions), you can use the [`BucketReader` interface](objstore.go): -```go mdox-exec="sed -n '71,106p' objstore.go" +```go mdox-exec="sed -n '89,124p' objstore.go" // BucketReader provides read access to an object storage bucket. type BucketReader interface { // Iter calls f for each entry in the given directory (not recursive.). The argument to f is the full diff --git a/client/factory.go b/client/factory.go index 5fe5a741..089fd843 100644 --- a/client/factory.go +++ b/client/factory.go @@ -27,25 +27,10 @@ import ( "gopkg.in/yaml.v2" ) -type ObjProvider string - -const ( - FILESYSTEM ObjProvider = "FILESYSTEM" - GCS ObjProvider = "GCS" - S3 ObjProvider = "S3" - AZURE ObjProvider = "AZURE" - SWIFT ObjProvider = "SWIFT" - COS ObjProvider = "COS" - ALIYUNOSS ObjProvider = "ALIYUNOSS" - BOS ObjProvider = "BOS" - OCI ObjProvider = "OCI" - OBS ObjProvider = "OBS" -) - type BucketConfig struct { - Type ObjProvider `yaml:"type"` - Config interface{} `yaml:"config"` - Prefix string `yaml:"prefix" default:""` + Type objstore.ObjProvider `yaml:"type"` + Config interface{} `yaml:"config"` + Prefix string `yaml:"prefix" default:""` } // NewBucket initializes and returns new object storage clients. @@ -64,25 +49,25 @@ func NewBucket(logger log.Logger, confContentYaml []byte, component string, wrap var bucket objstore.Bucket switch strings.ToUpper(string(bucketConf.Type)) { - case string(GCS): + case string(objstore.GCS): bucket, err = gcs.NewBucket(context.Background(), logger, config, component, wrapRoundtripper) - case string(S3): + case string(objstore.S3): bucket, err = s3.NewBucket(logger, config, component, wrapRoundtripper) - case string(AZURE): + case string(objstore.AZURE): bucket, err = azure.NewBucket(logger, config, component, wrapRoundtripper) - case string(SWIFT): + case string(objstore.SWIFT): bucket, err = swift.NewContainer(logger, config, wrapRoundtripper) - case string(COS): + case string(objstore.COS): bucket, err = cos.NewBucket(logger, config, component, wrapRoundtripper) - case string(ALIYUNOSS): + case string(objstore.ALIYUNOSS): bucket, err = oss.NewBucket(logger, config, component, wrapRoundtripper) - case string(FILESYSTEM): + case string(objstore.FILESYSTEM): bucket, err = filesystem.NewBucketFromConfig(config) - case string(BOS): + case string(objstore.BOS): bucket, err = bos.NewBucket(logger, config, component) - case string(OCI): + case string(objstore.OCI): bucket, err = oci.NewBucket(logger, config, wrapRoundtripper) - case string(OBS): + case string(objstore.OBS): bucket, err = obs.NewBucket(logger, config) default: return nil, errors.Errorf("bucket with type %s is not supported", bucketConf.Type) diff --git a/go.mod b/go.mod index 4555b6bb..12d617f2 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,8 @@ go 1.22 require ( cloud.google.com/go/storage v1.43.0 github.com/aliyun/aliyun-oss-go-sdk v2.2.2+incompatible - github.com/aws/aws-sdk-go-v2 v1.16.0 - github.com/aws/aws-sdk-go-v2/config v1.15.1 + github.com/aws/aws-sdk-go-v2 v1.30.4 + github.com/aws/aws-sdk-go-v2/config v1.27.30 github.com/baidubce/bce-sdk-go v0.9.111 github.com/efficientgo/core v1.0.0-rc.0.0.20221201130417-ba593f67d2a4 github.com/efficientgo/e2e v0.13.1-0.20220922081603-45de9fc588a8 @@ -43,15 +43,17 @@ require ( github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.11.0 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.1 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.7 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.1 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.8 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.1 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.11.1 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.16.1 // indirect - github.com/aws/smithy-go v1.11.1 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.29 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.18 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.22.5 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.30.5 // indirect + github.com/aws/smithy-go v1.20.4 // indirect github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bluele/gcache v0.0.2 // indirect diff --git a/go.sum b/go.sum index c010a36e..8ee7b244 100644 --- a/go.sum +++ b/go.sum @@ -33,28 +33,32 @@ github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAu github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/aliyun/aliyun-oss-go-sdk v2.2.2+incompatible h1:9gWa46nstkJ9miBReJcN8Gq34cBFbzSpQZVVT9N09TM= github.com/aliyun/aliyun-oss-go-sdk v2.2.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= -github.com/aws/aws-sdk-go-v2 v1.16.0 h1:cBAYjiiexRAg9v2z9vb6IdxAa7ef4KCtjW7w7e3GxGo= -github.com/aws/aws-sdk-go-v2 v1.16.0/go.mod h1:lJYcuZZEHWNIb6ugJjbQY1fykdoobWbOS7kJYb4APoI= -github.com/aws/aws-sdk-go-v2/config v1.15.1 h1:hTIZFepYESYyowQUBo47lu69WSxsYqGUILY9Nu8+7pY= -github.com/aws/aws-sdk-go-v2/config v1.15.1/go.mod h1:MZHGbuW2WnqIOQQBKu2ZkhTjuutZSTnn56TDq4QyydE= -github.com/aws/aws-sdk-go-v2/credentials v1.11.0 h1:gc4Uhs80s60nmLon5Z4JXWinX2BkAGT0YROoUT8h8U4= -github.com/aws/aws-sdk-go-v2/credentials v1.11.0/go.mod h1:EdV1ZFgtZ4XM5RDHWcRWK8H+xW5duNVBqWj2oLu7tRo= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.1 h1:F9Je1nq5YXfMOv6451NHvMf6U0iTWeMnsG0MMIQoUmk= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.1/go.mod h1:Yph0XsTbQ5GGZ2+mO1a03P/SO9fdX3t1nejIp2tq79g= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.7 h1:KUErSJgdqmqAPBWAp6Zx9CjL0YXfytXJeXcsWnuCM1c= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.7/go.mod h1:oB9nZcxH1cGq7NPGurVJwxrO2vmJ9mmEBayCwcAlmT8= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.1 h1:feVfa9eJonhJiss7g51ikjNB2DrUzbNZNvPL8pw/54k= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.1/go.mod h1:K4vz7lRYCyLYpYAMCLObODahFgARdD3YVa0MvQte9Co= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.8 h1:adr3PfiggFtqgFofAMUFCtdvwzpf3QxPES4ezK4M3iI= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.8/go.mod h1:wLbQYt36AJqaRZUQiCNXzbtkNigyPfKHrotHuIDiCy8= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.1 h1:B/SPX7J+Y0Yrcjv60Nhbh1gC2uBN47SfN8JYre6Mp4M= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.1/go.mod h1:2Hhr9Eh1gJzDatwACX/ozAZ/ljq5vzvPRu5cdu25tzc= -github.com/aws/aws-sdk-go-v2/service/sso v1.11.1 h1:DyHctRsJIAWIvom1Itb4T84D2jwpIu+KIi3d0SFaswg= -github.com/aws/aws-sdk-go-v2/service/sso v1.11.1/go.mod h1:CvFTucADIx7U/M44vjLs/ZttpQHdpxwK+62+dUGhDeY= -github.com/aws/aws-sdk-go-v2/service/sts v1.16.1 h1:xsOtPAvHqhvQvBza5ohaUcfq1LceH2lZKMUGZJKiZiM= -github.com/aws/aws-sdk-go-v2/service/sts v1.16.1/go.mod h1:Aq2/Qggh2oemSfyHH+EO4UBbgWG6zFCXLHYI4ILTY7w= -github.com/aws/smithy-go v1.11.1 h1:IQ+lPZVkSM3FRtyaDox41R8YS6iwPMYIreejOgPW49g= -github.com/aws/smithy-go v1.11.1/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnwub1bgM= +github.com/aws/aws-sdk-go-v2 v1.30.4 h1:frhcagrVNrzmT95RJImMHgabt99vkXGslubDaDagTk8= +github.com/aws/aws-sdk-go-v2 v1.30.4/go.mod h1:CT+ZPWXbYrci8chcARI3OmI/qgd+f6WtuLOoaIA8PR0= +github.com/aws/aws-sdk-go-v2/config v1.27.30 h1:AQF3/+rOgeJBQP3iI4vojlPib5X6eeOYoa/af7OxAYg= +github.com/aws/aws-sdk-go-v2/config v1.27.30/go.mod h1:yxqvuubha9Vw8stEgNiStO+yZpP68Wm9hLmcm+R/Qk4= +github.com/aws/aws-sdk-go-v2/credentials v1.17.29 h1:CwGsupsXIlAFYuDVHv1nnK0wnxO0wZ/g1L8DSK/xiIw= +github.com/aws/aws-sdk-go-v2/credentials v1.17.29/go.mod h1:BPJ/yXV92ZVq6G8uYvbU0gSl8q94UB63nMT5ctNO38g= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12 h1:yjwoSyDZF8Jth+mUk5lSPJCkMC0lMy6FaCD51jm6ayE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12/go.mod h1:fuR57fAgMk7ot3WcNQfb6rSEn+SUffl7ri+aa8uKysI= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16 h1:TNyt/+X43KJ9IJJMjKfa3bNTiZbUP7DeCxfbTROESwY= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16/go.mod h1:2DwJF39FlNAUiX5pAc0UNeiz16lK2t7IaFcm0LFHEgc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16 h1:jYfy8UPmd+6kJW5YhY0L1/KftReOGxI/4NtVSTh9O/I= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16/go.mod h1:7ZfEPZxkW42Afq4uQB8H2E2e6ebh6mXTueEpYzjCzcs= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 h1:KypMCbLPPHEmf9DgMGw51jMj77VfGPAN2Kv4cfhlfgI= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4/go.mod h1:Vz1JQXliGcQktFTN/LN6uGppAIRoLBR2bMvIMP0gOjc= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.18 h1:tJ5RnkHCiSH0jyd6gROjlJtNwov0eGYNz8s8nFcR0jQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.18/go.mod h1:++NHzT+nAF7ZPrHPsA+ENvsXkOO8wEu+C6RXltAG4/c= +github.com/aws/aws-sdk-go-v2/service/sso v1.22.5 h1:zCsFCKvbj25i7p1u94imVoO447I/sFv8qq+lGJhRN0c= +github.com/aws/aws-sdk-go-v2/service/sso v1.22.5/go.mod h1:ZeDX1SnKsVlejeuz41GiajjZpRSWR7/42q/EyA/QEiM= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.5 h1:SKvPgvdvmiTWoi0GAJ7AsJfOz3ngVkD/ERbs5pUnHNI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.5/go.mod h1:20sz31hv/WsPa3HhU3hfrIet2kxM4Pe0r20eBZ20Tac= +github.com/aws/aws-sdk-go-v2/service/sts v1.30.5 h1:OMsEmCyz2i89XwRwPouAJvhj81wINh+4UK+k/0Yo/q8= +github.com/aws/aws-sdk-go-v2/service/sts v1.30.5/go.mod h1:vmSqFK+BVIwVpDAGZB3CoCXHzurt4qBE8lf+I/kRTh0= +github.com/aws/smithy-go v1.20.4 h1:2HK1zBdPgRbjFOHlfeQZfpC4r72MOb9bZkiFwggKO+4= +github.com/aws/smithy-go v1.20.4/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/baidubce/bce-sdk-go v0.9.111 h1:yGgtPpZYUZW4uoVorQ4xnuEgVeddACydlcJKW87MDV4= github.com/baidubce/bce-sdk-go v0.9.111/go.mod h1:zbYJMQwE4IZuyrJiFO8tO8NbtYiKTFTbwh4eIsqjVdg= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA= @@ -134,6 +138,7 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -154,8 +159,6 @@ github.com/googleapis/gax-go/v2 v2.12.5 h1:8gw9KZK8TiVKB6q3zHY3SBzLnrGp6HQjyfYBY github.com/googleapis/gax-go/v2 v2.12.5/go.mod h1:BUDKcWo+RaKq5SC9vVYL0wLADa3VcfswbOMMRmB9H3E= github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.3+incompatible h1:tKTaPHNVwikS3I1rdyf1INNvgJXWSf/+TzqsiGbrgnQ= github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.3+incompatible/go.mod h1:l7VUhRbTKCzdOacdT4oWCwATKyvZqUOlOqr0Ous3k4s= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= @@ -326,7 +329,6 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/inmem.go b/inmem.go index d550e283..6a344066 100644 --- a/inmem.go +++ b/inmem.go @@ -34,6 +34,8 @@ func NewInMemBucket() *InMemBucket { } } +func (b *InMemBucket) Provider() ObjProvider { return MEMORY } + // Objects returns a copy of the internally stored objects. // NOTE: For assert purposes. func (b *InMemBucket) Objects() map[string][]byte { diff --git a/objstore.go b/objstore.go index 33c6e5e8..86ecfa26 100644 --- a/objstore.go +++ b/objstore.go @@ -26,6 +26,22 @@ import ( "golang.org/x/sync/errgroup" ) +type ObjProvider string + +const ( + MEMORY ObjProvider = "MEMORY" + FILESYSTEM ObjProvider = "FILESYSTEM" + GCS ObjProvider = "GCS" + S3 ObjProvider = "S3" + AZURE ObjProvider = "AZURE" + SWIFT ObjProvider = "SWIFT" + COS ObjProvider = "COS" + ALIYUNOSS ObjProvider = "ALIYUNOSS" + BOS ObjProvider = "BOS" + OCI ObjProvider = "OCI" + OBS ObjProvider = "OBS" +) + const ( OpIter = "iter" OpGet = "get" @@ -42,6 +58,8 @@ type Bucket interface { io.Closer BucketReader + Provider() ObjProvider + // Upload the contents of the reader as an object into the bucket. // Upload should be idempotent. Upload(ctx context.Context, name string, r io.Reader) error @@ -583,6 +601,10 @@ type metricBucket struct { metrics *Metrics } +func (b *metricBucket) Provider() ObjProvider { + return b.bkt.Provider() +} + func (b *metricBucket) WithExpectedErrs(fn IsOpFailureExpectedFunc) Bucket { return &metricBucket{ bkt: b.bkt, diff --git a/objtesting/foreach.go b/objtesting/foreach.go index 87d9ed1b..29d16c39 100644 --- a/objtesting/foreach.go +++ b/objtesting/foreach.go @@ -9,7 +9,6 @@ import ( "testing" "github.com/thanos-io/objstore" - "github.com/thanos-io/objstore/client" "github.com/thanos-io/objstore/providers/azure" "github.com/thanos-io/objstore/providers/bos" "github.com/thanos-io/objstore/providers/cos" @@ -26,7 +25,7 @@ import ( // IsObjStoreSkipped returns true if given provider ID is found in THANOS_TEST_OBJSTORE_SKIP array delimited by comma e.g: // THANOS_TEST_OBJSTORE_SKIP=GCS,S3,AZURE,SWIFT,COS,ALIYUNOSS,BOS,OCI. -func IsObjStoreSkipped(t *testing.T, provider client.ObjProvider) bool { +func IsObjStoreSkipped(t *testing.T, provider objstore.ObjProvider) bool { if e, ok := os.LookupEnv("THANOS_TEST_OBJSTORE_SKIP"); ok { obstores := strings.Split(e, ",") for _, objstore := range obstores { @@ -69,7 +68,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket)) }) // Optional GCS. - if !IsObjStoreSkipped(t, client.GCS) { + if !IsObjStoreSkipped(t, objstore.GCS) { t.Run("gcs", func(t *testing.T) { bkt, closeFn, err := gcs.NewTestBucket(t, os.Getenv("GCP_PROJECT")) testutil.Ok(t, err) @@ -84,7 +83,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket)) } // Optional S3. - if !IsObjStoreSkipped(t, client.S3) { + if !IsObjStoreSkipped(t, objstore.S3) { t.Run("aws s3", func(t *testing.T) { // TODO(bwplotka): Allow taking location from envvar. bkt, closeFn, err := s3.NewTestBucket(t, "us-west-2") @@ -103,7 +102,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket)) } // Optional Azure. - if !IsObjStoreSkipped(t, client.AZURE) { + if !IsObjStoreSkipped(t, objstore.AZURE) { t.Run("azure", func(t *testing.T) { bkt, closeFn, err := azure.NewTestBucket(t, "e2e-tests") testutil.Ok(t, err) @@ -117,7 +116,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket)) } // Optional SWIFT. - if !IsObjStoreSkipped(t, client.SWIFT) { + if !IsObjStoreSkipped(t, objstore.SWIFT) { t.Run("swift", func(t *testing.T) { container, closeFn, err := swift.NewTestContainer(t) testutil.Ok(t, err) @@ -131,7 +130,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket)) } // Optional COS. - if !IsObjStoreSkipped(t, client.COS) { + if !IsObjStoreSkipped(t, objstore.COS) { t.Run("Tencent cos", func(t *testing.T) { bkt, closeFn, err := cos.NewTestBucket(t) testutil.Ok(t, err) @@ -145,7 +144,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket)) } // Optional OSS. - if !IsObjStoreSkipped(t, client.ALIYUNOSS) { + if !IsObjStoreSkipped(t, objstore.ALIYUNOSS) { t.Run("AliYun oss", func(t *testing.T) { bkt, closeFn, err := oss.NewTestBucket(t) testutil.Ok(t, err) @@ -159,7 +158,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket)) } // Optional BOS. - if !IsObjStoreSkipped(t, client.BOS) { + if !IsObjStoreSkipped(t, objstore.BOS) { t.Run("Baidu BOS", func(t *testing.T) { bkt, closeFn, err := bos.NewTestBucket(t) testutil.Ok(t, err) @@ -173,7 +172,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket)) } // Optional OCI. - if !IsObjStoreSkipped(t, client.OCI) { + if !IsObjStoreSkipped(t, objstore.OCI) { t.Run("oci", func(t *testing.T) { bkt, closeFn, err := oci.NewTestBucket(t) testutil.Ok(t, err) @@ -186,7 +185,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket)) } // Optional OBS. - if !IsObjStoreSkipped(t, client.OBS) { + if !IsObjStoreSkipped(t, objstore.OBS) { t.Run("obs", func(t *testing.T) { bkt, closeFn, err := obs.NewTestBucket(t, "cn-south-1") testutil.Ok(t, err) diff --git a/prefixed_bucket.go b/prefixed_bucket.go index a76b34c3..a37450ca 100644 --- a/prefixed_bucket.go +++ b/prefixed_bucket.go @@ -39,6 +39,8 @@ func withPrefix(prefix, name string) string { return prefix + DirDelim + name } +func (p *PrefixedBucket) Provider() ObjProvider { return p.bkt.Provider() } + func (p *PrefixedBucket) Close() error { return p.bkt.Close() } @@ -93,7 +95,7 @@ func (p *PrefixedBucket) IsAccessDeniedErr(err error) bool { } // Attributes returns information about the specified object. -func (p PrefixedBucket) Attributes(ctx context.Context, name string) (ObjectAttributes, error) { +func (p *PrefixedBucket) Attributes(ctx context.Context, name string) (ObjectAttributes, error) { return p.bkt.Attributes(ctx, conditionalPrefix(p.prefix, name)) } diff --git a/providers/azure/azure.go b/providers/azure/azure.go index 387de9aa..1fdbd0e1 100644 --- a/providers/azure/azure.go +++ b/providers/azure/azure.go @@ -204,6 +204,8 @@ func NewBucketWithConfig(logger log.Logger, conf Config, component string, wrapR return bkt, nil } +func (b *Bucket) Provider() objstore.ObjProvider { return objstore.AZURE } + func (b *Bucket) SupportedIterOptions() []objstore.IterOptionType { return []objstore.IterOptionType{objstore.Recursive, objstore.UpdatedAt} } diff --git a/providers/bos/bos.go b/providers/bos/bos.go index 20c8dd3e..0cc4352c 100644 --- a/providers/bos/bos.go +++ b/providers/bos/bos.go @@ -100,6 +100,8 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B return bkt, nil } +func (b *Bucket) Provider() objstore.ObjProvider { return objstore.BOS } + // Name returns the bucket name for the provider. func (b *Bucket) Name() string { return b.name diff --git a/providers/cos/cos.go b/providers/cos/cos.go index d81150b1..1851c984 100644 --- a/providers/cos/cos.go +++ b/providers/cos/cos.go @@ -159,6 +159,8 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string, wra return bkt, nil } +func (b *Bucket) Provider() objstore.ObjProvider { return objstore.COS } + // Name returns the bucket name for COS. func (b *Bucket) Name() string { return b.name diff --git a/providers/filesystem/filesystem.go b/providers/filesystem/filesystem.go index 01dca4bb..df602877 100644 --- a/providers/filesystem/filesystem.go +++ b/providers/filesystem/filesystem.go @@ -50,6 +50,8 @@ func NewBucket(rootDir string) (*Bucket, error) { return &Bucket{rootDir: absDir}, nil } +func (b *Bucket) Provider() objstore.ObjProvider { return objstore.FILESYSTEM } + func (b *Bucket) SupportedIterOptions() []objstore.IterOptionType { return []objstore.IterOptionType{objstore.Recursive, objstore.UpdatedAt} } diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index d54a6782..b89f8735 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -186,6 +186,8 @@ func newBucket(ctx context.Context, logger log.Logger, gc Config, opts []option. return bkt, nil } +func (b *Bucket) Provider() objstore.ObjProvider { return objstore.GCS } + // Name returns the bucket name for gcs. func (b *Bucket) Name() string { return b.name diff --git a/providers/obs/obs.go b/providers/obs/obs.go index ecd235c9..20294cc6 100644 --- a/providers/obs/obs.go +++ b/providers/obs/obs.go @@ -122,6 +122,8 @@ func NewBucketWithConfig(logger log.Logger, config Config) (*Bucket, error) { return bkt, nil } +func (b *Bucket) Provider() objstore.ObjProvider { return objstore.OBS } + // Name returns the bucket name for the provider. func (b *Bucket) Name() string { return b.name diff --git a/providers/oci/oci.go b/providers/oci/oci.go index 062da7c1..6d5b6c02 100644 --- a/providers/oci/oci.go +++ b/providers/oci/oci.go @@ -96,6 +96,8 @@ type Bucket struct { requestMetadata common.RequestMetadata } +func (b *Bucket) Provider() objstore.ObjProvider { return objstore.OCI } + // Name returns the bucket name for the provider. func (b *Bucket) Name() string { return b.name @@ -353,7 +355,7 @@ func NewBucket(logger log.Logger, ociConfig []byte, wrapRoundtripper func(http.R return nil, errors.Wrapf(err, "unable to create OKE workload identity config provider") } default: - return nil, errors.Wrapf(err, fmt.Sprintf("unsupported OCI provider: %s", provider)) + return nil, fmt.Errorf("unsupported OCI provider: %s", provider) } client, err := objectstorage.NewObjectStorageClientWithConfigurationProvider(configurationProvider) diff --git a/providers/oss/oss.go b/providers/oss/oss.go index aee8c623..2a6cb219 100644 --- a/providers/oss/oss.go +++ b/providers/oss/oss.go @@ -68,6 +68,8 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) { return NewTestBucketFromConfig(t, c, false) } +func (b *Bucket) Provider() objstore.ObjProvider { return objstore.ALIYUNOSS } + // Upload the contents of the reader as an object into the bucket. func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) error { // TODO(https://github.com/thanos-io/thanos/issues/678): Remove guessing length when minio provider will support multipart upload without this. diff --git a/providers/s3/s3.go b/providers/s3/s3.go index 27e82ffb..58590486 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -345,6 +345,8 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string, wra return bkt, nil } +func (b *Bucket) Provider() objstore.ObjProvider { return objstore.S3 } + // Name returns the bucket name for s3. func (b *Bucket) Name() string { return b.name diff --git a/providers/swift/swift.go b/providers/swift/swift.go index 86caa0c1..19eb0d45 100644 --- a/providers/swift/swift.go +++ b/providers/swift/swift.go @@ -218,6 +218,8 @@ func NewContainerFromConfig(logger log.Logger, sc *Config, createContainer bool, }, nil } +func (c *Container) Provider() objstore.ObjProvider { return objstore.SWIFT } + // Name returns the container name for swift. func (c *Container) Name() string { return c.name diff --git a/scripts/cfggen/main.go b/scripts/cfggen/main.go index 424bf9b0..e1bc6410 100644 --- a/scripts/cfggen/main.go +++ b/scripts/cfggen/main.go @@ -5,6 +5,7 @@ package main import ( "fmt" + "github.com/thanos-io/objstore" "io" "os" "path/filepath" @@ -35,17 +36,17 @@ var ( configs map[string]interface{} possibleValues []string - bucketConfigs = map[client.ObjProvider]interface{}{ - client.AZURE: azure.Config{}, - client.GCS: gcs.Config{}, - client.S3: s3.DefaultConfig, - client.SWIFT: swift.DefaultConfig, - client.COS: cos.DefaultConfig, - client.ALIYUNOSS: oss.Config{}, - client.FILESYSTEM: filesystem.Config{}, - client.BOS: bos.Config{}, - client.OCI: oci.Config{}, - client.OBS: obs.DefaultConfig, + bucketConfigs = map[objstore.ObjProvider]interface{}{ + objstore.AZURE: azure.Config{}, + objstore.GCS: gcs.Config{}, + objstore.S3: s3.DefaultConfig, + objstore.SWIFT: swift.DefaultConfig, + objstore.COS: cos.DefaultConfig, + objstore.ALIYUNOSS: oss.Config{}, + objstore.FILESYSTEM: filesystem.Config{}, + objstore.BOS: bos.Config{}, + objstore.OCI: oci.Config{}, + objstore.OBS: obs.DefaultConfig, } ) diff --git a/testing.go b/testing.go index d3fa1def..80f1e198 100644 --- a/testing.go +++ b/testing.go @@ -280,6 +280,8 @@ func WithDelay(bkt Bucket, delay time.Duration) Bucket { return &delayingBucket{bkt: bkt, delay: delay} } +func (d *delayingBucket) Provider() ObjProvider { return d.bkt.Provider() } + func (d *delayingBucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { time.Sleep(d.delay) return d.bkt.Get(ctx, name) diff --git a/tracing/opentelemetry/opentelemetry.go b/tracing/opentelemetry/opentelemetry.go index dad71e39..d5f9bd8c 100644 --- a/tracing/opentelemetry/opentelemetry.go +++ b/tracing/opentelemetry/opentelemetry.go @@ -23,6 +23,8 @@ func WrapWithTraces(bkt objstore.Bucket, tracer trace.Tracer) objstore.Instrumen return TracingBucket{tracer: tracer, bkt: bkt} } +func (t TracingBucket) Provider() objstore.ObjProvider { return t.bkt.Provider() } + func (t TracingBucket) Iter(ctx context.Context, dir string, f func(string) error, options ...objstore.IterOption) (err error) { ctx, span := t.tracer.Start(ctx, "bucket_iter") defer span.End() diff --git a/tracing/opentracing/opentracing.go b/tracing/opentracing/opentracing.go index cabe07b2..58bdea07 100644 --- a/tracing/opentracing/opentracing.go +++ b/tracing/opentracing/opentracing.go @@ -44,6 +44,8 @@ func WrapWithTraces(bkt objstore.Bucket) objstore.InstrumentedBucket { return TracingBucket{bkt: bkt} } +func (t TracingBucket) Provider() objstore.ObjProvider { return t.bkt.Provider() } + func (t TracingBucket) Iter(ctx context.Context, dir string, f func(string) error, options ...objstore.IterOption) (err error) { doWithSpan(ctx, "bucket_iter", func(spanCtx context.Context, span opentracing.Span) { span.LogKV("dir", dir)