From f06322381f09b1e031fc464eacf79b7b5106d9ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Wed, 28 Aug 2024 17:47:49 +0300 Subject: [PATCH 01/41] go.mod: update gRPC dependency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update gRPC dependency to a version that includes memory pooling. Remove old methods in `gcs` as pooling is always enabled now. Signed-off-by: Giedrius Statkevičius --- .github/workflows/docs.yaml | 2 +- go.mod | 16 +++++------ go.sum | 55 ++++++++++--------------------------- providers/gcs/gcs.go | 3 -- 4 files changed, 22 insertions(+), 54 deletions(-) diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 6be30e4d..98f16bba 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -20,7 +20,7 @@ jobs: - name: Install Go uses: actions/setup-go@v2 with: - go-version: 1.20.x + go-version: 1.21.x - uses: actions/cache@v1 with: diff --git a/go.mod b/go.mod index c76fa357..4ec29639 100644 --- a/go.mod +++ b/go.mod @@ -25,18 +25,17 @@ require ( go.opentelemetry.io/otel v1.24.0 go.opentelemetry.io/otel/trace v1.24.0 go.uber.org/atomic v1.9.0 - golang.org/x/oauth2 v0.18.0 + golang.org/x/oauth2 v0.21.0 golang.org/x/sync v0.7.0 google.golang.org/api v0.172.0 - google.golang.org/grpc v1.62.1 + google.golang.org/grpc v1.66.0 gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/yaml.v2 v2.4.0 ) require ( cloud.google.com/go v0.112.1 // indirect - cloud.google.com/go/compute v1.24.0 // indirect - cloud.google.com/go/compute/metadata v0.2.3 // indirect + cloud.google.com/go/compute/metadata v0.3.0 // indirect cloud.google.com/go/iam v1.1.7 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect @@ -54,7 +53,7 @@ require ( github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bluele/gcache v0.0.2 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/clbanning/mxj v1.8.4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect @@ -95,11 +94,10 @@ require ( golang.org/x/sys v0.21.0 // indirect golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.5.0 // indirect - google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect - google.golang.org/protobuf v1.33.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect + google.golang.org/protobuf v1.34.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) diff --git a/go.sum b/go.sum index f111af51..f2ecc7c5 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.112.1 h1:uJSeirPke5UNZHIb4SxfZklVSiWWVqW4oXlETwZziwM= cloud.google.com/go v0.112.1/go.mod h1:+Vbu+Y1UU+I1rjmzeMOb/8RfkKJK2Gyxi1X6jJCZLo4= -cloud.google.com/go/compute v1.24.0 h1:phWcR2eWzRJaL/kOiJwfFsPs4BaKq1j6vnpZrc1YlVg= -cloud.google.com/go/compute v1.24.0/go.mod h1:kw1/T+h/+tK2LJK0wiPPx1intgdAM3j/g3hFDlscY40= -cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= -cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= +cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/iam v1.1.7 h1:z4VHOhwKLF/+UYXAJDFwGtNF0b6gjsW1Pk9Ml0U/IoM= cloud.google.com/go/iam v1.1.7/go.mod h1:J4PMPg8TtyurAUvSmPj8FF3EDgY1SPRZxcUGrn7WXGA= cloud.google.com/go/storage v1.40.0 h1:VEpDQV5CJxFmJ6ueWNsKxcr1QAYOXEgxDa+sBbJahPw= @@ -60,8 +58,8 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I= github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -117,8 +115,6 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= @@ -130,7 +126,6 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -223,7 +218,6 @@ github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.194/go.mod github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.194/go.mod h1:yrBKWhChnDqNz1xuXdSbWXG56XawEq0G5j1lg4VwBD4= github.com/tencentyun/cos-go-sdk-v5 v0.7.40 h1:W6vDGKCHe4wBACI1d2UgE6+50sJFhRWU4O8IB2ozzxM= github.com/tencentyun/cos-go-sdk-v5 v0.7.40/go.mod h1:4dCEtLHGh8QPxHEkgq+nFaky7yZxQuYwgSJM87icDaw= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= @@ -242,54 +236,40 @@ go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= -golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= @@ -299,9 +279,6 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= @@ -309,24 +286,22 @@ google.golang.org/api v0.172.0 h1:/1OcMZGPmW1rX2LCu2CmGUD1KXK1+pfzxotxyRUCCdk= google.golang.org/api v0.172.0/go.mod h1:+fJZq6QXWfa9pXhnIzsjx4yI22d4aI9ZpLb58gvXjis= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= -google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 h1:9+tzLLstTlPTRyJTh+ah5wIMsBW5c4tQwGTN3thOW9Y= google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:mqHbVIp48Muh7Ywss/AD6I5kNVKZMmAa/QEW58Gxp2s= -google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c h1:kaI7oewGK5YnVwj+Y+EJBO/YN1ht8iTL9XkFHtVZLsc= -google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c/go.mod h1:VQW3tUculP/D4B+xVCo+VgSq8As6wA9ZjHl//pmk+6s= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 h1:+rdxYoE3E5htTEWIe15GlN6IfvbURM//Jt0mmkmm6ZU= +google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117/go.mod h1:OimBR/bc1wPO9iV4NC2bpyjy3VnAwZh5EBPQdtaE5oo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 h1:1GBuWVLM/KMVUv1t1En5Gs+gFZCNd360GGb4sSxtrhU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= -google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= +google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= +google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -336,10 +311,8 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index a697e597..87e31f66 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -22,9 +22,7 @@ import ( "google.golang.org/api/iterator" "google.golang.org/api/option" htransport "google.golang.org/api/transport/http" - "google.golang.org/grpc" "google.golang.org/grpc/codes" - "google.golang.org/grpc/experimental" "google.golang.org/grpc/status" "gopkg.in/yaml.v2" @@ -155,7 +153,6 @@ func newBucket(ctx context.Context, logger log.Logger, gc Config, opts []option. ) if gc.UseGRPC { opts = append(opts, - option.WithGRPCDialOption(experimental.WithRecvBufferPool(grpc.NewSharedBufferPool())), option.WithGRPCConnectionPool(gc.GRPCConnPoolSize), ) gcsClient, err = storage.NewGRPCClient(ctx, opts...) From ef1de0fb0857cf8bf9d64fce5fa7efcedc70a1a8 Mon Sep 17 00:00:00 2001 From: Filip Petkovski Date: Tue, 11 Jun 2024 08:30:48 +0200 Subject: [PATCH 02/41] Add support for WriterTo in TimingReader This commit adds support for the WriterTo interface in the TimingReader so that it can be used efficiently in io.Copy. Signed-off-by: Filip Petkovski --- objstore.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/objstore.go b/objstore.go index 87ec9e98..924b2948 100644 --- a/objstore.go +++ b/objstore.go @@ -728,7 +728,7 @@ func newTimingReader(r io.Reader, closeReader bool, op string, dur *prometheus.H _, isSeeker := r.(io.Seeker) _, isReaderAt := r.(io.ReaderAt) - + _, isWriterTo := r.(io.WriterTo) if isSeeker && isReaderAt { // The assumption is that in most cases when io.ReaderAt() is implemented then // io.Seeker is implemented too (e.g. os.File). @@ -737,6 +737,9 @@ func newTimingReader(r io.Reader, closeReader bool, op string, dur *prometheus.H if isSeeker { return &timingReaderSeeker{timingReader: trc} } + if isWriterTo { + return &timingReaderWriterTo{timingReader: trc} + } return &trc } @@ -802,3 +805,11 @@ type timingReaderSeekerReaderAt struct { func (rsc *timingReaderSeekerReaderAt) ReadAt(p []byte, off int64) (int, error) { return (rsc.Reader).(io.ReaderAt).ReadAt(p, off) } + +type timingReaderWriterTo struct { + timingReader +} + +func (t *timingReaderWriterTo) WriteTo(w io.Writer) (n int64, err error) { + return (t.Reader).(io.WriterTo).WriteTo(w) +} From 46e7fe0322a083d4a520f2c80a8562794ef8380c Mon Sep 17 00:00:00 2001 From: Filip Petkovski Date: Tue, 11 Jun 2024 09:44:21 +0200 Subject: [PATCH 03/41] Fix metrics Signed-off-by: Filip Petkovski --- objstore.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/objstore.go b/objstore.go index 924b2948..a9bffca7 100644 --- a/objstore.go +++ b/objstore.go @@ -728,7 +728,6 @@ func newTimingReader(r io.Reader, closeReader bool, op string, dur *prometheus.H _, isSeeker := r.(io.Seeker) _, isReaderAt := r.(io.ReaderAt) - _, isWriterTo := r.(io.WriterTo) if isSeeker && isReaderAt { // The assumption is that in most cases when io.ReaderAt() is implemented then // io.Seeker is implemented too (e.g. os.File). @@ -737,7 +736,7 @@ func newTimingReader(r io.Reader, closeReader bool, op string, dur *prometheus.H if isSeeker { return &timingReaderSeeker{timingReader: trc} } - if isWriterTo { + if _, isWriterTo := r.(io.WriterTo); isWriterTo { return &timingReaderWriterTo{timingReader: trc} } @@ -775,11 +774,16 @@ func (r *timingReader) Close() error { func (r *timingReader) Read(b []byte) (n int, err error) { n, err = r.Reader.Read(b) + r.updateMetrics(n, err) + return n, err +} + +func (r *timingReader) updateMetrics(n int, err error) { if r.fetchedBytes != nil { r.fetchedBytes.WithLabelValues(r.op).Add(float64(n)) } - r.readBytes += int64(n) + // Report metric just once. if !r.alreadyGotErr && err != nil && err != io.EOF { if !r.isFailureExpected(err) && !errors.Is(err, context.Canceled) { @@ -787,7 +791,6 @@ func (r *timingReader) Read(b []byte) (n int, err error) { } r.alreadyGotErr = true } - return n, err } type timingReaderSeeker struct { @@ -811,5 +814,7 @@ type timingReaderWriterTo struct { } func (t *timingReaderWriterTo) WriteTo(w io.Writer) (n int64, err error) { - return (t.Reader).(io.WriterTo).WriteTo(w) + n, err = (t.Reader).(io.WriterTo).WriteTo(w) + t.timingReader.updateMetrics(int(n), err) + return n, err } From 1a62b9e910bfa2477710e42df0374ad36f50bfdc Mon Sep 17 00:00:00 2001 From: Michael Hoffmann Date: Thu, 12 Sep 2024 10:06:32 +0200 Subject: [PATCH 04/41] fix: timing reader ignores initial get_range Some objstore providers like s3 do one initial read that already talks to object storage, this read might fail due to timeout, which is invisible in the current instrumentation since only subsequent reads are instrumented. This PR fixes this. Signed-off-by: Michael Hoffmann --- objstore.go | 15 +++++++++++++-- objstore_test.go | 11 ++++++----- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/objstore.go b/objstore.go index a9bffca7..bfee7503 100644 --- a/objstore.go +++ b/objstore.go @@ -566,14 +566,18 @@ func (b *metricBucket) Get(ctx context.Context, name string) (io.ReadCloser, err const op = OpGet b.metrics.ops.WithLabelValues(op).Inc() + start := time.Now() + rc, err := b.bkt.Get(ctx, name) if err != nil { if !b.metrics.isOpFailureExpected(err) && ctx.Err() != context.Canceled { b.metrics.opsFailures.WithLabelValues(op).Inc() } + b.metrics.opsDuration.WithLabelValues(op).Observe(float64(time.Since(start))) return nil, err } return newTimingReader( + start, rc, true, op, @@ -589,14 +593,18 @@ func (b *metricBucket) GetRange(ctx context.Context, name string, off, length in const op = OpGetRange b.metrics.ops.WithLabelValues(op).Inc() + start := time.Now() + rc, err := b.bkt.GetRange(ctx, name, off, length) if err != nil { if !b.metrics.isOpFailureExpected(err) && ctx.Err() != context.Canceled { b.metrics.opsFailures.WithLabelValues(op).Inc() } + b.metrics.opsDuration.WithLabelValues(op).Observe(float64(time.Since(start))) return nil, err } return newTimingReader( + start, rc, true, op, @@ -628,7 +636,10 @@ func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) err const op = OpUpload b.metrics.ops.WithLabelValues(op).Inc() + start := time.Now() + trc := newTimingReader( + start, r, false, op, @@ -705,7 +716,7 @@ type timingReader struct { transferredBytes *prometheus.HistogramVec } -func newTimingReader(r io.Reader, closeReader bool, op string, dur *prometheus.HistogramVec, failed *prometheus.CounterVec, isFailureExpected IsOpFailureExpectedFunc, fetchedBytes *prometheus.CounterVec, transferredBytes *prometheus.HistogramVec) io.ReadCloser { +func newTimingReader(start time.Time, r io.Reader, closeReader bool, op string, dur *prometheus.HistogramVec, failed *prometheus.CounterVec, isFailureExpected IsOpFailureExpectedFunc, fetchedBytes *prometheus.CounterVec, transferredBytes *prometheus.HistogramVec) io.ReadCloser { // Initialize the metrics with 0. dur.WithLabelValues(op) failed.WithLabelValues(op) @@ -716,7 +727,7 @@ func newTimingReader(r io.Reader, closeReader bool, op string, dur *prometheus.H closeReader: closeReader, objSize: objSize, objSizeErr: objSizeErr, - start: time.Now(), + start: start, op: op, duration: dur, failed: failed, diff --git a/objstore_test.go b/objstore_test.go index b62858fb..51486d23 100644 --- a/objstore_test.go +++ b/objstore_test.go @@ -11,6 +11,7 @@ import ( "path/filepath" "strings" "testing" + "time" "github.com/efficientgo/core/testutil" "github.com/go-kit/log" @@ -412,7 +413,7 @@ func TestDownloadUploadDirConcurrency(t *testing.T) { func TestTimingReader(t *testing.T) { m := WrapWithMetrics(NewInMemBucket(), nil, "") r := bytes.NewReader([]byte("hello world")) - tr := newTimingReader(r, true, OpGet, m.metrics.opsDuration, m.metrics.opsFailures, func(err error) bool { + tr := newTimingReader(time.Now(), r, true, OpGet, m.metrics.opsDuration, m.metrics.opsFailures, func(err error) bool { return false }, m.metrics.opsFetchedBytes, m.metrics.opsTransferredBytes) @@ -447,7 +448,7 @@ func TestTimingReader_ExpectedError(t *testing.T) { m := WrapWithMetrics(NewInMemBucket(), nil, "") r := dummyReader{readerErr} - tr := newTimingReader(r, true, OpGet, m.metrics.opsDuration, m.metrics.opsFailures, func(err error) bool { return errors.Is(err, readerErr) }, m.metrics.opsFetchedBytes, m.metrics.opsTransferredBytes) + tr := newTimingReader(time.Now(), r, true, OpGet, m.metrics.opsDuration, m.metrics.opsFailures, func(err error) bool { return errors.Is(err, readerErr) }, m.metrics.opsFetchedBytes, m.metrics.opsTransferredBytes) buf := make([]byte, 1) _, err := io.ReadFull(tr, buf) @@ -461,7 +462,7 @@ func TestTimingReader_UnexpectedError(t *testing.T) { m := WrapWithMetrics(NewInMemBucket(), nil, "") r := dummyReader{readerErr} - tr := newTimingReader(r, true, OpGet, m.metrics.opsDuration, m.metrics.opsFailures, func(err error) bool { return false }, m.metrics.opsFetchedBytes, m.metrics.opsTransferredBytes) + tr := newTimingReader(time.Now(), r, true, OpGet, m.metrics.opsDuration, m.metrics.opsFailures, func(err error) bool { return false }, m.metrics.opsFetchedBytes, m.metrics.opsTransferredBytes) buf := make([]byte, 1) _, err := io.ReadFull(tr, buf) @@ -476,7 +477,7 @@ func TestTimingReader_ContextCancellation(t *testing.T) { m := WrapWithMetrics(NewInMemBucket(), nil, "") r := dummyReader{ctx.Err()} - tr := newTimingReader(r, true, OpGet, m.metrics.opsDuration, m.metrics.opsFailures, func(err error) bool { return false }, m.metrics.opsFetchedBytes, m.metrics.opsTransferredBytes) + tr := newTimingReader(time.Now(), r, true, OpGet, m.metrics.opsDuration, m.metrics.opsFailures, func(err error) bool { return false }, m.metrics.opsFetchedBytes, m.metrics.opsTransferredBytes) buf := make([]byte, 1) _, err := io.ReadFull(tr, buf) @@ -506,7 +507,7 @@ func TestTimingReader_ShouldCorrectlyWrapFile(t *testing.T) { }) m := WrapWithMetrics(NewInMemBucket(), nil, "") - r := newTimingReader(file, true, "", m.metrics.opsDuration, m.metrics.opsFailures, func(err error) bool { + r := newTimingReader(time.Now(), file, true, "", m.metrics.opsDuration, m.metrics.opsFailures, func(err error) bool { return false }, m.metrics.opsFetchedBytes, m.metrics.opsTransferredBytes) From f735308ff69b039c35a24d342e13e402143bf5f6 Mon Sep 17 00:00:00 2001 From: Michael Hoffmann Date: Thu, 12 Sep 2024 20:16:07 +0200 Subject: [PATCH 05/41] s3: stat object instead of empty read Minio Client issues a HEAD request against the object which for some object storage providers might be easier then a GET with Range header set to zero bytes. Signed-off-by: Michael Hoffmann --- providers/s3/s3.go | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/providers/s3/s3.go b/providers/s3/s3.go index dad89e66..afd0982d 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -15,7 +15,6 @@ import ( "strings" "testing" - "github.com/efficientgo/core/logerrcapture" "github.com/go-kit/log" "github.com/go-kit/log/level" "github.com/minio/minio-go/v7" @@ -438,21 +437,12 @@ func (b *Bucket) getRange(ctx context.Context, name string, off, length int64) ( return nil, err } } - r, err := b.client.GetObject(ctx, b.name, name, *opts) - if err != nil { - return nil, err - } - // NotFoundObject error is revealed only after first Read. This does the initial GetRequest. Prefetch this here - // for convenience. - if _, err := r.Read(nil); err != nil { - defer logerrcapture.Do(b.logger, r.Close, "s3 get range obj close") - - // First GET Object request error. + // StatObject to see if the object exists and we have permissions to read it + if _, err := b.client.StatObject(ctx, b.name, name, *opts); err != nil { return nil, err } - - return r, nil + return b.client.GetObject(ctx, b.name, name, *opts) } // Get returns a reader for the given object name. From ba86bbae5a4a23a33f47717d53ad19460432c33a Mon Sep 17 00:00:00 2001 From: Filip Petkovski Date: Fri, 13 Sep 2024 11:14:14 -0400 Subject: [PATCH 06/41] Update the GCS client to v1.43.0 The commit updates the GCS client to the latest released version. Signed-off-by: Filip Petkovski --- CHANGELOG.md | 1 + go.mod | 20 ++++++++++--------- go.sum | 56 ++++++++++++++++++++++++++++------------------------ 3 files changed, 42 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6aae677a..12375cbf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan We use *breaking :warning:* to mark changes that are not backward compatible (relates only to v0.y.z releases.) ## Unreleased +- [#38](https://github.com/thanos-io/objstore/pull/38) GCS: Upgrade cloud.google.com/go/storage version to `v1.43.0`. ### Fixed - [#117](https://github.com/thanos-io/objstore/pull/117) Metrics: Fix `objstore_bucket_operation_failures_total` incorrectly incremented if context is cancelled while reading object contents. diff --git a/go.mod b/go.mod index 4ec29639..cb95d387 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/thanos-io/objstore go 1.21 require ( - cloud.google.com/go/storage v1.40.0 + cloud.google.com/go/storage v1.43.0 github.com/aliyun/aliyun-oss-go-sdk v2.2.2+incompatible github.com/aws/aws-sdk-go-v2 v1.16.0 github.com/aws/aws-sdk-go-v2/config v1.15.1 @@ -27,16 +27,18 @@ require ( go.uber.org/atomic v1.9.0 golang.org/x/oauth2 v0.21.0 golang.org/x/sync v0.7.0 - google.golang.org/api v0.172.0 + google.golang.org/api v0.187.0 google.golang.org/grpc v1.66.0 gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/yaml.v2 v2.4.0 ) require ( - cloud.google.com/go v0.112.1 // indirect + cloud.google.com/go v0.115.0 // indirect + cloud.google.com/go/auth v0.6.1 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect cloud.google.com/go/compute/metadata v0.3.0 // indirect - cloud.google.com/go/iam v1.1.7 // indirect + cloud.google.com/go/iam v1.1.8 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect @@ -71,7 +73,7 @@ require ( github.com/google/s2a-go v0.1.7 // indirect github.com/google/uuid v1.6.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect - github.com/googleapis/gax-go/v2 v2.12.3 // indirect + github.com/googleapis/gax-go/v2 v2.12.5 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/klauspost/compress v1.17.9 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect @@ -94,10 +96,10 @@ require ( golang.org/x/sys v0.21.0 // indirect golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.5.0 // indirect - google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect - google.golang.org/protobuf v1.34.1 // indirect + google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) diff --git a/go.sum b/go.sum index f2ecc7c5..0b5b860b 100644 --- a/go.sum +++ b/go.sum @@ -1,12 +1,18 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.112.1 h1:uJSeirPke5UNZHIb4SxfZklVSiWWVqW4oXlETwZziwM= -cloud.google.com/go v0.112.1/go.mod h1:+Vbu+Y1UU+I1rjmzeMOb/8RfkKJK2Gyxi1X6jJCZLo4= +cloud.google.com/go v0.115.0 h1:CnFSK6Xo3lDYRoBKEcAtia6VSC837/ZkJuRduSFnr14= +cloud.google.com/go v0.115.0/go.mod h1:8jIM5vVgoAEoiVxQ/O4BFTfHqulPZgs/ufEzMcFMdWU= +cloud.google.com/go/auth v0.6.1 h1:T0Zw1XM5c1GlpN2HYr2s+m3vr1p2wy+8VN+Z1FKxW38= +cloud.google.com/go/auth v0.6.1/go.mod h1:eFHG7zDzbXHKmjJddFG/rBlcGp6t25SwRUiEQSlO4x4= +cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKFt//oWu7HX4= +cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q= cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= -cloud.google.com/go/iam v1.1.7 h1:z4VHOhwKLF/+UYXAJDFwGtNF0b6gjsW1Pk9Ml0U/IoM= -cloud.google.com/go/iam v1.1.7/go.mod h1:J4PMPg8TtyurAUvSmPj8FF3EDgY1SPRZxcUGrn7WXGA= -cloud.google.com/go/storage v1.40.0 h1:VEpDQV5CJxFmJ6ueWNsKxcr1QAYOXEgxDa+sBbJahPw= -cloud.google.com/go/storage v1.40.0/go.mod h1:Rrj7/hKlG87BLqDJYtwR0fbPld8uJPbQ2ucUMY7Ir0g= +cloud.google.com/go/iam v1.1.8 h1:r7umDwhj+BQyz0ScZMp4QrGXjSTI3ZINnpgU2nlB/K0= +cloud.google.com/go/iam v1.1.8/go.mod h1:GvE6lyMmfxXauzNq8NbgJbeVQNspG+tcdL/W8QO1+zE= +cloud.google.com/go/longrunning v0.5.7 h1:WLbHekDbjK1fVFD3ibpFFVoyizlLRl73I7YKuAKilhU= +cloud.google.com/go/longrunning v0.5.7/go.mod h1:8GClkudohy1Fxm3owmBGid8W0pSgodEMwEAztp38Xng= +cloud.google.com/go/storage v1.43.0 h1:CcxnSohZwizt4LCzQHWvBf1/kvtHUn7gk9QERXPyXFs= +cloud.google.com/go/storage v1.43.0/go.mod h1:ajvxEa7WmZS1PxvKRq4bq0tFT3vMd502JwstCcYv0Q0= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2 h1:c4k2FIYIh4xtwqrQwV0Ct1v5+ehlNXj5NI/MWVsiTkQ= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2/go.mod h1:5FDJtLEO/GxwNgUxbwrY3LP0pEoThTQJtk2oysdXHxM= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 h1:sO0/P7g68FrryJzljemN+6GTssUXdANk6aJ7T1ZxnsQ= @@ -132,8 +138,8 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= -github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= -github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -142,8 +148,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= -github.com/googleapis/gax-go/v2 v2.12.3 h1:5/zPPDvw8Q1SuXjrqrZslrqT7dL/uJT2CQii/cLCKqA= -github.com/googleapis/gax-go/v2 v2.12.3/go.mod h1:AKloxT6GtNbaLm8QTNSidHUVsHYcBHwWRvkNFJUQcS4= +github.com/googleapis/gax-go/v2 v2.12.5 h1:8gw9KZK8TiVKB6q3zHY3SBzLnrGp6HQjyfYBYGmXdxA= +github.com/googleapis/gax-go/v2 v2.12.5/go.mod h1:BUDKcWo+RaKq5SC9vVYL0wLADa3VcfswbOMMRmB9H3E= github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.3+incompatible h1:tKTaPHNVwikS3I1rdyf1INNvgJXWSf/+TzqsiGbrgnQ= github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.3+incompatible/go.mod h1:l7VUhRbTKCzdOacdT4oWCwATKyvZqUOlOqr0Ous3k4s= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= @@ -212,8 +218,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.194/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.194/go.mod h1:yrBKWhChnDqNz1xuXdSbWXG56XawEq0G5j1lg4VwBD4= github.com/tencentyun/cos-go-sdk-v5 v0.7.40 h1:W6vDGKCHe4wBACI1d2UgE6+50sJFhRWU4O8IB2ozzxM= @@ -228,8 +234,8 @@ go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= -go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw= -go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc= +go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= +go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= @@ -280,21 +286,19 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= -google.golang.org/api v0.172.0 h1:/1OcMZGPmW1rX2LCu2CmGUD1KXK1+pfzxotxyRUCCdk= -google.golang.org/api v0.172.0/go.mod h1:+fJZq6QXWfa9pXhnIzsjx4yI22d4aI9ZpLb58gvXjis= +google.golang.org/api v0.187.0 h1:Mxs7VATVC2v7CY+7Xwm4ndkX71hpElcvx0D1Ji/p1eo= +google.golang.org/api v0.187.0/go.mod h1:KIHlTc4x7N7gKKuVsdmfBXN13yEEWXWFURWY6SBp2gk= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 h1:9+tzLLstTlPTRyJTh+ah5wIMsBW5c4tQwGTN3thOW9Y= -google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:mqHbVIp48Muh7Ywss/AD6I5kNVKZMmAa/QEW58Gxp2s= -google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 h1:+rdxYoE3E5htTEWIe15GlN6IfvbURM//Jt0mmkmm6ZU= -google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117/go.mod h1:OimBR/bc1wPO9iV4NC2bpyjy3VnAwZh5EBPQdtaE5oo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 h1:1GBuWVLM/KMVUv1t1En5Gs+gFZCNd360GGb4sSxtrhU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d h1:PksQg4dV6Sem3/HkBX+Ltq8T0ke0PKIRBNBatoDTVls= +google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d/go.mod h1:s7iA721uChleev562UJO2OYB0PPT9CMFjV+Ce7VJH5M= +google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 h1:MuYw1wJzT+ZkybKfaOXKp5hJiZDn2iHaXRw0mRYdHSc= +google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4/go.mod h1:px9SlOOZBg1wM1zdnr8jEL4CNGUBZ+ZKYtNPApNQc4c= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d h1:k3zyW3BYYR30e8v3x0bTDdE9vpYFjZHK+HcyqkrppWk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= @@ -311,8 +315,8 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= -google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 03fd703d46f5cdfc2c31bcf02b09a93fad656615 Mon Sep 17 00:00:00 2001 From: milinddethe15 Date: Fri, 13 Sep 2024 21:42:18 +0530 Subject: [PATCH 07/41] add roundtripper param in NewBucket Signed-off-by: milinddethe15 --- client/factory.go | 19 +++++++++-------- client/factory_test.go | 6 +++--- go.mod | 3 +++ go.sum | 2 ++ providers/azure/azure.go | 12 +++++++---- providers/azure/azure_test.go | 27 +++++++++++++++++++++++- providers/azure/helpers.go | 17 +++++++++++---- providers/cos/cos.go | 27 +++++++++++++++++------- providers/cos/cos_test.go | 31 ++++++++++++++++++++++++++++ providers/gcs/gcs.go | 12 ++++++----- providers/gcs/gcs_test.go | 25 +++++++++++++++++++++- providers/obs/obs.go | 28 ++++++++++++++++--------- providers/oci/oci.go | 26 ++++++++++++++--------- providers/oss/oss.go | 22 +++++++++++++------- providers/s3/s3.go | 20 ++++++++++-------- providers/s3/s3_e2e_test.go | 1 + providers/s3/s3_test.go | 39 ++++++++++++++++++++++++++++------- providers/swift/swift.go | 22 +++++++++++--------- providers/swift/swift_test.go | 24 +++++++++++++++++++++ 19 files changed, 275 insertions(+), 88 deletions(-) diff --git a/client/factory.go b/client/factory.go index 9e1adec9..cfecd67c 100644 --- a/client/factory.go +++ b/client/factory.go @@ -6,6 +6,7 @@ package client import ( "context" "fmt" + "net/http" "strings" "github.com/thanos-io/objstore" @@ -49,7 +50,7 @@ type BucketConfig struct { // NewBucket initializes and returns new object storage clients. // NOTE: confContentYaml can contain secrets. -func NewBucket(logger log.Logger, confContentYaml []byte, component string) (objstore.Bucket, error) { +func NewBucket(logger log.Logger, confContentYaml []byte, component string, rt http.RoundTripper) (objstore.Bucket, error) { level.Info(logger).Log("msg", "loading bucket configuration") bucketConf := &BucketConfig{} if err := yaml.UnmarshalStrict(confContentYaml, bucketConf); err != nil { @@ -64,25 +65,25 @@ func NewBucket(logger log.Logger, confContentYaml []byte, component string) (obj var bucket objstore.Bucket switch strings.ToUpper(string(bucketConf.Type)) { case string(GCS): - bucket, err = gcs.NewBucket(context.Background(), logger, config, component) + bucket, err = gcs.NewBucket(context.Background(), logger, config, component, rt) case string(S3): - bucket, err = s3.NewBucket(logger, config, component) + bucket, err = s3.NewBucket(logger, config, component, rt) case string(AZURE): - bucket, err = azure.NewBucket(logger, config, component) + bucket, err = azure.NewBucket(logger, config, component, rt) case string(SWIFT): - bucket, err = swift.NewContainer(logger, config) + bucket, err = swift.NewContainer(logger, config, rt) case string(COS): - bucket, err = cos.NewBucket(logger, config, component) + bucket, err = cos.NewBucket(logger, config, component, rt) case string(ALIYUNOSS): - bucket, err = oss.NewBucket(logger, config, component) + bucket, err = oss.NewBucket(logger, config, component, rt) case string(FILESYSTEM): bucket, err = filesystem.NewBucketFromConfig(config) case string(BOS): bucket, err = bos.NewBucket(logger, config, component) case string(OCI): - bucket, err = oci.NewBucket(logger, config) + bucket, err = oci.NewBucket(logger, config, rt) case string(OBS): - bucket, err = obs.NewBucket(logger, config) + bucket, err = obs.NewBucket(logger, config, rt) default: return nil, errors.Errorf("bucket with type %s is not supported", bucketConf.Type) } diff --git a/client/factory_test.go b/client/factory_test.go index 4a9cf879..cd008add 100644 --- a/client/factory_test.go +++ b/client/factory_test.go @@ -23,7 +23,7 @@ func ExampleBucket() { } // Create a new bucket. - bucket, err := NewBucket(log.NewNopLogger(), confContentYaml, "example") + bucket, err := NewBucket(log.NewNopLogger(), confContentYaml, "example", nil) if err != nil { panic(err) } @@ -46,7 +46,7 @@ func ExampleTracingBucketUsingOpenTracing() { //nolint:govet } // Create a new bucket. - bucket, err := NewBucket(log.NewNopLogger(), confContentYaml, "example") + bucket, err := NewBucket(log.NewNopLogger(), confContentYaml, "example", nil) if err != nil { panic(err) } @@ -72,7 +72,7 @@ func ExampleTracingBucketUsingOpenTelemetry() { //nolint:govet } // Create a new bucket. - bucket, err := NewBucket(log.NewNopLogger(), confContentYaml, "example") + bucket, err := NewBucket(log.NewNopLogger(), confContentYaml, "example", nil) if err != nil { panic(err) } diff --git a/go.mod b/go.mod index 4ec29639..5a98916f 100644 --- a/go.mod +++ b/go.mod @@ -82,6 +82,7 @@ require ( github.com/mozillazg/go-httpheader v0.2.1 // indirect github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect github.com/prometheus/procfs v0.11.1 // indirect github.com/rs/xid v1.5.0 // indirect @@ -99,6 +100,7 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect google.golang.org/protobuf v1.34.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( @@ -107,5 +109,6 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.0 github.com/kr/text v0.2.0 // indirect github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b // indirect + github.com/stretchr/testify v1.9.0 golang.org/x/crypto v0.24.0 // indirect ) diff --git a/go.sum b/go.sum index f2ecc7c5..f015ecc4 100644 --- a/go.sum +++ b/go.sum @@ -214,6 +214,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.194/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.194/go.mod h1:yrBKWhChnDqNz1xuXdSbWXG56XawEq0G5j1lg4VwBD4= github.com/tencentyun/cos-go-sdk-v5 v0.7.40 h1:W6vDGKCHe4wBACI1d2UgE6+50sJFhRWU4O8IB2ozzxM= diff --git a/providers/azure/azure.go b/providers/azure/azure.go index 63bd2c1a..c2f3adb5 100644 --- a/providers/azure/azure.go +++ b/providers/azure/azure.go @@ -6,6 +6,7 @@ package azure import ( "context" "io" + "net/http" "os" "strings" "testing" @@ -145,7 +146,7 @@ type Bucket struct { } // NewBucket returns a new Bucket using the provided Azure config. -func NewBucket(logger log.Logger, azureConfig []byte, component string) (*Bucket, error) { +func NewBucket(logger log.Logger, azureConfig []byte, component string, rt http.RoundTripper) (*Bucket, error) { level.Debug(logger).Log("msg", "creating new Azure bucket connection", "component", component) conf, err := parseConfig(azureConfig) if err != nil { @@ -154,11 +155,14 @@ func NewBucket(logger log.Logger, azureConfig []byte, component string) (*Bucket if conf.MSIResource != "" { level.Warn(logger).Log("msg", "The field msi_resource has been deprecated and should no longer be set") } - return NewBucketWithConfig(logger, conf, component) + return NewBucketWithConfig(logger, conf, component, rt) } // NewBucketWithConfig returns a new Bucket using the provided Azure config struct. -func NewBucketWithConfig(logger log.Logger, conf Config, component string) (*Bucket, error) { +func NewBucketWithConfig(logger log.Logger, conf Config, component string, rt http.RoundTripper) (*Bucket, error) { + if rt != nil { + conf.HTTPConfig.Transport = rt + } if err := conf.validate(); err != nil { return nil, err } @@ -355,7 +359,7 @@ func NewTestBucket(t testing.TB, component string) (objstore.Bucket, func(), err if err != nil { return nil, nil, err } - bkt, err := NewBucket(log.NewNopLogger(), bc, component) + bkt, err := NewBucket(log.NewNopLogger(), bc, component, nil) if err != nil { t.Errorf("Cannot create Azure storage container:") return nil, nil, err diff --git a/providers/azure/azure_test.go b/providers/azure/azure_test.go index c49695ab..cc9c1317 100644 --- a/providers/azure/azure_test.go +++ b/providers/azure/azure_test.go @@ -4,10 +4,13 @@ package azure import ( + "net/http" "testing" "time" "github.com/efficientgo/core/testutil" + "github.com/go-kit/log" + "github.com/pkg/errors" "github.com/thanos-io/objstore/exthttp" ) @@ -20,7 +23,7 @@ type TestCase struct { } var validConfig = []byte(`storage_account: "myStorageAccount" -storage_account_key: "abc123" +storage_account_key: "bXlTdXBlclNlY3JldEtleTEyMyFAIw==" container: "MyContainer" endpoint: "blob.core.windows.net" reader_config: @@ -222,3 +225,25 @@ http_config: testutil.Ok(t, err) testutil.Equals(t, true, transport.TLSClientConfig.InsecureSkipVerify) } + +// ErrorRoundTripper is a custom RoundTripper that always returns an error +type ErrorRoundTripper struct { + Err error +} + +func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { + return nil, ert.Err +} + +func TestNewBucketWithErrorRoundTripper(t *testing.T) { + cfg, err := parseConfig(validConfig) + testutil.Ok(t, err) + + rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} + + _, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test", rt) + + // We expect an error from the RoundTripper + testutil.NotOk(t, err) + testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) +} diff --git a/providers/azure/helpers.go b/providers/azure/helpers.go index 846394a0..5ced695b 100644 --- a/providers/azure/helpers.go +++ b/providers/azure/helpers.go @@ -20,9 +20,17 @@ import ( const DirDelim = "/" func getContainerClient(conf Config) (*container.Client, error) { - dt, err := exthttp.DefaultTransport(conf.HTTPConfig) - if err != nil { - return nil, err + // Check if a roundtripper has been set in the config + // otherwise build the default transport. + var rt http.RoundTripper + if conf.HTTPConfig.Transport != nil { + rt = conf.HTTPConfig.Transport + } else { + var err error + rt, err = exthttp.DefaultTransport(conf.HTTPConfig) + if err != nil { + return nil, err + } } opt := &container.ClientOptions{ ClientOptions: azcore.ClientOptions{ @@ -35,7 +43,7 @@ func getContainerClient(conf Config) (*container.Client, error) { Telemetry: policy.TelemetryOptions{ ApplicationID: "Thanos", }, - Transport: &http.Client{Transport: dt}, + Transport: &http.Client{Transport: rt}, }, } @@ -65,6 +73,7 @@ func getContainerClient(conf Config) (*container.Client, error) { // Otherwise use a token credential var cred azcore.TokenCredential + var err error // Use Managed Identity Credential if a user assigned ID is set if conf.UserAssignedID != "" { diff --git a/providers/cos/cos.go b/providers/cos/cos.go index e518cae2..ce61ca87 100644 --- a/providers/cos/cos.go +++ b/providers/cos/cos.go @@ -95,7 +95,7 @@ func parseConfig(conf []byte) (Config, error) { } // NewBucket returns a new Bucket using the provided cos configuration. -func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error) { +func NewBucket(logger log.Logger, conf []byte, component string, rt http.RoundTripper) (*Bucket, error) { if logger == nil { logger = log.NewNopLogger() } @@ -104,12 +104,11 @@ func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error if err != nil { return nil, errors.Wrap(err, "parsing cos configuration") } - - return NewBucketWithConfig(logger, config, component) + return NewBucketWithConfig(logger, config, component, rt) } // NewBucketWithConfig returns a new Bucket using the provided cos config values. -func NewBucketWithConfig(logger log.Logger, config Config, component string) (*Bucket, error) { +func NewBucketWithConfig(logger log.Logger, config Config, component string, rt http.RoundTripper) (*Bucket, error) { if err := config.validate(); err != nil { return nil, errors.Wrap(err, "validate cos configuration") } @@ -128,7 +127,21 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B } } b := &cos.BaseURL{BucketURL: bucketURL} - tpt, _ := exthttp.DefaultTransport(config.HTTPConfig) + if rt != nil { + config.HTTPConfig.Transport = rt + } + // Check if a roundtripper has been set in the config + // otherwise build the default transport. + var tpt http.RoundTripper + if config.HTTPConfig.Transport != nil { + tpt = config.HTTPConfig.Transport + } else { + var err error + tpt, err = exthttp.DefaultTransport(config.HTTPConfig) + if err != nil { + return nil, err + } + } client := cos.NewClient(b, &http.Client{ Transport: &cos.AuthorizationTransport{ SecretID: config.SecretId, @@ -485,7 +498,7 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) { return nil, nil, err } - b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test") + b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test", nil) if err != nil { return nil, nil, err } @@ -506,7 +519,7 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) { return nil, nil, err } - b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test") + b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test", nil) if err != nil { return nil, nil, err } diff --git a/providers/cos/cos_test.go b/providers/cos/cos_test.go index 6d4316f2..c0f713b1 100644 --- a/providers/cos/cos_test.go +++ b/providers/cos/cos_test.go @@ -4,10 +4,14 @@ package cos import ( + "context" + "net/http" "testing" "time" "github.com/efficientgo/core/testutil" + "github.com/go-kit/log" + "github.com/pkg/errors" "github.com/prometheus/common/model" "github.com/thanos-io/objstore/exthttp" @@ -137,3 +141,30 @@ func TestConfig_validate(t *testing.T) { }) } } + +// ErrorRoundTripper is a custom RoundTripper that always returns an error +type ErrorRoundTripper struct { + Err error +} + +func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { + return nil, ert.Err +} + +func TestNewBucketWithErrorRoundTripper(t *testing.T) { + config := Config{ + Bucket: "bucket", + AppId: "123", + Region: "ap-beijing", + SecretId: "sid", + SecretKey: "skey", + } + rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} + + bkt, err := NewBucketWithConfig(log.NewNopLogger(), config, "test", rt) + testutil.Ok(t, err) + _, err = bkt.Get(context.Background(), "Test") + // We expect an error from the RoundTripper + testutil.NotOk(t, err) + testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) +} diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index 87e31f66..e07ca05b 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -76,20 +76,22 @@ func parseConfig(conf []byte) (Config, error) { } // NewBucket returns a new Bucket against the given bucket handle. -func NewBucket(ctx context.Context, logger log.Logger, conf []byte, component string) (*Bucket, error) { +func NewBucket(ctx context.Context, logger log.Logger, conf []byte, component string, rt http.RoundTripper) (*Bucket, error) { config, err := parseConfig(conf) if err != nil { return nil, err } - return NewBucketWithConfig(ctx, logger, config, component) + return NewBucketWithConfig(ctx, logger, config, component, rt) } // NewBucketWithConfig returns a new Bucket with gcs Config struct. -func NewBucketWithConfig(ctx context.Context, logger log.Logger, gc Config, component string) (*Bucket, error) { +func NewBucketWithConfig(ctx context.Context, logger log.Logger, gc Config, component string, rt http.RoundTripper) (*Bucket, error) { if gc.Bucket == "" { return nil, errors.New("missing Google Cloud Storage bucket name for stored blocks") } - + if rt != nil { + gc.HTTPConfig.Transport = rt + } var opts []option.ClientOption // If ServiceAccount is provided, use them in GCS client, otherwise fallback to Google default logic. @@ -312,7 +314,7 @@ func NewTestBucket(t testing.TB, project string) (objstore.Bucket, func(), error return nil, nil, err } - b, err := NewBucket(ctx, log.NewNopLogger(), bc, "thanos-e2e-test") + b, err := NewBucket(ctx, log.NewNopLogger(), bc, "thanos-e2e-test", nil) if err != nil { return nil, nil, err } diff --git a/providers/gcs/gcs_test.go b/providers/gcs/gcs_test.go index d03c5dd9..76e2b99b 100644 --- a/providers/gcs/gcs_test.go +++ b/providers/gcs/gcs_test.go @@ -66,7 +66,7 @@ func TestNewBucketWithConfig_ShouldCreateGRPC(t *testing.T) { err = os.Setenv("STORAGE_EMULATOR_HOST_GRPC", svr.Addr) testutil.Ok(t, err) - bkt, err := NewBucketWithConfig(context.Background(), log.NewNopLogger(), cfg, "test-bucket") + bkt, err := NewBucketWithConfig(context.Background(), log.NewNopLogger(), cfg, "test-bucket", nil) testutil.Ok(t, err) // Check if the bucket is created. @@ -157,3 +157,26 @@ http_config: }) } } + +// ErrorRoundTripper is a custom RoundTripper that always returns an error +type ErrorRoundTripper struct { + Err error +} + +func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { + return nil, ert.Err +} + +// func TestNewBucketWithErrorRoundTripper(t *testing.T) { +// cfg := Config{ +// Bucket: "test-bucket", +// ServiceAccount: "", +// } +// // Create an error RoundTripper +// rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} + +// // Create the bucket with the custom RoundTripper +// _, err := NewBucketWithConfig(context.Background(), log.NewNopLogger(), cfg, "test-bucket", rt) +// testutil.NotOk(t, err) // Expect an error when using the error RoundTripper +// testutil.Assert(t, err.Error() == "RoundTripper error" || errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) +// } diff --git a/providers/obs/obs.go b/providers/obs/obs.go index 7bd9666b..dcd342d5 100644 --- a/providers/obs/obs.go +++ b/providers/obs/obs.go @@ -7,6 +7,7 @@ import ( "context" "io" "math" + "net/http" "os" "strings" "testing" @@ -74,13 +75,12 @@ type Bucket struct { name string } -func NewBucket(logger log.Logger, conf []byte) (*Bucket, error) { +func NewBucket(logger log.Logger, conf []byte, rt http.RoundTripper) (*Bucket, error) { config, err := parseConfig(conf) if err != nil { return nil, errors.Wrap(err, "parsing cos configuration") } - - return NewBucketWithConfig(logger, config) + return NewBucketWithConfig(logger, config, rt) } func parseConfig(conf []byte) (Config, error) { @@ -92,17 +92,25 @@ func parseConfig(conf []byte) (Config, error) { return config, nil } -func NewBucketWithConfig(logger log.Logger, config Config) (*Bucket, error) { +func NewBucketWithConfig(logger log.Logger, config Config, rt http.RoundTripper) (*Bucket, error) { if err := config.validate(); err != nil { return nil, errors.Wrap(err, "validate obs config err") } - - rt, err := exthttp.DefaultTransport(config.HTTPConfig) - if err != nil { - return nil, errors.Wrap(err, "get http transport err") + if rt != nil { + config.HTTPConfig.Transport = rt + } + var tpt *http.Transport + if config.HTTPConfig.Transport != nil { + tpt = config.HTTPConfig.Transport.(*http.Transport) + } else { + var err error + tpt, err = exthttp.DefaultTransport(config.HTTPConfig) + if err != nil { + return nil, errors.Wrap(err, "get http transport err") + } } - client, err := obs.New(config.AccessKey, config.SecretKey, config.Endpoint, obs.WithHttpTransport(rt)) + client, err := obs.New(config.AccessKey, config.SecretKey, config.Endpoint, obs.WithHttpTransport(tpt)) if err != nil { return nil, errors.Wrap(err, "initialize obs client err") } @@ -369,7 +377,7 @@ func NewTestBucketFromConfig(t testing.TB, c Config, reuseBucket bool, location if err != nil { return nil, nil, err } - b, err := NewBucket(log.NewNopLogger(), bc) + b, err := NewBucket(log.NewNopLogger(), bc, nil) if err != nil { return nil, nil, err } diff --git a/providers/oci/oci.go b/providers/oci/oci.go index 2db35461..e8f8e0c5 100644 --- a/providers/oci/oci.go +++ b/providers/oci/oci.go @@ -58,13 +58,14 @@ type HTTPConfig struct { ResponseHeaderTimeout model.Duration `yaml:"response_header_timeout"` InsecureSkipVerify bool `yaml:"insecure_skip_verify"` - TLSHandshakeTimeout model.Duration `yaml:"tls_handshake_timeout"` - ExpectContinueTimeout model.Duration `yaml:"expect_continue_timeout"` - MaxIdleConns int `yaml:"max_idle_conns"` - MaxIdleConnsPerHost int `yaml:"max_idle_conns_per_host"` - MaxConnsPerHost int `yaml:"max_conns_per_host"` - DisableCompression bool `yaml:"disable_compression"` - ClientTimeout time.Duration `yaml:"client_timeout"` + TLSHandshakeTimeout model.Duration `yaml:"tls_handshake_timeout"` + ExpectContinueTimeout model.Duration `yaml:"expect_continue_timeout"` + MaxIdleConns int `yaml:"max_idle_conns"` + MaxIdleConnsPerHost int `yaml:"max_idle_conns_per_host"` + MaxConnsPerHost int `yaml:"max_conns_per_host"` + DisableCompression bool `yaml:"disable_compression"` + ClientTimeout time.Duration `yaml:"client_timeout"` + Transport http.RoundTripper `yaml:"-"` } // Config stores the configuration for oci bucket. @@ -288,7 +289,7 @@ func (b *Bucket) deleteBucket(ctx context.Context) (err error) { } // NewBucket returns a new Bucket using the provided oci config values. -func NewBucket(logger log.Logger, ociConfig []byte) (*Bucket, error) { +func NewBucket(logger log.Logger, ociConfig []byte, rt http.RoundTripper) (*Bucket, error) { level.Debug(logger).Log("msg", "creating new oci bucket connection") var config = DefaultConfig var configurationProvider common.ConfigurationProvider @@ -335,8 +336,13 @@ func NewBucket(logger log.Logger, ociConfig []byte) (*Bucket, error) { return nil, errors.Wrapf(err, "unable to create ObjectStorage client with the given oci configurations") } + if rt != nil { + config.HTTPConfig.Transport = rt + } else { + config.HTTPConfig.Transport = CustomTransport(config) + } httpClient := http.Client{ - Transport: CustomTransport(config), + Transport: config.HTTPConfig.Transport, Timeout: config.HTTPConfig.ClientTimeout, } client.HTTPClient = &httpClient @@ -375,7 +381,7 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) { return nil, nil, err } - bkt, err := NewBucket(log.NewNopLogger(), ociConfig) + bkt, err := NewBucket(log.NewNopLogger(), ociConfig, nil) if err != nil { return nil, nil, err } diff --git a/providers/oss/oss.go b/providers/oss/oss.go index e7e3a648..572778d1 100644 --- a/providers/oss/oss.go +++ b/providers/oss/oss.go @@ -158,22 +158,30 @@ func (b *Bucket) Attributes(ctx context.Context, name string) (objstore.ObjectAt } // NewBucket returns a new Bucket using the provided oss config values. -func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error) { +func NewBucket(logger log.Logger, conf []byte, component string, rt http.RoundTripper) (*Bucket, error) { var config Config if err := yaml.Unmarshal(conf, &config); err != nil { return nil, errors.Wrap(err, "parse aliyun oss config file failed") } - - return NewBucketWithConfig(logger, config, component) + return NewBucketWithConfig(logger, config, component, rt) } // NewBucketWithConfig returns a new Bucket using the provided oss config struct. -func NewBucketWithConfig(logger log.Logger, config Config, component string) (*Bucket, error) { +func NewBucketWithConfig(logger log.Logger, config Config, component string, rt http.RoundTripper) (*Bucket, error) { if err := validate(config); err != nil { return nil, err } - - client, err := alioss.New(config.Endpoint, config.AccessKeyID, config.AccessKeySecret) + var client *alioss.Client + var err error + if rt != nil { + // custom RoundTripper + clientOption := func(client *alioss.Client) { + client.HTTPClient = &http.Client{Transport: rt} + } + client, err = alioss.New(config.Endpoint, config.AccessKeyID, config.AccessKeySecret, clientOption) + } else { + client, err = alioss.New(config.Endpoint, config.AccessKeyID, config.AccessKeySecret) + } if err != nil { return nil, errors.Wrap(err, "create aliyun oss client failed") } @@ -274,7 +282,7 @@ func NewTestBucketFromConfig(t testing.TB, c Config, reuseBucket bool) (objstore return nil, nil, err } - b, err := NewBucket(log.NewNopLogger(), bc, "thanos-aliyun-oss-test") + b, err := NewBucket(log.NewNopLogger(), bc, "thanos-aliyun-oss-test", nil) if err != nil { return nil, nil, err } diff --git a/providers/s3/s3.go b/providers/s3/s3.go index dad89e66..5fc5f188 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -176,13 +176,13 @@ func parseConfig(conf []byte) (Config, error) { } // NewBucket returns a new Bucket using the provided s3 config values. -func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error) { +func NewBucket(logger log.Logger, conf []byte, component string, rt http.RoundTripper) (*Bucket, error) { config, err := parseConfig(conf) if err != nil { return nil, err } - return NewBucketWithConfig(logger, config, component) + return NewBucketWithConfig(logger, config, component, rt) } type overrideSignerType struct { @@ -202,7 +202,7 @@ func (s *overrideSignerType) Retrieve() (credentials.Value, error) { } // NewBucketWithConfig returns a new Bucket using the provided s3 config values. -func NewBucketWithConfig(logger log.Logger, config Config, component string) (*Bucket, error) { +func NewBucketWithConfig(logger log.Logger, config Config, component string, rt http.RoundTripper) (*Bucket, error) { var chain []credentials.Provider // TODO(bwplotka): Don't do flags as they won't scale, use actual params like v2, v4 instead @@ -242,15 +242,17 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B }), } } - + if rt != nil { + config.HTTPConfig.Transport = rt + } // Check if a roundtripper has been set in the config // otherwise build the default transport. - var rt http.RoundTripper + var tpt http.RoundTripper if config.HTTPConfig.Transport != nil { - rt = config.HTTPConfig.Transport + tpt = config.HTTPConfig.Transport } else { var err error - rt, err = exthttp.DefaultTransport(config.HTTPConfig) + tpt, err = exthttp.DefaultTransport(config.HTTPConfig) if err != nil { return nil, err } @@ -260,7 +262,7 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B Creds: credentials.NewChainCredentials(chain), Secure: !config.Insecure, Region: config.Region, - Transport: rt, + Transport: tpt, BucketLookup: config.BucketLookupType.MinioType(), }) if err != nil { @@ -611,7 +613,7 @@ func NewTestBucketFromConfig(t testing.TB, location string, c Config, reuseBucke if err != nil { return nil, nil, err } - b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test") + b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test", nil) if err != nil { return nil, nil, err } diff --git a/providers/s3/s3_e2e_test.go b/providers/s3/s3_e2e_test.go index 4b75a014..ac9ec261 100644 --- a/providers/s3/s3_e2e_test.go +++ b/providers/s3/s3_e2e_test.go @@ -37,6 +37,7 @@ func BenchmarkUpload(b *testing.B) { log.NewNopLogger(), e2ethanos.NewS3Config(bucket, m.Endpoint("https"), m.Dir()), "test-feed", + nil, ) testutil.Ok(b, err) diff --git a/providers/s3/s3_test.go b/providers/s3/s3_test.go index cdab39c3..049b330b 100644 --- a/providers/s3/s3_test.go +++ b/providers/s3/s3_test.go @@ -7,6 +7,7 @@ import ( "context" "encoding/base64" "encoding/json" + "errors" "io" "net/http" "net/http/httptest" @@ -324,7 +325,7 @@ func TestBucket_getServerSideEncryption(t *testing.T) { // Default config should return no SSE config. cfg := DefaultConfig cfg.Endpoint = endpoint - bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test") + bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test", nil) testutil.Ok(t, err) sse, err := bkt.getServerSideEncryption(context.Background()) @@ -335,7 +336,7 @@ func TestBucket_getServerSideEncryption(t *testing.T) { cfg = DefaultConfig cfg.Endpoint = endpoint cfg.SSEConfig = SSEConfig{Type: SSES3} - bkt, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test") + bkt, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test", nil) testutil.Ok(t, err) sse, err = bkt.getServerSideEncryption(context.Background()) @@ -351,7 +352,7 @@ func TestBucket_getServerSideEncryption(t *testing.T) { Type: SSEKMS, KMSKeyID: "key", } - bkt, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test") + bkt, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test", nil) testutil.Ok(t, err) sse, err = bkt.getServerSideEncryption(context.Background()) @@ -375,7 +376,7 @@ func TestBucket_getServerSideEncryption(t *testing.T) { KMSKeyID: "key", KMSEncryptionContext: map[string]string{"foo": "bar"}, } - bkt, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test") + bkt, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test", nil) testutil.Ok(t, err) sse, err = bkt.getServerSideEncryption(context.Background()) @@ -396,7 +397,7 @@ func TestBucket_getServerSideEncryption(t *testing.T) { override, err := encrypt.NewSSEKMS("test", nil) testutil.Ok(t, err) - bkt, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test") + bkt, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test", nil) testutil.Ok(t, err) sse, err = bkt.getServerSideEncryption(context.WithValue(context.Background(), sseConfigKey, override)) @@ -423,7 +424,7 @@ func TestBucket_Get_ShouldReturnErrorIfServerTruncateResponse(t *testing.T) { cfg.AccessKey = "test" cfg.SecretKey = "test" - bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test") + bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test", nil) testutil.Ok(t, err) reader, err := bkt.Get(context.Background(), "test") @@ -448,7 +449,7 @@ func TestParseConfig_CustomStorageClass(t *testing.T) { cfg.Endpoint = endpoint storageClass := "STANDARD_IA" cfg.PutUserMetadata[testCase.storageClassKey] = storageClass - bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test") + bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test", nil) testutil.Ok(t, err) testutil.Equals(t, storageClass, bkt.storageClass) }) @@ -458,7 +459,29 @@ func TestParseConfig_CustomStorageClass(t *testing.T) { func TestParseConfig_DefaultStorageClassIsZero(t *testing.T) { cfg := DefaultConfig cfg.Endpoint = endpoint - bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test") + bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test", nil) testutil.Ok(t, err) testutil.Equals(t, "", bkt.storageClass) } + +// ErrorRoundTripper is a custom RoundTripper that always returns an error +type ErrorRoundTripper struct { + Err error +} + +func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { + return nil, ert.Err +} + +func TestNewBucketWithErrorRoundTripper(t *testing.T) { + cfg := DefaultConfig + cfg.Endpoint = endpoint + cfg.Bucket = "test" + rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} + bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test", rt) + testutil.Ok(t, err) + _, err = bkt.Get(context.Background(), "test") + // We expect an error from the RoundTripper + testutil.NotOk(t, err) + testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) +} diff --git a/providers/swift/swift.go b/providers/swift/swift.go index a8c56c55..f287f551 100644 --- a/providers/swift/swift.go +++ b/providers/swift/swift.go @@ -154,12 +154,12 @@ type Container struct { segmentsContainer string } -func NewContainer(logger log.Logger, conf []byte) (*Container, error) { +func NewContainer(logger log.Logger, conf []byte, rt http.RoundTripper) (*Container, error) { sc, err := parseConfig(conf) if err != nil { return nil, errors.Wrap(err, "parse config") } - return NewContainerFromConfig(logger, sc, false) + return NewContainerFromConfig(logger, sc, false, rt) } func ensureContainer(connection *swift.Connection, name string, createIfNotExist bool) error { @@ -178,22 +178,24 @@ func ensureContainer(connection *swift.Connection, name string, createIfNotExist return nil } -func NewContainerFromConfig(logger log.Logger, sc *Config, createContainer bool) (*Container, error) { - +func NewContainerFromConfig(logger log.Logger, sc *Config, createContainer bool, rt http.RoundTripper) (*Container, error) { + if rt != nil { + sc.HTTPConfig.Transport = rt + } // Check if a roundtripper has been set in the config // otherwise build the default transport. - var rt http.RoundTripper + var tpt http.RoundTripper if sc.HTTPConfig.Transport != nil { - rt = sc.HTTPConfig.Transport + tpt = sc.HTTPConfig.Transport } else { var err error - rt, err = exthttp.DefaultTransport(sc.HTTPConfig) + tpt, err = exthttp.DefaultTransport(sc.HTTPConfig) if err != nil { return nil, err } } - connection := connectionFromConfig(sc, rt) + connection := connectionFromConfig(sc, tpt) if err := connection.Authenticate(); err != nil { return nil, errors.Wrap(err, "authentication") } @@ -378,7 +380,7 @@ func NewTestContainer(t testing.TB) (objstore.Bucket, func(), error) { "needs to be manually cleared. This means that it is only useful to run one test in a time. This is due " + "to safety (accidentally pointing prod container for test) as well as swift not being fully strong consistent.") } - c, err := NewContainerFromConfig(log.NewNopLogger(), config, false) + c, err := NewContainerFromConfig(log.NewNopLogger(), config, false, nil) if err != nil { return nil, nil, errors.Wrap(err, "initializing new container") } @@ -392,7 +394,7 @@ func NewTestContainer(t testing.TB) (objstore.Bucket, func(), error) { } config.ContainerName = objstore.CreateTemporaryTestBucketName(t) config.SegmentContainerName = config.ContainerName - c, err := NewContainerFromConfig(log.NewNopLogger(), config, true) + c, err := NewContainerFromConfig(log.NewNopLogger(), config, true, nil) if err != nil { return nil, nil, errors.Wrap(err, "initializing new container") } diff --git a/providers/swift/swift_test.go b/providers/swift/swift_test.go index 656e7756..a203378c 100644 --- a/providers/swift/swift_test.go +++ b/providers/swift/swift_test.go @@ -4,10 +4,13 @@ package swift import ( + "net/http" "testing" "time" "github.com/efficientgo/core/testutil" + "github.com/go-kit/log" + "github.com/pkg/errors" "github.com/prometheus/common/model" ) @@ -64,3 +67,24 @@ http_config: testutil.Equals(t, false, cfg.HTTPConfig.InsecureSkipVerify) } + +// ErrorRoundTripper is a custom RoundTripper that always returns an error +type ErrorRoundTripper struct { + Err error +} + +func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { + return nil, ert.Err +} + +func TestNewBucketWithErrorRoundTripper(t *testing.T) { + logger := log.NewNopLogger() + rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} + config := DefaultConfig + config.AuthUrl = "http://identity.something.com/v3" + _, err := NewContainerFromConfig(logger, &config, false, rt) + + // We expect an error from the RoundTripper + testutil.NotOk(t, err) + testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) +} From 528b2c24299388f74c1a2c56f699fa39b8c95a23 Mon Sep 17 00:00:00 2001 From: milinddethe15 Date: Sun, 15 Sep 2024 05:26:56 +0530 Subject: [PATCH 08/41] add tests for custom rt Signed-off-by: milinddethe15 --- go.mod | 4 +-- go.sum | 5 ++-- providers/cos/cos_test.go | 2 +- providers/gcs/gcs_test.go | 34 ++++++++++++++--------- providers/obs/obs_test.go | 33 +++++++++++++++++++++++ providers/oci/oci_test.go | 57 +++++++++++++++++++++++++++++++++++++++ providers/oss/oss_test.go | 39 +++++++++++++++++++++++++++ 7 files changed, 154 insertions(+), 20 deletions(-) create mode 100644 providers/obs/obs_test.go create mode 100644 providers/oci/oci_test.go create mode 100644 providers/oss/oss_test.go diff --git a/go.mod b/go.mod index 5a98916f..1d10fb58 100644 --- a/go.mod +++ b/go.mod @@ -82,7 +82,6 @@ require ( github.com/mozillazg/go-httpheader v0.2.1 // indirect github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect github.com/prometheus/procfs v0.11.1 // indirect github.com/rs/xid v1.5.0 // indirect @@ -100,7 +99,6 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect google.golang.org/protobuf v1.34.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( @@ -109,6 +107,6 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.0 github.com/kr/text v0.2.0 // indirect github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b // indirect - github.com/stretchr/testify v1.9.0 + github.com/stretchr/testify v1.9.0 // indirect golang.org/x/crypto v0.24.0 // indirect ) diff --git a/go.sum b/go.sum index f015ecc4..61b88553 100644 --- a/go.sum +++ b/go.sum @@ -204,16 +204,15 @@ github.com/sony/gobreaker v0.5.0 h1:dRCvqm0P490vZPmy7ppEk2qCnCieBooFJ+YoXGYB+yg= github.com/sony/gobreaker v0.5.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.194/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y= diff --git a/providers/cos/cos_test.go b/providers/cos/cos_test.go index c0f713b1..d67e2dfd 100644 --- a/providers/cos/cos_test.go +++ b/providers/cos/cos_test.go @@ -155,7 +155,7 @@ func TestNewBucketWithErrorRoundTripper(t *testing.T) { config := Config{ Bucket: "bucket", AppId: "123", - Region: "ap-beijing", + Region: "test", SecretId: "sid", SecretKey: "skey", } diff --git a/providers/gcs/gcs_test.go b/providers/gcs/gcs_test.go index 76e2b99b..120e3450 100644 --- a/providers/gcs/gcs_test.go +++ b/providers/gcs/gcs_test.go @@ -15,6 +15,7 @@ import ( "github.com/efficientgo/core/testutil" "github.com/fullstorydev/emulators/storage/gcsemu" "github.com/go-kit/log" + "github.com/pkg/errors" "github.com/prometheus/common/model" "google.golang.org/api/option" ) @@ -167,16 +168,23 @@ func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { return nil, ert.Err } -// func TestNewBucketWithErrorRoundTripper(t *testing.T) { -// cfg := Config{ -// Bucket: "test-bucket", -// ServiceAccount: "", -// } -// // Create an error RoundTripper -// rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} - -// // Create the bucket with the custom RoundTripper -// _, err := NewBucketWithConfig(context.Background(), log.NewNopLogger(), cfg, "test-bucket", rt) -// testutil.NotOk(t, err) // Expect an error when using the error RoundTripper -// testutil.Assert(t, err.Error() == "RoundTripper error" || errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) -// } +func TestNewBucketWithErrorRoundTripper(t *testing.T) { + // Create an error RoundTripper + rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} + cfg := Config{ + Bucket: "test-bucket", + ServiceAccount: "", + UseGRPC: false, + } + svr, err := gcsemu.NewServer("127.0.0.1:0", gcsemu.Options{}) + testutil.Ok(t, err) + err = os.Setenv("STORAGE_EMULATOR_HOST", svr.Addr) + testutil.Ok(t, err) + err = os.Setenv("GCS_EMULATOR_HOST", svr.Addr) + testutil.Ok(t, err) + defer svr.Close() + + _, err = NewBucketWithConfig(context.Background(), log.NewNopLogger(), cfg, "test-bucket", rt) + testutil.NotOk(t, err) // Expect an error when using the error RoundTripper + testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) +} diff --git a/providers/obs/obs_test.go b/providers/obs/obs_test.go new file mode 100644 index 00000000..26563b3c --- /dev/null +++ b/providers/obs/obs_test.go @@ -0,0 +1,33 @@ +package obs + +import ( + "net/http" + "testing" + + "github.com/efficientgo/core/testutil" + "github.com/go-kit/log" + "github.com/pkg/errors" +) + +// ErrorRoundTripper is a custom RoundTripper that always returns an error +type ErrorRoundTripper struct { + Err error + Transport *http.Transport +} + +func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { + return nil, ert.Err +} + +func TestNewBucketWithErrorRoundTripper(t *testing.T) { + config := DefaultConfig + config.Endpoint = "https://test.com" + config.SecretKey = "test123" + config.AccessKey = "test123" + + rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} + _, err := NewBucketWithConfig(log.NewNopLogger(), config, rt) + // We expect an error from the RoundTripper + testutil.NotOk(t, err) + testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) +} diff --git a/providers/oci/oci_test.go b/providers/oci/oci_test.go new file mode 100644 index 00000000..87288cee --- /dev/null +++ b/providers/oci/oci_test.go @@ -0,0 +1,57 @@ +package oci + +import ( + "net/http" + "testing" + + "github.com/efficientgo/core/testutil" + "github.com/go-kit/log" + "github.com/pkg/errors" + "gopkg.in/yaml.v2" +) + +// ErrorRoundTripper is a custom RoundTripper that always returns an error +type ErrorRoundTripper struct { + Err error +} + +func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { + return nil, ert.Err +} + +func TestNewBucketWithErrorRoundTripper(t *testing.T) { + + mockPrivateKey := `-----BEGIN RSA PRIVATE KEY----- +MIICXgIBAAKBgQDCFENGw33yGihy92pDjZQhl0C36rPJj+CvfSC8+q28hxA161QF +NUd13wuCTUcq0Qd2qsBe/2hFyc2DCJJg0h1L78+6Z4UMR7EOcpfdUE9Hf3m/hs+F +UR45uBJeDK1HSFHD8bHKD6kv8FPGfJTotc+2xjJwoYi+1hqp1fIekaxsyQIDAQAB +AoGBAJR8ZkCUvx5kzv+utdl7T5MnordT1TvoXXJGXK7ZZ+UuvMNUCdN2QPc4sBiA +QWvLw1cSKt5DsKZ8UETpYPy8pPYnnDEz2dDYiaew9+xEpubyeW2oH4Zx71wqBtOK +kqwrXa/pzdpiucRRjk6vE6YY7EBBs/g7uanVpGibOVAEsqH1AkEA7DkjVH28WDUg +f1nqvfn2Kj6CT7nIcE3jGJsZZ7zlZmBmHFDONMLUrXR/Zm3pR5m0tCmBqa5RK95u +412jt1dPIwJBANJT3v8pnkth48bQo/fKel6uEYyboRtA5/uHuHkZ6FQF7OUkGogc +mSJluOdc5t6hI1VsLn0QZEjQZMEOWr+wKSMCQQCC4kXJEsHAve77oP6HtG/IiEn7 +kpyUXRNvFsDE0czpJJBvL/aRFUJxuRK91jhjC68sA7NsKMGg5OXb5I5Jj36xAkEA +gIT7aFOYBFwGgQAQkWNKLvySgKbAZRTeLBacpHMuQdl1DfdntvAyqpAZ0lY0RKmW +G6aFKaqQfOXKCyWoUiVknQJAXrlgySFci/2ueKlIE1QqIiLSZ8V8OlpFLRnb1pzI +7U1yQXnTAEFYM560yJlzUpOb1V4cScGd365tiSMvxLOvTA== +-----END RSA PRIVATE KEY-----` + + config := DefaultConfig + config.Provider = "raw" + config.Tenancy = "test" + config.User = "test" + config.Region = "test" + config.Fingerprint = "123" + config.PrivateKey = mockPrivateKey + config.Passphrase = "123" + ociConfig, err := yaml.Marshal(config) + testutil.Ok(t, err) + + rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} + + _, err = NewBucket(log.NewNopLogger(), ociConfig, rt) + // We expect an error from the RoundTripper + testutil.NotOk(t, err) + testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) +} diff --git a/providers/oss/oss_test.go b/providers/oss/oss_test.go new file mode 100644 index 00000000..23f174d5 --- /dev/null +++ b/providers/oss/oss_test.go @@ -0,0 +1,39 @@ +package oss + +import ( + "context" + "net/http" + "testing" + + "github.com/efficientgo/core/testutil" + "github.com/go-kit/log" + "github.com/pkg/errors" +) + +// ErrorRoundTripper is a custom RoundTripper that always returns an error +type ErrorRoundTripper struct { + Err error +} + +func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { + return nil, ert.Err +} + +func TestNewBucketWithErrorRoundTripper(t *testing.T) { + + var config Config + config.Endpoint = "http://test.com/" + config.AccessKeyID = "123" + config.AccessKeySecret = "123" + config.Bucket = "test" + config.AccessKeySecret = "123" + + rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} + + bkt, err := NewBucketWithConfig(log.NewNopLogger(), config, "test", rt) + // We expect an error from the RoundTripper + testutil.Ok(t, err) + _, err = bkt.Get(context.Background(), "test") + testutil.NotOk(t, err) + testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) +} From f2f0217f183f5d4a1db10a86cbb48919a61b5646 Mon Sep 17 00:00:00 2001 From: Milind Dethe <99114125+milinddethe15@users.noreply.github.com> Date: Sun, 15 Sep 2024 00:26:25 +0530 Subject: [PATCH 09/41] fix lint Signed-off-by: Milind Dethe <99114125+milinddethe15@users.noreply.github.com> --- providers/s3/s3_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/providers/s3/s3_test.go b/providers/s3/s3_test.go index 049b330b..e34e19bb 100644 --- a/providers/s3/s3_test.go +++ b/providers/s3/s3_test.go @@ -7,7 +7,6 @@ import ( "context" "encoding/base64" "encoding/json" - "errors" "io" "net/http" "net/http/httptest" @@ -17,7 +16,8 @@ import ( "github.com/efficientgo/core/testutil" "github.com/go-kit/log" "github.com/minio/minio-go/v7/pkg/encrypt" - + "github.com/pkg/errors" + "github.com/thanos-io/objstore/exthttp" ) From 85504b96f6bbadeefda99d28cb00b07a423f3d09 Mon Sep 17 00:00:00 2001 From: milinddethe15 Date: Tue, 1 Oct 2024 11:22:44 +0530 Subject: [PATCH 10/41] refactor code Signed-off-by: milinddethe15 --- client/factory.go | 2 +- errutil/rt_error.go | 12 ++++++++++++ providers/azure/azure_test.go | 13 ++----------- providers/azure/helpers.go | 11 ++++------- providers/bos/bos.go | 1 + providers/cos/cos.go | 19 ++++++------------- providers/cos/cos_test.go | 13 ++----------- providers/gcs/gcs.go | 10 ++++------ providers/gcs/gcs_test.go | 13 +++---------- providers/obs/obs.go | 28 ++++++++++------------------ providers/obs/obs_test.go | 33 --------------------------------- providers/oci/oci.go | 3 +-- providers/oci/oci_test.go | 13 ++----------- providers/oss/oss.go | 5 +---- providers/oss/oss_test.go | 13 ++----------- providers/s3/s3.go | 10 ++++------ providers/s3/s3_test.go | 14 +++----------- providers/swift/swift.go | 10 ++++------ providers/swift/swift_test.go | 13 ++----------- 19 files changed, 64 insertions(+), 172 deletions(-) create mode 100644 errutil/rt_error.go delete mode 100644 providers/obs/obs_test.go diff --git a/client/factory.go b/client/factory.go index cfecd67c..bd345024 100644 --- a/client/factory.go +++ b/client/factory.go @@ -83,7 +83,7 @@ func NewBucket(logger log.Logger, confContentYaml []byte, component string, rt h case string(OCI): bucket, err = oci.NewBucket(logger, config, rt) case string(OBS): - bucket, err = obs.NewBucket(logger, config, rt) + bucket, err = obs.NewBucket(logger, config) default: return nil, errors.Errorf("bucket with type %s is not supported", bucketConf.Type) } diff --git a/errutil/rt_error.go b/errutil/rt_error.go new file mode 100644 index 00000000..ad1309e1 --- /dev/null +++ b/errutil/rt_error.go @@ -0,0 +1,12 @@ +package errutil + +import "net/http" + +// ErrorRoundTripper is a custom RoundTripper that always returns an error. +type ErrorRoundTripper struct { + Err error +} + +func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { + return nil, ert.Err +} diff --git a/providers/azure/azure_test.go b/providers/azure/azure_test.go index cc9c1317..85533eaa 100644 --- a/providers/azure/azure_test.go +++ b/providers/azure/azure_test.go @@ -4,7 +4,6 @@ package azure import ( - "net/http" "testing" "time" @@ -12,6 +11,7 @@ import ( "github.com/go-kit/log" "github.com/pkg/errors" + "github.com/thanos-io/objstore/errutil" "github.com/thanos-io/objstore/exthttp" ) @@ -226,20 +226,11 @@ http_config: testutil.Equals(t, true, transport.TLSClientConfig.InsecureSkipVerify) } -// ErrorRoundTripper is a custom RoundTripper that always returns an error -type ErrorRoundTripper struct { - Err error -} - -func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { - return nil, ert.Err -} - func TestNewBucketWithErrorRoundTripper(t *testing.T) { cfg, err := parseConfig(validConfig) testutil.Ok(t, err) - rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} + rt := &errutil.ErrorRoundTripper{Err: errors.New("RoundTripper error")} _, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test", rt) diff --git a/providers/azure/helpers.go b/providers/azure/helpers.go index 5ced695b..1f9dde08 100644 --- a/providers/azure/helpers.go +++ b/providers/azure/helpers.go @@ -23,14 +23,12 @@ func getContainerClient(conf Config) (*container.Client, error) { // Check if a roundtripper has been set in the config // otherwise build the default transport. var rt http.RoundTripper + rt, err := exthttp.DefaultTransport(conf.HTTPConfig) + if err != nil { + return nil, err + } if conf.HTTPConfig.Transport != nil { rt = conf.HTTPConfig.Transport - } else { - var err error - rt, err = exthttp.DefaultTransport(conf.HTTPConfig) - if err != nil { - return nil, err - } } opt := &container.ClientOptions{ ClientOptions: azcore.ClientOptions{ @@ -73,7 +71,6 @@ func getContainerClient(conf Config) (*container.Client, error) { // Otherwise use a token credential var cred azcore.TokenCredential - var err error // Use Managed Identity Credential if a user assigned ID is set if conf.UserAssignedID != "" { diff --git a/providers/bos/bos.go b/providers/bos/bos.go index 72e1b1e0..f88b7a21 100644 --- a/providers/bos/bos.go +++ b/providers/bos/bos.go @@ -66,6 +66,7 @@ func parseConfig(conf []byte) (Config, error) { // NewBucket new bos bucket. func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error) { + //TODO: Add support for custom roundtripper. if logger == nil { logger = log.NewNopLogger() } diff --git a/providers/cos/cos.go b/providers/cos/cos.go index ce61ca87..a8b853e3 100644 --- a/providers/cos/cos.go +++ b/providers/cos/cos.go @@ -127,20 +127,13 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string, rt } } b := &cos.BaseURL{BucketURL: bucketURL} - if rt != nil { - config.HTTPConfig.Transport = rt - } - // Check if a roundtripper has been set in the config - // otherwise build the default transport. var tpt http.RoundTripper - if config.HTTPConfig.Transport != nil { - tpt = config.HTTPConfig.Transport - } else { - var err error - tpt, err = exthttp.DefaultTransport(config.HTTPConfig) - if err != nil { - return nil, err - } + tpt, err = exthttp.DefaultTransport(config.HTTPConfig) + if err != nil { + return nil, err + } + if rt != nil { + tpt = rt } client := cos.NewClient(b, &http.Client{ Transport: &cos.AuthorizationTransport{ diff --git a/providers/cos/cos_test.go b/providers/cos/cos_test.go index d67e2dfd..f682aee9 100644 --- a/providers/cos/cos_test.go +++ b/providers/cos/cos_test.go @@ -5,7 +5,6 @@ package cos import ( "context" - "net/http" "testing" "time" @@ -14,6 +13,7 @@ import ( "github.com/pkg/errors" "github.com/prometheus/common/model" + "github.com/thanos-io/objstore/errutil" "github.com/thanos-io/objstore/exthttp" ) @@ -142,15 +142,6 @@ func TestConfig_validate(t *testing.T) { } } -// ErrorRoundTripper is a custom RoundTripper that always returns an error -type ErrorRoundTripper struct { - Err error -} - -func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { - return nil, ert.Err -} - func TestNewBucketWithErrorRoundTripper(t *testing.T) { config := Config{ Bucket: "bucket", @@ -159,7 +150,7 @@ func TestNewBucketWithErrorRoundTripper(t *testing.T) { SecretId: "sid", SecretKey: "skey", } - rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} + rt := &errutil.ErrorRoundTripper{Err: errors.New("RoundTripper error")} bkt, err := NewBucketWithConfig(log.NewNopLogger(), config, "test", rt) testutil.Ok(t, err) diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index e07ca05b..4617e0b7 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -122,14 +122,12 @@ func appendHttpOptions(gc Config, opts []option.ClientOption) ([]option.ClientOp // Check if a roundtripper has been set in the config // otherwise build the default transport. var rt http.RoundTripper + rt, err := exthttp.DefaultTransport(gc.HTTPConfig) + if err != nil { + return nil, err + } if gc.HTTPConfig.Transport != nil { rt = gc.HTTPConfig.Transport - } else { - var err error - rt, err = exthttp.DefaultTransport(gc.HTTPConfig) - if err != nil { - return nil, err - } } // GCS uses some defaults when "options.WithHTTPClient" is not used that are important when we call diff --git a/providers/gcs/gcs_test.go b/providers/gcs/gcs_test.go index 120e3450..66ea41cf 100644 --- a/providers/gcs/gcs_test.go +++ b/providers/gcs/gcs_test.go @@ -17,6 +17,7 @@ import ( "github.com/go-kit/log" "github.com/pkg/errors" "github.com/prometheus/common/model" + "github.com/thanos-io/objstore/errutil" "google.golang.org/api/option" ) @@ -159,18 +160,10 @@ http_config: } } -// ErrorRoundTripper is a custom RoundTripper that always returns an error -type ErrorRoundTripper struct { - Err error -} - -func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { - return nil, ert.Err -} - func TestNewBucketWithErrorRoundTripper(t *testing.T) { // Create an error RoundTripper - rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} + rt := &errutil.ErrorRoundTripper{Err: errors.New("RoundTripper error")} + cfg := Config{ Bucket: "test-bucket", ServiceAccount: "", diff --git a/providers/obs/obs.go b/providers/obs/obs.go index dcd342d5..eb6ca7ca 100644 --- a/providers/obs/obs.go +++ b/providers/obs/obs.go @@ -7,7 +7,6 @@ import ( "context" "io" "math" - "net/http" "os" "strings" "testing" @@ -75,12 +74,13 @@ type Bucket struct { name string } -func NewBucket(logger log.Logger, conf []byte, rt http.RoundTripper) (*Bucket, error) { +func NewBucket(logger log.Logger, conf []byte) (*Bucket, error) { + //TODO: Add support for custom roundtripper. config, err := parseConfig(conf) if err != nil { return nil, errors.Wrap(err, "parsing cos configuration") } - return NewBucketWithConfig(logger, config, rt) + return NewBucketWithConfig(logger, config) } func parseConfig(conf []byte) (Config, error) { @@ -92,25 +92,17 @@ func parseConfig(conf []byte) (Config, error) { return config, nil } -func NewBucketWithConfig(logger log.Logger, config Config, rt http.RoundTripper) (*Bucket, error) { +func NewBucketWithConfig(logger log.Logger, config Config) (*Bucket, error) { if err := config.validate(); err != nil { return nil, errors.Wrap(err, "validate obs config err") } - if rt != nil { - config.HTTPConfig.Transport = rt - } - var tpt *http.Transport - if config.HTTPConfig.Transport != nil { - tpt = config.HTTPConfig.Transport.(*http.Transport) - } else { - var err error - tpt, err = exthttp.DefaultTransport(config.HTTPConfig) - if err != nil { - return nil, errors.Wrap(err, "get http transport err") - } + + rt, err := exthttp.DefaultTransport(config.HTTPConfig) + if err != nil { + return nil, errors.Wrap(err, "get http transport err") } - client, err := obs.New(config.AccessKey, config.SecretKey, config.Endpoint, obs.WithHttpTransport(tpt)) + client, err := obs.New(config.AccessKey, config.SecretKey, config.Endpoint, obs.WithHttpTransport(rt)) if err != nil { return nil, errors.Wrap(err, "initialize obs client err") } @@ -377,7 +369,7 @@ func NewTestBucketFromConfig(t testing.TB, c Config, reuseBucket bool, location if err != nil { return nil, nil, err } - b, err := NewBucket(log.NewNopLogger(), bc, nil) + b, err := NewBucket(log.NewNopLogger(), bc) if err != nil { return nil, nil, err } diff --git a/providers/obs/obs_test.go b/providers/obs/obs_test.go deleted file mode 100644 index 26563b3c..00000000 --- a/providers/obs/obs_test.go +++ /dev/null @@ -1,33 +0,0 @@ -package obs - -import ( - "net/http" - "testing" - - "github.com/efficientgo/core/testutil" - "github.com/go-kit/log" - "github.com/pkg/errors" -) - -// ErrorRoundTripper is a custom RoundTripper that always returns an error -type ErrorRoundTripper struct { - Err error - Transport *http.Transport -} - -func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { - return nil, ert.Err -} - -func TestNewBucketWithErrorRoundTripper(t *testing.T) { - config := DefaultConfig - config.Endpoint = "https://test.com" - config.SecretKey = "test123" - config.AccessKey = "test123" - - rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} - _, err := NewBucketWithConfig(log.NewNopLogger(), config, rt) - // We expect an error from the RoundTripper - testutil.NotOk(t, err) - testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) -} diff --git a/providers/oci/oci.go b/providers/oci/oci.go index e8f8e0c5..e2f9e98b 100644 --- a/providers/oci/oci.go +++ b/providers/oci/oci.go @@ -336,10 +336,9 @@ func NewBucket(logger log.Logger, ociConfig []byte, rt http.RoundTripper) (*Buck return nil, errors.Wrapf(err, "unable to create ObjectStorage client with the given oci configurations") } + config.HTTPConfig.Transport = CustomTransport(config) if rt != nil { config.HTTPConfig.Transport = rt - } else { - config.HTTPConfig.Transport = CustomTransport(config) } httpClient := http.Client{ Transport: config.HTTPConfig.Transport, diff --git a/providers/oci/oci_test.go b/providers/oci/oci_test.go index 87288cee..7ea292fd 100644 --- a/providers/oci/oci_test.go +++ b/providers/oci/oci_test.go @@ -1,24 +1,15 @@ package oci import ( - "net/http" "testing" "github.com/efficientgo/core/testutil" "github.com/go-kit/log" "github.com/pkg/errors" + "github.com/thanos-io/objstore/errutil" "gopkg.in/yaml.v2" ) -// ErrorRoundTripper is a custom RoundTripper that always returns an error -type ErrorRoundTripper struct { - Err error -} - -func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { - return nil, ert.Err -} - func TestNewBucketWithErrorRoundTripper(t *testing.T) { mockPrivateKey := `-----BEGIN RSA PRIVATE KEY----- @@ -48,7 +39,7 @@ G6aFKaqQfOXKCyWoUiVknQJAXrlgySFci/2ueKlIE1QqIiLSZ8V8OlpFLRnb1pzI ociConfig, err := yaml.Marshal(config) testutil.Ok(t, err) - rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} + rt := &errutil.ErrorRoundTripper{Err: errors.New("RoundTripper error")} _, err = NewBucket(log.NewNopLogger(), ociConfig, rt) // We expect an error from the RoundTripper diff --git a/providers/oss/oss.go b/providers/oss/oss.go index 572778d1..0bf5bd4b 100644 --- a/providers/oss/oss.go +++ b/providers/oss/oss.go @@ -171,16 +171,13 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string, rt if err := validate(config); err != nil { return nil, err } - var client *alioss.Client - var err error + client, err := alioss.New(config.Endpoint, config.AccessKeyID, config.AccessKeySecret) if rt != nil { // custom RoundTripper clientOption := func(client *alioss.Client) { client.HTTPClient = &http.Client{Transport: rt} } client, err = alioss.New(config.Endpoint, config.AccessKeyID, config.AccessKeySecret, clientOption) - } else { - client, err = alioss.New(config.Endpoint, config.AccessKeyID, config.AccessKeySecret) } if err != nil { return nil, errors.Wrap(err, "create aliyun oss client failed") diff --git a/providers/oss/oss_test.go b/providers/oss/oss_test.go index 23f174d5..f3380e71 100644 --- a/providers/oss/oss_test.go +++ b/providers/oss/oss_test.go @@ -2,23 +2,14 @@ package oss import ( "context" - "net/http" "testing" "github.com/efficientgo/core/testutil" "github.com/go-kit/log" "github.com/pkg/errors" + "github.com/thanos-io/objstore/errutil" ) -// ErrorRoundTripper is a custom RoundTripper that always returns an error -type ErrorRoundTripper struct { - Err error -} - -func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { - return nil, ert.Err -} - func TestNewBucketWithErrorRoundTripper(t *testing.T) { var config Config @@ -28,7 +19,7 @@ func TestNewBucketWithErrorRoundTripper(t *testing.T) { config.Bucket = "test" config.AccessKeySecret = "123" - rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} + rt := &errutil.ErrorRoundTripper{Err: errors.New("RoundTripper error")} bkt, err := NewBucketWithConfig(log.NewNopLogger(), config, "test", rt) // We expect an error from the RoundTripper diff --git a/providers/s3/s3.go b/providers/s3/s3.go index c85c0970..3d9ba213 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -247,14 +247,12 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string, rt // Check if a roundtripper has been set in the config // otherwise build the default transport. var tpt http.RoundTripper + tpt, err := exthttp.DefaultTransport(config.HTTPConfig) + if err != nil { + return nil, err + } if config.HTTPConfig.Transport != nil { tpt = config.HTTPConfig.Transport - } else { - var err error - tpt, err = exthttp.DefaultTransport(config.HTTPConfig) - if err != nil { - return nil, err - } } client, err := minio.New(config.Endpoint, &minio.Options{ diff --git a/providers/s3/s3_test.go b/providers/s3/s3_test.go index e34e19bb..2a44f0e0 100644 --- a/providers/s3/s3_test.go +++ b/providers/s3/s3_test.go @@ -17,7 +17,8 @@ import ( "github.com/go-kit/log" "github.com/minio/minio-go/v7/pkg/encrypt" "github.com/pkg/errors" - + + "github.com/thanos-io/objstore/errutil" "github.com/thanos-io/objstore/exthttp" ) @@ -464,20 +465,11 @@ func TestParseConfig_DefaultStorageClassIsZero(t *testing.T) { testutil.Equals(t, "", bkt.storageClass) } -// ErrorRoundTripper is a custom RoundTripper that always returns an error -type ErrorRoundTripper struct { - Err error -} - -func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { - return nil, ert.Err -} - func TestNewBucketWithErrorRoundTripper(t *testing.T) { cfg := DefaultConfig cfg.Endpoint = endpoint cfg.Bucket = "test" - rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} + rt := &errutil.ErrorRoundTripper{Err: errors.New("RoundTripper error")} bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test", rt) testutil.Ok(t, err) _, err = bkt.Get(context.Background(), "test") diff --git a/providers/swift/swift.go b/providers/swift/swift.go index f287f551..682c494c 100644 --- a/providers/swift/swift.go +++ b/providers/swift/swift.go @@ -185,14 +185,12 @@ func NewContainerFromConfig(logger log.Logger, sc *Config, createContainer bool, // Check if a roundtripper has been set in the config // otherwise build the default transport. var tpt http.RoundTripper + tpt, err := exthttp.DefaultTransport(sc.HTTPConfig) + if err != nil { + return nil, err + } if sc.HTTPConfig.Transport != nil { tpt = sc.HTTPConfig.Transport - } else { - var err error - tpt, err = exthttp.DefaultTransport(sc.HTTPConfig) - if err != nil { - return nil, err - } } connection := connectionFromConfig(sc, tpt) diff --git a/providers/swift/swift_test.go b/providers/swift/swift_test.go index a203378c..629dfd77 100644 --- a/providers/swift/swift_test.go +++ b/providers/swift/swift_test.go @@ -4,7 +4,6 @@ package swift import ( - "net/http" "testing" "time" @@ -12,6 +11,7 @@ import ( "github.com/go-kit/log" "github.com/pkg/errors" "github.com/prometheus/common/model" + "github.com/thanos-io/objstore/errutil" ) func TestParseConfig(t *testing.T) { @@ -68,18 +68,9 @@ http_config: } -// ErrorRoundTripper is a custom RoundTripper that always returns an error -type ErrorRoundTripper struct { - Err error -} - -func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { - return nil, ert.Err -} - func TestNewBucketWithErrorRoundTripper(t *testing.T) { logger := log.NewNopLogger() - rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} + rt := &errutil.ErrorRoundTripper{Err: errors.New("RoundTripper error")} config := DefaultConfig config.AuthUrl = "http://identity.something.com/v3" _, err := NewContainerFromConfig(logger, &config, false, rt) From 6eb7db97d6d7b1ba60f2d68ea271c8f3737d619d Mon Sep 17 00:00:00 2001 From: Suraj Patil <31805557+patilsuraj767@users.noreply.github.com> Date: Tue, 1 Oct 2024 12:57:06 +0530 Subject: [PATCH 11/41] S3: exporting minio client (#143) Signed-off-by: Suraj Patil --- providers/s3/s3.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/providers/s3/s3.go b/providers/s3/s3.go index afd0982d..0cc36f69 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -154,7 +154,7 @@ type TraceConfig struct { type Bucket struct { logger log.Logger name string - client *minio.Client + Client *minio.Client defaultSSE encrypt.ServerSide putUserMetadata map[string]string storageClass string @@ -329,7 +329,7 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B bkt := &Bucket{ logger: logger, name: config.Bucket, - client: client, + Client: client, defaultSSE: sse, putUserMetadata: config.PutUserMetadata, storageClass: storageClass, @@ -400,7 +400,7 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt UseV1: b.listObjectsV1, } - for object := range b.client.ListObjects(ctx, b.name, opts) { + for object := range b.Client.ListObjects(ctx, b.name, opts) { // Catch the error when failed to list objects. if object.Err != nil { return object.Err @@ -439,10 +439,10 @@ func (b *Bucket) getRange(ctx context.Context, name string, off, length int64) ( } // StatObject to see if the object exists and we have permissions to read it - if _, err := b.client.StatObject(ctx, b.name, name, *opts); err != nil { + if _, err := b.Client.StatObject(ctx, b.name, name, *opts); err != nil { return nil, err } - return b.client.GetObject(ctx, b.name, name, *opts) + return b.Client.GetObject(ctx, b.name, name, *opts) } // Get returns a reader for the given object name. @@ -457,7 +457,7 @@ func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) ( // Exists checks if the given object exists. func (b *Bucket) Exists(ctx context.Context, name string) (bool, error) { - _, err := b.client.StatObject(ctx, b.name, name, minio.StatObjectOptions{}) + _, err := b.Client.StatObject(ctx, b.name, name, minio.StatObjectOptions{}) if err != nil { if b.IsObjNotFoundErr(err) { return false, nil @@ -493,7 +493,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { userMetadata[k] = v } - if _, err := b.client.PutObject( + if _, err := b.Client.PutObject( ctx, b.name, name, @@ -520,7 +520,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { // Attributes returns information about the specified object. func (b *Bucket) Attributes(ctx context.Context, name string) (objstore.ObjectAttributes, error) { - objInfo, err := b.client.StatObject(ctx, b.name, name, minio.StatObjectOptions{}) + objInfo, err := b.Client.StatObject(ctx, b.name, name, minio.StatObjectOptions{}) if err != nil { return objstore.ObjectAttributes{}, err } @@ -533,7 +533,7 @@ func (b *Bucket) Attributes(ctx context.Context, name string) (objstore.ObjectAt // Delete removes the object with the given name. func (b *Bucket) Delete(ctx context.Context, name string) error { - return b.client.RemoveObject(ctx, b.name, name, minio.RemoveObjectOptions{}) + return b.Client.RemoveObject(ctx, b.name, name, minio.RemoveObjectOptions{}) } // IsObjNotFoundErr returns true if error means that object is not found. Relevant to Get operations. @@ -622,7 +622,7 @@ func NewTestBucketFromConfig(t testing.TB, location string, c Config, reuseBucke bktToCreate = objstore.CreateTemporaryTestBucketName(t) } - if err := b.client.MakeBucket(ctx, bktToCreate, minio.MakeBucketOptions{Region: location}); err != nil { + if err := b.Client.MakeBucket(ctx, bktToCreate, minio.MakeBucketOptions{Region: location}); err != nil { return nil, nil, err } b.name = bktToCreate @@ -630,7 +630,7 @@ func NewTestBucketFromConfig(t testing.TB, location string, c Config, reuseBucke return b, func() { objstore.EmptyBucket(t, ctx, b) - if err := b.client.RemoveBucket(ctx, bktToCreate); err != nil { + if err := b.Client.RemoveBucket(ctx, bktToCreate); err != nil { t.Logf("deleting bucket %s failed: %s", bktToCreate, err) } }, nil From 075bf61a90bc7383aa90af2c5fda458af5e9fd30 Mon Sep 17 00:00:00 2001 From: Suraj Patil <31805557+patilsuraj767@users.noreply.github.com> Date: Tue, 1 Oct 2024 17:47:38 +0530 Subject: [PATCH 12/41] Revert #143 (#144) Signed-off-by: Suraj Patil --- providers/s3/s3.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/providers/s3/s3.go b/providers/s3/s3.go index 0cc36f69..afd0982d 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -154,7 +154,7 @@ type TraceConfig struct { type Bucket struct { logger log.Logger name string - Client *minio.Client + client *minio.Client defaultSSE encrypt.ServerSide putUserMetadata map[string]string storageClass string @@ -329,7 +329,7 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B bkt := &Bucket{ logger: logger, name: config.Bucket, - Client: client, + client: client, defaultSSE: sse, putUserMetadata: config.PutUserMetadata, storageClass: storageClass, @@ -400,7 +400,7 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt UseV1: b.listObjectsV1, } - for object := range b.Client.ListObjects(ctx, b.name, opts) { + for object := range b.client.ListObjects(ctx, b.name, opts) { // Catch the error when failed to list objects. if object.Err != nil { return object.Err @@ -439,10 +439,10 @@ func (b *Bucket) getRange(ctx context.Context, name string, off, length int64) ( } // StatObject to see if the object exists and we have permissions to read it - if _, err := b.Client.StatObject(ctx, b.name, name, *opts); err != nil { + if _, err := b.client.StatObject(ctx, b.name, name, *opts); err != nil { return nil, err } - return b.Client.GetObject(ctx, b.name, name, *opts) + return b.client.GetObject(ctx, b.name, name, *opts) } // Get returns a reader for the given object name. @@ -457,7 +457,7 @@ func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) ( // Exists checks if the given object exists. func (b *Bucket) Exists(ctx context.Context, name string) (bool, error) { - _, err := b.Client.StatObject(ctx, b.name, name, minio.StatObjectOptions{}) + _, err := b.client.StatObject(ctx, b.name, name, minio.StatObjectOptions{}) if err != nil { if b.IsObjNotFoundErr(err) { return false, nil @@ -493,7 +493,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { userMetadata[k] = v } - if _, err := b.Client.PutObject( + if _, err := b.client.PutObject( ctx, b.name, name, @@ -520,7 +520,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { // Attributes returns information about the specified object. func (b *Bucket) Attributes(ctx context.Context, name string) (objstore.ObjectAttributes, error) { - objInfo, err := b.Client.StatObject(ctx, b.name, name, minio.StatObjectOptions{}) + objInfo, err := b.client.StatObject(ctx, b.name, name, minio.StatObjectOptions{}) if err != nil { return objstore.ObjectAttributes{}, err } @@ -533,7 +533,7 @@ func (b *Bucket) Attributes(ctx context.Context, name string) (objstore.ObjectAt // Delete removes the object with the given name. func (b *Bucket) Delete(ctx context.Context, name string) error { - return b.Client.RemoveObject(ctx, b.name, name, minio.RemoveObjectOptions{}) + return b.client.RemoveObject(ctx, b.name, name, minio.RemoveObjectOptions{}) } // IsObjNotFoundErr returns true if error means that object is not found. Relevant to Get operations. @@ -622,7 +622,7 @@ func NewTestBucketFromConfig(t testing.TB, location string, c Config, reuseBucke bktToCreate = objstore.CreateTemporaryTestBucketName(t) } - if err := b.Client.MakeBucket(ctx, bktToCreate, minio.MakeBucketOptions{Region: location}); err != nil { + if err := b.client.MakeBucket(ctx, bktToCreate, minio.MakeBucketOptions{Region: location}); err != nil { return nil, nil, err } b.name = bktToCreate @@ -630,7 +630,7 @@ func NewTestBucketFromConfig(t testing.TB, location string, c Config, reuseBucke return b, func() { objstore.EmptyBucket(t, ctx, b) - if err := b.Client.RemoveBucket(ctx, bktToCreate); err != nil { + if err := b.client.RemoveBucket(ctx, bktToCreate); err != nil { t.Logf("deleting bucket %s failed: %s", bktToCreate, err) } }, nil From b469d7451d6d9b7b3a0df9dde1f48e02dc712079 Mon Sep 17 00:00:00 2001 From: milinddethe15 Date: Wed, 2 Oct 2024 00:36:38 +0530 Subject: [PATCH 13/41] gcs error-rt test Signed-off-by: milinddethe15 --- go.mod | 2 +- providers/gcs/fakecred.json | 9 +++++++++ providers/gcs/gcs_test.go | 18 +++++++++--------- 3 files changed, 19 insertions(+), 10 deletions(-) create mode 100644 providers/gcs/fakecred.json diff --git a/go.mod b/go.mod index af1cc40e..db189c20 100644 --- a/go.mod +++ b/go.mod @@ -88,6 +88,7 @@ require ( github.com/prometheus/procfs v0.11.1 // indirect github.com/rs/xid v1.5.0 // indirect github.com/sony/gobreaker v0.5.0 // indirect + github.com/stretchr/objx v0.5.2 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect @@ -109,6 +110,5 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.0 github.com/kr/text v0.2.0 // indirect github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b // indirect - github.com/stretchr/testify v1.9.0 // indirect golang.org/x/crypto v0.24.0 // indirect ) diff --git a/providers/gcs/fakecred.json b/providers/gcs/fakecred.json new file mode 100644 index 00000000..fe130026 --- /dev/null +++ b/providers/gcs/fakecred.json @@ -0,0 +1,9 @@ +{ + "type": "service_account", + "private_key_id": "abc", + "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDY3E8o1NEFcjMM\nHW/5ZfFJw29/8NEqpViNjQIx95Xx5KDtJ+nWn9+OW0uqsSqKlKGhAdAo+Q6bjx2c\nuXVsXTu7XrZUY5Kltvj94DvUa1wjNXs606r/RxWTJ58bfdC+gLLxBfGnB6CwK0YQ\nxnfpjNbkUfVVzO0MQD7UP0Hl5ZcY0Puvxd/yHuONQn/rIAieTHH1pqgW+zrH/y3c\n59IGThC9PPtugI9ea8RSnVj3PWz1bX2UkCDpy9IRh9LzJLaYYX9RUd7++dULUlat\nAaXBh1U6emUDzhrIsgApjDVtimOPbmQWmX1S60mqQikRpVYZ8u+NDD+LNw+/Eovn\nxCj2Y3z1AgMBAAECggEAWDBzoqO1IvVXjBA2lqId10T6hXmN3j1ifyH+aAqK+FVl\nGjyWjDj0xWQcJ9ync7bQ6fSeTeNGzP0M6kzDU1+w6FgyZqwdmXWI2VmEizRjwk+/\n/uLQUcL7I55Dxn7KUoZs/rZPmQDxmGLoue60Gg6z3yLzVcKiDc7cnhzhdBgDc8vd\nQorNAlqGPRnm3EqKQ6VQp6fyQmCAxrr45kspRXNLddat3AMsuqImDkqGKBmF3Q1y\nxWGe81LphUiRqvqbyUlh6cdSZ8pLBpc9m0c3qWPKs9paqBIvgUPlvOZMqec6x4S6\nChbdkkTRLnbsRr0Yg/nDeEPlkhRBhasXpxpMUBgPywKBgQDs2axNkFjbU94uXvd5\nznUhDVxPFBuxyUHtsJNqW4p/ujLNimGet5E/YthCnQeC2P3Ym7c3fiz68amM6hiA\nOnW7HYPZ+jKFnefpAtjyOOs46AkftEg07T9XjwWNPt8+8l0DYawPoJgbM5iE0L2O\nx8TU1Vs4mXc+ql9F90GzI0x3VwKBgQDqZOOqWw3hTnNT07Ixqnmd3dugV9S7eW6o\nU9OoUgJB4rYTpG+yFqNqbRT8bkx37iKBMEReppqonOqGm4wtuRR6LSLlgcIU9Iwx\nyfH12UWqVmFSHsgZFqM/cK3wGev38h1WBIOx3/djKn7BdlKVh8kWyx6uC8bmV+E6\nOoK0vJD6kwKBgHAySOnROBZlqzkiKW8c+uU2VATtzJSydrWm0J4wUPJifNBa/hVW\ndcqmAzXC9xznt5AVa3wxHBOfyKaE+ig8CSsjNyNZ3vbmr0X04FoV1m91k2TeXNod\njMTobkPThaNm4eLJMN2SQJuaHGTGERWC0l3T18t+/zrDMDCPiSLX1NAvAoGBAN1T\nVLJYdjvIMxf1bm59VYcepbK7HLHFkRq6xMJMZbtG0ryraZjUzYvB4q4VjHk2UDiC\nlhx13tXWDZH7MJtABzjyg+AI7XWSEQs2cBXACos0M4Myc6lU+eL+iA+OuoUOhmrh\nqmT8YYGu76/IBWUSqWuvcpHPpwl7871i4Ga/I3qnAoGBANNkKAcMoeAbJQK7a/Rn\nwPEJB+dPgNDIaboAsh1nZhVhN5cvdvCWuEYgOGCPQLYQF0zmTLcM+sVxOYgfy8mV\nfbNgPgsP5xmu6dw2COBKdtozw0HrWSRjACd1N4yGu75+wPCcX/gQarcjRcXXZeEa\nNtBLSfcqPULqD+h7br9lEJio\n-----END PRIVATE KEY-----\n", + "client_email": "123-abc@developer.gserviceaccount.com", + "client_id": "123-abc.apps.googleusercontent.com", + "auth_uri": "http://127.0.0.1:43649/", + "token_uri": "http://127.0.0.1:43649/" + } \ No newline at end of file diff --git a/providers/gcs/gcs_test.go b/providers/gcs/gcs_test.go index 66ea41cf..f627ff04 100644 --- a/providers/gcs/gcs_test.go +++ b/providers/gcs/gcs_test.go @@ -161,23 +161,23 @@ http_config: } func TestNewBucketWithErrorRoundTripper(t *testing.T) { - // Create an error RoundTripper + // Create an error RoundTripper. rt := &errutil.ErrorRoundTripper{Err: errors.New("RoundTripper error")} - cfg := Config{ Bucket: "test-bucket", ServiceAccount: "", UseGRPC: false, } - svr, err := gcsemu.NewServer("127.0.0.1:0", gcsemu.Options{}) - testutil.Ok(t, err) - err = os.Setenv("STORAGE_EMULATOR_HOST", svr.Addr) + svr, err := gcsemu.NewServer("127.0.0.1:43649", gcsemu.Options{}) testutil.Ok(t, err) + defer svr.Close() err = os.Setenv("GCS_EMULATOR_HOST", svr.Addr) testutil.Ok(t, err) - defer svr.Close() - - _, err = NewBucketWithConfig(context.Background(), log.NewNopLogger(), cfg, "test-bucket", rt) - testutil.NotOk(t, err) // Expect an error when using the error RoundTripper + os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", "fakecred.json") + bkt, err := NewBucketWithConfig(context.Background(), log.NewNopLogger(), cfg, "test-bucket", rt) + testutil.Ok(t, err) + testutil.Assert(t, bkt.name == "test-bucket") + _, err = bkt.Get(context.Background(), "test-bucket") + testutil.NotOk(t, err) testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) } From baca2f75dab03a338759c9ddb3b1b0ab7a92b0a4 Mon Sep 17 00:00:00 2001 From: milinddethe15 Date: Wed, 2 Oct 2024 15:39:49 +0530 Subject: [PATCH 14/41] added error-rt gcs test Signed-off-by: milinddethe15 --- providers/gcs/fakecred.json | 9 --------- providers/gcs/gcs.go | 7 +++++-- providers/gcs/gcs_test.go | 8 ++++---- 3 files changed, 9 insertions(+), 15 deletions(-) delete mode 100644 providers/gcs/fakecred.json diff --git a/providers/gcs/fakecred.json b/providers/gcs/fakecred.json deleted file mode 100644 index fe130026..00000000 --- a/providers/gcs/fakecred.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "type": "service_account", - "private_key_id": "abc", - "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDY3E8o1NEFcjMM\nHW/5ZfFJw29/8NEqpViNjQIx95Xx5KDtJ+nWn9+OW0uqsSqKlKGhAdAo+Q6bjx2c\nuXVsXTu7XrZUY5Kltvj94DvUa1wjNXs606r/RxWTJ58bfdC+gLLxBfGnB6CwK0YQ\nxnfpjNbkUfVVzO0MQD7UP0Hl5ZcY0Puvxd/yHuONQn/rIAieTHH1pqgW+zrH/y3c\n59IGThC9PPtugI9ea8RSnVj3PWz1bX2UkCDpy9IRh9LzJLaYYX9RUd7++dULUlat\nAaXBh1U6emUDzhrIsgApjDVtimOPbmQWmX1S60mqQikRpVYZ8u+NDD+LNw+/Eovn\nxCj2Y3z1AgMBAAECggEAWDBzoqO1IvVXjBA2lqId10T6hXmN3j1ifyH+aAqK+FVl\nGjyWjDj0xWQcJ9ync7bQ6fSeTeNGzP0M6kzDU1+w6FgyZqwdmXWI2VmEizRjwk+/\n/uLQUcL7I55Dxn7KUoZs/rZPmQDxmGLoue60Gg6z3yLzVcKiDc7cnhzhdBgDc8vd\nQorNAlqGPRnm3EqKQ6VQp6fyQmCAxrr45kspRXNLddat3AMsuqImDkqGKBmF3Q1y\nxWGe81LphUiRqvqbyUlh6cdSZ8pLBpc9m0c3qWPKs9paqBIvgUPlvOZMqec6x4S6\nChbdkkTRLnbsRr0Yg/nDeEPlkhRBhasXpxpMUBgPywKBgQDs2axNkFjbU94uXvd5\nznUhDVxPFBuxyUHtsJNqW4p/ujLNimGet5E/YthCnQeC2P3Ym7c3fiz68amM6hiA\nOnW7HYPZ+jKFnefpAtjyOOs46AkftEg07T9XjwWNPt8+8l0DYawPoJgbM5iE0L2O\nx8TU1Vs4mXc+ql9F90GzI0x3VwKBgQDqZOOqWw3hTnNT07Ixqnmd3dugV9S7eW6o\nU9OoUgJB4rYTpG+yFqNqbRT8bkx37iKBMEReppqonOqGm4wtuRR6LSLlgcIU9Iwx\nyfH12UWqVmFSHsgZFqM/cK3wGev38h1WBIOx3/djKn7BdlKVh8kWyx6uC8bmV+E6\nOoK0vJD6kwKBgHAySOnROBZlqzkiKW8c+uU2VATtzJSydrWm0J4wUPJifNBa/hVW\ndcqmAzXC9xznt5AVa3wxHBOfyKaE+ig8CSsjNyNZ3vbmr0X04FoV1m91k2TeXNod\njMTobkPThaNm4eLJMN2SQJuaHGTGERWC0l3T18t+/zrDMDCPiSLX1NAvAoGBAN1T\nVLJYdjvIMxf1bm59VYcepbK7HLHFkRq6xMJMZbtG0ryraZjUzYvB4q4VjHk2UDiC\nlhx13tXWDZH7MJtABzjyg+AI7XWSEQs2cBXACos0M4Myc6lU+eL+iA+OuoUOhmrh\nqmT8YYGu76/IBWUSqWuvcpHPpwl7871i4Ga/I3qnAoGBANNkKAcMoeAbJQK7a/Rn\nwPEJB+dPgNDIaboAsh1nZhVhN5cvdvCWuEYgOGCPQLYQF0zmTLcM+sVxOYgfy8mV\nfbNgPgsP5xmu6dw2COBKdtozw0HrWSRjACd1N4yGu75+wPCcX/gQarcjRcXXZeEa\nNtBLSfcqPULqD+h7br9lEJio\n-----END PRIVATE KEY-----\n", - "client_email": "123-abc@developer.gserviceaccount.com", - "client_id": "123-abc.apps.googleusercontent.com", - "auth_uri": "http://127.0.0.1:43649/", - "token_uri": "http://127.0.0.1:43649/" - } \ No newline at end of file diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index 4617e0b7..b5dae200 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -52,7 +52,8 @@ type Config struct { // ChunkSizeBytes controls the maximum number of bytes of the object that the // Writer will attempt to send to the server in a single request // Used as storage.Writer.ChunkSize of https://pkg.go.dev/google.golang.org/cloud/storage#Writer - ChunkSizeBytes int `yaml:"chunk_size_bytes"` + ChunkSizeBytes int `yaml:"chunk_size_bytes"` + noAuth bool `yaml:"no_auth"` } // Bucket implements the store.Bucket and shipper.Bucket interfaces against GCS. @@ -102,7 +103,9 @@ func NewBucketWithConfig(ctx context.Context, logger log.Logger, gc Config, comp } opts = append(opts, option.WithCredentials(credentials)) } - + if gc.noAuth { + opts = append(opts, option.WithoutAuthentication()) + } opts = append(opts, option.WithUserAgent(fmt.Sprintf("thanos-%s/%s (%s)", component, version.Version, runtime.Version())), ) diff --git a/providers/gcs/gcs_test.go b/providers/gcs/gcs_test.go index f627ff04..a06b3a16 100644 --- a/providers/gcs/gcs_test.go +++ b/providers/gcs/gcs_test.go @@ -167,16 +167,16 @@ func TestNewBucketWithErrorRoundTripper(t *testing.T) { Bucket: "test-bucket", ServiceAccount: "", UseGRPC: false, + noAuth: true, } - svr, err := gcsemu.NewServer("127.0.0.1:43649", gcsemu.Options{}) + svr, err := gcsemu.NewServer("127.0.0.1:0", gcsemu.Options{}) testutil.Ok(t, err) defer svr.Close() - err = os.Setenv("GCS_EMULATOR_HOST", svr.Addr) + err = os.Setenv("STORAGE_EMULATOR_HOST", svr.Addr) testutil.Ok(t, err) - os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", "fakecred.json") + bkt, err := NewBucketWithConfig(context.Background(), log.NewNopLogger(), cfg, "test-bucket", rt) testutil.Ok(t, err) - testutil.Assert(t, bkt.name == "test-bucket") _, err = bkt.Get(context.Background(), "test-bucket") testutil.NotOk(t, err) testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) From 7f505a232e86119497b4224583f6fabe14b29d9c Mon Sep 17 00:00:00 2001 From: milinddethe15 Date: Wed, 2 Oct 2024 17:17:34 +0530 Subject: [PATCH 15/41] refactor code Signed-off-by: milinddethe15 --- providers/azure/helpers.go | 2 -- providers/bos/bos.go | 2 +- providers/gcs/gcs_test.go | 1 - providers/obs/obs.go | 2 +- providers/oci/oci_test.go | 3 +-- providers/oss/oss.go | 1 - providers/oss/oss_test.go | 14 ++++++-------- 7 files changed, 9 insertions(+), 16 deletions(-) diff --git a/providers/azure/helpers.go b/providers/azure/helpers.go index 1f9dde08..2915fbbb 100644 --- a/providers/azure/helpers.go +++ b/providers/azure/helpers.go @@ -20,8 +20,6 @@ import ( const DirDelim = "/" func getContainerClient(conf Config) (*container.Client, error) { - // Check if a roundtripper has been set in the config - // otherwise build the default transport. var rt http.RoundTripper rt, err := exthttp.DefaultTransport(conf.HTTPConfig) if err != nil { diff --git a/providers/bos/bos.go b/providers/bos/bos.go index f88b7a21..74f9688e 100644 --- a/providers/bos/bos.go +++ b/providers/bos/bos.go @@ -66,7 +66,7 @@ func parseConfig(conf []byte) (Config, error) { // NewBucket new bos bucket. func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error) { - //TODO: Add support for custom roundtripper. + // TODO(https://github.com/thanos-io/objstore/pull/140): Add support for custom roundtripper. if logger == nil { logger = log.NewNopLogger() } diff --git a/providers/gcs/gcs_test.go b/providers/gcs/gcs_test.go index a06b3a16..39b55041 100644 --- a/providers/gcs/gcs_test.go +++ b/providers/gcs/gcs_test.go @@ -161,7 +161,6 @@ http_config: } func TestNewBucketWithErrorRoundTripper(t *testing.T) { - // Create an error RoundTripper. rt := &errutil.ErrorRoundTripper{Err: errors.New("RoundTripper error")} cfg := Config{ Bucket: "test-bucket", diff --git a/providers/obs/obs.go b/providers/obs/obs.go index eb6ca7ca..7de70633 100644 --- a/providers/obs/obs.go +++ b/providers/obs/obs.go @@ -75,7 +75,7 @@ type Bucket struct { } func NewBucket(logger log.Logger, conf []byte) (*Bucket, error) { - //TODO: Add support for custom roundtripper. + // TODO(https://github.com/thanos-io/objstore/pull/140): Add support for custom roundtripper. config, err := parseConfig(conf) if err != nil { return nil, errors.Wrap(err, "parsing cos configuration") diff --git a/providers/oci/oci_test.go b/providers/oci/oci_test.go index 7ea292fd..0fc81ccf 100644 --- a/providers/oci/oci_test.go +++ b/providers/oci/oci_test.go @@ -11,8 +11,7 @@ import ( ) func TestNewBucketWithErrorRoundTripper(t *testing.T) { - - mockPrivateKey := `-----BEGIN RSA PRIVATE KEY----- + const mockPrivateKey = `-----BEGIN RSA PRIVATE KEY----- MIICXgIBAAKBgQDCFENGw33yGihy92pDjZQhl0C36rPJj+CvfSC8+q28hxA161QF NUd13wuCTUcq0Qd2qsBe/2hFyc2DCJJg0h1L78+6Z4UMR7EOcpfdUE9Hf3m/hs+F UR45uBJeDK1HSFHD8bHKD6kv8FPGfJTotc+2xjJwoYi+1hqp1fIekaxsyQIDAQAB diff --git a/providers/oss/oss.go b/providers/oss/oss.go index 0bf5bd4b..0a4cc76e 100644 --- a/providers/oss/oss.go +++ b/providers/oss/oss.go @@ -173,7 +173,6 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string, rt } client, err := alioss.New(config.Endpoint, config.AccessKeyID, config.AccessKeySecret) if rt != nil { - // custom RoundTripper clientOption := func(client *alioss.Client) { client.HTTPClient = &http.Client{Transport: rt} } diff --git a/providers/oss/oss_test.go b/providers/oss/oss_test.go index f3380e71..99ad68a3 100644 --- a/providers/oss/oss_test.go +++ b/providers/oss/oss_test.go @@ -11,14 +11,12 @@ import ( ) func TestNewBucketWithErrorRoundTripper(t *testing.T) { - - var config Config - config.Endpoint = "http://test.com/" - config.AccessKeyID = "123" - config.AccessKeySecret = "123" - config.Bucket = "test" - config.AccessKeySecret = "123" - + config := Config{ + Endpoint: "http://test.com/", + AccessKeyID: "123", + AccessKeySecret: "123", + Bucket: "test", + } rt := &errutil.ErrorRoundTripper{Err: errors.New("RoundTripper error")} bkt, err := NewBucketWithConfig(log.NewNopLogger(), config, "test", rt) From 3abd692dc442003a475b688f63a4b75005438aaf Mon Sep 17 00:00:00 2001 From: Ashwanth Goli Date: Thu, 10 Oct 2024 19:01:49 +0530 Subject: [PATCH 16/41] gcs: add config to disable retries Signed-off-by: Ashwanth Goli --- providers/gcs/gcs.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index b5dae200..0a5d89df 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -54,6 +54,9 @@ type Config struct { // Used as storage.Writer.ChunkSize of https://pkg.go.dev/google.golang.org/cloud/storage#Writer ChunkSizeBytes int `yaml:"chunk_size_bytes"` noAuth bool `yaml:"no_auth"` + + // gcs client retries idempotent operations by default, this option disables retries. + DisableRetries bool `yaml:"disable_retries"` } // Bucket implements the store.Bucket and shipper.Bucket interfaces against GCS. @@ -172,6 +175,11 @@ func newBucket(ctx context.Context, logger log.Logger, gc Config, opts []option. name: gc.Bucket, chunkSize: gc.ChunkSizeBytes, } + + if gc.DisableRetries { + bkt.bkt = bkt.bkt.Retryer(storage.WithPolicy(storage.RetryNever)) + } + return bkt, nil } From 220d250d7f1e9097aa067b34af045f87c47aed84 Mon Sep 17 00:00:00 2001 From: Ashwanth Goli Date: Thu, 10 Oct 2024 19:09:17 +0530 Subject: [PATCH 17/41] add changelog Signed-off-by: Ashwanth Goli --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12375cbf..5deef6bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#116](https://github.com/thanos-io/objstore/pull/116) Azure: Add new storage_create_container configuration property - [#128](https://github.com/thanos-io/objstore/pull/128) GCS: Add support for `ChunkSize` for writer. - [#130](https://github.com/thanos-io/objstore/pull/130) feat: Decouple creating bucket metrics from instrumenting the bucket +- [#147](https://github.com/thanos-io/objstore/pull/147) gcs: Add config to disable retries. ### Changed - [#38](https://github.com/thanos-io/objstore/pull/38) *: Upgrade minio-go version to `v7.0.45`. From 25c6a369f264d21c50b122137d94363d1708a0a7 Mon Sep 17 00:00:00 2001 From: Ake van der Meer Date: Thu, 10 Oct 2024 17:58:36 +0200 Subject: [PATCH 18/41] Revert "s3: stat object instead of empty read" This reverts commit f735308ff69b039c35a24d342e13e402143bf5f6. Signed-off-by: Ake van der Meer --- providers/s3/s3.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/providers/s3/s3.go b/providers/s3/s3.go index 3d9ba213..2f0447a7 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -15,6 +15,7 @@ import ( "strings" "testing" + "github.com/efficientgo/core/logerrcapture" "github.com/go-kit/log" "github.com/go-kit/log/level" "github.com/minio/minio-go/v7" @@ -437,12 +438,21 @@ func (b *Bucket) getRange(ctx context.Context, name string, off, length int64) ( return nil, err } } + r, err := b.client.GetObject(ctx, b.name, name, *opts) + if err != nil { + return nil, err + } - // StatObject to see if the object exists and we have permissions to read it - if _, err := b.client.StatObject(ctx, b.name, name, *opts); err != nil { + // NotFoundObject error is revealed only after first Read. This does the initial GetRequest. Prefetch this here + // for convenience. + if _, err := r.Read(nil); err != nil { + defer logerrcapture.Do(b.logger, r.Close, "s3 get range obj close") + + // First GET Object request error. return nil, err } - return b.client.GetObject(ctx, b.name, name, *opts) + + return r, nil } // Get returns a reader for the given object name. From f15a3f89666ab99d76712d66672bbc897bdf01ab Mon Sep 17 00:00:00 2001 From: Ashwanth Goli Date: Tue, 15 Oct 2024 11:32:43 +0530 Subject: [PATCH 19/41] Include content length in the response of Get and GetRange Signed-off-by: Ashwanth Goli --- objstore.go | 10 ++++++++++ providers/azure/azure.go | 2 +- providers/bos/bos.go | 2 +- providers/cos/cos.go | 12 +----------- providers/filesystem/filesystem.go | 13 ++++++++++--- providers/gcs/gcs.go | 14 ++++++++++++-- providers/obs/obs.go | 2 +- providers/oci/oci.go | 4 ++-- providers/s3/s3.go | 5 ++++- 9 files changed, 42 insertions(+), 22 deletions(-) diff --git a/objstore.go b/objstore.go index bfee7503..dcb65e1d 100644 --- a/objstore.go +++ b/objstore.go @@ -829,3 +829,13 @@ func (t *timingReaderWriterTo) WriteTo(w io.Writer) (n int64, err error) { t.timingReader.updateMetrics(int(n), err) return n, err } + +type ObjectSizerReadCloser struct { + io.ReadCloser + Size int64 +} + +// ObjectSize implement ObjectSizer. +func (o ObjectSizerReadCloser) ObjectSize() (int64, error) { + return o.Size, nil +} diff --git a/providers/azure/azure.go b/providers/azure/azure.go index c2f3adb5..957ccdd6 100644 --- a/providers/azure/azure.go +++ b/providers/azure/azure.go @@ -273,7 +273,7 @@ func (b *Bucket) getBlobReader(ctx context.Context, name string, httpRange blob. return nil, errors.Wrapf(err, "cannot download blob, address: %s", blobClient.URL()) } retryOpts := azblob.RetryReaderOptions{MaxRetries: int32(b.readerMaxRetries)} - return resp.NewRetryReader(ctx, &retryOpts), nil + return objstore.ObjectSizerReadCloser{ReadCloser: resp.NewRetryReader(ctx, &retryOpts), Size: *resp.ContentLength}, nil } // Get returns a reader for the given object name. diff --git a/providers/bos/bos.go b/providers/bos/bos.go index 74f9688e..6a111fe2 100644 --- a/providers/bos/bos.go +++ b/providers/bos/bos.go @@ -308,7 +308,7 @@ func (b *Bucket) getRange(_ context.Context, bucketName, objectKey string, off, return nil, err } - return obj.Body, nil + return objstore.ObjectSizerReadCloser{ReadCloser: obj.Body, Size: obj.ContentLength}, err } func configFromEnv() Config { diff --git a/providers/cos/cos.go b/providers/cos/cos.go index a8b853e3..2c1c07fe 100644 --- a/providers/cos/cos.go +++ b/providers/cos/cos.go @@ -320,20 +320,10 @@ func (b *Bucket) getRange(ctx context.Context, name string, off, length int64) ( return nil, err } // Add size info into reader to pass it to Upload function. - r := objectSizerReadCloser{ReadCloser: resp.Body, size: resp.ContentLength} + r := objstore.ObjectSizerReadCloser{ReadCloser: resp.Body, Size: resp.ContentLength} return r, nil } -type objectSizerReadCloser struct { - io.ReadCloser - size int64 -} - -// ObjectSize implement objstore.ObjectSizer. -func (o objectSizerReadCloser) ObjectSize() (int64, error) { - return o.size, nil -} - // Get returns a reader for the given object name. func (b *Bucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { return b.getRange(ctx, name, 0, -1) diff --git a/providers/filesystem/filesystem.go b/providers/filesystem/filesystem.go index 21c70485..be49c5f3 100644 --- a/providers/filesystem/filesystem.go +++ b/providers/filesystem/filesystem.go @@ -150,8 +150,12 @@ func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) ( return nil, errors.New("object name is empty") } - file := filepath.Join(b.rootDir, name) - if _, err := os.Stat(file); err != nil { + var ( + file = filepath.Join(b.rootDir, name) + stat os.FileInfo + err error + ) + if stat, err = os.Stat(file); err != nil { return nil, errors.Wrapf(err, "stat %s", file) } @@ -171,7 +175,10 @@ func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) ( return f, nil } - return &rangeReaderCloser{Reader: io.LimitReader(f, length), f: f}, nil + return objstore.ObjectSizerReadCloser{ + ReadCloser: &rangeReaderCloser{Reader: io.LimitReader(f, length), + f: f, + }, Size: stat.Size()}, nil } // Exists checks if the given directory exists in memory. diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index b5dae200..89582774 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -226,12 +226,22 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt // Get returns a reader for the given object name. func (b *Bucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { - return b.bkt.Object(name).NewReader(ctx) + r, err := b.bkt.Object(name).NewReader(ctx) + if err != nil { + return r, err + } + + return objstore.ObjectSizerReadCloser{ReadCloser: r, Size: r.Attrs.Size}, nil } // GetRange returns a new range reader for the given object name and range. func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error) { - return b.bkt.Object(name).NewRangeReader(ctx, off, length) + r, err := b.bkt.Object(name).NewRangeReader(ctx, off, length) + if err != nil { + return r, err + } + + return objstore.ObjectSizerReadCloser{ReadCloser: r, Size: r.Attrs.Size}, nil } // Attributes returns information about the specified object. diff --git a/providers/obs/obs.go b/providers/obs/obs.go index 7de70633..ec513f90 100644 --- a/providers/obs/obs.go +++ b/providers/obs/obs.go @@ -299,7 +299,7 @@ func (b *Bucket) getRange(_ context.Context, name string, off, length int64) (io if err != nil { return nil, errors.Wrap(err, "failed to get object") } - return output.Body, nil + return objstore.ObjectSizerReadCloser{ReadCloser: output.Body, Size: output.ContentLength}, nil } // Exists checks if the given object exists in the bucket. diff --git a/providers/oci/oci.go b/providers/oci/oci.go index e2f9e98b..b5c3bed1 100644 --- a/providers/oci/oci.go +++ b/providers/oci/oci.go @@ -134,7 +134,7 @@ func (b *Bucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { if err != nil { return nil, err } - return response.Content, nil + return objstore.ObjectSizerReadCloser{ReadCloser: response.Content, Size: *response.ContentLength}, nil } // GetRange returns a new range reader for the given object name and range. @@ -164,7 +164,7 @@ func (b *Bucket) GetRange(ctx context.Context, name string, offset, length int64 if err != nil { return nil, err } - return response.Content, nil + return objstore.ObjectSizerReadCloser{ReadCloser: response.Content, Size: *response.ContentLength}, nil } // Upload the contents of the reader as an object into the bucket. diff --git a/providers/s3/s3.go b/providers/s3/s3.go index 2f0447a7..ebed16ee 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -452,7 +452,10 @@ func (b *Bucket) getRange(ctx context.Context, name string, off, length int64) ( return nil, err } - return r, nil + return objstore.ObjectSizerReadCloser{ + ReadCloser: r, + Size: 0, + }, nil } // Get returns a reader for the given object name. From 76887a7c572dfab8e65561781724045d14f37bc1 Mon Sep 17 00:00:00 2001 From: Ashwanth Goli Date: Tue, 8 Oct 2024 19:42:09 +0530 Subject: [PATCH 20/41] add for oss Signed-off-by: Ashwanth Goli --- providers/oss/oss.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/providers/oss/oss.go b/providers/oss/oss.go index 0a4cc76e..1f59d7f3 100644 --- a/providers/oss/oss.go +++ b/providers/oss/oss.go @@ -16,6 +16,7 @@ import ( "testing" "time" + "github.com/aliyun/aliyun-oss-go-sdk/oss" alioss "github.com/aliyun/aliyun-oss-go-sdk/oss" "github.com/go-kit/log" "github.com/pkg/errors" @@ -342,12 +343,17 @@ func (b *Bucket) getRange(_ context.Context, name string, off, length int64) (io opts = append(opts, opt) } - resp, err := b.bucket.GetObject(name, opts...) + resp, err := b.bucket.DoGetObject(&oss.GetObjectRequest{ObjectKey: name}, opts) if err != nil { return nil, err } - return resp, nil + size, err := clientutil.ParseContentLength(resp.Response.Headers) + if err == nil { + return objstore.ObjectSizerReadCloser{ReadCloser: resp.Response, Size: size}, nil + } + + return resp.Response, nil } // Get returns a reader for the given object name. From 7fdd46598d15d4eaa33cf057e345241b9dd205f2 Mon Sep 17 00:00:00 2001 From: Ashwanth Goli Date: Tue, 8 Oct 2024 19:49:01 +0530 Subject: [PATCH 21/41] add changelog Signed-off-by: Ashwanth Goli --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12375cbf..4686a470 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re ## Unreleased - [#38](https://github.com/thanos-io/objstore/pull/38) GCS: Upgrade cloud.google.com/go/storage version to `v1.43.0`. +- [#145](https://github.com/thanos-io/objstore/pull/145) Include content length in the response of Get and GetRange. ### Fixed - [#117](https://github.com/thanos-io/objstore/pull/117) Metrics: Fix `objstore_bucket_operation_failures_total` incorrectly incremented if context is cancelled while reading object contents. From 37b701c09aaea0648bfaf6fb493008ab33657d17 Mon Sep 17 00:00:00 2001 From: Ashwanth Goli Date: Wed, 9 Oct 2024 18:38:45 +0530 Subject: [PATCH 22/41] update getRange for fs Signed-off-by: Ashwanth Goli --- providers/filesystem/filesystem.go | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/providers/filesystem/filesystem.go b/providers/filesystem/filesystem.go index be49c5f3..423abb90 100644 --- a/providers/filesystem/filesystem.go +++ b/providers/filesystem/filesystem.go @@ -164,21 +164,29 @@ func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) ( return nil, err } + var newOffset int64 if off > 0 { - _, err := f.Seek(off, 0) + newOffset, err = f.Seek(off, 0) if err != nil { return nil, errors.Wrapf(err, "seek %v", off) } } + size := stat.Size() - newOffset if length == -1 { - return f, nil + return objstore.ObjectSizerReadCloser{ + ReadCloser: f, + Size: size, + }, nil } return objstore.ObjectSizerReadCloser{ - ReadCloser: &rangeReaderCloser{Reader: io.LimitReader(f, length), - f: f, - }, Size: stat.Size()}, nil + ReadCloser: &rangeReaderCloser{ + Reader: io.LimitReader(f, length), + f: f, + }, + Size: min(length, size), + }, nil } // Exists checks if the given directory exists in memory. From ddca4de27326b4fb5958eb8ec097edc71763c545 Mon Sep 17 00:00:00 2001 From: Ashwanth Goli Date: Tue, 15 Oct 2024 11:33:50 +0530 Subject: [PATCH 23/41] add acceptance test + refactor Signed-off-by: Ashwanth Goli --- objstore.go | 4 ++-- providers/azure/azure.go | 8 +++++++- providers/bos/bos.go | 7 ++++++- providers/cos/cos.go | 8 ++++++-- providers/filesystem/filesystem.go | 8 ++++++-- providers/gcs/gcs.go | 15 +++++++++++++-- providers/obs/obs.go | 7 ++++++- providers/oci/oci.go | 13 +++++++++++-- providers/oss/oss.go | 7 ++++++- providers/s3/s3.go | 9 ++++++++- providers/swift/swift.go | 6 +++++- testing.go | 20 ++++++++++++++++++++ 12 files changed, 96 insertions(+), 16 deletions(-) diff --git a/objstore.go b/objstore.go index dcb65e1d..116538ca 100644 --- a/objstore.go +++ b/objstore.go @@ -832,10 +832,10 @@ func (t *timingReaderWriterTo) WriteTo(w io.Writer) (n int64, err error) { type ObjectSizerReadCloser struct { io.ReadCloser - Size int64 + Size func() (int64, error) } // ObjectSize implement ObjectSizer. func (o ObjectSizerReadCloser) ObjectSize() (int64, error) { - return o.Size, nil + return o.Size() } diff --git a/providers/azure/azure.go b/providers/azure/azure.go index 957ccdd6..9a4e8518 100644 --- a/providers/azure/azure.go +++ b/providers/azure/azure.go @@ -273,7 +273,13 @@ func (b *Bucket) getBlobReader(ctx context.Context, name string, httpRange blob. return nil, errors.Wrapf(err, "cannot download blob, address: %s", blobClient.URL()) } retryOpts := azblob.RetryReaderOptions{MaxRetries: int32(b.readerMaxRetries)} - return objstore.ObjectSizerReadCloser{ReadCloser: resp.NewRetryReader(ctx, &retryOpts), Size: *resp.ContentLength}, nil + + return objstore.ObjectSizerReadCloser{ + ReadCloser: resp.NewRetryReader(ctx, &retryOpts), + Size: func() (int64, error) { + return *resp.ContentLength, nil + }, + }, nil } // Get returns a reader for the given object name. diff --git a/providers/bos/bos.go b/providers/bos/bos.go index 6a111fe2..1f81e920 100644 --- a/providers/bos/bos.go +++ b/providers/bos/bos.go @@ -308,7 +308,12 @@ func (b *Bucket) getRange(_ context.Context, bucketName, objectKey string, off, return nil, err } - return objstore.ObjectSizerReadCloser{ReadCloser: obj.Body, Size: obj.ContentLength}, err + return objstore.ObjectSizerReadCloser{ + ReadCloser: obj.Body, + Size: func() (int64, error) { + return obj.ContentLength, nil + }, + }, err } func configFromEnv() Config { diff --git a/providers/cos/cos.go b/providers/cos/cos.go index 2c1c07fe..f88a8e76 100644 --- a/providers/cos/cos.go +++ b/providers/cos/cos.go @@ -320,8 +320,12 @@ func (b *Bucket) getRange(ctx context.Context, name string, off, length int64) ( return nil, err } // Add size info into reader to pass it to Upload function. - r := objstore.ObjectSizerReadCloser{ReadCloser: resp.Body, Size: resp.ContentLength} - return r, nil + return objstore.ObjectSizerReadCloser{ + ReadCloser: resp.Body, + Size: func() (int64, error) { + return resp.ContentLength, nil + }, + }, nil } // Get returns a reader for the given object name. diff --git a/providers/filesystem/filesystem.go b/providers/filesystem/filesystem.go index 423abb90..2ed42ee8 100644 --- a/providers/filesystem/filesystem.go +++ b/providers/filesystem/filesystem.go @@ -176,7 +176,9 @@ func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) ( if length == -1 { return objstore.ObjectSizerReadCloser{ ReadCloser: f, - Size: size, + Size: func() (int64, error) { + return size, nil + }, }, nil } @@ -185,7 +187,9 @@ func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) ( Reader: io.LimitReader(f, length), f: f, }, - Size: min(length, size), + Size: func() (int64, error) { + return min(length, size), nil + }, }, nil } diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index 89582774..e022b14f 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -231,7 +231,12 @@ func (b *Bucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { return r, err } - return objstore.ObjectSizerReadCloser{ReadCloser: r, Size: r.Attrs.Size}, nil + return objstore.ObjectSizerReadCloser{ + ReadCloser: r, + Size: func() (int64, error) { + return r.Attrs.Size, nil + }, + }, nil } // GetRange returns a new range reader for the given object name and range. @@ -241,7 +246,13 @@ func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) ( return r, err } - return objstore.ObjectSizerReadCloser{ReadCloser: r, Size: r.Attrs.Size}, nil + sz := r.Remain() + return objstore.ObjectSizerReadCloser{ + ReadCloser: r, + Size: func() (int64, error) { + return sz, nil + }, + }, nil } // Attributes returns information about the specified object. diff --git a/providers/obs/obs.go b/providers/obs/obs.go index ec513f90..cb450365 100644 --- a/providers/obs/obs.go +++ b/providers/obs/obs.go @@ -299,7 +299,12 @@ func (b *Bucket) getRange(_ context.Context, name string, off, length int64) (io if err != nil { return nil, errors.Wrap(err, "failed to get object") } - return objstore.ObjectSizerReadCloser{ReadCloser: output.Body, Size: output.ContentLength}, nil + return objstore.ObjectSizerReadCloser{ + ReadCloser: output.Body, + Size: func() (int64, error) { + return output.ContentLength, nil + }, + }, nil } // Exists checks if the given object exists in the bucket. diff --git a/providers/oci/oci.go b/providers/oci/oci.go index b5c3bed1..3bdf80f3 100644 --- a/providers/oci/oci.go +++ b/providers/oci/oci.go @@ -134,7 +134,12 @@ func (b *Bucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { if err != nil { return nil, err } - return objstore.ObjectSizerReadCloser{ReadCloser: response.Content, Size: *response.ContentLength}, nil + return objstore.ObjectSizerReadCloser{ + ReadCloser: response.Content, + Size: func() (int64, error) { + return *response.ContentLength, nil + }, + }, nil } // GetRange returns a new range reader for the given object name and range. @@ -164,7 +169,11 @@ func (b *Bucket) GetRange(ctx context.Context, name string, offset, length int64 if err != nil { return nil, err } - return objstore.ObjectSizerReadCloser{ReadCloser: response.Content, Size: *response.ContentLength}, nil + return objstore.ObjectSizerReadCloser{ReadCloser: response.Content, + Size: func() (int64, error) { + return *response.ContentLength, nil + }, + }, nil } // Upload the contents of the reader as an object into the bucket. diff --git a/providers/oss/oss.go b/providers/oss/oss.go index 1f59d7f3..d6e1bbf5 100644 --- a/providers/oss/oss.go +++ b/providers/oss/oss.go @@ -350,7 +350,12 @@ func (b *Bucket) getRange(_ context.Context, name string, off, length int64) (io size, err := clientutil.ParseContentLength(resp.Response.Headers) if err == nil { - return objstore.ObjectSizerReadCloser{ReadCloser: resp.Response, Size: size}, nil + return objstore.ObjectSizerReadCloser{ + ReadCloser: resp.Response, + Size: func() (int64, error) { + return size, nil + }, + }, nil } return resp.Response, nil diff --git a/providers/s3/s3.go b/providers/s3/s3.go index ebed16ee..eac8191a 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -454,7 +454,14 @@ func (b *Bucket) getRange(ctx context.Context, name string, off, length int64) ( return objstore.ObjectSizerReadCloser{ ReadCloser: r, - Size: 0, + Size: func() (int64, error) { + stat, err := r.Stat() + if err != nil { + return 0, err + } + + return stat.Size, nil + }, }, nil } diff --git a/providers/swift/swift.go b/providers/swift/swift.go index 682c494c..44fa6ed0 100644 --- a/providers/swift/swift.go +++ b/providers/swift/swift.go @@ -262,7 +262,11 @@ func (c *Container) get(name string, headers swift.Headers, checkHash bool) (io. if err != nil { return nil, errors.Wrap(err, "open object") } - return file, err + + return objstore.ObjectSizerReadCloser{ + ReadCloser: file, + Size: file.Length, + }, nil } // Get returns a reader for the given object name. diff --git a/testing.go b/testing.go index b8e3744c..28cbd658 100644 --- a/testing.go +++ b/testing.go @@ -106,6 +106,11 @@ func AcceptanceTest(t *testing.T, bkt Bucket) { rc1, err := bkt.Get(ctx, "id1/obj_1.some") testutil.Ok(t, err) defer func() { testutil.Ok(t, rc1.Close()) }() + + sz, err := TryToGetSize(rc1) + testutil.Ok(t, err) + testutil.Equals(t, int64(11), sz, "expected size to be equal to 11") + content, err := io.ReadAll(rc1) testutil.Ok(t, err) testutil.Equals(t, "@test-data@", string(content)) @@ -118,6 +123,11 @@ func AcceptanceTest(t *testing.T, bkt Bucket) { rc2, err := bkt.GetRange(ctx, "id1/obj_1.some", 1, 3) testutil.Ok(t, err) defer func() { testutil.Ok(t, rc2.Close()) }() + + sz, err = TryToGetSize(rc2) + testutil.Ok(t, err) + testutil.Equals(t, int64(3), sz, "expected size to be equal to 3") + content, err = io.ReadAll(rc2) testutil.Ok(t, err) testutil.Equals(t, "tes", string(content)) @@ -126,6 +136,11 @@ func AcceptanceTest(t *testing.T, bkt Bucket) { rcUnspecifiedLen, err := bkt.GetRange(ctx, "id1/obj_1.some", 1, -1) testutil.Ok(t, err) defer func() { testutil.Ok(t, rcUnspecifiedLen.Close()) }() + + sz, err = TryToGetSize(rcUnspecifiedLen) + testutil.Ok(t, err) + testutil.Equals(t, int64(10), sz, "expected size to be equal to 10") + content, err = io.ReadAll(rcUnspecifiedLen) testutil.Ok(t, err) testutil.Equals(t, "test-data@", string(content)) @@ -141,6 +156,11 @@ func AcceptanceTest(t *testing.T, bkt Bucket) { rcLength, err := bkt.GetRange(ctx, "id1/obj_1.some", 3, 9999) testutil.Ok(t, err) defer func() { testutil.Ok(t, rcLength.Close()) }() + + sz, err = TryToGetSize(rcLength) + testutil.Ok(t, err) + testutil.Equals(t, int64(8), sz, "expected size to be equal to 8") + content, err = io.ReadAll(rcLength) testutil.Ok(t, err) testutil.Equals(t, "st-data@", string(content)) From 2947125233e6b3edace706149b6862e911458058 Mon Sep 17 00:00:00 2001 From: Ashwanth Goli Date: Thu, 10 Oct 2024 12:07:04 +0530 Subject: [PATCH 24/41] add size for inmem Signed-off-by: Ashwanth Goli --- inmem.go | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/inmem.go b/inmem.go index 3f6f35e9..ed256c9c 100644 --- a/inmem.go +++ b/inmem.go @@ -119,7 +119,12 @@ func (b *InMemBucket) Get(_ context.Context, name string) (io.ReadCloser, error) return nil, errNotFound } - return io.NopCloser(bytes.NewReader(file)), nil + return ObjectSizerReadCloser{ + ReadCloser: io.NopCloser(bytes.NewReader(file)), + Size: func() (int64, error) { + return int64(len(file)), nil + }, + }, nil } // GetRange returns a new range reader for the given object name and range. @@ -136,15 +141,27 @@ func (b *InMemBucket) GetRange(_ context.Context, name string, off, length int64 } if int64(len(file)) < off { - return io.NopCloser(bytes.NewReader(nil)), nil + return ObjectSizerReadCloser{ + ReadCloser: io.NopCloser(bytes.NewReader(nil)), + Size: func() (int64, error) { return 0, nil }, + }, nil } if length == -1 { - return io.NopCloser(bytes.NewReader(file[off:])), nil + return ObjectSizerReadCloser{ + ReadCloser: io.NopCloser(bytes.NewReader(file[off:])), + Size: func() (int64, error) { + return int64(len(file[off:])), nil + }, + }, nil } if length <= 0 { - return io.NopCloser(bytes.NewReader(nil)), errors.New("length cannot be smaller or equal 0") + // wrap with ObjectSizerReadCloser to return 0 size. + return ObjectSizerReadCloser{ + ReadCloser: io.NopCloser(bytes.NewReader(nil)), + Size: func() (int64, error) { return 0, nil }, + }, errors.New("length cannot be smaller or equal 0") } if int64(len(file)) <= off+length { @@ -152,7 +169,12 @@ func (b *InMemBucket) GetRange(_ context.Context, name string, off, length int64 length = int64(len(file)) - off } - return io.NopCloser(bytes.NewReader(file[off : off+length])), nil + return ObjectSizerReadCloser{ + ReadCloser: io.NopCloser(bytes.NewReader(file[off : off+length])), + Size: func() (int64, error) { + return length, nil + }, + }, nil } // Exists checks if the given directory exists in memory. From dfed39a7c13e36f49ad115c92f386307c170dd09 Mon Sep 17 00:00:00 2001 From: Ashwanth Goli Date: Thu, 10 Oct 2024 12:40:30 +0530 Subject: [PATCH 25/41] return err if Size is not set Signed-off-by: Ashwanth Goli --- objstore.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/objstore.go b/objstore.go index 116538ca..5bce3ef6 100644 --- a/objstore.go +++ b/objstore.go @@ -837,5 +837,9 @@ type ObjectSizerReadCloser struct { // ObjectSize implement ObjectSizer. func (o ObjectSizerReadCloser) ObjectSize() (int64, error) { + if o.Size == nil { + return 0, errors.New("unknown size") + } + return o.Size() } From c7d91edfa809f55afc2dd96eb4e791c220677249 Mon Sep 17 00:00:00 2001 From: Ashwanth Goli Date: Wed, 23 Oct 2024 19:13:07 +0530 Subject: [PATCH 26/41] add max_retries to gcs, obs, cos Signed-off-by: Ashwanth Goli --- providers/cos/cos.go | 5 +++++ providers/gcs/gcs.go | 10 ++++++---- providers/obs/obs.go | 9 ++++++++- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/providers/cos/cos.go b/providers/cos/cos.go index f88a8e76..23ed3b7a 100644 --- a/providers/cos/cos.go +++ b/providers/cos/cos.go @@ -59,6 +59,7 @@ type Config struct { Endpoint string `yaml:"endpoint"` SecretKey string `yaml:"secret_key"` SecretId string `yaml:"secret_id"` + MaxRetries int `yaml:"max_retries"` HTTPConfig exthttp.HTTPConfig `yaml:"http_config"` } @@ -143,6 +144,10 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string, rt }, }) + if config.MaxRetries > 0 { + client.Conf.RetryOpt.Count = config.MaxRetries + } + bkt := &Bucket{ logger: logger, client: client, diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index f4411177..87ce1230 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -55,8 +55,10 @@ type Config struct { ChunkSizeBytes int `yaml:"chunk_size_bytes"` noAuth bool `yaml:"no_auth"` - // gcs client retries idempotent operations by default, this option disables retries. - DisableRetries bool `yaml:"disable_retries"` + // MaxRetries controls the number of retries for idempotent operations. + // Overrides the default gcs storage client behaviour if this value is greater than 0. + // Set this to 1 to disable retries. + MaxRetries int `yaml:"max_retries"` } // Bucket implements the store.Bucket and shipper.Bucket interfaces against GCS. @@ -176,8 +178,8 @@ func newBucket(ctx context.Context, logger log.Logger, gc Config, opts []option. chunkSize: gc.ChunkSizeBytes, } - if gc.DisableRetries { - bkt.bkt = bkt.bkt.Retryer(storage.WithPolicy(storage.RetryNever)) + if gc.MaxRetries > 0 { + bkt.bkt = bkt.bkt.Retryer(storage.WithMaxAttempts(gc.MaxRetries)) } return bkt, nil diff --git a/providers/obs/obs.go b/providers/obs/obs.go index cb450365..8bc3e91a 100644 --- a/providers/obs/obs.go +++ b/providers/obs/obs.go @@ -46,6 +46,7 @@ type Config struct { Endpoint string `yaml:"endpoint"` AccessKey string `yaml:"access_key"` SecretKey string `yaml:"secret_key"` + MaxRetries int `yaml:"max_retries"` HTTPConfig exthttp.HTTPConfig `yaml:"http_config"` } @@ -102,7 +103,13 @@ func NewBucketWithConfig(logger log.Logger, config Config) (*Bucket, error) { return nil, errors.Wrap(err, "get http transport err") } - client, err := obs.New(config.AccessKey, config.SecretKey, config.Endpoint, obs.WithHttpTransport(rt)) + var client *obs.ObsClient + if config.MaxRetries > 0 { + client, err = obs.New(config.AccessKey, config.SecretKey, config.Endpoint, obs.WithHttpTransport(rt), obs.WithMaxRetryCount(config.MaxRetries)) + } else { + client, err = obs.New(config.AccessKey, config.SecretKey, config.Endpoint, obs.WithHttpTransport(rt)) + } + if err != nil { return nil, errors.Wrap(err, "initialize obs client err") } From e812454f31d27e900c38320ad99d59b19230c567 Mon Sep 17 00:00:00 2001 From: Ashwanth Goli Date: Wed, 23 Oct 2024 19:16:24 +0530 Subject: [PATCH 27/41] update changelog Signed-off-by: Ashwanth Goli --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e693ee7..9fc83213 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,7 +52,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#116](https://github.com/thanos-io/objstore/pull/116) Azure: Add new storage_create_container configuration property - [#128](https://github.com/thanos-io/objstore/pull/128) GCS: Add support for `ChunkSize` for writer. - [#130](https://github.com/thanos-io/objstore/pull/130) feat: Decouple creating bucket metrics from instrumenting the bucket -- [#147](https://github.com/thanos-io/objstore/pull/147) gcs: Add config to disable retries. +- [#147](https://github.com/thanos-io/objstore/pull/147) gcs: Add MaxRetries config to cos, gcs and oci. ### Changed - [#38](https://github.com/thanos-io/objstore/pull/38) *: Upgrade minio-go version to `v7.0.45`. From 165faeace5a0a91f29b485d5c20b5f5f6e69a148 Mon Sep 17 00:00:00 2001 From: Ashwanth Goli Date: Wed, 23 Oct 2024 19:24:20 +0530 Subject: [PATCH 28/41] lint Signed-off-by: Ashwanth Goli --- providers/gcs/gcs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index 87ce1230..6a9977bc 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -56,7 +56,7 @@ type Config struct { noAuth bool `yaml:"no_auth"` // MaxRetries controls the number of retries for idempotent operations. - // Overrides the default gcs storage client behaviour if this value is greater than 0. + // Overrides the default gcs storage client behavior if this value is greater than 0. // Set this to 1 to disable retries. MaxRetries int `yaml:"max_retries"` } From 0d5108b1dd47754c99b33af148aff2de81ee36d1 Mon Sep 17 00:00:00 2001 From: Ashwanth Goli Date: Wed, 23 Oct 2024 19:27:05 +0530 Subject: [PATCH 29/41] fixup! update changelog Signed-off-by: Ashwanth Goli --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fc83213..c4946470 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,7 +52,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#116](https://github.com/thanos-io/objstore/pull/116) Azure: Add new storage_create_container configuration property - [#128](https://github.com/thanos-io/objstore/pull/128) GCS: Add support for `ChunkSize` for writer. - [#130](https://github.com/thanos-io/objstore/pull/130) feat: Decouple creating bucket metrics from instrumenting the bucket -- [#147](https://github.com/thanos-io/objstore/pull/147) gcs: Add MaxRetries config to cos, gcs and oci. +- [#147](https://github.com/thanos-io/objstore/pull/147) gcs: Add MaxRetries config to cos, gcs and obs. ### Changed - [#38](https://github.com/thanos-io/objstore/pull/38) *: Upgrade minio-go version to `v7.0.45`. From b54d9eaf858b6a906db44737809a3ed9f9956e1f Mon Sep 17 00:00:00 2001 From: Ashwanth Goli Date: Wed, 23 Oct 2024 19:27:40 +0530 Subject: [PATCH 30/41] fixup! fixup! update changelog Signed-off-by: Ashwanth Goli --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4946470..a4e0fae0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,7 +52,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#116](https://github.com/thanos-io/objstore/pull/116) Azure: Add new storage_create_container configuration property - [#128](https://github.com/thanos-io/objstore/pull/128) GCS: Add support for `ChunkSize` for writer. - [#130](https://github.com/thanos-io/objstore/pull/130) feat: Decouple creating bucket metrics from instrumenting the bucket -- [#147](https://github.com/thanos-io/objstore/pull/147) gcs: Add MaxRetries config to cos, gcs and obs. +- [#147](https://github.com/thanos-io/objstore/pull/147) feat: Add MaxRetries config to cos, gcs and obs. ### Changed - [#38](https://github.com/thanos-io/objstore/pull/38) *: Upgrade minio-go version to `v7.0.45`. From 168679cbbf200a7d9091065d7239ee14e5561190 Mon Sep 17 00:00:00 2001 From: Milind Dethe <99114125+milinddethe15@users.noreply.github.com> Date: Thu, 24 Oct 2024 17:37:00 +0530 Subject: [PATCH 31/41] Added roundtripper wrapper as param (#150) * added roundtripper wrapper as param Signed-off-by: milinddethe15 * CHANGELOG and TODO Signed-off-by: milinddethe15 * fix lint Signed-off-by: milinddethe15 * Update CHANGELOG.md Signed-off-by: Milind Dethe <99114125+milinddethe15@users.noreply.github.com> * refactor code Signed-off-by: milinddethe15 --------- Signed-off-by: milinddethe15 Signed-off-by: Milind Dethe <99114125+milinddethe15@users.noreply.github.com> --- CHANGELOG.md | 1 + client/factory.go | 16 ++++++++-------- errutil/rt_error.go | 16 +++++++++++++++- providers/azure/azure.go | 11 ++++------- providers/azure/azure_test.go | 7 ++----- providers/azure/helpers.go | 5 ++++- providers/bos/bos.go | 2 +- providers/cos/cos.go | 19 +++++++++++-------- providers/cos/cos_test.go | 6 ++---- providers/gcs/gcs.go | 17 +++++++++-------- providers/gcs/gcs_test.go | 6 ++---- providers/obs/obs.go | 2 +- providers/oci/oci.go | 15 +++++++++------ providers/oci/oci_test.go | 7 ++----- providers/oss/oss.go | 23 +++++++++++++++-------- providers/oss/oss_test.go | 6 ++---- providers/s3/s3.go | 13 +++++++------ providers/s3/s3_test.go | 6 ++---- providers/swift/swift.go | 20 ++++++++++---------- providers/swift/swift_test.go | 7 ++----- 20 files changed, 109 insertions(+), 96 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4686a470..75cbd834 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,6 +52,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#116](https://github.com/thanos-io/objstore/pull/116) Azure: Add new storage_create_container configuration property - [#128](https://github.com/thanos-io/objstore/pull/128) GCS: Add support for `ChunkSize` for writer. - [#130](https://github.com/thanos-io/objstore/pull/130) feat: Decouple creating bucket metrics from instrumenting the bucket +- [#150](https://github.com/thanos-io/objstore/pull/150) Add support for roundtripper wrapper. ### Changed - [#38](https://github.com/thanos-io/objstore/pull/38) *: Upgrade minio-go version to `v7.0.45`. diff --git a/client/factory.go b/client/factory.go index bd345024..5fe5a741 100644 --- a/client/factory.go +++ b/client/factory.go @@ -50,7 +50,7 @@ type BucketConfig struct { // NewBucket initializes and returns new object storage clients. // NOTE: confContentYaml can contain secrets. -func NewBucket(logger log.Logger, confContentYaml []byte, component string, rt http.RoundTripper) (objstore.Bucket, error) { +func NewBucket(logger log.Logger, confContentYaml []byte, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (objstore.Bucket, error) { level.Info(logger).Log("msg", "loading bucket configuration") bucketConf := &BucketConfig{} if err := yaml.UnmarshalStrict(confContentYaml, bucketConf); err != nil { @@ -65,23 +65,23 @@ func NewBucket(logger log.Logger, confContentYaml []byte, component string, rt h var bucket objstore.Bucket switch strings.ToUpper(string(bucketConf.Type)) { case string(GCS): - bucket, err = gcs.NewBucket(context.Background(), logger, config, component, rt) + bucket, err = gcs.NewBucket(context.Background(), logger, config, component, wrapRoundtripper) case string(S3): - bucket, err = s3.NewBucket(logger, config, component, rt) + bucket, err = s3.NewBucket(logger, config, component, wrapRoundtripper) case string(AZURE): - bucket, err = azure.NewBucket(logger, config, component, rt) + bucket, err = azure.NewBucket(logger, config, component, wrapRoundtripper) case string(SWIFT): - bucket, err = swift.NewContainer(logger, config, rt) + bucket, err = swift.NewContainer(logger, config, wrapRoundtripper) case string(COS): - bucket, err = cos.NewBucket(logger, config, component, rt) + bucket, err = cos.NewBucket(logger, config, component, wrapRoundtripper) case string(ALIYUNOSS): - bucket, err = oss.NewBucket(logger, config, component, rt) + bucket, err = oss.NewBucket(logger, config, component, wrapRoundtripper) case string(FILESYSTEM): bucket, err = filesystem.NewBucketFromConfig(config) case string(BOS): bucket, err = bos.NewBucket(logger, config, component) case string(OCI): - bucket, err = oci.NewBucket(logger, config, rt) + bucket, err = oci.NewBucket(logger, config, wrapRoundtripper) case string(OBS): bucket, err = obs.NewBucket(logger, config) default: diff --git a/errutil/rt_error.go b/errutil/rt_error.go index ad1309e1..b6b2e9c9 100644 --- a/errutil/rt_error.go +++ b/errutil/rt_error.go @@ -1,6 +1,16 @@ package errutil -import "net/http" +import ( + "net/http" + + "github.com/pkg/errors" +) + +var rtErr = errors.New("RoundTripper error") + +func IsMockedError(err error) bool { + return errors.Is(err, rtErr) +} // ErrorRoundTripper is a custom RoundTripper that always returns an error. type ErrorRoundTripper struct { @@ -10,3 +20,7 @@ type ErrorRoundTripper struct { func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { return nil, ert.Err } + +func WrapWithErrRoundtripper(rt http.RoundTripper) http.RoundTripper { + return &ErrorRoundTripper{Err: rtErr} +} diff --git a/providers/azure/azure.go b/providers/azure/azure.go index 9a4e8518..e125ca35 100644 --- a/providers/azure/azure.go +++ b/providers/azure/azure.go @@ -146,7 +146,7 @@ type Bucket struct { } // NewBucket returns a new Bucket using the provided Azure config. -func NewBucket(logger log.Logger, azureConfig []byte, component string, rt http.RoundTripper) (*Bucket, error) { +func NewBucket(logger log.Logger, azureConfig []byte, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error) { level.Debug(logger).Log("msg", "creating new Azure bucket connection", "component", component) conf, err := parseConfig(azureConfig) if err != nil { @@ -155,19 +155,16 @@ func NewBucket(logger log.Logger, azureConfig []byte, component string, rt http. if conf.MSIResource != "" { level.Warn(logger).Log("msg", "The field msi_resource has been deprecated and should no longer be set") } - return NewBucketWithConfig(logger, conf, component, rt) + return NewBucketWithConfig(logger, conf, component, wrapRoundtripper) } // NewBucketWithConfig returns a new Bucket using the provided Azure config struct. -func NewBucketWithConfig(logger log.Logger, conf Config, component string, rt http.RoundTripper) (*Bucket, error) { - if rt != nil { - conf.HTTPConfig.Transport = rt - } +func NewBucketWithConfig(logger log.Logger, conf Config, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error) { if err := conf.validate(); err != nil { return nil, err } - containerClient, err := getContainerClient(conf) + containerClient, err := getContainerClient(conf, wrapRoundtripper) if err != nil { return nil, err } diff --git a/providers/azure/azure_test.go b/providers/azure/azure_test.go index 85533eaa..a96dcefb 100644 --- a/providers/azure/azure_test.go +++ b/providers/azure/azure_test.go @@ -9,7 +9,6 @@ import ( "github.com/efficientgo/core/testutil" "github.com/go-kit/log" - "github.com/pkg/errors" "github.com/thanos-io/objstore/errutil" "github.com/thanos-io/objstore/exthttp" @@ -230,11 +229,9 @@ func TestNewBucketWithErrorRoundTripper(t *testing.T) { cfg, err := parseConfig(validConfig) testutil.Ok(t, err) - rt := &errutil.ErrorRoundTripper{Err: errors.New("RoundTripper error")} - - _, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test", rt) + _, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test", errutil.WrapWithErrRoundtripper) // We expect an error from the RoundTripper testutil.NotOk(t, err) - testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) + testutil.Assert(t, errutil.IsMockedError(err), "Expected RoundTripper error, got: %v", err) } diff --git a/providers/azure/helpers.go b/providers/azure/helpers.go index 2915fbbb..deb86d03 100644 --- a/providers/azure/helpers.go +++ b/providers/azure/helpers.go @@ -19,7 +19,7 @@ import ( // DirDelim is the delimiter used to model a directory structure in an object store bucket. const DirDelim = "/" -func getContainerClient(conf Config) (*container.Client, error) { +func getContainerClient(conf Config, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*container.Client, error) { var rt http.RoundTripper rt, err := exthttp.DefaultTransport(conf.HTTPConfig) if err != nil { @@ -28,6 +28,9 @@ func getContainerClient(conf Config) (*container.Client, error) { if conf.HTTPConfig.Transport != nil { rt = conf.HTTPConfig.Transport } + if wrapRoundtripper != nil { + rt = wrapRoundtripper(rt) + } opt := &container.ClientOptions{ ClientOptions: azcore.ClientOptions{ Retry: policy.RetryOptions{ diff --git a/providers/bos/bos.go b/providers/bos/bos.go index 1f81e920..9faa93f3 100644 --- a/providers/bos/bos.go +++ b/providers/bos/bos.go @@ -66,7 +66,7 @@ func parseConfig(conf []byte) (Config, error) { // NewBucket new bos bucket. func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error) { - // TODO(https://github.com/thanos-io/objstore/pull/140): Add support for custom roundtripper. + // TODO(https://github.com/thanos-io/objstore/pull/150): Add support for roundtripper wrapper. if logger == nil { logger = log.NewNopLogger() } diff --git a/providers/cos/cos.go b/providers/cos/cos.go index f88a8e76..6bd39caa 100644 --- a/providers/cos/cos.go +++ b/providers/cos/cos.go @@ -95,7 +95,7 @@ func parseConfig(conf []byte) (Config, error) { } // NewBucket returns a new Bucket using the provided cos configuration. -func NewBucket(logger log.Logger, conf []byte, component string, rt http.RoundTripper) (*Bucket, error) { +func NewBucket(logger log.Logger, conf []byte, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error) { if logger == nil { logger = log.NewNopLogger() } @@ -104,11 +104,11 @@ func NewBucket(logger log.Logger, conf []byte, component string, rt http.RoundTr if err != nil { return nil, errors.Wrap(err, "parsing cos configuration") } - return NewBucketWithConfig(logger, config, component, rt) + return NewBucketWithConfig(logger, config, component, wrapRoundtripper) } // NewBucketWithConfig returns a new Bucket using the provided cos config values. -func NewBucketWithConfig(logger log.Logger, config Config, component string, rt http.RoundTripper) (*Bucket, error) { +func NewBucketWithConfig(logger log.Logger, config Config, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error) { if err := config.validate(); err != nil { return nil, errors.Wrap(err, "validate cos configuration") } @@ -127,19 +127,22 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string, rt } } b := &cos.BaseURL{BucketURL: bucketURL} - var tpt http.RoundTripper - tpt, err = exthttp.DefaultTransport(config.HTTPConfig) + var rt http.RoundTripper + rt, err = exthttp.DefaultTransport(config.HTTPConfig) if err != nil { return nil, err } - if rt != nil { - tpt = rt + if config.HTTPConfig.Transport != nil { + rt = config.HTTPConfig.Transport + } + if wrapRoundtripper != nil { + rt = wrapRoundtripper(rt) } client := cos.NewClient(b, &http.Client{ Transport: &cos.AuthorizationTransport{ SecretID: config.SecretId, SecretKey: config.SecretKey, - Transport: tpt, + Transport: rt, }, }) diff --git a/providers/cos/cos_test.go b/providers/cos/cos_test.go index f682aee9..4f7a56d6 100644 --- a/providers/cos/cos_test.go +++ b/providers/cos/cos_test.go @@ -10,7 +10,6 @@ import ( "github.com/efficientgo/core/testutil" "github.com/go-kit/log" - "github.com/pkg/errors" "github.com/prometheus/common/model" "github.com/thanos-io/objstore/errutil" @@ -150,12 +149,11 @@ func TestNewBucketWithErrorRoundTripper(t *testing.T) { SecretId: "sid", SecretKey: "skey", } - rt := &errutil.ErrorRoundTripper{Err: errors.New("RoundTripper error")} - bkt, err := NewBucketWithConfig(log.NewNopLogger(), config, "test", rt) + bkt, err := NewBucketWithConfig(log.NewNopLogger(), config, "test", errutil.WrapWithErrRoundtripper) testutil.Ok(t, err) _, err = bkt.Get(context.Background(), "Test") // We expect an error from the RoundTripper testutil.NotOk(t, err) - testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) + testutil.Assert(t, errutil.IsMockedError(err), "Expected RoundTripper error, got: %v", err) } diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index e022b14f..efb208e6 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -77,22 +77,20 @@ func parseConfig(conf []byte) (Config, error) { } // NewBucket returns a new Bucket against the given bucket handle. -func NewBucket(ctx context.Context, logger log.Logger, conf []byte, component string, rt http.RoundTripper) (*Bucket, error) { +func NewBucket(ctx context.Context, logger log.Logger, conf []byte, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error) { config, err := parseConfig(conf) if err != nil { return nil, err } - return NewBucketWithConfig(ctx, logger, config, component, rt) + return NewBucketWithConfig(ctx, logger, config, component, wrapRoundtripper) } // NewBucketWithConfig returns a new Bucket with gcs Config struct. -func NewBucketWithConfig(ctx context.Context, logger log.Logger, gc Config, component string, rt http.RoundTripper) (*Bucket, error) { +func NewBucketWithConfig(ctx context.Context, logger log.Logger, gc Config, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error) { if gc.Bucket == "" { return nil, errors.New("missing Google Cloud Storage bucket name for stored blocks") } - if rt != nil { - gc.HTTPConfig.Transport = rt - } + var opts []option.ClientOption // If ServiceAccount is provided, use them in GCS client, otherwise fallback to Google default logic. @@ -112,7 +110,7 @@ func NewBucketWithConfig(ctx context.Context, logger log.Logger, gc Config, comp if !gc.UseGRPC { var err error - opts, err = appendHttpOptions(gc, opts) + opts, err = appendHttpOptions(gc, opts, wrapRoundtripper) if err != nil { return nil, err } @@ -121,7 +119,7 @@ func NewBucketWithConfig(ctx context.Context, logger log.Logger, gc Config, comp return newBucket(ctx, logger, gc, opts) } -func appendHttpOptions(gc Config, opts []option.ClientOption) ([]option.ClientOption, error) { +func appendHttpOptions(gc Config, opts []option.ClientOption, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) ([]option.ClientOption, error) { // Check if a roundtripper has been set in the config // otherwise build the default transport. var rt http.RoundTripper @@ -132,6 +130,9 @@ func appendHttpOptions(gc Config, opts []option.ClientOption) ([]option.ClientOp if gc.HTTPConfig.Transport != nil { rt = gc.HTTPConfig.Transport } + if wrapRoundtripper != nil { + rt = wrapRoundtripper(rt) + } // GCS uses some defaults when "options.WithHTTPClient" is not used that are important when we call // htransport.NewTransport namely the scopes that are then used for OAth authentication. So to build our own diff --git a/providers/gcs/gcs_test.go b/providers/gcs/gcs_test.go index 39b55041..80951d7a 100644 --- a/providers/gcs/gcs_test.go +++ b/providers/gcs/gcs_test.go @@ -15,7 +15,6 @@ import ( "github.com/efficientgo/core/testutil" "github.com/fullstorydev/emulators/storage/gcsemu" "github.com/go-kit/log" - "github.com/pkg/errors" "github.com/prometheus/common/model" "github.com/thanos-io/objstore/errutil" "google.golang.org/api/option" @@ -161,7 +160,6 @@ http_config: } func TestNewBucketWithErrorRoundTripper(t *testing.T) { - rt := &errutil.ErrorRoundTripper{Err: errors.New("RoundTripper error")} cfg := Config{ Bucket: "test-bucket", ServiceAccount: "", @@ -174,9 +172,9 @@ func TestNewBucketWithErrorRoundTripper(t *testing.T) { err = os.Setenv("STORAGE_EMULATOR_HOST", svr.Addr) testutil.Ok(t, err) - bkt, err := NewBucketWithConfig(context.Background(), log.NewNopLogger(), cfg, "test-bucket", rt) + bkt, err := NewBucketWithConfig(context.Background(), log.NewNopLogger(), cfg, "test-bucket", errutil.WrapWithErrRoundtripper) testutil.Ok(t, err) _, err = bkt.Get(context.Background(), "test-bucket") testutil.NotOk(t, err) - testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) + testutil.Assert(t, errutil.IsMockedError(err), "Expected RoundTripper error, got: %v", err) } diff --git a/providers/obs/obs.go b/providers/obs/obs.go index cb450365..4fb17baa 100644 --- a/providers/obs/obs.go +++ b/providers/obs/obs.go @@ -75,7 +75,7 @@ type Bucket struct { } func NewBucket(logger log.Logger, conf []byte) (*Bucket, error) { - // TODO(https://github.com/thanos-io/objstore/pull/140): Add support for custom roundtripper. + // TODO(https://github.com/thanos-io/objstore/pull/150): Add support for roundtripper wrapper. config, err := parseConfig(conf) if err != nil { return nil, errors.Wrap(err, "parsing cos configuration") diff --git a/providers/oci/oci.go b/providers/oci/oci.go index 3bdf80f3..bc8a8bd9 100644 --- a/providers/oci/oci.go +++ b/providers/oci/oci.go @@ -298,7 +298,7 @@ func (b *Bucket) deleteBucket(ctx context.Context) (err error) { } // NewBucket returns a new Bucket using the provided oci config values. -func NewBucket(logger log.Logger, ociConfig []byte, rt http.RoundTripper) (*Bucket, error) { +func NewBucket(logger log.Logger, ociConfig []byte, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error) { level.Debug(logger).Log("msg", "creating new oci bucket connection") var config = DefaultConfig var configurationProvider common.ConfigurationProvider @@ -344,13 +344,16 @@ func NewBucket(logger log.Logger, ociConfig []byte, rt http.RoundTripper) (*Buck if err != nil { return nil, errors.Wrapf(err, "unable to create ObjectStorage client with the given oci configurations") } - - config.HTTPConfig.Transport = CustomTransport(config) - if rt != nil { - config.HTTPConfig.Transport = rt + var rt http.RoundTripper + rt = CustomTransport(config) + if config.HTTPConfig.Transport != nil { + rt = config.HTTPConfig.Transport + } + if wrapRoundtripper != nil { + rt = wrapRoundtripper(rt) } httpClient := http.Client{ - Transport: config.HTTPConfig.Transport, + Transport: rt, Timeout: config.HTTPConfig.ClientTimeout, } client.HTTPClient = &httpClient diff --git a/providers/oci/oci_test.go b/providers/oci/oci_test.go index 0fc81ccf..6b44a0e7 100644 --- a/providers/oci/oci_test.go +++ b/providers/oci/oci_test.go @@ -5,7 +5,6 @@ import ( "github.com/efficientgo/core/testutil" "github.com/go-kit/log" - "github.com/pkg/errors" "github.com/thanos-io/objstore/errutil" "gopkg.in/yaml.v2" ) @@ -38,10 +37,8 @@ G6aFKaqQfOXKCyWoUiVknQJAXrlgySFci/2ueKlIE1QqIiLSZ8V8OlpFLRnb1pzI ociConfig, err := yaml.Marshal(config) testutil.Ok(t, err) - rt := &errutil.ErrorRoundTripper{Err: errors.New("RoundTripper error")} - - _, err = NewBucket(log.NewNopLogger(), ociConfig, rt) + _, err = NewBucket(log.NewNopLogger(), ociConfig, errutil.WrapWithErrRoundtripper) // We expect an error from the RoundTripper testutil.NotOk(t, err) - testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) + testutil.Assert(t, errutil.IsMockedError(err), "Expected RoundTripper error, got: %v", err) } diff --git a/providers/oss/oss.go b/providers/oss/oss.go index d6e1bbf5..e01aff2e 100644 --- a/providers/oss/oss.go +++ b/providers/oss/oss.go @@ -23,6 +23,7 @@ import ( "gopkg.in/yaml.v2" "github.com/thanos-io/objstore/clientutil" + "github.com/thanos-io/objstore/exthttp" "github.com/thanos-io/objstore" ) @@ -159,26 +160,32 @@ func (b *Bucket) Attributes(ctx context.Context, name string) (objstore.ObjectAt } // NewBucket returns a new Bucket using the provided oss config values. -func NewBucket(logger log.Logger, conf []byte, component string, rt http.RoundTripper) (*Bucket, error) { +func NewBucket(logger log.Logger, conf []byte, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error) { var config Config if err := yaml.Unmarshal(conf, &config); err != nil { return nil, errors.Wrap(err, "parse aliyun oss config file failed") } - return NewBucketWithConfig(logger, config, component, rt) + return NewBucketWithConfig(logger, config, component, wrapRoundtripper) } // NewBucketWithConfig returns a new Bucket using the provided oss config struct. -func NewBucketWithConfig(logger log.Logger, config Config, component string, rt http.RoundTripper) (*Bucket, error) { +func NewBucketWithConfig(logger log.Logger, config Config, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error) { if err := validate(config); err != nil { return nil, err } - client, err := alioss.New(config.Endpoint, config.AccessKeyID, config.AccessKeySecret) - if rt != nil { - clientOption := func(client *alioss.Client) { - client.HTTPClient = &http.Client{Transport: rt} + var clientOptions []alioss.ClientOption + if wrapRoundtripper != nil { + rt, err := exthttp.DefaultTransport(exthttp.DefaultHTTPConfig) + if err != nil { + return nil, err } - client, err = alioss.New(config.Endpoint, config.AccessKeyID, config.AccessKeySecret, clientOption) + clientOptions = append(clientOptions, func(client *alioss.Client) { + client.HTTPClient = &http.Client{ + Transport: wrapRoundtripper(rt), + } + }) } + client, err := alioss.New(config.Endpoint, config.AccessKeyID, config.AccessKeySecret, clientOptions...) if err != nil { return nil, errors.Wrap(err, "create aliyun oss client failed") } diff --git a/providers/oss/oss_test.go b/providers/oss/oss_test.go index 99ad68a3..b43d3077 100644 --- a/providers/oss/oss_test.go +++ b/providers/oss/oss_test.go @@ -6,7 +6,6 @@ import ( "github.com/efficientgo/core/testutil" "github.com/go-kit/log" - "github.com/pkg/errors" "github.com/thanos-io/objstore/errutil" ) @@ -17,12 +16,11 @@ func TestNewBucketWithErrorRoundTripper(t *testing.T) { AccessKeySecret: "123", Bucket: "test", } - rt := &errutil.ErrorRoundTripper{Err: errors.New("RoundTripper error")} - bkt, err := NewBucketWithConfig(log.NewNopLogger(), config, "test", rt) + bkt, err := NewBucketWithConfig(log.NewNopLogger(), config, "test", errutil.WrapWithErrRoundtripper) // We expect an error from the RoundTripper testutil.Ok(t, err) _, err = bkt.Get(context.Background(), "test") testutil.NotOk(t, err) - testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) + testutil.Assert(t, errutil.IsMockedError(err), "Expected RoundTripper error, got: %v", err) } diff --git a/providers/s3/s3.go b/providers/s3/s3.go index eac8191a..8e5b8b56 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -176,13 +176,13 @@ func parseConfig(conf []byte) (Config, error) { } // NewBucket returns a new Bucket using the provided s3 config values. -func NewBucket(logger log.Logger, conf []byte, component string, rt http.RoundTripper) (*Bucket, error) { +func NewBucket(logger log.Logger, conf []byte, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error) { config, err := parseConfig(conf) if err != nil { return nil, err } - return NewBucketWithConfig(logger, config, component, rt) + return NewBucketWithConfig(logger, config, component, wrapRoundtripper) } type overrideSignerType struct { @@ -202,7 +202,7 @@ func (s *overrideSignerType) Retrieve() (credentials.Value, error) { } // NewBucketWithConfig returns a new Bucket using the provided s3 config values. -func NewBucketWithConfig(logger log.Logger, config Config, component string, rt http.RoundTripper) (*Bucket, error) { +func NewBucketWithConfig(logger log.Logger, config Config, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error) { var chain []credentials.Provider // TODO(bwplotka): Don't do flags as they won't scale, use actual params like v2, v4 instead @@ -242,9 +242,7 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string, rt }), } } - if rt != nil { - config.HTTPConfig.Transport = rt - } + // Check if a roundtripper has been set in the config // otherwise build the default transport. var tpt http.RoundTripper @@ -255,6 +253,9 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string, rt if config.HTTPConfig.Transport != nil { tpt = config.HTTPConfig.Transport } + if wrapRoundtripper != nil { + tpt = wrapRoundtripper(tpt) + } client, err := minio.New(config.Endpoint, &minio.Options{ Creds: credentials.NewChainCredentials(chain), diff --git a/providers/s3/s3_test.go b/providers/s3/s3_test.go index 2a44f0e0..3040cd81 100644 --- a/providers/s3/s3_test.go +++ b/providers/s3/s3_test.go @@ -16,7 +16,6 @@ import ( "github.com/efficientgo/core/testutil" "github.com/go-kit/log" "github.com/minio/minio-go/v7/pkg/encrypt" - "github.com/pkg/errors" "github.com/thanos-io/objstore/errutil" "github.com/thanos-io/objstore/exthttp" @@ -469,11 +468,10 @@ func TestNewBucketWithErrorRoundTripper(t *testing.T) { cfg := DefaultConfig cfg.Endpoint = endpoint cfg.Bucket = "test" - rt := &errutil.ErrorRoundTripper{Err: errors.New("RoundTripper error")} - bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test", rt) + bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test", errutil.WrapWithErrRoundtripper) testutil.Ok(t, err) _, err = bkt.Get(context.Background(), "test") // We expect an error from the RoundTripper testutil.NotOk(t, err) - testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) + testutil.Assert(t, errutil.IsMockedError(err), "Expected RoundTripper error, got: %v", err) } diff --git a/providers/swift/swift.go b/providers/swift/swift.go index 44fa6ed0..e872728e 100644 --- a/providers/swift/swift.go +++ b/providers/swift/swift.go @@ -154,12 +154,12 @@ type Container struct { segmentsContainer string } -func NewContainer(logger log.Logger, conf []byte, rt http.RoundTripper) (*Container, error) { +func NewContainer(logger log.Logger, conf []byte, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Container, error) { sc, err := parseConfig(conf) if err != nil { return nil, errors.Wrap(err, "parse config") } - return NewContainerFromConfig(logger, sc, false, rt) + return NewContainerFromConfig(logger, sc, false, wrapRoundtripper) } func ensureContainer(connection *swift.Connection, name string, createIfNotExist bool) error { @@ -178,22 +178,22 @@ func ensureContainer(connection *swift.Connection, name string, createIfNotExist return nil } -func NewContainerFromConfig(logger log.Logger, sc *Config, createContainer bool, rt http.RoundTripper) (*Container, error) { - if rt != nil { - sc.HTTPConfig.Transport = rt - } +func NewContainerFromConfig(logger log.Logger, sc *Config, createContainer bool, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Container, error) { // Check if a roundtripper has been set in the config // otherwise build the default transport. - var tpt http.RoundTripper - tpt, err := exthttp.DefaultTransport(sc.HTTPConfig) + var rt http.RoundTripper + rt, err := exthttp.DefaultTransport(sc.HTTPConfig) if err != nil { return nil, err } if sc.HTTPConfig.Transport != nil { - tpt = sc.HTTPConfig.Transport + rt = sc.HTTPConfig.Transport + } + if wrapRoundtripper != nil { + rt = wrapRoundtripper(rt) } - connection := connectionFromConfig(sc, tpt) + connection := connectionFromConfig(sc, rt) if err := connection.Authenticate(); err != nil { return nil, errors.Wrap(err, "authentication") } diff --git a/providers/swift/swift_test.go b/providers/swift/swift_test.go index 629dfd77..b17a5e2b 100644 --- a/providers/swift/swift_test.go +++ b/providers/swift/swift_test.go @@ -9,7 +9,6 @@ import ( "github.com/efficientgo/core/testutil" "github.com/go-kit/log" - "github.com/pkg/errors" "github.com/prometheus/common/model" "github.com/thanos-io/objstore/errutil" ) @@ -69,13 +68,11 @@ http_config: } func TestNewBucketWithErrorRoundTripper(t *testing.T) { - logger := log.NewNopLogger() - rt := &errutil.ErrorRoundTripper{Err: errors.New("RoundTripper error")} config := DefaultConfig config.AuthUrl = "http://identity.something.com/v3" - _, err := NewContainerFromConfig(logger, &config, false, rt) + _, err := NewContainerFromConfig(log.NewNopLogger(), &config, false, errutil.WrapWithErrRoundtripper) // We expect an error from the RoundTripper testutil.NotOk(t, err) - testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) + testutil.Assert(t, errutil.IsMockedError(err), "Expected RoundTripper error, got: %v", err) } From 31c0873d5266a6c58fe8fb5d6de01b66dcd50f5f Mon Sep 17 00:00:00 2001 From: Siavash Safi Date: Mon, 28 Oct 2024 15:45:11 +0100 Subject: [PATCH 32/41] fix objstore_bucket_operation_duration_seconds metrics fix histogram metrics for `get` and `getrange` operations. Signed-off-by: Siavash Safi --- objstore.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/objstore.go b/objstore.go index 5bce3ef6..62f1c655 100644 --- a/objstore.go +++ b/objstore.go @@ -573,7 +573,7 @@ func (b *metricBucket) Get(ctx context.Context, name string) (io.ReadCloser, err if !b.metrics.isOpFailureExpected(err) && ctx.Err() != context.Canceled { b.metrics.opsFailures.WithLabelValues(op).Inc() } - b.metrics.opsDuration.WithLabelValues(op).Observe(float64(time.Since(start))) + b.metrics.opsDuration.WithLabelValues(op).Observe(time.Since(start).Seconds()) return nil, err } return newTimingReader( @@ -600,7 +600,7 @@ func (b *metricBucket) GetRange(ctx context.Context, name string, off, length in if !b.metrics.isOpFailureExpected(err) && ctx.Err() != context.Canceled { b.metrics.opsFailures.WithLabelValues(op).Inc() } - b.metrics.opsDuration.WithLabelValues(op).Observe(float64(time.Since(start))) + b.metrics.opsDuration.WithLabelValues(op).Observe(time.Since(start).Seconds()) return nil, err } return newTimingReader( From cf96e342ed0d5977b1269c08d8c34ea9bd5ff442 Mon Sep 17 00:00:00 2001 From: Siavash Safi Date: Mon, 28 Oct 2024 15:59:44 +0100 Subject: [PATCH 33/41] add changelog Signed-off-by: Siavash Safi --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75cbd834..d2b1aaab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#145](https://github.com/thanos-io/objstore/pull/145) Include content length in the response of Get and GetRange. ### Fixed +- [#153](https://github.com/thanos-io/objstore/pull/153) Metrics: Fix `objstore_bucket_operation_duration_seconds_*` for `get` and `get_range` operations. - [#117](https://github.com/thanos-io/objstore/pull/117) Metrics: Fix `objstore_bucket_operation_failures_total` incorrectly incremented if context is cancelled while reading object contents. - [#115](https://github.com/thanos-io/objstore/pull/115) GCS: Fix creation of bucket with GRPC connections. Also update storage client to `v1.40.0`. - [#102](https://github.com/thanos-io/objstore/pull/102) Azure: bump azblob sdk to get concurrency fixes. From 3b23d359578bb4f7a72d6dc02bd96415c67b6f6e Mon Sep 17 00:00:00 2001 From: Filip Petkovski Date: Tue, 29 Oct 2024 16:31:18 +0100 Subject: [PATCH 34/41] Add support for IterWithAttributes This commit adds support for an IterWithAttributes on the bucket client. The method allows iterating through objects and getting multiple attributes into the callback function, removing the need to do an Iter followed by Attrs. For now, we only support getting the last updated time as an attribute, but the implementation allows adding more in the future. Not all buckets support this method. The client can check whether the bucket has support by calling the SupportedIterOptions method on the client. Co-authored-by: Ashwanth Goli Co-authored-by: Filip Petkovski Signed-off-by: Filip Petkovski --- README.md | 23 +++++- inmem.go | 14 ++++ objstore.go | 100 ++++++++++++++++++++++-- prefixed_bucket.go | 13 +++ prefixed_bucket_test.go | 2 +- providers/azure/azure.go | 47 +++++++++-- providers/bos/bos.go | 48 ++++++++++-- providers/cos/cos.go | 18 ++++- providers/filesystem/filesystem.go | 45 +++++++++-- providers/filesystem/filesystem_test.go | 58 ++++++++++++++ providers/gcs/gcs.go | 50 +++++++++--- providers/obs/obs.go | 16 +++- providers/oci/oci.go | 20 ++++- providers/oss/oss.go | 21 ++++- providers/s3/s3.go | 43 ++++++++-- providers/swift/swift.go | 18 ++++- testing.go | 15 +++- tracing/opentelemetry/opentelemetry.go | 18 +++++ tracing/opentracing/opentracing.go | 12 +++ 19 files changed, 520 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index 6d848e79..d8f58023 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ See [MAINTAINERS.md](https://github.com/thanos-io/thanos/blob/main/MAINTAINERS.m The core this module is the [`Bucket` interface](objstore.go): -```go mdox-exec="sed -n '37,50p' objstore.go" +```go mdox-exec="sed -n '39,55p' objstore.go" // Bucket provides read and write access to an object storage bucket. // NOTE: We assume strong consistency for write-read flow. type Bucket interface { @@ -63,18 +63,31 @@ type Bucket interface { // If object does not exist in the moment of deletion, Delete should throw error. Delete(ctx context.Context, name string) error + // Name returns the bucket name for the provider. + Name() string +} ``` All [provider implementations](providers) have to implement `Bucket` interface that allows common read and write operations that all supported by all object providers. If you want to limit the code that will do bucket operation to only read access (smart idea, allowing to limit access permissions), you can use the [`BucketReader` interface](objstore.go): -```go mdox-exec="sed -n '68,93p' objstore.go" - +```go mdox-exec="sed -n '71,106p' objstore.go" // BucketReader provides read access to an object storage bucket. type BucketReader interface { // Iter calls f for each entry in the given directory (not recursive.). The argument to f is the full // object name including the prefix of the inspected directory. + // Entries are passed to function in sorted order. - Iter(ctx context.Context, dir string, f func(string) error, options ...IterOption) error + Iter(ctx context.Context, dir string, f func(name string) error, options ...IterOption) error + + // IterWithAttributes calls f for each entry in the given directory similar to Iter. + // In addition to Name, it also includes requested object attributes in the argument to f. + // + // Attributes can be requested using IterOption. + // Not all IterOptions are supported by all providers, requesting for an unsupported option will fail with ErrOptionNotSupported. + IterWithAttributes(ctx context.Context, dir string, f func(attrs IterObjectAttributes) error, options ...IterOption) error + + // SupportedIterOptions returns a list of supported IterOptions by the underlying provider. + SupportedIterOptions() []IterOptionType // Get returns a reader for the given object name. Get(ctx context.Context, name string) (io.ReadCloser, error) @@ -374,6 +387,7 @@ config: server_name: "" insecure_skip_verify: false disable_compression: false + chunk_size_bytes: 0 prefix: "" ``` @@ -447,6 +461,7 @@ config: storage_account: "" storage_account_key: "" storage_connection_string: "" + storage_create_container: false container: "" endpoint: "" user_assigned_id: "" diff --git a/inmem.go b/inmem.go index ed256c9c..d550e283 100644 --- a/inmem.go +++ b/inmem.go @@ -106,6 +106,20 @@ func (b *InMemBucket) Iter(_ context.Context, dir string, f func(string) error, return nil } +func (i *InMemBucket) SupportedIterOptions() []IterOptionType { + return []IterOptionType{Recursive} +} + +func (b *InMemBucket) IterWithAttributes(ctx context.Context, dir string, f func(attrs IterObjectAttributes) error, options ...IterOption) error { + if err := ValidateIterOptions(b.SupportedIterOptions(), options...); err != nil { + return err + } + + return b.Iter(ctx, dir, func(name string) error { + return f(IterObjectAttributes{Name: name}) + }, options...) +} + // Get returns a reader for the given object name. func (b *InMemBucket) Get(_ context.Context, name string) (io.ReadCloser, error) { if name == "" { diff --git a/objstore.go b/objstore.go index 62f1c655..80916888 100644 --- a/objstore.go +++ b/objstore.go @@ -6,11 +6,13 @@ package objstore import ( "bytes" "context" + "fmt" "io" "io/fs" "os" "path" "path/filepath" + "slices" "strings" "sync" "time" @@ -70,8 +72,19 @@ type InstrumentedBucket interface { type BucketReader interface { // Iter calls f for each entry in the given directory (not recursive.). The argument to f is the full // object name including the prefix of the inspected directory. + // Entries are passed to function in sorted order. - Iter(ctx context.Context, dir string, f func(string) error, options ...IterOption) error + Iter(ctx context.Context, dir string, f func(name string) error, options ...IterOption) error + + // IterWithAttributes calls f for each entry in the given directory similar to Iter. + // In addition to Name, it also includes requested object attributes in the argument to f. + // + // Attributes can be requested using IterOption. + // Not all IterOptions are supported by all providers, requesting for an unsupported option will fail with ErrOptionNotSupported. + IterWithAttributes(ctx context.Context, dir string, f func(attrs IterObjectAttributes) error, options ...IterOption) error + + // SupportedIterOptions returns a list of supported IterOptions by the underlying provider. + SupportedIterOptions() []IterOptionType // Get returns a reader for the given object name. Get(ctx context.Context, name string) (io.ReadCloser, error) @@ -101,24 +114,66 @@ type InstrumentedBucketReader interface { ReaderWithExpectedErrs(IsOpFailureExpectedFunc) BucketReader } +var ErrOptionNotSupported = errors.New("iter option is not supported") + +// IterOptionType is used for type-safe option support checking. +type IterOptionType int + +const ( + Recursive IterOptionType = iota + UpdatedAt +) + // IterOption configures the provided params. -type IterOption func(params *IterParams) +type IterOption struct { + Type IterOptionType + Apply func(params *IterParams) +} // WithRecursiveIter is an option that can be applied to Iter() to recursively list objects // in the bucket. -func WithRecursiveIter(params *IterParams) { - params.Recursive = true +func WithRecursiveIter() IterOption { + return IterOption{ + Type: Recursive, + Apply: func(params *IterParams) { + params.Recursive = true + }, + } +} + +// WithUpdatedAt is an option that can be applied to Iter() to +// include the last modified time in the attributes. +// NB: Prefixes may not report last modified time. +// This option is currently supported for the azure, aws, bos, gcs and filesystem providers. +func WithUpdatedAt() IterOption { + return IterOption{ + Type: UpdatedAt, + Apply: func(params *IterParams) { + params.LastModified = true + }, + } } // IterParams holds the Iter() parameters and is used by objstore clients implementations. type IterParams struct { - Recursive bool + Recursive bool + LastModified bool +} + +func ValidateIterOptions(supportedOptions []IterOptionType, options ...IterOption) error { + for _, opt := range options { + if !slices.Contains(supportedOptions, opt.Type) { + return fmt.Errorf("%w: %v", ErrOptionNotSupported, opt.Type) + } + } + + return nil } func ApplyIterOptions(options ...IterOption) IterParams { out := IterParams{} for _, opt := range options { - opt(&out) + opt.Apply(&out) } return out } @@ -189,6 +244,20 @@ type ObjectAttributes struct { LastModified time.Time `json:"last_modified"` } +type IterObjectAttributes struct { + Name string + lastModified time.Time +} + +func (i *IterObjectAttributes) SetLastModified(t time.Time) { + i.lastModified = t +} + +// LastModified returns the timestamp the object was last modified. Returns false if the timestamp is not available. +func (i *IterObjectAttributes) LastModified() (time.Time, bool) { + return i.lastModified, !i.lastModified.IsZero() +} + // TryToGetSize tries to get upfront size from reader. // Some implementations may return only size of unread data in the reader, so it's best to call this method before // doing any reading. @@ -531,7 +600,7 @@ func (b *metricBucket) ReaderWithExpectedErrs(fn IsOpFailureExpectedFunc) Bucket return b.WithExpectedErrs(fn) } -func (b *metricBucket) Iter(ctx context.Context, dir string, f func(name string) error, options ...IterOption) error { +func (b *metricBucket) Iter(ctx context.Context, dir string, f func(string) error, options ...IterOption) error { const op = OpIter b.metrics.ops.WithLabelValues(op).Inc() @@ -546,6 +615,23 @@ func (b *metricBucket) Iter(ctx context.Context, dir string, f func(name string) return err } +func (b *metricBucket) IterWithAttributes(ctx context.Context, dir string, f func(IterObjectAttributes) error, options ...IterOption) error { + const op = OpIter + b.metrics.ops.WithLabelValues(op).Inc() + + err := b.bkt.IterWithAttributes(ctx, dir, f, options...) + if err != nil { + if !b.metrics.isOpFailureExpected(err) && ctx.Err() != context.Canceled { + b.metrics.opsFailures.WithLabelValues(op).Inc() + } + } + return err +} + +func (b *metricBucket) SupportedIterOptions() []IterOptionType { + return b.bkt.SupportedIterOptions() +} + func (b *metricBucket) Attributes(ctx context.Context, name string) (ObjectAttributes, error) { const op = OpAttributes b.metrics.ops.WithLabelValues(op).Inc() diff --git a/prefixed_bucket.go b/prefixed_bucket.go index f2b71434..a76b34c3 100644 --- a/prefixed_bucket.go +++ b/prefixed_bucket.go @@ -54,6 +54,19 @@ func (p *PrefixedBucket) Iter(ctx context.Context, dir string, f func(string) er }, options...) } +func (p *PrefixedBucket) IterWithAttributes(ctx context.Context, dir string, f func(IterObjectAttributes) error, options ...IterOption) error { + pdir := withPrefix(p.prefix, dir) + + return p.bkt.IterWithAttributes(ctx, pdir, func(attrs IterObjectAttributes) error { + attrs.Name = strings.TrimPrefix(attrs.Name, p.prefix+DirDelim) + return f(attrs) + }, options...) +} + +func (p *PrefixedBucket) SupportedIterOptions() []IterOptionType { + return p.bkt.SupportedIterOptions() +} + // Get returns a reader for the given object name. func (p *PrefixedBucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { return p.bkt.Get(ctx, conditionalPrefix(p.prefix, name)) diff --git a/prefixed_bucket_test.go b/prefixed_bucket_test.go index f93c8580..6252e05d 100644 --- a/prefixed_bucket_test.go +++ b/prefixed_bucket_test.go @@ -74,7 +74,7 @@ func UsesPrefixTest(t *testing.T, bkt Bucket, prefix string) { testutil.Ok(t, pBkt.Iter(context.Background(), "", func(fn string) error { seen = append(seen, fn) return nil - }, WithRecursiveIter)) + }, WithRecursiveIter())) expected := []string{"dir/file1.jpg", "file1.jpg"} sort.Strings(expected) sort.Strings(seen) diff --git a/providers/azure/azure.go b/providers/azure/azure.go index e125ca35..5689dc62 100644 --- a/providers/azure/azure.go +++ b/providers/azure/azure.go @@ -193,9 +193,15 @@ func NewBucketWithConfig(logger log.Logger, conf Config, component string, wrapR return bkt, nil } -// Iter calls f for each entry in the given directory. The argument to f is the full -// object name including the prefix of the inspected directory. -func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, options ...objstore.IterOption) error { +func (b *Bucket) SupportedIterOptions() []objstore.IterOptionType { + return []objstore.IterOptionType{objstore.Recursive, objstore.UpdatedAt} +} + +func (b *Bucket) IterWithAttributes(ctx context.Context, dir string, f func(attrs objstore.IterObjectAttributes) error, options ...objstore.IterOption) error { + if err := objstore.ValidateIterOptions(b.SupportedIterOptions(), options...); err != nil { + return err + } + prefix := dir if prefix != "" && !strings.HasSuffix(prefix, DirDelim) { prefix += DirDelim @@ -211,7 +217,13 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt return err } for _, blob := range resp.Segment.BlobItems { - if err := f(*blob.Name); err != nil { + attrs := objstore.IterObjectAttributes{ + Name: *blob.Name, + } + if params.LastModified { + attrs.SetLastModified(*blob.Properties.LastModified) + } + if err := f(attrs); err != nil { return err } } @@ -227,12 +239,18 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt return err } for _, blobItem := range resp.Segment.BlobItems { - if err := f(*blobItem.Name); err != nil { + attrs := objstore.IterObjectAttributes{ + Name: *blobItem.Name, + } + if params.LastModified { + attrs.SetLastModified(*blobItem.Properties.LastModified) + } + if err := f(attrs); err != nil { return err } } for _, blobPrefix := range resp.Segment.BlobPrefixes { - if err := f(*blobPrefix.Name); err != nil { + if err := f(objstore.IterObjectAttributes{Name: *blobPrefix.Name}); err != nil { return err } } @@ -240,6 +258,23 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt return nil } +// Iter calls f for each entry in the given directory. The argument to f is the full +// object name including the prefix of the inspected directory. +func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opts ...objstore.IterOption) error { + // Only include recursive option since attributes are not used in this method. + var filteredOpts []objstore.IterOption + for _, opt := range opts { + if opt.Type == objstore.Recursive { + filteredOpts = append(filteredOpts, opt) + break + } + } + + return b.IterWithAttributes(ctx, dir, func(attrs objstore.IterObjectAttributes) error { + return f(attrs.Name) + }, filteredOpts...) +} + // IsObjNotFoundErr returns true if error means that object is not found. Relevant to Get operations. func (b *Bucket) IsObjNotFoundErr(err error) bool { if err == nil { diff --git a/providers/bos/bos.go b/providers/bos/bos.go index 9faa93f3..20c8dd3e 100644 --- a/providers/bos/bos.go +++ b/providers/bos/bos.go @@ -176,16 +176,23 @@ func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) error { return nil } -// Iter calls f for each entry in the given directory (not recursive). The argument to f is the full -// object name including the prefix of the inspected directory. -func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt ...objstore.IterOption) error { +func (b *Bucket) SupportedIterOptions() []objstore.IterOptionType { + return []objstore.IterOptionType{objstore.Recursive, objstore.UpdatedAt} +} + +func (b *Bucket) IterWithAttributes(ctx context.Context, dir string, f func(attrs objstore.IterObjectAttributes) error, options ...objstore.IterOption) error { + if err := objstore.ValidateIterOptions(b.SupportedIterOptions(), options...); err != nil { + return err + } + if dir != "" { dir = strings.TrimSuffix(dir, objstore.DirDelim) + objstore.DirDelim } delimiter := objstore.DirDelim - if objstore.ApplyIterOptions(opt...).Recursive { + params := objstore.ApplyIterOptions(options...) + if params.Recursive { delimiter = "" } @@ -207,13 +214,25 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt marker = objects.NextMarker for _, object := range objects.Contents { - if err := f(object.Key); err != nil { + attrs := objstore.IterObjectAttributes{ + Name: object.Key, + } + + if params.LastModified && object.LastModified != "" { + lastModified, err := time.Parse(time.RFC1123, object.LastModified) + if err != nil { + return fmt.Errorf("iter: get last modified: %w", err) + } + attrs.SetLastModified(lastModified) + } + + if err := f(attrs); err != nil { return err } } for _, object := range objects.CommonPrefixes { - if err := f(object.Prefix); err != nil { + if err := f(objstore.IterObjectAttributes{Name: object.Prefix}); err != nil { return err } } @@ -224,6 +243,23 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt return nil } +// Iter calls f for each entry in the given directory. The argument to f is the full +// object name including the prefix of the inspected directory. +func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opts ...objstore.IterOption) error { + // Only include recursive option since attributes are not used in this method. + var filteredOpts []objstore.IterOption + for _, opt := range opts { + if opt.Type == objstore.Recursive { + filteredOpts = append(filteredOpts, opt) + break + } + } + + return b.IterWithAttributes(ctx, dir, func(attrs objstore.IterObjectAttributes) error { + return f(attrs.Name) + }, filteredOpts...) +} + // Get returns a reader for the given object name. func (b *Bucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { return b.getRange(ctx, b.name, name, 0, -1) diff --git a/providers/cos/cos.go b/providers/cos/cos.go index 6bd39caa..9caf567e 100644 --- a/providers/cos/cos.go +++ b/providers/cos/cos.go @@ -276,7 +276,11 @@ func (b *Bucket) Delete(ctx context.Context, name string) error { return nil } -// Iter calls f for each entry in the given directory (not recursive.). The argument to f is the full +func (b *Bucket) SupportedIterOptions() []objstore.IterOptionType { + return []objstore.IterOptionType{objstore.Recursive} +} + +// Iter calls f for each entry in the given directory. The argument to f is the full // object name including the prefix of the inspected directory. func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, options ...objstore.IterOption) error { if dir != "" { @@ -298,6 +302,16 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt return nil } +func (b *Bucket) IterWithAttributes(ctx context.Context, dir string, f func(attrs objstore.IterObjectAttributes) error, options ...objstore.IterOption) error { + if err := objstore.ValidateIterOptions(b.SupportedIterOptions(), options...); err != nil { + return err + } + + return b.Iter(ctx, dir, func(name string) error { + return f(objstore.IterObjectAttributes{Name: name}) + }, options...) +} + func (b *Bucket) getRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error) { if name == "" { return nil, errors.New("given object name should not empty") @@ -493,7 +507,7 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) { return nil, nil, err } - if err := b.Iter(context.Background(), "", func(f string) error { + if err := b.Iter(context.Background(), "", func(_ string) error { return errors.Errorf("bucket %s is not empty", c.Bucket) }); err != nil { return nil, nil, errors.Wrapf(err, "cos check bucket %s", c.Bucket) diff --git a/providers/filesystem/filesystem.go b/providers/filesystem/filesystem.go index 2ed42ee8..01dca4bb 100644 --- a/providers/filesystem/filesystem.go +++ b/providers/filesystem/filesystem.go @@ -50,13 +50,19 @@ func NewBucket(rootDir string) (*Bucket, error) { return &Bucket{rootDir: absDir}, nil } -// Iter calls f for each entry in the given directory. The argument to f is the full -// object name including the prefix of the inspected directory. -func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, options ...objstore.IterOption) error { +func (b *Bucket) SupportedIterOptions() []objstore.IterOptionType { + return []objstore.IterOptionType{objstore.Recursive, objstore.UpdatedAt} +} + +func (b *Bucket) IterWithAttributes(ctx context.Context, dir string, f func(attrs objstore.IterObjectAttributes) error, options ...objstore.IterOption) error { if ctx.Err() != nil { return ctx.Err() } + if err := objstore.ValidateIterOptions(b.SupportedIterOptions(), options...); err != nil { + return err + } + params := objstore.ApplyIterOptions(options...) absDir := filepath.Join(b.rootDir, dir) info, err := os.Stat(absDir) @@ -92,7 +98,7 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt if params.Recursive { // Recursively list files in the subdirectory. - if err := b.Iter(ctx, name, f, options...); err != nil { + if err := b.IterWithAttributes(ctx, name, f, options...); err != nil { return err } @@ -101,13 +107,42 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt continue } } - if err := f(name); err != nil { + + attrs := objstore.IterObjectAttributes{ + Name: name, + } + if params.LastModified { + absPath := filepath.Join(absDir, file.Name()) + stat, err := os.Stat(absPath) + if err != nil { + return errors.Wrapf(err, "stat %s", name) + } + attrs.SetLastModified(stat.ModTime()) + } + if err := f(attrs); err != nil { return err } } return nil } +// Iter calls f for each entry in the given directory. The argument to f is the full +// object name including the prefix of the inspected directory. +func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opts ...objstore.IterOption) error { + // Only include recursive option since attributes are not used in this method. + var filteredOpts []objstore.IterOption + for _, opt := range opts { + if opt.Type == objstore.Recursive { + filteredOpts = append(filteredOpts, opt) + break + } + } + + return b.IterWithAttributes(ctx, dir, func(attrs objstore.IterObjectAttributes) error { + return f(attrs.Name) + }, filteredOpts...) +} + // Get returns a reader for the given object name. func (b *Bucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { return b.GetRange(ctx, name, 0, -1) diff --git a/providers/filesystem/filesystem_test.go b/providers/filesystem/filesystem_test.go index c3621fe0..105aab8e 100644 --- a/providers/filesystem/filesystem_test.go +++ b/providers/filesystem/filesystem_test.go @@ -6,11 +6,15 @@ package filesystem import ( "bytes" "context" + "os" "strings" "sync" "testing" + "time" "github.com/efficientgo/core/testutil" + + "github.com/thanos-io/objstore" ) func TestDelete_EmptyDirDeletionRaceCondition(t *testing.T) { @@ -61,6 +65,60 @@ func TestIter_CancelledContext(t *testing.T) { testutil.Equals(t, context.Canceled, err) } +func TestIterWithAttributes(t *testing.T) { + dir := t.TempDir() + f, err := os.CreateTemp(dir, "test") + testutil.Ok(t, err) + defer f.Close() + + stat, err := f.Stat() + testutil.Ok(t, err) + + cases := []struct { + name string + opts []objstore.IterOption + expectedUpdatedAt time.Time + }{ + { + name: "no options", + opts: nil, + }, + { + name: "with updated at", + opts: []objstore.IterOption{ + objstore.WithUpdatedAt(), + }, + expectedUpdatedAt: stat.ModTime(), + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + b, err := NewBucket(dir) + testutil.Ok(t, err) + + var attrs objstore.IterObjectAttributes + + ctx := context.Background() + err = b.IterWithAttributes(ctx, "", func(objectAttrs objstore.IterObjectAttributes) error { + attrs = objectAttrs + return nil + }, tc.opts...) + + testutil.Ok(t, err) + + lastModified, ok := attrs.LastModified() + if zero := tc.expectedUpdatedAt.IsZero(); zero { + testutil.Equals(t, false, ok) + } else { + testutil.Equals(t, true, ok) + testutil.Equals(t, tc.expectedUpdatedAt, lastModified) + } + }) + + } +} + func TestGet_CancelledContext(t *testing.T) { b, err := NewBucket(t.TempDir()) testutil.Ok(t, err) diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index efb208e6..0d4690c2 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -181,31 +181,33 @@ func (b *Bucket) Name() string { return b.name } -// Iter calls f for each entry in the given directory. The argument to f is the full -// object name including the prefix of the inspected directory. -func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, options ...objstore.IterOption) error { +func (b *Bucket) SupportedIterOptions() []objstore.IterOptionType { + return []objstore.IterOptionType{objstore.Recursive, objstore.UpdatedAt} +} + +func (b *Bucket) IterWithAttributes(ctx context.Context, dir string, f func(attrs objstore.IterObjectAttributes) error, options ...objstore.IterOption) error { + if err := objstore.ValidateIterOptions(b.SupportedIterOptions(), options...); err != nil { + return err + } + // Ensure the object name actually ends with a dir suffix. Otherwise we'll just iterate the // object itself as one prefix item. if dir != "" { dir = strings.TrimSuffix(dir, DirDelim) + DirDelim } + appliedOpts := objstore.ApplyIterOptions(options...) + // If recursive iteration is enabled we should pass an empty delimiter. delimiter := DirDelim - if objstore.ApplyIterOptions(options...).Recursive { + if appliedOpts.Recursive { delimiter = "" } - query := &storage.Query{ + it := b.bkt.Objects(ctx, &storage.Query{ Prefix: dir, Delimiter: delimiter, - } - err := query.SetAttrSelection([]string{"Name"}) - if err != nil { - return err - } - - it := b.bkt.Objects(ctx, query) + }) for { select { case <-ctx.Done(): @@ -219,12 +221,34 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt if err != nil { return err } - if err := f(attrs.Prefix + attrs.Name); err != nil { + + objAttrs := objstore.IterObjectAttributes{Name: attrs.Prefix + attrs.Name} + if appliedOpts.LastModified { + objAttrs.SetLastModified(attrs.Updated) + } + if err := f(objAttrs); err != nil { return err } } } +// Iter calls f for each entry in the given directory. The argument to f is the full +// object name including the prefix of the inspected directory. +func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opts ...objstore.IterOption) error { + // Only include recursive option since attributes are not used in this method. + var filteredOpts []objstore.IterOption + for _, opt := range opts { + if opt.Type == objstore.Recursive { + filteredOpts = append(filteredOpts, opt) + break + } + } + + return b.IterWithAttributes(ctx, dir, func(attrs objstore.IterObjectAttributes) error { + return f(attrs.Name) + }, filteredOpts...) +} + // Get returns a reader for the given object name. func (b *Bucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { r, err := b.bkt.Object(name).NewReader(ctx) diff --git a/providers/obs/obs.go b/providers/obs/obs.go index 4fb17baa..4ee9a227 100644 --- a/providers/obs/obs.go +++ b/providers/obs/obs.go @@ -232,6 +232,10 @@ func (b *Bucket) multipartUpload(size int64, key, uploadId string, body io.Reade func (b *Bucket) Close() error { return nil } +func (b *Bucket) SupportedIterOptions() []objstore.IterOptionType { + return []objstore.IterOptionType{objstore.Recursive} +} + // Iter calls f for each entry in the given directory (not recursive.) func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, options ...objstore.IterOption) error { if dir != "" { @@ -270,6 +274,16 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt return nil } +func (b *Bucket) IterWithAttributes(ctx context.Context, dir string, f func(attrs objstore.IterObjectAttributes) error, options ...objstore.IterOption) error { + if err := objstore.ValidateIterOptions(b.SupportedIterOptions(), options...); err != nil { + return err + } + + return b.Iter(ctx, dir, func(name string) error { + return f(objstore.IterObjectAttributes{Name: name}) + }, options...) +} + // Get returns a reader for the given object name. func (b *Bucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { return b.getRange(ctx, name, 0, -1) @@ -381,7 +395,7 @@ func NewTestBucketFromConfig(t testing.TB, c Config, reuseBucket bool, location bktToCreate := c.Bucket if c.Bucket != "" && reuseBucket { - if err := b.Iter(ctx, "", func(f string) error { + if err := b.Iter(ctx, "", func(_ string) error { return errors.Errorf("bucket %s is not empty", c.Bucket) }); err != nil { return nil, nil, err diff --git a/providers/oci/oci.go b/providers/oci/oci.go index bc8a8bd9..062da7c1 100644 --- a/providers/oci/oci.go +++ b/providers/oci/oci.go @@ -21,8 +21,9 @@ import ( "github.com/oracle/oci-go-sdk/v65/objectstorage/transfer" "github.com/pkg/errors" "github.com/prometheus/common/model" - "github.com/thanos-io/objstore" "gopkg.in/yaml.v2" + + "github.com/thanos-io/objstore" ) // DirDelim is the delimiter used to model a directory structure in an object store bucket. @@ -100,7 +101,11 @@ func (b *Bucket) Name() string { return b.name } -// Iter calls f for each entry in the given directory (not recursive). The argument to f is the full +func (b *Bucket) SupportedIterOptions() []objstore.IterOptionType { + return []objstore.IterOptionType{objstore.Recursive} +} + +// Iter calls f for each entry in the given directory. The argument to f is the full // object name including the prefix of the inspected directory. func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, options ...objstore.IterOption) error { // Ensure the object name actually ends with a dir suffix. Otherwise we'll just iterate the @@ -120,6 +125,7 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt if objectName == "" || objectName == dir { continue } + if err := f(objectName); err != nil { return err } @@ -128,6 +134,16 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt return nil } +func (b *Bucket) IterWithAttributes(ctx context.Context, dir string, f func(attrs objstore.IterObjectAttributes) error, options ...objstore.IterOption) error { + if err := objstore.ValidateIterOptions(b.SupportedIterOptions(), options...); err != nil { + return err + } + + return b.Iter(ctx, dir, func(name string) error { + return f(objstore.IterObjectAttributes{Name: name}) + }, options...) +} + // Get returns a reader for the given object name. func (b *Bucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { response, err := getObject(ctx, *b, name, "") diff --git a/providers/oss/oss.go b/providers/oss/oss.go index e01aff2e..aee8c623 100644 --- a/providers/oss/oss.go +++ b/providers/oss/oss.go @@ -22,10 +22,9 @@ import ( "github.com/pkg/errors" "gopkg.in/yaml.v2" + "github.com/thanos-io/objstore" "github.com/thanos-io/objstore/clientutil" "github.com/thanos-io/objstore/exthttp" - - "github.com/thanos-io/objstore" ) // PartSize is a part size for multi part upload. @@ -216,7 +215,11 @@ func validate(config Config) error { return nil } -// Iter calls f for each entry in the given directory (not recursive). The argument to f is the full +func (b *Bucket) SupportedIterOptions() []objstore.IterOptionType { + return []objstore.IterOptionType{objstore.Recursive} +} + +// Iter calls f for each entry in the given directory. The argument to f is the full // object name including the prefix of the inspected directory. func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, options ...objstore.IterOption) error { if dir != "" { @@ -258,6 +261,16 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt return nil } +func (b *Bucket) IterWithAttributes(ctx context.Context, dir string, f func(attrs objstore.IterObjectAttributes) error, options ...objstore.IterOption) error { + if err := objstore.ValidateIterOptions(b.SupportedIterOptions(), options...); err != nil { + return err + } + + return b.Iter(ctx, dir, func(name string) error { + return f(objstore.IterObjectAttributes{Name: name}) + }, options...) +} + func (b *Bucket) Name() string { return b.name } @@ -292,7 +305,7 @@ func NewTestBucketFromConfig(t testing.TB, c Config, reuseBucket bool) (objstore } if reuseBucket { - if err := b.Iter(context.Background(), "", func(f string) error { + if err := b.Iter(context.Background(), "", func(_ string) error { return errors.Errorf("bucket %s is not empty", c.Bucket) }); err != nil { return nil, nil, errors.Wrapf(err, "oss check bucket %s", c.Bucket) diff --git a/providers/s3/s3.go b/providers/s3/s3.go index 8e5b8b56..fc8da7b3 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -387,18 +387,26 @@ func ValidateForTests(conf Config) error { return nil } -// Iter calls f for each entry in the given directory. The argument to f is the full -// object name including the prefix of the inspected directory. -func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, options ...objstore.IterOption) error { +func (b *Bucket) SupportedIterOptions() []objstore.IterOptionType { + return []objstore.IterOptionType{objstore.Recursive, objstore.UpdatedAt} +} + +func (b *Bucket) IterWithAttributes(ctx context.Context, dir string, f func(attrs objstore.IterObjectAttributes) error, options ...objstore.IterOption) error { + if err := objstore.ValidateIterOptions(b.SupportedIterOptions(), options...); err != nil { + return err + } + // Ensure the object name actually ends with a dir suffix. Otherwise we'll just iterate the // object itself as one prefix item. if dir != "" { dir = strings.TrimSuffix(dir, DirDelim) + DirDelim } + appliedOpts := objstore.ApplyIterOptions(options...) + opts := minio.ListObjectsOptions{ Prefix: dir, - Recursive: objstore.ApplyIterOptions(options...).Recursive, + Recursive: appliedOpts.Recursive, UseV1: b.listObjectsV1, } @@ -415,7 +423,15 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt if object.Key == dir { continue } - if err := f(object.Key); err != nil { + + attr := objstore.IterObjectAttributes{ + Name: object.Key, + } + if appliedOpts.LastModified { + attr.SetLastModified(object.LastModified) + } + + if err := f(attr); err != nil { return err } } @@ -423,6 +439,21 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt return ctx.Err() } +func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opts ...objstore.IterOption) error { + // Only include recursive option since attributes are not used in this method. + var filteredOpts []objstore.IterOption + for _, opt := range opts { + if opt.Type == objstore.Recursive { + filteredOpts = append(filteredOpts, opt) + break + } + } + + return b.IterWithAttributes(ctx, dir, func(attrs objstore.IterObjectAttributes) error { + return f(attrs.Name) + }, filteredOpts...) +} + func (b *Bucket) getRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error) { sse, err := b.getServerSideEncryption(ctx) if err != nil { @@ -629,7 +660,7 @@ func NewTestBucketFromConfig(t testing.TB, location string, c Config, reuseBucke bktToCreate := c.Bucket if c.Bucket != "" && reuseBucket { - if err := b.Iter(ctx, "", func(f string) error { + if err := b.Iter(ctx, "", func(string) error { return errors.Errorf("bucket %s is not empty", c.Bucket) }); err != nil { return nil, nil, errors.Wrapf(err, "s3 check bucket %s", c.Bucket) diff --git a/providers/swift/swift.go b/providers/swift/swift.go index e872728e..86caa0c1 100644 --- a/providers/swift/swift.go +++ b/providers/swift/swift.go @@ -21,6 +21,7 @@ import ( "github.com/ncw/swift" "github.com/pkg/errors" "github.com/prometheus/common/model" + "github.com/thanos-io/objstore" "github.com/thanos-io/objstore/exthttp" "gopkg.in/yaml.v2" @@ -222,9 +223,13 @@ func (c *Container) Name() string { return c.name } +func (c *Container) SupportedIterOptions() []objstore.IterOptionType { + return []objstore.IterOptionType{objstore.Recursive} +} + // Iter calls f for each entry in the given directory. The argument to f is the full // object name including the prefix of the inspected directory. -func (c *Container) Iter(_ context.Context, dir string, f func(string) error, options ...objstore.IterOption) error { +func (c *Container) Iter(ctx context.Context, dir string, f func(string) error, options ...objstore.IterOption) error { if dir != "" { dir = strings.TrimSuffix(dir, string(DirDelim)) + string(DirDelim) } @@ -242,6 +247,7 @@ func (c *Container) Iter(_ context.Context, dir string, f func(string) error, op if err != nil { return objects, errors.Wrap(err, "list object names") } + for _, object := range objects { if object == SegmentsDir { continue @@ -254,6 +260,16 @@ func (c *Container) Iter(_ context.Context, dir string, f func(string) error, op }) } +func (c *Container) IterWithAttributes(ctx context.Context, dir string, f func(attrs objstore.IterObjectAttributes) error, options ...objstore.IterOption) error { + if err := objstore.ValidateIterOptions(c.SupportedIterOptions(), options...); err != nil { + return err + } + + return c.Iter(ctx, dir, func(name string) error { + return f(objstore.IterObjectAttributes{Name: name}) + }, options...) +} + func (c *Container) get(name string, headers swift.Headers, checkHash bool) (io.ReadCloser, error) { if name == "" { return nil, errors.New("object name cannot be empty") diff --git a/testing.go b/testing.go index 28cbd658..d3fa1def 100644 --- a/testing.go +++ b/testing.go @@ -195,7 +195,7 @@ func AcceptanceTest(t *testing.T, bkt Bucket) { testutil.Ok(t, bkt.Iter(ctx, "", func(fn string) error { seen = append(seen, fn) return nil - }, WithRecursiveIter)) + }, WithRecursiveIter())) expected = []string{"id1/obj_1.some", "id1/obj_2.some", "id1/obj_3.some", "id1/sub/subobj_1.some", "id1/sub/subobj_2.some", "id2/obj_4.some", "obj_5.some"} sort.Strings(expected) sort.Strings(seen) @@ -214,7 +214,7 @@ func AcceptanceTest(t *testing.T, bkt Bucket) { testutil.Ok(t, bkt.Iter(ctx, "id1/", func(fn string) error { seen = append(seen, fn) return nil - }, WithRecursiveIter)) + }, WithRecursiveIter())) testutil.Equals(t, []string{"id1/obj_1.some", "id1/obj_2.some", "id1/obj_3.some", "id1/sub/subobj_1.some", "id1/sub/subobj_2.some"}, seen) // Can we iter over items from id1 dir? @@ -230,7 +230,7 @@ func AcceptanceTest(t *testing.T, bkt Bucket) { testutil.Ok(t, bkt.Iter(ctx, "id1", func(fn string) error { seen = append(seen, fn) return nil - }, WithRecursiveIter)) + }, WithRecursiveIter())) testutil.Equals(t, []string{"id1/obj_1.some", "id1/obj_2.some", "id1/obj_3.some", "id1/sub/subobj_1.some", "id1/sub/subobj_2.some"}, seen) // Can we iter over items from not existing dir? @@ -295,6 +295,15 @@ func (d *delayingBucket) Iter(ctx context.Context, dir string, f func(string) er return d.bkt.Iter(ctx, dir, f, options...) } +func (d *delayingBucket) IterWithAttributes(ctx context.Context, dir string, f func(IterObjectAttributes) error, options ...IterOption) error { + time.Sleep(d.delay) + return d.bkt.IterWithAttributes(ctx, dir, f, options...) +} + +func (d *delayingBucket) SupportedIterOptions() []IterOptionType { + return d.bkt.SupportedIterOptions() +} + func (d *delayingBucket) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error) { time.Sleep(d.delay) return d.bkt.GetRange(ctx, name, off, length) diff --git a/tracing/opentelemetry/opentelemetry.go b/tracing/opentelemetry/opentelemetry.go index f65b0f0e..dad71e39 100644 --- a/tracing/opentelemetry/opentelemetry.go +++ b/tracing/opentelemetry/opentelemetry.go @@ -36,6 +36,24 @@ func (t TracingBucket) Iter(ctx context.Context, dir string, f func(string) erro return t.bkt.Iter(ctx, dir, f, options...) } +func (t TracingBucket) IterWithAttributes(ctx context.Context, dir string, f func(attrs objstore.IterObjectAttributes) error, options ...objstore.IterOption) (err error) { + ctx, span := t.tracer.Start(ctx, "bucket_iter_with_attrs") + defer span.End() + span.SetAttributes(attribute.String("dir", dir)) + + defer func() { + if err != nil { + span.RecordError(err) + } + }() + return t.bkt.IterWithAttributes(ctx, dir, f, options...) +} + +// SupportedIterOptions returns a list of supported IterOptions by the underlying provider. +func (t TracingBucket) SupportedIterOptions() []objstore.IterOptionType { + return t.bkt.SupportedIterOptions() +} + func (t TracingBucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { ctx, span := t.tracer.Start(ctx, "bucket_get") defer span.End() diff --git a/tracing/opentracing/opentracing.go b/tracing/opentracing/opentracing.go index 0a26ceeb..cabe07b2 100644 --- a/tracing/opentracing/opentracing.go +++ b/tracing/opentracing/opentracing.go @@ -52,6 +52,18 @@ func (t TracingBucket) Iter(ctx context.Context, dir string, f func(string) erro return } +func (t TracingBucket) IterWithAttributes(ctx context.Context, dir string, f func(attrs objstore.IterObjectAttributes) error, options ...objstore.IterOption) (err error) { + doWithSpan(ctx, "bucket_iter_with_attrs", func(spanCtx context.Context, span opentracing.Span) { + span.LogKV("dir", dir) + err = t.bkt.IterWithAttributes(spanCtx, dir, f, options...) + }) + return +} + +func (t TracingBucket) SupportedIterOptions() []objstore.IterOptionType { + return t.bkt.SupportedIterOptions() +} + func (t TracingBucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { span, spanCtx := startSpan(ctx, "bucket_get") span.LogKV("name", name) From 2d25e83cdce7dbe5b3a5ce56ad9b890af88a017a Mon Sep 17 00:00:00 2001 From: Filip Petkovski Date: Wed, 30 Oct 2024 07:37:14 +0100 Subject: [PATCH 35/41] Request name for GCS Signed-off-by: Filip Petkovski --- objstore.go | 2 +- providers/gcs/gcs.go | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/objstore.go b/objstore.go index 80916888..359107d6 100644 --- a/objstore.go +++ b/objstore.go @@ -144,7 +144,7 @@ func WithRecursiveIter() IterOption { // WithUpdatedAt is an option that can be applied to Iter() to // include the last modified time in the attributes. // NB: Prefixes may not report last modified time. -// This option is currently supported for the azure, aws, bos, gcs and filesystem providers. +// This option is currently supported for the azure, s3, bos, gcs and filesystem providers. func WithUpdatedAt() IterOption { return IterOption{ Type: UpdatedAt, diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index 0d4690c2..1a3edfd2 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -204,10 +204,20 @@ func (b *Bucket) IterWithAttributes(ctx context.Context, dir string, f func(attr delimiter = "" } - it := b.bkt.Objects(ctx, &storage.Query{ + query := &storage.Query{ Prefix: dir, Delimiter: delimiter, - }) + } + if appliedOpts.LastModified { + if err := query.SetAttrSelection([]string{"Name", "Updated"}); err != nil { + return err + } + } else { + if err := query.SetAttrSelection([]string{"Name"}); err != nil { + return err + } + } + it := b.bkt.Objects(ctx, query) for { select { case <-ctx.Done(): From ac8779f7cb498c73a650360499e8d304b076b5f7 Mon Sep 17 00:00:00 2001 From: Filip Petkovski Date: Wed, 30 Oct 2024 07:39:40 +0100 Subject: [PATCH 36/41] Add duration metric Signed-off-by: Filip Petkovski --- objstore.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/objstore.go b/objstore.go index 359107d6..d05bc019 100644 --- a/objstore.go +++ b/objstore.go @@ -604,14 +604,15 @@ func (b *metricBucket) Iter(ctx context.Context, dir string, f func(string) erro const op = OpIter b.metrics.ops.WithLabelValues(op).Inc() - start := time.Now() + timer := prometheus.NewTimer(b.metrics.opsDuration.WithLabelValues(op)) + defer timer.ObserveDuration() + err := b.bkt.Iter(ctx, dir, f, options...) if err != nil { if !b.metrics.isOpFailureExpected(err) && ctx.Err() != context.Canceled { b.metrics.opsFailures.WithLabelValues(op).Inc() } } - b.metrics.opsDuration.WithLabelValues(op).Observe(time.Since(start).Seconds()) return err } @@ -619,12 +620,16 @@ func (b *metricBucket) IterWithAttributes(ctx context.Context, dir string, f fun const op = OpIter b.metrics.ops.WithLabelValues(op).Inc() + timer := prometheus.NewTimer(b.metrics.opsDuration.WithLabelValues(op)) + defer timer.ObserveDuration() + err := b.bkt.IterWithAttributes(ctx, dir, f, options...) if err != nil { if !b.metrics.isOpFailureExpected(err) && ctx.Err() != context.Canceled { b.metrics.opsFailures.WithLabelValues(op).Inc() } } + return err } From 8648e8ed7794c446f8bd4cb18eeefb8be67f04a7 Mon Sep 17 00:00:00 2001 From: Thor <8681572+thorfour@users.noreply.github.com> Date: Thu, 31 Oct 2024 05:28:01 -0500 Subject: [PATCH 37/41] Support io.LimitedReader in object size check (#152) * Support io.LimitedReader in object size check Signed-off-by: thorfour * unit test Signed-off-by: thorfour --------- Signed-off-by: thorfour --- objstore.go | 2 ++ objstore_test.go | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/objstore.go b/objstore.go index 62f1c655..ab663ef0 100644 --- a/objstore.go +++ b/objstore.go @@ -211,6 +211,8 @@ func TryToGetSize(r io.Reader) (int64, error) { return f.Size(), nil case ObjectSizer: return f.ObjectSize() + case *io.LimitedReader: + return f.N, nil } return 0, errors.Errorf("unsupported type of io.Reader: %T", r) } diff --git a/objstore_test.go b/objstore_test.go index 51486d23..b1f82922 100644 --- a/objstore_test.go +++ b/objstore_test.go @@ -595,3 +595,11 @@ func (b *mockBucket) GetRange(ctx context.Context, name string, off, length int6 } return nil, errors.New("GetRange has not been mocked") } + +func Test_TryToGetSizeLimitedReader(t *testing.T) { + b := &bytes.Buffer{} + r := io.LimitReader(b, 1024) + size, err := TryToGetSize(r) + testutil.Ok(t, err) + testutil.Equals(t, int64(1024), size) +} From 2c4ff97c95c7ff9b953ecb231fbde162202b2091 Mon Sep 17 00:00:00 2001 From: Filip Petkovski Date: Thu, 31 Oct 2024 12:06:28 +0100 Subject: [PATCH 38/41] Add changelog Signed-off-by: Filip Petkovski --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2b1aaab..099f83ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#79](https://github.com/thanos-io/objstore/pull/79) Metrics: Fix `objstore_bucket_operation_duration_seconds` for `iter` operations. ### Added +- [#63](https://github.com/thanos-io/objstore/pull/63) Implement a `IterWithAttributes` method on the bucket client. - [#15](https://github.com/thanos-io/objstore/pull/15) Add Oracle Cloud Infrastructure Object Storage Bucket support. - [#25](https://github.com/thanos-io/objstore/pull/25) S3: Support specifying S3 storage class. - [#32](https://github.com/thanos-io/objstore/pull/32) Swift: Support authentication using application credentials. From ac8c1e14b6a50fdc1936c6e9d1cf4aa6a7201f9d Mon Sep 17 00:00:00 2001 From: Ashwanth Goli Date: Mon, 11 Nov 2024 11:45:49 +0530 Subject: [PATCH 39/41] s3: configure max_retries Signed-off-by: Ashwanth Goli --- go.mod | 20 ++++++++++---------- go.sum | 36 ++++++++++++++++++------------------ providers/s3/s3.go | 2 ++ 3 files changed, 30 insertions(+), 28 deletions(-) diff --git a/go.mod b/go.mod index db189c20..4555b6bb 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/thanos-io/objstore -go 1.21 +go 1.22 require ( cloud.google.com/go/storage v1.43.0 @@ -14,7 +14,7 @@ require ( github.com/fullstorydev/emulators/storage v0.0.0-20240401123056-edc69752f474 github.com/go-kit/log v0.2.1 github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.3+incompatible - github.com/minio/minio-go/v7 v7.0.72 + github.com/minio/minio-go/v7 v7.0.80 github.com/ncw/swift v1.0.53 github.com/opentracing/opentracing-go v1.2.0 github.com/oracle/oci-go-sdk/v65 v65.41.1 @@ -26,7 +26,7 @@ require ( go.opentelemetry.io/otel/trace v1.24.0 go.uber.org/atomic v1.9.0 golang.org/x/oauth2 v0.21.0 - golang.org/x/sync v0.7.0 + golang.org/x/sync v0.8.0 google.golang.org/api v0.187.0 google.golang.org/grpc v1.66.0 gopkg.in/alecthomas/kingpin.v2 v2.2.6 @@ -60,6 +60,7 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-ini/ini v1.67.0 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -75,7 +76,7 @@ require ( github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.12.5 // indirect github.com/jpillora/backoff v1.0.0 // indirect - github.com/klauspost/compress v1.17.9 // indirect + github.com/klauspost/compress v1.17.11 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect @@ -86,22 +87,21 @@ require ( github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect github.com/prometheus/procfs v0.11.1 // indirect - github.com/rs/xid v1.5.0 // indirect + github.com/rs/xid v1.6.0 // indirect github.com/sony/gobreaker v0.5.0 // indirect github.com/stretchr/objx v0.5.2 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect go.opentelemetry.io/otel/metric v1.24.0 // indirect - golang.org/x/net v0.26.0 // indirect - golang.org/x/sys v0.21.0 // indirect - golang.org/x/text v0.16.0 // indirect + golang.org/x/net v0.30.0 // indirect + golang.org/x/sys v0.26.0 // indirect + golang.org/x/text v0.19.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d // indirect google.golang.org/protobuf v1.34.2 // indirect - gopkg.in/ini.v1 v1.67.0 // indirect ) require ( @@ -110,5 +110,5 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.0 github.com/kr/text v0.2.0 // indirect github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b // indirect - golang.org/x/crypto v0.24.0 // indirect + golang.org/x/crypto v0.28.0 // indirect ) diff --git a/go.sum b/go.sum index 6e22dfab..c010a36e 100644 --- a/go.sum +++ b/go.sum @@ -92,6 +92,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fullstorydev/emulators/storage v0.0.0-20240401123056-edc69752f474 h1:TufioMBjkJ6/Oqmlye/ReuxHFS35HyLmypj/BNy/8GY= github.com/fullstorydev/emulators/storage v0.0.0-20240401123056-edc69752f474/go.mod h1:PQwxF4UU8wuL+srGxr3BOhIW5zXqgucwVlO/nPZLsxw= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= @@ -156,8 +158,8 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= -github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= @@ -171,8 +173,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zk github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.72 h1:ZSbxs2BfJensLyHdVOgHv+pfmvxYraaUy07ER04dWnA= -github.com/minio/minio-go/v7 v7.0.72/go.mod h1:4yBA8v80xGA30cfM3fz0DKYMXunWl/AV/6tWEs9ryzo= +github.com/minio/minio-go/v7 v7.0.80 h1:2mdUHXEykRdY/BigLt3Iuu1otL0JTogT0Nmltg0wujk= +github.com/minio/minio-go/v7 v7.0.80/go.mod h1:84gmIilaX4zcvAWWzJ5Z1WI5axN+hAbM5w25xf8xvC0= github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ= @@ -202,8 +204,8 @@ github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwa github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= -github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b h1:gQZ0qzfKHQIybLANtM3mBXNUtOfsCFXeTsnBqCsx1KM= github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sony/gobreaker v0.5.0 h1:dRCvqm0P490vZPmy7ppEk2qCnCieBooFJ+YoXGYB+yg= @@ -243,8 +245,8 @@ go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= -golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -255,8 +257,8 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= @@ -264,8 +266,8 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -273,12 +275,12 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -323,8 +325,6 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= -gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= diff --git a/providers/s3/s3.go b/providers/s3/s3.go index fc8da7b3..27e82ffb 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -136,6 +136,7 @@ type Config struct { PartSize uint64 `yaml:"part_size"` SSEConfig SSEConfig `yaml:"sse_config"` STSEndpoint string `yaml:"sts_endpoint"` + MaxRetries int `yaml:"max_retries"` } // SSEConfig deals with the configuration of SSE for Minio. The following options are valid: @@ -263,6 +264,7 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string, wra Region: config.Region, Transport: tpt, BucketLookup: config.BucketLookupType.MinioType(), + MaxRetries: config.MaxRetries, }) if err != nil { return nil, errors.Wrap(err, "initialize s3 client") From bd774b3b1efa9202c311bb1acfd0fdf8d5bd79d8 Mon Sep 17 00:00:00 2001 From: Filip Petkovski Date: Mon, 18 Nov 2024 13:53:40 +0100 Subject: [PATCH 40/41] Add Provider() method to objstore.Client This commit adds a Provider() method to the objstore.Client interface. The method is useful when the client was created from a config file using the client factory. Signed-off-by: Filip Petkovski --- README.md | 6 ++-- client/factory.go | 41 ++++++++------------------ inmem.go | 2 ++ objstore.go | 22 ++++++++++++++ objtesting/foreach.go | 21 +++++++------ prefixed_bucket.go | 4 ++- providers/azure/azure.go | 2 ++ providers/bos/bos.go | 2 ++ providers/cos/cos.go | 2 ++ providers/filesystem/filesystem.go | 2 ++ providers/gcs/gcs.go | 2 ++ providers/obs/obs.go | 2 ++ providers/oci/oci.go | 2 ++ providers/oss/oss.go | 2 ++ providers/s3/s3.go | 2 ++ providers/swift/swift.go | 2 ++ scripts/cfggen/main.go | 23 ++++++++------- testing.go | 2 ++ tracing/opentelemetry/opentelemetry.go | 2 ++ tracing/opentracing/opentracing.go | 2 ++ 20 files changed, 92 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index d8f58023..516d1070 100644 --- a/README.md +++ b/README.md @@ -48,13 +48,15 @@ See [MAINTAINERS.md](https://github.com/thanos-io/thanos/blob/main/MAINTAINERS.m The core this module is the [`Bucket` interface](objstore.go): -```go mdox-exec="sed -n '39,55p' objstore.go" +```go mdox-exec="sed -n '55,73p' objstore.go" // Bucket provides read and write access to an object storage bucket. // NOTE: We assume strong consistency for write-read flow. type Bucket interface { io.Closer BucketReader + Provider() ObjProvider + // Upload the contents of the reader as an object into the bucket. // Upload should be idempotent. Upload(ctx context.Context, name string, r io.Reader) error @@ -70,7 +72,7 @@ type Bucket interface { All [provider implementations](providers) have to implement `Bucket` interface that allows common read and write operations that all supported by all object providers. If you want to limit the code that will do bucket operation to only read access (smart idea, allowing to limit access permissions), you can use the [`BucketReader` interface](objstore.go): -```go mdox-exec="sed -n '71,106p' objstore.go" +```go mdox-exec="sed -n '89,124p' objstore.go" // BucketReader provides read access to an object storage bucket. type BucketReader interface { // Iter calls f for each entry in the given directory (not recursive.). The argument to f is the full diff --git a/client/factory.go b/client/factory.go index 5fe5a741..089fd843 100644 --- a/client/factory.go +++ b/client/factory.go @@ -27,25 +27,10 @@ import ( "gopkg.in/yaml.v2" ) -type ObjProvider string - -const ( - FILESYSTEM ObjProvider = "FILESYSTEM" - GCS ObjProvider = "GCS" - S3 ObjProvider = "S3" - AZURE ObjProvider = "AZURE" - SWIFT ObjProvider = "SWIFT" - COS ObjProvider = "COS" - ALIYUNOSS ObjProvider = "ALIYUNOSS" - BOS ObjProvider = "BOS" - OCI ObjProvider = "OCI" - OBS ObjProvider = "OBS" -) - type BucketConfig struct { - Type ObjProvider `yaml:"type"` - Config interface{} `yaml:"config"` - Prefix string `yaml:"prefix" default:""` + Type objstore.ObjProvider `yaml:"type"` + Config interface{} `yaml:"config"` + Prefix string `yaml:"prefix" default:""` } // NewBucket initializes and returns new object storage clients. @@ -64,25 +49,25 @@ func NewBucket(logger log.Logger, confContentYaml []byte, component string, wrap var bucket objstore.Bucket switch strings.ToUpper(string(bucketConf.Type)) { - case string(GCS): + case string(objstore.GCS): bucket, err = gcs.NewBucket(context.Background(), logger, config, component, wrapRoundtripper) - case string(S3): + case string(objstore.S3): bucket, err = s3.NewBucket(logger, config, component, wrapRoundtripper) - case string(AZURE): + case string(objstore.AZURE): bucket, err = azure.NewBucket(logger, config, component, wrapRoundtripper) - case string(SWIFT): + case string(objstore.SWIFT): bucket, err = swift.NewContainer(logger, config, wrapRoundtripper) - case string(COS): + case string(objstore.COS): bucket, err = cos.NewBucket(logger, config, component, wrapRoundtripper) - case string(ALIYUNOSS): + case string(objstore.ALIYUNOSS): bucket, err = oss.NewBucket(logger, config, component, wrapRoundtripper) - case string(FILESYSTEM): + case string(objstore.FILESYSTEM): bucket, err = filesystem.NewBucketFromConfig(config) - case string(BOS): + case string(objstore.BOS): bucket, err = bos.NewBucket(logger, config, component) - case string(OCI): + case string(objstore.OCI): bucket, err = oci.NewBucket(logger, config, wrapRoundtripper) - case string(OBS): + case string(objstore.OBS): bucket, err = obs.NewBucket(logger, config) default: return nil, errors.Errorf("bucket with type %s is not supported", bucketConf.Type) diff --git a/inmem.go b/inmem.go index d550e283..6a344066 100644 --- a/inmem.go +++ b/inmem.go @@ -34,6 +34,8 @@ func NewInMemBucket() *InMemBucket { } } +func (b *InMemBucket) Provider() ObjProvider { return MEMORY } + // Objects returns a copy of the internally stored objects. // NOTE: For assert purposes. func (b *InMemBucket) Objects() map[string][]byte { diff --git a/objstore.go b/objstore.go index 33c6e5e8..86ecfa26 100644 --- a/objstore.go +++ b/objstore.go @@ -26,6 +26,22 @@ import ( "golang.org/x/sync/errgroup" ) +type ObjProvider string + +const ( + MEMORY ObjProvider = "MEMORY" + FILESYSTEM ObjProvider = "FILESYSTEM" + GCS ObjProvider = "GCS" + S3 ObjProvider = "S3" + AZURE ObjProvider = "AZURE" + SWIFT ObjProvider = "SWIFT" + COS ObjProvider = "COS" + ALIYUNOSS ObjProvider = "ALIYUNOSS" + BOS ObjProvider = "BOS" + OCI ObjProvider = "OCI" + OBS ObjProvider = "OBS" +) + const ( OpIter = "iter" OpGet = "get" @@ -42,6 +58,8 @@ type Bucket interface { io.Closer BucketReader + Provider() ObjProvider + // Upload the contents of the reader as an object into the bucket. // Upload should be idempotent. Upload(ctx context.Context, name string, r io.Reader) error @@ -583,6 +601,10 @@ type metricBucket struct { metrics *Metrics } +func (b *metricBucket) Provider() ObjProvider { + return b.bkt.Provider() +} + func (b *metricBucket) WithExpectedErrs(fn IsOpFailureExpectedFunc) Bucket { return &metricBucket{ bkt: b.bkt, diff --git a/objtesting/foreach.go b/objtesting/foreach.go index 87d9ed1b..29d16c39 100644 --- a/objtesting/foreach.go +++ b/objtesting/foreach.go @@ -9,7 +9,6 @@ import ( "testing" "github.com/thanos-io/objstore" - "github.com/thanos-io/objstore/client" "github.com/thanos-io/objstore/providers/azure" "github.com/thanos-io/objstore/providers/bos" "github.com/thanos-io/objstore/providers/cos" @@ -26,7 +25,7 @@ import ( // IsObjStoreSkipped returns true if given provider ID is found in THANOS_TEST_OBJSTORE_SKIP array delimited by comma e.g: // THANOS_TEST_OBJSTORE_SKIP=GCS,S3,AZURE,SWIFT,COS,ALIYUNOSS,BOS,OCI. -func IsObjStoreSkipped(t *testing.T, provider client.ObjProvider) bool { +func IsObjStoreSkipped(t *testing.T, provider objstore.ObjProvider) bool { if e, ok := os.LookupEnv("THANOS_TEST_OBJSTORE_SKIP"); ok { obstores := strings.Split(e, ",") for _, objstore := range obstores { @@ -69,7 +68,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket)) }) // Optional GCS. - if !IsObjStoreSkipped(t, client.GCS) { + if !IsObjStoreSkipped(t, objstore.GCS) { t.Run("gcs", func(t *testing.T) { bkt, closeFn, err := gcs.NewTestBucket(t, os.Getenv("GCP_PROJECT")) testutil.Ok(t, err) @@ -84,7 +83,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket)) } // Optional S3. - if !IsObjStoreSkipped(t, client.S3) { + if !IsObjStoreSkipped(t, objstore.S3) { t.Run("aws s3", func(t *testing.T) { // TODO(bwplotka): Allow taking location from envvar. bkt, closeFn, err := s3.NewTestBucket(t, "us-west-2") @@ -103,7 +102,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket)) } // Optional Azure. - if !IsObjStoreSkipped(t, client.AZURE) { + if !IsObjStoreSkipped(t, objstore.AZURE) { t.Run("azure", func(t *testing.T) { bkt, closeFn, err := azure.NewTestBucket(t, "e2e-tests") testutil.Ok(t, err) @@ -117,7 +116,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket)) } // Optional SWIFT. - if !IsObjStoreSkipped(t, client.SWIFT) { + if !IsObjStoreSkipped(t, objstore.SWIFT) { t.Run("swift", func(t *testing.T) { container, closeFn, err := swift.NewTestContainer(t) testutil.Ok(t, err) @@ -131,7 +130,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket)) } // Optional COS. - if !IsObjStoreSkipped(t, client.COS) { + if !IsObjStoreSkipped(t, objstore.COS) { t.Run("Tencent cos", func(t *testing.T) { bkt, closeFn, err := cos.NewTestBucket(t) testutil.Ok(t, err) @@ -145,7 +144,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket)) } // Optional OSS. - if !IsObjStoreSkipped(t, client.ALIYUNOSS) { + if !IsObjStoreSkipped(t, objstore.ALIYUNOSS) { t.Run("AliYun oss", func(t *testing.T) { bkt, closeFn, err := oss.NewTestBucket(t) testutil.Ok(t, err) @@ -159,7 +158,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket)) } // Optional BOS. - if !IsObjStoreSkipped(t, client.BOS) { + if !IsObjStoreSkipped(t, objstore.BOS) { t.Run("Baidu BOS", func(t *testing.T) { bkt, closeFn, err := bos.NewTestBucket(t) testutil.Ok(t, err) @@ -173,7 +172,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket)) } // Optional OCI. - if !IsObjStoreSkipped(t, client.OCI) { + if !IsObjStoreSkipped(t, objstore.OCI) { t.Run("oci", func(t *testing.T) { bkt, closeFn, err := oci.NewTestBucket(t) testutil.Ok(t, err) @@ -186,7 +185,7 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket)) } // Optional OBS. - if !IsObjStoreSkipped(t, client.OBS) { + if !IsObjStoreSkipped(t, objstore.OBS) { t.Run("obs", func(t *testing.T) { bkt, closeFn, err := obs.NewTestBucket(t, "cn-south-1") testutil.Ok(t, err) diff --git a/prefixed_bucket.go b/prefixed_bucket.go index a76b34c3..a37450ca 100644 --- a/prefixed_bucket.go +++ b/prefixed_bucket.go @@ -39,6 +39,8 @@ func withPrefix(prefix, name string) string { return prefix + DirDelim + name } +func (p *PrefixedBucket) Provider() ObjProvider { return p.bkt.Provider() } + func (p *PrefixedBucket) Close() error { return p.bkt.Close() } @@ -93,7 +95,7 @@ func (p *PrefixedBucket) IsAccessDeniedErr(err error) bool { } // Attributes returns information about the specified object. -func (p PrefixedBucket) Attributes(ctx context.Context, name string) (ObjectAttributes, error) { +func (p *PrefixedBucket) Attributes(ctx context.Context, name string) (ObjectAttributes, error) { return p.bkt.Attributes(ctx, conditionalPrefix(p.prefix, name)) } diff --git a/providers/azure/azure.go b/providers/azure/azure.go index 5689dc62..8d055e77 100644 --- a/providers/azure/azure.go +++ b/providers/azure/azure.go @@ -193,6 +193,8 @@ func NewBucketWithConfig(logger log.Logger, conf Config, component string, wrapR return bkt, nil } +func (b *Bucket) Provider() objstore.ObjProvider { return objstore.AZURE } + func (b *Bucket) SupportedIterOptions() []objstore.IterOptionType { return []objstore.IterOptionType{objstore.Recursive, objstore.UpdatedAt} } diff --git a/providers/bos/bos.go b/providers/bos/bos.go index 20c8dd3e..0cc4352c 100644 --- a/providers/bos/bos.go +++ b/providers/bos/bos.go @@ -100,6 +100,8 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B return bkt, nil } +func (b *Bucket) Provider() objstore.ObjProvider { return objstore.BOS } + // Name returns the bucket name for the provider. func (b *Bucket) Name() string { return b.name diff --git a/providers/cos/cos.go b/providers/cos/cos.go index d81150b1..1851c984 100644 --- a/providers/cos/cos.go +++ b/providers/cos/cos.go @@ -159,6 +159,8 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string, wra return bkt, nil } +func (b *Bucket) Provider() objstore.ObjProvider { return objstore.COS } + // Name returns the bucket name for COS. func (b *Bucket) Name() string { return b.name diff --git a/providers/filesystem/filesystem.go b/providers/filesystem/filesystem.go index 01dca4bb..df602877 100644 --- a/providers/filesystem/filesystem.go +++ b/providers/filesystem/filesystem.go @@ -50,6 +50,8 @@ func NewBucket(rootDir string) (*Bucket, error) { return &Bucket{rootDir: absDir}, nil } +func (b *Bucket) Provider() objstore.ObjProvider { return objstore.FILESYSTEM } + func (b *Bucket) SupportedIterOptions() []objstore.IterOptionType { return []objstore.IterOptionType{objstore.Recursive, objstore.UpdatedAt} } diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index d54a6782..b89f8735 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -186,6 +186,8 @@ func newBucket(ctx context.Context, logger log.Logger, gc Config, opts []option. return bkt, nil } +func (b *Bucket) Provider() objstore.ObjProvider { return objstore.GCS } + // Name returns the bucket name for gcs. func (b *Bucket) Name() string { return b.name diff --git a/providers/obs/obs.go b/providers/obs/obs.go index ecd235c9..20294cc6 100644 --- a/providers/obs/obs.go +++ b/providers/obs/obs.go @@ -122,6 +122,8 @@ func NewBucketWithConfig(logger log.Logger, config Config) (*Bucket, error) { return bkt, nil } +func (b *Bucket) Provider() objstore.ObjProvider { return objstore.OBS } + // Name returns the bucket name for the provider. func (b *Bucket) Name() string { return b.name diff --git a/providers/oci/oci.go b/providers/oci/oci.go index 062da7c1..7b4230ec 100644 --- a/providers/oci/oci.go +++ b/providers/oci/oci.go @@ -96,6 +96,8 @@ type Bucket struct { requestMetadata common.RequestMetadata } +func (b *Bucket) Provider() objstore.ObjProvider { return objstore.OCI } + // Name returns the bucket name for the provider. func (b *Bucket) Name() string { return b.name diff --git a/providers/oss/oss.go b/providers/oss/oss.go index aee8c623..2a6cb219 100644 --- a/providers/oss/oss.go +++ b/providers/oss/oss.go @@ -68,6 +68,8 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) { return NewTestBucketFromConfig(t, c, false) } +func (b *Bucket) Provider() objstore.ObjProvider { return objstore.ALIYUNOSS } + // Upload the contents of the reader as an object into the bucket. func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) error { // TODO(https://github.com/thanos-io/thanos/issues/678): Remove guessing length when minio provider will support multipart upload without this. diff --git a/providers/s3/s3.go b/providers/s3/s3.go index 27e82ffb..58590486 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -345,6 +345,8 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string, wra return bkt, nil } +func (b *Bucket) Provider() objstore.ObjProvider { return objstore.S3 } + // Name returns the bucket name for s3. func (b *Bucket) Name() string { return b.name diff --git a/providers/swift/swift.go b/providers/swift/swift.go index 86caa0c1..19eb0d45 100644 --- a/providers/swift/swift.go +++ b/providers/swift/swift.go @@ -218,6 +218,8 @@ func NewContainerFromConfig(logger log.Logger, sc *Config, createContainer bool, }, nil } +func (c *Container) Provider() objstore.ObjProvider { return objstore.SWIFT } + // Name returns the container name for swift. func (c *Container) Name() string { return c.name diff --git a/scripts/cfggen/main.go b/scripts/cfggen/main.go index 424bf9b0..e1bc6410 100644 --- a/scripts/cfggen/main.go +++ b/scripts/cfggen/main.go @@ -5,6 +5,7 @@ package main import ( "fmt" + "github.com/thanos-io/objstore" "io" "os" "path/filepath" @@ -35,17 +36,17 @@ var ( configs map[string]interface{} possibleValues []string - bucketConfigs = map[client.ObjProvider]interface{}{ - client.AZURE: azure.Config{}, - client.GCS: gcs.Config{}, - client.S3: s3.DefaultConfig, - client.SWIFT: swift.DefaultConfig, - client.COS: cos.DefaultConfig, - client.ALIYUNOSS: oss.Config{}, - client.FILESYSTEM: filesystem.Config{}, - client.BOS: bos.Config{}, - client.OCI: oci.Config{}, - client.OBS: obs.DefaultConfig, + bucketConfigs = map[objstore.ObjProvider]interface{}{ + objstore.AZURE: azure.Config{}, + objstore.GCS: gcs.Config{}, + objstore.S3: s3.DefaultConfig, + objstore.SWIFT: swift.DefaultConfig, + objstore.COS: cos.DefaultConfig, + objstore.ALIYUNOSS: oss.Config{}, + objstore.FILESYSTEM: filesystem.Config{}, + objstore.BOS: bos.Config{}, + objstore.OCI: oci.Config{}, + objstore.OBS: obs.DefaultConfig, } ) diff --git a/testing.go b/testing.go index d3fa1def..80f1e198 100644 --- a/testing.go +++ b/testing.go @@ -280,6 +280,8 @@ func WithDelay(bkt Bucket, delay time.Duration) Bucket { return &delayingBucket{bkt: bkt, delay: delay} } +func (d *delayingBucket) Provider() ObjProvider { return d.bkt.Provider() } + func (d *delayingBucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { time.Sleep(d.delay) return d.bkt.Get(ctx, name) diff --git a/tracing/opentelemetry/opentelemetry.go b/tracing/opentelemetry/opentelemetry.go index dad71e39..d5f9bd8c 100644 --- a/tracing/opentelemetry/opentelemetry.go +++ b/tracing/opentelemetry/opentelemetry.go @@ -23,6 +23,8 @@ func WrapWithTraces(bkt objstore.Bucket, tracer trace.Tracer) objstore.Instrumen return TracingBucket{tracer: tracer, bkt: bkt} } +func (t TracingBucket) Provider() objstore.ObjProvider { return t.bkt.Provider() } + func (t TracingBucket) Iter(ctx context.Context, dir string, f func(string) error, options ...objstore.IterOption) (err error) { ctx, span := t.tracer.Start(ctx, "bucket_iter") defer span.End() diff --git a/tracing/opentracing/opentracing.go b/tracing/opentracing/opentracing.go index cabe07b2..58bdea07 100644 --- a/tracing/opentracing/opentracing.go +++ b/tracing/opentracing/opentracing.go @@ -44,6 +44,8 @@ func WrapWithTraces(bkt objstore.Bucket) objstore.InstrumentedBucket { return TracingBucket{bkt: bkt} } +func (t TracingBucket) Provider() objstore.ObjProvider { return t.bkt.Provider() } + func (t TracingBucket) Iter(ctx context.Context, dir string, f func(string) error, options ...objstore.IterOption) (err error) { doWithSpan(ctx, "bucket_iter", func(spanCtx context.Context, span opentracing.Span) { span.LogKV("dir", dir) From 01c67f7534070611c573df64d467689369c50011 Mon Sep 17 00:00:00 2001 From: Filip Petkovski Date: Mon, 18 Nov 2024 14:02:05 +0100 Subject: [PATCH 41/41] Add CHANGELOG entry Signed-off-by: Filip Petkovski --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f0904faa..d0779d62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,7 +25,6 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#79](https://github.com/thanos-io/objstore/pull/79) Metrics: Fix `objstore_bucket_operation_duration_seconds` for `iter` operations. ### Added -- [#63](https://github.com/thanos-io/objstore/pull/63) Implement a `IterWithAttributes` method on the bucket client. - [#15](https://github.com/thanos-io/objstore/pull/15) Add Oracle Cloud Infrastructure Object Storage Bucket support. - [#25](https://github.com/thanos-io/objstore/pull/25) S3: Support specifying S3 storage class. - [#32](https://github.com/thanos-io/objstore/pull/32) Swift: Support authentication using application credentials. @@ -56,6 +55,9 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#130](https://github.com/thanos-io/objstore/pull/130) feat: Decouple creating bucket metrics from instrumenting the bucket - [#147](https://github.com/thanos-io/objstore/pull/147) feat: Add MaxRetries config to cos, gcs and obs. - [#150](https://github.com/thanos-io/objstore/pull/150) Add support for roundtripper wrapper. +- [#63](https://github.com/thanos-io/objstore/pull/63) Implement a `IterWithAttributes` method on the bucket client. +- [#155](https://github.com/thanos-io/objstore/pull/155) Add a `Provider` method on `objstore.Client`. + ### Changed - [#38](https://github.com/thanos-io/objstore/pull/38) *: Upgrade minio-go version to `v7.0.45`.