From 1ef291103a7909fe2cc3c15a049aa3e05fba05ba Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Sat, 23 Sep 2023 12:34:12 +0200 Subject: [PATCH 01/63] Move AwsCredentials into a separate header --- tensorstore/kvstore/s3/BUILD | 7 ++- .../kvstore/s3/aws_credential_provider.h | 19 +------ tensorstore/kvstore/s3/aws_credentials.h | 49 +++++++++++++++++++ 3 files changed, 55 insertions(+), 20 deletions(-) create mode 100644 tensorstore/kvstore/s3/aws_credentials.h diff --git a/tensorstore/kvstore/s3/BUILD b/tensorstore/kvstore/s3/BUILD index c87ddc6f9..7b1def517 100644 --- a/tensorstore/kvstore/s3/BUILD +++ b/tensorstore/kvstore/s3/BUILD @@ -230,7 +230,9 @@ tensorstore_cc_test( tensorstore_cc_library( name = "aws_credential_provider", srcs = ["aws_credential_provider.cc"], - hdrs = ["aws_credential_provider.h"], + hdrs = [ + "aws_credentials.h", + "aws_credential_provider.h"], deps = [ "//tensorstore/internal:env", "//tensorstore/internal:no_destructor", @@ -241,7 +243,8 @@ tensorstore_cc_library( "@com_google_absl//absl/status", "@com_google_absl//absl/strings", "@com_google_absl//absl/synchronization", - ], + "@com_google_absl//absl/time", + ], ) tensorstore_cc_test( diff --git a/tensorstore/kvstore/s3/aws_credential_provider.h b/tensorstore/kvstore/s3/aws_credential_provider.h index d8a088abe..0613bfd38 100644 --- a/tensorstore/kvstore/s3/aws_credential_provider.h +++ b/tensorstore/kvstore/s3/aws_credential_provider.h @@ -23,30 +23,13 @@ #include "absl/status/status.h" #include "absl/synchronization/mutex.h" +#include "tensorstore/kvstore/s3/aws_credentials.h" #include "tensorstore/internal/http/http_transport.h" #include "tensorstore/util/result.h" namespace tensorstore { namespace internal_kvstore_s3 { -/// Holds S3 credentials -/// -/// Contains the access key, secret key and session token. -/// An empty access key implies anonymous access, -/// while the presence of a session token implies the use of -/// short-lived STS credentials -/// https://docs.aws.amazon.com/STS/latest/APIReference/welcome.html -struct AwsCredentials { - /// AWS_ACCESS_KEY_ID - std::string access_key; - /// AWS_SECRET_KEY_ID - std::string secret_key; - /// AWS_SESSION_TOKEN - std::string session_token; - - bool IsAnonymous() const { return access_key.empty(); } -}; - /// Base class for S3 Credential Providers /// /// Implementers should override GetCredentials diff --git a/tensorstore/kvstore/s3/aws_credentials.h b/tensorstore/kvstore/s3/aws_credentials.h new file mode 100644 index 000000000..77484827a --- /dev/null +++ b/tensorstore/kvstore/s3/aws_credentials.h @@ -0,0 +1,49 @@ +// Copyright 2023 The TensorStore Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef TENSORSTORE_KVSTORE_S3_S3_CREDENTIALS_H +#define TENSORSTORE_KVSTORE_S3_S3_CREDENTIALS_H + +#include + +#include "absl/time/time.h" + +namespace tensorstore { +namespace internal_kvstore_s3 { + +/// Holds AWS credentials +/// +/// Contains the access key, secret key and session token. +/// An empty access key implies anonymous access, +/// while the presence of a session token implies the use of +/// short-lived STS credentials +/// https://docs.aws.amazon.com/STS/latest/APIReference/welcome.html +struct AwsCredentials { + /// AWS_ACCESS_KEY_ID + std::string access_key; + /// AWS_SECRET_KEY_ID + std::string secret_key; + /// AWS_SESSION_TOKEN + std::string session_token; + /// Expiry Date + absl::Time expiry_date; + + bool IsAnonymous() const { return access_key.empty(); } + bool isExpired() const { return expiry_date < absl::Now(); } +}; + +} // internal_kvstore_s3 +} // tensorstore + +#endif // TENSORSTORE_KVSTORE_S3_S3_CREDENTIALS_H \ No newline at end of file From 3ac4c5e856f8746d19802db451ac977ce6f23d0c Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Sat, 23 Sep 2023 13:02:10 +0200 Subject: [PATCH 02/63] Split environment credential provider out into separate file --- tensorstore/kvstore/s3/BUILD | 7 ++- .../kvstore/s3/aws_credential_provider.cc | 22 ++----- .../kvstore/s3/aws_credential_provider.h | 12 ---- tensorstore/kvstore/s3/aws_credentials.h | 1 + .../s3/aws_environment_credential_provider.cc | 60 +++++++++++++++++++ .../s3/aws_environment_credential_provider.h | 38 ++++++++++++ 6 files changed, 108 insertions(+), 32 deletions(-) create mode 100644 tensorstore/kvstore/s3/aws_environment_credential_provider.cc create mode 100644 tensorstore/kvstore/s3/aws_environment_credential_provider.h diff --git a/tensorstore/kvstore/s3/BUILD b/tensorstore/kvstore/s3/BUILD index 7b1def517..c70ffc29d 100644 --- a/tensorstore/kvstore/s3/BUILD +++ b/tensorstore/kvstore/s3/BUILD @@ -229,10 +229,13 @@ tensorstore_cc_test( tensorstore_cc_library( name = "aws_credential_provider", - srcs = ["aws_credential_provider.cc"], + srcs = [ + "aws_credential_provider.cc", + "aws_environment_credential_provider.cc"], hdrs = [ "aws_credentials.h", - "aws_credential_provider.h"], + "aws_credential_provider.h", + "aws_environment_credential_provider.h"], deps = [ "//tensorstore/internal:env", "//tensorstore/internal:no_destructor", diff --git a/tensorstore/kvstore/s3/aws_credential_provider.cc b/tensorstore/kvstore/s3/aws_credential_provider.cc index 9caa145fc..1ec24097e 100644 --- a/tensorstore/kvstore/s3/aws_credential_provider.cc +++ b/tensorstore/kvstore/s3/aws_credential_provider.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "tensorstore/kvstore/s3/aws_credential_provider.h" +#include "tensorstore/kvstore/s3/aws_environment_credential_provider.h" #include #include @@ -99,25 +100,10 @@ Result> GetDefaultAwsCredentialProvider( std::string_view profile, std::shared_ptr transport) { // 1. Obtain credentials from environment variables - if (auto access_key = GetEnv(kEnvAwsAccessKeyId); access_key) { - ABSL_LOG_FIRST_N(INFO, 1) - << "Using Environment Variable " << kEnvAwsAccessKeyId; - AwsCredentials credentials; - credentials.access_key = *access_key; - auto secret_key = GetEnv(kEnvAwsSecretAccessKey); - - if (secret_key.has_value()) { - credentials.secret_key = *secret_key; - } - - auto session_token = GetEnv(kEnvAwsSessionToken); - - if (session_token.has_value()) { - credentials.session_token = *session_token; - } - return std::make_unique( - std::move(credentials)); + auto env_creds = std::make_unique(); + if(env_creds->GetCredentials().ok()) { + return env_creds; } // 2. Obtain credentials from AWS_SHARED_CREDENTIALS_FILE or diff --git a/tensorstore/kvstore/s3/aws_credential_provider.h b/tensorstore/kvstore/s3/aws_credential_provider.h index 0613bfd38..07405d61e 100644 --- a/tensorstore/kvstore/s3/aws_credential_provider.h +++ b/tensorstore/kvstore/s3/aws_credential_provider.h @@ -39,18 +39,6 @@ class AwsCredentialProvider { virtual Result GetCredentials() = 0; }; -/// Provides credentials from the following environment variables: -/// AWS_ACCESS_KEY_ID, AWS_SECRET_KEY_ID, AWS_SESSION_TOKEN -class EnvironmentCredentialProvider : public AwsCredentialProvider { - private: - AwsCredentials credentials_; - - public: - EnvironmentCredentialProvider(const AwsCredentials& credentials) - : credentials_(credentials) {} - - Result GetCredentials() override { return credentials_; } -}; /// Obtains S3 credentials from a profile in a file, usually /// `~/.aws/credentials` or a file specified in AWS_SHARED_CREDENTIALS_FILE. A diff --git a/tensorstore/kvstore/s3/aws_credentials.h b/tensorstore/kvstore/s3/aws_credentials.h index 77484827a..72354416b 100644 --- a/tensorstore/kvstore/s3/aws_credentials.h +++ b/tensorstore/kvstore/s3/aws_credentials.h @@ -18,6 +18,7 @@ #include #include "absl/time/time.h" +#include "absl/time/clock.h" namespace tensorstore { namespace internal_kvstore_s3 { diff --git a/tensorstore/kvstore/s3/aws_environment_credential_provider.cc b/tensorstore/kvstore/s3/aws_environment_credential_provider.cc new file mode 100644 index 000000000..843dbcb86 --- /dev/null +++ b/tensorstore/kvstore/s3/aws_environment_credential_provider.cc @@ -0,0 +1,60 @@ +// Copyright 2023 The TensorStore Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "tensorstore/kvstore/s3/aws_environment_credential_provider.h" +#include "tensorstore/internal/env.h" + +using ::tensorstore::internal::GetEnv; + +namespace tensorstore { +namespace internal_kvstore_s3 { + +namespace { + +// AWS user identifier +constexpr char kEnvAwsAccessKeyId[] = "AWS_ACCESS_KEY_ID"; +// AWS user password +constexpr char kEnvAwsSecretAccessKey[] = "AWS_SECRET_ACCESS_KEY"; +// AWS session token +constexpr char kEnvAwsSessionToken[] = "AWS_SESSION_TOKEN"; + +} // namespace + +Result EnvironmentCredentialProvider::GetCredentials() { +// 1. Obtain credentials from environment variables + if (auto access_key = GetEnv(kEnvAwsAccessKeyId); access_key) { + ABSL_LOG_FIRST_N(INFO, 1) + << "Using Environment Variable " << kEnvAwsAccessKeyId; + AwsCredentials credentials; + credentials.access_key = *access_key; + auto secret_key = GetEnv(kEnvAwsSecretAccessKey); + + if (secret_key.has_value()) { + credentials.secret_key = *secret_key; + } + + auto session_token = GetEnv(kEnvAwsSessionToken); + + if (session_token.has_value()) { + credentials.session_token = *session_token; + } + + return credentials; + } + + return absl::NotFoundError("AWS Credentials not found in environment variables"); +} + +} // namespace internal_kvstore_s3 +} // namespace tensorstore diff --git a/tensorstore/kvstore/s3/aws_environment_credential_provider.h b/tensorstore/kvstore/s3/aws_environment_credential_provider.h new file mode 100644 index 000000000..a9bb44e72 --- /dev/null +++ b/tensorstore/kvstore/s3/aws_environment_credential_provider.h @@ -0,0 +1,38 @@ +// Copyright 2023 The TensorStore Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef TENSORSTORE_KVSTORE_S3_AWS_ENVIRONMENT_CREDENTIAL_PROVIDER_H +#define TENSORSTORE_KVSTORE_S3_AWS_ENVIRONMENT_CREDENTIAL_PROVIDER_H + +#include "tensorstore/kvstore/s3/aws_credentials.h" +#include "tensorstore/kvstore/s3/aws_credential_provider.h" +#include "tensorstore/util/result.h" + +namespace tensorstore { +namespace internal_kvstore_s3 { + +/// Provides credentials from the following environment variables: +/// AWS_ACCESS_KEY_ID, AWS_SECRET_KEY_ID, AWS_SESSION_TOKEN +class EnvironmentCredentialProvider : public AwsCredentialProvider { + private: + AwsCredentials credentials_; + + public: + Result GetCredentials() override; +}; + +} // namespace internal_kvstore_s3 +} // namespace tensorstore + +#endif // TENSORSTORE_KVSTORE_S3_AWS_ENVIRONMENT_CREDENTIAL_PROVIDER_H \ No newline at end of file From 9edd078a591690b3abe73d43d78b6750ddb1bfd2 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Sat, 23 Sep 2023 13:48:39 +0200 Subject: [PATCH 03/63] Split file credential provider out into separate files --- tensorstore/kvstore/s3/BUILD | 6 +- .../kvstore/s3/aws_credential_provider.cc | 118 +-------------- .../kvstore/s3/aws_credential_provider.h | 20 --- .../s3/aws_credential_provider_test.cc | 41 ++++-- .../s3/aws_file_credential_provider.cc | 137 ++++++++++++++++++ .../kvstore/s3/aws_file_credential_provider.h | 45 ++++++ 6 files changed, 217 insertions(+), 150 deletions(-) create mode 100644 tensorstore/kvstore/s3/aws_file_credential_provider.cc create mode 100644 tensorstore/kvstore/s3/aws_file_credential_provider.h diff --git a/tensorstore/kvstore/s3/BUILD b/tensorstore/kvstore/s3/BUILD index c70ffc29d..19d24c9cf 100644 --- a/tensorstore/kvstore/s3/BUILD +++ b/tensorstore/kvstore/s3/BUILD @@ -231,11 +231,13 @@ tensorstore_cc_library( name = "aws_credential_provider", srcs = [ "aws_credential_provider.cc", - "aws_environment_credential_provider.cc"], + "aws_environment_credential_provider.cc", + "aws_file_credential_provider.cc"], hdrs = [ "aws_credentials.h", "aws_credential_provider.h", - "aws_environment_credential_provider.h"], + "aws_environment_credential_provider.h", + "aws_file_credential_provider.h"], deps = [ "//tensorstore/internal:env", "//tensorstore/internal:no_destructor", diff --git a/tensorstore/kvstore/s3/aws_credential_provider.cc b/tensorstore/kvstore/s3/aws_credential_provider.cc index 1ec24097e..2cd562dde 100644 --- a/tensorstore/kvstore/s3/aws_credential_provider.cc +++ b/tensorstore/kvstore/s3/aws_credential_provider.cc @@ -14,6 +14,7 @@ #include "tensorstore/kvstore/s3/aws_credential_provider.h" #include "tensorstore/kvstore/s3/aws_environment_credential_provider.h" +#include "tensorstore/kvstore/s3/aws_file_credential_provider.h" #include #include @@ -46,55 +47,6 @@ namespace { // For reference, see the latest AWS environment variables used by the cli: // https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html -// AWS user identifier -constexpr char kEnvAwsAccessKeyId[] = "AWS_ACCESS_KEY_ID"; -constexpr char kCfgAwsAccessKeyId[] = "aws_access_key_id"; - -// AWS user password -constexpr char kEnvAwsSecretAccessKey[] = "AWS_SECRET_ACCESS_KEY"; -constexpr char kCfgAwsSecretAccessKeyId[] = "aws_secret_access_key"; - -// AWS session token -constexpr char kEnvAwsSessionToken[] = "AWS_SESSION_TOKEN"; -constexpr char kCfgAwsSessionToken[] = "aws_session_token"; - -// AWS Profile environment variables -constexpr char kEnvAwsProfile[] = "AWS_PROFILE"; -constexpr char kDefaultProfile[] = "default"; - -// Credentials file environment variable -constexpr char kEnvAwsCredentialsFile[] = "AWS_SHARED_CREDENTIALS_FILE"; - -// Default path to the AWS credentials file, relative to the home folder -constexpr char kDefaultAwsCredentialsFilePath[] = ".aws/credentials"; - -/// Returns whether the given path points to a readable file. -bool IsFile(const std::string& filename) { - std::ifstream fstream(filename.c_str()); - return fstream.good(); -} - -Result GetAwsCredentialsFileName() { - std::string result; - - auto credentials_file = GetEnv(kEnvAwsCredentialsFile); - if (!credentials_file) { - auto home_dir = GetEnv("HOME"); - if (!home_dir) { - return absl::NotFoundError("Could not read $HOME"); - } - result = JoinPath(*home_dir, kDefaultAwsCredentialsFilePath); - } else { - result = *credentials_file; - } - if (!IsFile(result)) { - return absl::NotFoundError( - absl::StrCat("Could not find the credentials file at " - "location [", - result, "]")); - } - return result; -} Result> GetDefaultAwsCredentialProvider( std::string_view profile, @@ -106,19 +58,11 @@ Result> GetDefaultAwsCredentialProvider( return env_creds; } - // 2. Obtain credentials from AWS_SHARED_CREDENTIALS_FILE or - // ~/.aws/credentials - if (auto credentials_file = GetAwsCredentialsFileName(); - credentials_file.ok()) { - std::string env_profile; // value must not outlive view - if (profile.empty()) { - env_profile = GetEnv(kEnvAwsProfile).value_or(kDefaultProfile); - profile = std::string_view(env_profile); - } + auto file_creds = std::make_unique(std::string(profile)); + if(file_creds->GetCredentials().ok()) { ABSL_LOG(INFO) << "Using File AwsCredentialProvider with profile " << profile; - return std::make_unique( - std::move(credentials_file).value(), std::string(profile)); + return file_creds; } // 3. Obtain credentials from EC2 Metadata server @@ -144,60 +88,6 @@ AwsCredentialProviderRegistry& GetAwsProviderRegistry() { } // namespace -/// https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html#cli-configure-files-format -Result FileCredentialProvider::GetCredentials() { - absl::ReaderMutexLock lock(&mutex_); - std::ifstream ifs(filename_); - - if (!ifs) { - return absl::NotFoundError( - absl::StrCat("Could not open the credentials file [", filename_, "]")); - } - - AwsCredentials credentials; - std::string section_name; - std::string line; - bool profile_found = false; - - while (std::getline(ifs, line)) { - auto sline = absl::StripAsciiWhitespace(line); - // Ignore empty and commented out lines - if (sline.empty() || sline[0] == '#') continue; - - // A configuration section name has been encountered - if (sline[0] == '[' && sline[sline.size() - 1] == ']') { - section_name = - absl::StripAsciiWhitespace(sline.substr(1, sline.size() - 2)); - continue; - } - - // Look for key=value pairs if we're in the appropriate profile - if (section_name == profile_) { - profile_found = true; - if (auto pos = sline.find('='); pos != std::string::npos) { - auto key = absl::StripAsciiWhitespace(sline.substr(0, pos)); - auto value = absl::StripAsciiWhitespace(sline.substr(pos + 1)); - - if (key == kCfgAwsAccessKeyId) { - credentials.access_key = value; - } else if (key == kCfgAwsSecretAccessKeyId) { - credentials.secret_key = value; - } else if (key == kCfgAwsSessionToken) { - credentials.session_token = value; - } - } - } - } - - if (!profile_found) { - return absl::NotFoundError(absl::StrCat("Profile [", profile_, - "] not found " - "in credentials file [", - filename_, "]")); - } - - return credentials; -} void RegisterAwsCredentialProviderProvider(AwsCredentialProviderFn provider, int priority) { diff --git a/tensorstore/kvstore/s3/aws_credential_provider.h b/tensorstore/kvstore/s3/aws_credential_provider.h index 07405d61e..9d5aba331 100644 --- a/tensorstore/kvstore/s3/aws_credential_provider.h +++ b/tensorstore/kvstore/s3/aws_credential_provider.h @@ -40,26 +40,6 @@ class AwsCredentialProvider { }; -/// Obtains S3 credentials from a profile in a file, usually -/// `~/.aws/credentials` or a file specified in AWS_SHARED_CREDENTIALS_FILE. A -/// desired profile may be specified in the constructor: This value should be -/// derived from the s3 json spec. -/// However, if profile is passed as an empty string, the profile is obtained -/// from AWS_DEFAULT_PROFILE, AWS_PROFILE before finally defaulting to -/// "default". -class FileCredentialProvider : public AwsCredentialProvider { - private: - absl::Mutex mutex_; - std::string filename_; - std::string profile_; - - public: - FileCredentialProvider(std::string filename, std::string profile) - : filename_(std::move(filename)), profile_(std::move(profile)) {} - - Result GetCredentials() override; -}; - /// Provides S3 credentials from the EC2 Metadata server /// if running within AWS class EC2MetadataCredentialProvider : public AwsCredentialProvider { diff --git a/tensorstore/kvstore/s3/aws_credential_provider_test.cc b/tensorstore/kvstore/s3/aws_credential_provider_test.cc index 04553aefb..a342d1dfd 100644 --- a/tensorstore/kvstore/s3/aws_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/aws_credential_provider_test.cc @@ -28,11 +28,14 @@ namespace { +using ::tensorstore::Future; using ::tensorstore::internal::GetEnv; using ::tensorstore::internal::JoinPath; using ::tensorstore::internal::SetEnv; using ::tensorstore::internal::UnsetEnv; -using ::tensorstore::internal_http::GetDefaultHttpTransport; +using ::tensorstore::internal_http::HttpRequest; +using ::tensorstore::internal_http::HttpResponse; +using ::tensorstore::internal_http::HttpTransport; using ::tensorstore::internal_kvstore_s3::GetAwsCredentialProvider; class TestData : public tensorstore::internal::ScopedTemporaryDirectory { @@ -57,8 +60,22 @@ class TestData : public tensorstore::internal::ScopedTemporaryDirectory { } }; +/// Cause EC2Metadata to always fail +class NotFoundTransport : public HttpTransport { +public: + Future IssueRequest(const HttpRequest& request, + absl::Cord payload, + absl::Duration request_timeout, + absl::Duration connect_timeout) override { + return HttpResponse{404, absl::Cord(), {}}; + } +}; + + class AwsCredentialProviderTest : public ::testing::Test { protected: + std::shared_ptr transport_; + void SetUp() override { // Make sure that env vars are not set. for (const char* var : @@ -66,14 +83,16 @@ class AwsCredentialProviderTest : public ::testing::Test { "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "AWS_PROFILE"}) { UnsetEnv(var); } + + transport_ = std::make_shared(); } }; TEST_F(AwsCredentialProviderTest, ProviderNoCredentials) { - ASSERT_FALSE(GetAwsCredentialProvider("", GetDefaultHttpTransport()).ok()); + ASSERT_FALSE(GetAwsCredentialProvider("", transport_).ok()); SetEnv("AWS_ACCESS_KEY_ID", "foo"); TENSORSTORE_ASSERT_OK_AND_ASSIGN( - auto provider, GetAwsCredentialProvider("", GetDefaultHttpTransport())); + auto provider, GetAwsCredentialProvider("", transport_)); TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, provider->GetCredentials()); ASSERT_EQ(credentials.access_key, "foo"); @@ -86,7 +105,7 @@ TEST_F(AwsCredentialProviderTest, ProviderAwsCredentialsFromEnv) { SetEnv("AWS_SECRET_ACCESS_KEY", "bar"); SetEnv("AWS_SESSION_TOKEN", "qux"); TENSORSTORE_ASSERT_OK_AND_ASSIGN( - auto provider, GetAwsCredentialProvider("", GetDefaultHttpTransport())); + auto provider, GetAwsCredentialProvider("", transport_)); TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, provider->GetCredentials()); ASSERT_EQ(credentials.access_key, "foo"); @@ -100,7 +119,7 @@ TEST_F(AwsCredentialProviderTest, ProviderAwsCredentialsFromFileDefault) { SetEnv("AWS_SHARED_CREDENTIALS_FILE", credentials_filename.c_str()); TENSORSTORE_ASSERT_OK_AND_ASSIGN( - auto provider, GetAwsCredentialProvider("", GetDefaultHttpTransport())); + auto provider, GetAwsCredentialProvider("", transport_)); TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, provider->GetCredentials()); ASSERT_EQ(credentials.access_key, "AKIAIOSFODNN7EXAMPLE"); @@ -116,7 +135,7 @@ TEST_F(AwsCredentialProviderTest, SetEnv("AWS_SHARED_CREDENTIALS_FILE", credentials_filename.c_str()); TENSORSTORE_ASSERT_OK_AND_ASSIGN( auto provider, - GetAwsCredentialProvider("alice", GetDefaultHttpTransport())); + GetAwsCredentialProvider("alice", transport_)); TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, provider->GetCredentials()); ASSERT_EQ(credentials.access_key, "AKIAIOSFODNN6EXAMPLE"); @@ -131,7 +150,7 @@ TEST_F(AwsCredentialProviderTest, ProviderAwsCredentialsFromFileProfileEnv) { SetEnv("AWS_SHARED_CREDENTIALS_FILE", credentials_filename.c_str()); SetEnv("AWS_PROFILE", "alice"); TENSORSTORE_ASSERT_OK_AND_ASSIGN( - auto provider, GetAwsCredentialProvider("", GetDefaultHttpTransport())); + auto provider, GetAwsCredentialProvider("", transport_)); TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, provider->GetCredentials()); ASSERT_EQ(credentials.access_key, "AKIAIOSFODNN6EXAMPLE"); @@ -146,13 +165,7 @@ TEST_F(AwsCredentialProviderTest, SetEnv("AWS_SHARED_CREDENTIALS_FILE", credentials_filename.c_str()); SetEnv("AWS_PROFILE", "bob"); - TENSORSTORE_ASSERT_OK_AND_ASSIGN( - auto provider, GetAwsCredentialProvider("", GetDefaultHttpTransport())); - auto result = provider->GetCredentials(); - ASSERT_FALSE(result.ok()); - EXPECT_THAT( - result.status().message(), - ::testing::HasSubstr("Profile [bob] not found in credentials file")); + ASSERT_FALSE(GetAwsCredentialProvider("", transport_).ok()); } } // namespace diff --git a/tensorstore/kvstore/s3/aws_file_credential_provider.cc b/tensorstore/kvstore/s3/aws_file_credential_provider.cc new file mode 100644 index 000000000..abd68921e --- /dev/null +++ b/tensorstore/kvstore/s3/aws_file_credential_provider.cc @@ -0,0 +1,137 @@ +// Copyright 2023 The TensorStore Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "absl/strings/strip.h" +#include "tensorstore/kvstore/s3/aws_file_credential_provider.h" +#include "tensorstore/internal/env.h" +#include "tensorstore/internal/path.h" + +using ::tensorstore::internal::GetEnv; +using ::tensorstore::internal::JoinPath; + + +namespace tensorstore { +namespace internal_kvstore_s3 { + +namespace { + +// Credentials file environment variable +static constexpr char kEnvAwsCredentialsFile[] = "AWS_SHARED_CREDENTIALS_FILE"; +// Default path to the AWS credentials file, relative to the home folder +static constexpr char kDefaultAwsCredentialsFilePath[] = ".aws/credentials"; +// AWS user identifier +static constexpr char kCfgAwsAccessKeyId[] = "aws_access_key_id"; +// AWS user password +static constexpr char kCfgAwsSecretAccessKeyId[] = "aws_secret_access_key"; +// AWS session token +static constexpr char kCfgAwsSessionToken[] = "aws_session_token"; +// Discover AWS profile in environment variables +static constexpr char kEnvAwsProfile[] = "AWS_PROFILE"; +// Default profile +static constexpr char kDefaultProfile[] = "default"; + +/// Returns whether the given path points to a readable file. +bool IsFile(const std::string& filename) { + std::ifstream fstream(filename.c_str()); + return fstream.good(); +} + +Result GetAwsCredentialsFileName() { + std::string result; + + auto credentials_file = GetEnv(kEnvAwsCredentialsFile); + if (!credentials_file) { + auto home_dir = GetEnv("HOME"); + if (!home_dir) { + return absl::NotFoundError("Could not read $HOME"); + } + result = JoinPath(*home_dir, kDefaultAwsCredentialsFilePath); + } else { + result = *credentials_file; + } + if (!IsFile(result)) { + return absl::NotFoundError( + absl::StrCat("Could not find the credentials file at " + "location [", + result, "]")); + } + return result; +} + +} // namespace + +/// https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html#cli-configure-files-format +Result FileCredentialProvider::GetCredentials() { + absl::ReaderMutexLock lock(&mutex_); + TENSORSTORE_ASSIGN_OR_RETURN(auto filename, GetAwsCredentialsFileName()); + std::ifstream ifs(filename); + std::string profile = profile_.empty() ? + GetEnv(kEnvAwsProfile).value_or(kDefaultProfile) : + profile_; + + + if (!ifs) { + return absl::NotFoundError( + absl::StrCat("Could not open the credentials file [", filename, "]")); + } + + AwsCredentials credentials; + std::string section_name; + std::string line; + bool profile_found = false; + + while (std::getline(ifs, line)) { + auto sline = absl::StripAsciiWhitespace(line); + // Ignore empty and commented out lines + if (sline.empty() || sline[0] == '#') continue; + + // A configuration section name has been encountered + if (sline[0] == '[' && sline[sline.size() - 1] == ']') { + section_name = + absl::StripAsciiWhitespace(sline.substr(1, sline.size() - 2)); + continue; + } + + // Look for key=value pairs if we're in the appropriate profile + if (section_name == profile) { + profile_found = true; + if (auto pos = sline.find('='); pos != std::string::npos) { + auto key = absl::StripAsciiWhitespace(sline.substr(0, pos)); + auto value = absl::StripAsciiWhitespace(sline.substr(pos + 1)); + + if (key == kCfgAwsAccessKeyId) { + credentials.access_key = value; + } else if (key == kCfgAwsSecretAccessKeyId) { + credentials.secret_key = value; + } else if (key == kCfgAwsSessionToken) { + credentials.session_token = value; + } + } + } + } + + if (!profile_found) { + return absl::NotFoundError(absl::StrCat("Profile [", profile, + "] not found " + "in credentials file [", + filename, "]")); + } + + return credentials; +} + +} // namespace internal_kvstore_s3 +} // namespace tensorstore diff --git a/tensorstore/kvstore/s3/aws_file_credential_provider.h b/tensorstore/kvstore/s3/aws_file_credential_provider.h new file mode 100644 index 000000000..e38d2f29b --- /dev/null +++ b/tensorstore/kvstore/s3/aws_file_credential_provider.h @@ -0,0 +1,45 @@ +// Copyright 2023 The TensorStore Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef TENSORSTORE_KVSTORE_S3_AWS_FILE_CREDENTIAL_PROVIDER_H +#define TENSORSTORE_KVSTORE_S3_AWS_FILE_CREDENTIAL_PROVIDER_H + +#include "tensorstore/kvstore/s3/aws_credentials.h" +#include "tensorstore/kvstore/s3/aws_credential_provider.h" +#include "tensorstore/util/result.h" + +namespace tensorstore { +namespace internal_kvstore_s3 { + +/// Obtains S3 credentials from a profile in a file, usually +/// `~/.aws/credentials` or a file specified in AWS_SHARED_CREDENTIALS_FILE. A +/// desired profile may be specified in the constructor: This value should be +/// derived from the s3 json spec. +/// However, if profile is passed as an empty string, the profile is obtained +/// from AWS_DEFAULT_PROFILE, AWS_PROFILE before finally defaulting to +/// "default". +class FileCredentialProvider : public AwsCredentialProvider { + private: + absl::Mutex mutex_; + std::string profile_; + + public: + FileCredentialProvider(std::string profile) : profile_(std::move(profile)) {} + Result GetCredentials() override; +}; + +} // namespace internal_kvstore_s3 +} // namespace tensorstore + +#endif // TENSORSTORE_KVSTORE_S3_AWS_FILE_CREDENTIAL_PROVIDER_H \ No newline at end of file From d5aeed6f00a00705f18905d7b0204ead3188762e Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Sat, 23 Sep 2023 13:51:31 +0200 Subject: [PATCH 04/63] Shorter filenames --- tensorstore/kvstore/s3/BUILD | 8 ++++---- tensorstore/kvstore/s3/aws_credential_provider.cc | 4 ++-- ...ial_provider.cc => environment_credential_provider.cc} | 2 +- ...ntial_provider.h => environment_credential_provider.h} | 6 +++--- ...credential_provider.cc => file_credential_provider.cc} | 2 +- ...e_credential_provider.h => file_credential_provider.h} | 6 +++--- 6 files changed, 14 insertions(+), 14 deletions(-) rename tensorstore/kvstore/s3/{aws_environment_credential_provider.cc => environment_credential_provider.cc} (96%) rename tensorstore/kvstore/s3/{aws_environment_credential_provider.h => environment_credential_provider.h} (84%) rename tensorstore/kvstore/s3/{aws_file_credential_provider.cc => file_credential_provider.cc} (98%) rename tensorstore/kvstore/s3/{aws_file_credential_provider.h => file_credential_provider.h} (89%) diff --git a/tensorstore/kvstore/s3/BUILD b/tensorstore/kvstore/s3/BUILD index 19d24c9cf..c6c8f18dd 100644 --- a/tensorstore/kvstore/s3/BUILD +++ b/tensorstore/kvstore/s3/BUILD @@ -231,13 +231,13 @@ tensorstore_cc_library( name = "aws_credential_provider", srcs = [ "aws_credential_provider.cc", - "aws_environment_credential_provider.cc", - "aws_file_credential_provider.cc"], + "environment_credential_provider.cc", + "file_credential_provider.cc"], hdrs = [ "aws_credentials.h", "aws_credential_provider.h", - "aws_environment_credential_provider.h", - "aws_file_credential_provider.h"], + "environment_credential_provider.h", + "file_credential_provider.h"], deps = [ "//tensorstore/internal:env", "//tensorstore/internal:no_destructor", diff --git a/tensorstore/kvstore/s3/aws_credential_provider.cc b/tensorstore/kvstore/s3/aws_credential_provider.cc index 2cd562dde..af9842210 100644 --- a/tensorstore/kvstore/s3/aws_credential_provider.cc +++ b/tensorstore/kvstore/s3/aws_credential_provider.cc @@ -13,8 +13,8 @@ // limitations under the License. #include "tensorstore/kvstore/s3/aws_credential_provider.h" -#include "tensorstore/kvstore/s3/aws_environment_credential_provider.h" -#include "tensorstore/kvstore/s3/aws_file_credential_provider.h" +#include "tensorstore/kvstore/s3/environment_credential_provider.h" +#include "tensorstore/kvstore/s3/file_credential_provider.h" #include #include diff --git a/tensorstore/kvstore/s3/aws_environment_credential_provider.cc b/tensorstore/kvstore/s3/environment_credential_provider.cc similarity index 96% rename from tensorstore/kvstore/s3/aws_environment_credential_provider.cc rename to tensorstore/kvstore/s3/environment_credential_provider.cc index 843dbcb86..2f7593fb6 100644 --- a/tensorstore/kvstore/s3/aws_environment_credential_provider.cc +++ b/tensorstore/kvstore/s3/environment_credential_provider.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "tensorstore/kvstore/s3/aws_environment_credential_provider.h" +#include "tensorstore/kvstore/s3/environment_credential_provider.h" #include "tensorstore/internal/env.h" using ::tensorstore::internal::GetEnv; diff --git a/tensorstore/kvstore/s3/aws_environment_credential_provider.h b/tensorstore/kvstore/s3/environment_credential_provider.h similarity index 84% rename from tensorstore/kvstore/s3/aws_environment_credential_provider.h rename to tensorstore/kvstore/s3/environment_credential_provider.h index a9bb44e72..211391a42 100644 --- a/tensorstore/kvstore/s3/aws_environment_credential_provider.h +++ b/tensorstore/kvstore/s3/environment_credential_provider.h @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef TENSORSTORE_KVSTORE_S3_AWS_ENVIRONMENT_CREDENTIAL_PROVIDER_H -#define TENSORSTORE_KVSTORE_S3_AWS_ENVIRONMENT_CREDENTIAL_PROVIDER_H +#ifndef TENSORSTORE_KVSTORE_S3_ENVIRONMENT_CREDENTIAL_PROVIDER_H +#define TENSORSTORE_KVSTORE_S3_ENVIRONMENT_CREDENTIAL_PROVIDER_H #include "tensorstore/kvstore/s3/aws_credentials.h" #include "tensorstore/kvstore/s3/aws_credential_provider.h" @@ -35,4 +35,4 @@ class EnvironmentCredentialProvider : public AwsCredentialProvider { } // namespace internal_kvstore_s3 } // namespace tensorstore -#endif // TENSORSTORE_KVSTORE_S3_AWS_ENVIRONMENT_CREDENTIAL_PROVIDER_H \ No newline at end of file +#endif // TENSORSTORE_KVSTORE_S3_ENVIRONMENT_CREDENTIAL_PROVIDER_H \ No newline at end of file diff --git a/tensorstore/kvstore/s3/aws_file_credential_provider.cc b/tensorstore/kvstore/s3/file_credential_provider.cc similarity index 98% rename from tensorstore/kvstore/s3/aws_file_credential_provider.cc rename to tensorstore/kvstore/s3/file_credential_provider.cc index abd68921e..becd67556 100644 --- a/tensorstore/kvstore/s3/aws_file_credential_provider.cc +++ b/tensorstore/kvstore/s3/file_credential_provider.cc @@ -15,7 +15,7 @@ #include #include "absl/strings/strip.h" -#include "tensorstore/kvstore/s3/aws_file_credential_provider.h" +#include "tensorstore/kvstore/s3/file_credential_provider.h" #include "tensorstore/internal/env.h" #include "tensorstore/internal/path.h" diff --git a/tensorstore/kvstore/s3/aws_file_credential_provider.h b/tensorstore/kvstore/s3/file_credential_provider.h similarity index 89% rename from tensorstore/kvstore/s3/aws_file_credential_provider.h rename to tensorstore/kvstore/s3/file_credential_provider.h index e38d2f29b..e6b3780a9 100644 --- a/tensorstore/kvstore/s3/aws_file_credential_provider.h +++ b/tensorstore/kvstore/s3/file_credential_provider.h @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef TENSORSTORE_KVSTORE_S3_AWS_FILE_CREDENTIAL_PROVIDER_H -#define TENSORSTORE_KVSTORE_S3_AWS_FILE_CREDENTIAL_PROVIDER_H +#ifndef TENSORSTORE_KVSTORE_S3_FILE_CREDENTIAL_PROVIDER_H +#define TENSORSTORE_KVSTORE_S3_FILE_CREDENTIAL_PROVIDER_H #include "tensorstore/kvstore/s3/aws_credentials.h" #include "tensorstore/kvstore/s3/aws_credential_provider.h" @@ -42,4 +42,4 @@ class FileCredentialProvider : public AwsCredentialProvider { } // namespace internal_kvstore_s3 } // namespace tensorstore -#endif // TENSORSTORE_KVSTORE_S3_AWS_FILE_CREDENTIAL_PROVIDER_H \ No newline at end of file +#endif // TENSORSTORE_KVSTORE_S3_FILE_CREDENTIAL_PROVIDER_H \ No newline at end of file From 9fea6cbe7be862462a50bb3f5c6a59726f953dd0 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Sat, 23 Sep 2023 14:12:22 +0200 Subject: [PATCH 05/63] Split EC2 Metadata credential provider into separate files --- tensorstore/kvstore/s3/BUILD | 6 ++- .../kvstore/s3/aws_credential_provider.cc | 7 ++-- .../kvstore/s3/aws_credential_provider.h | 16 -------- .../kvstore/s3/ec2_credential_provider.cc | 28 +++++++++++++ .../kvstore/s3/ec2_credential_provider.h | 41 +++++++++++++++++++ 5 files changed, 77 insertions(+), 21 deletions(-) create mode 100644 tensorstore/kvstore/s3/ec2_credential_provider.cc create mode 100644 tensorstore/kvstore/s3/ec2_credential_provider.h diff --git a/tensorstore/kvstore/s3/BUILD b/tensorstore/kvstore/s3/BUILD index c6c8f18dd..b024d3025 100644 --- a/tensorstore/kvstore/s3/BUILD +++ b/tensorstore/kvstore/s3/BUILD @@ -232,12 +232,14 @@ tensorstore_cc_library( srcs = [ "aws_credential_provider.cc", "environment_credential_provider.cc", - "file_credential_provider.cc"], + "file_credential_provider.cc", + "ec2_credential_provider.cc"], hdrs = [ "aws_credentials.h", "aws_credential_provider.h", "environment_credential_provider.h", - "file_credential_provider.h"], + "file_credential_provider.h", + "ec2_credential_provider.h"], deps = [ "//tensorstore/internal:env", "//tensorstore/internal:no_destructor", diff --git a/tensorstore/kvstore/s3/aws_credential_provider.cc b/tensorstore/kvstore/s3/aws_credential_provider.cc index af9842210..8d7ce9e43 100644 --- a/tensorstore/kvstore/s3/aws_credential_provider.cc +++ b/tensorstore/kvstore/s3/aws_credential_provider.cc @@ -15,6 +15,7 @@ #include "tensorstore/kvstore/s3/aws_credential_provider.h" #include "tensorstore/kvstore/s3/environment_credential_provider.h" #include "tensorstore/kvstore/s3/file_credential_provider.h" +#include "tensorstore/kvstore/s3/ec2_credential_provider.h" #include #include @@ -65,10 +66,10 @@ Result> GetDefaultAwsCredentialProvider( return file_creds; } - // 3. Obtain credentials from EC2 Metadata server - if (false) { + auto ec2_creds = std::make_unique(transport); + if(ec2_creds->GetCredentials().ok()) { ABSL_LOG(INFO) << "Using EC2 Metadata Service AwsCredentialProvider"; - return std::make_unique(transport); + return ec2_creds; } return absl::NotFoundError( diff --git a/tensorstore/kvstore/s3/aws_credential_provider.h b/tensorstore/kvstore/s3/aws_credential_provider.h index 9d5aba331..499612efb 100644 --- a/tensorstore/kvstore/s3/aws_credential_provider.h +++ b/tensorstore/kvstore/s3/aws_credential_provider.h @@ -40,22 +40,6 @@ class AwsCredentialProvider { }; -/// Provides S3 credentials from the EC2 Metadata server -/// if running within AWS -class EC2MetadataCredentialProvider : public AwsCredentialProvider { - private: - std::shared_ptr transport_; - - public: - EC2MetadataCredentialProvider( - std::shared_ptr transport) - : transport_(std::move(transport)) {} - - Result GetCredentials() override { - return absl::UnimplementedError("EC2 Metadata Server"); - } -}; - using AwsCredentialProviderFn = std::function>()>; diff --git a/tensorstore/kvstore/s3/ec2_credential_provider.cc b/tensorstore/kvstore/s3/ec2_credential_provider.cc new file mode 100644 index 000000000..6cb3bf8bb --- /dev/null +++ b/tensorstore/kvstore/s3/ec2_credential_provider.cc @@ -0,0 +1,28 @@ +// Copyright 2023 The TensorStore Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/status/status.h" +#include "tensorstore/kvstore/s3/ec2_credential_provider.h" + +namespace tensorstore { +namespace internal_kvstore_s3 { + +Result EC2MetadataCredentialProvider::GetCredentials() +{ + return absl::UnimplementedError("EC2 Metadata Server"); +} + + +} // namespace internal_kvstore_s3 +} // namespace tensorstore diff --git a/tensorstore/kvstore/s3/ec2_credential_provider.h b/tensorstore/kvstore/s3/ec2_credential_provider.h new file mode 100644 index 000000000..741be26d5 --- /dev/null +++ b/tensorstore/kvstore/s3/ec2_credential_provider.h @@ -0,0 +1,41 @@ +// Copyright 2023 The TensorStore Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef TENSORSTORE_KVSTORE_S3_EC2_CREDENTIAL_PROVIDER_H +#define TENSORSTORE_KVSTORE_S3_EC2_CREDENTIAL_PROVIDER_H + +#include "tensorstore/kvstore/s3/aws_credential_provider.h" + +namespace tensorstore { +namespace internal_kvstore_s3 { + +/// Provides S3 credentials from the EC2 Metadata server +/// if running within AWS +class EC2MetadataCredentialProvider : public AwsCredentialProvider { + private: + std::shared_ptr transport_; + + public: + EC2MetadataCredentialProvider( + std::shared_ptr transport) + : transport_(std::move(transport)) {} + + Result GetCredentials() override; +}; + +} // internal_kvstore_s3 +} // tensorstore + + +#endif // TENSORSTORE_KVSTORE_S3_EC2_CREDENTIAL_PROVIDER_H \ No newline at end of file From e02941d458b7d0e1731e75dd644e93a13127052b Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Sat, 23 Sep 2023 14:15:20 +0200 Subject: [PATCH 06/63] Cleanup imports --- tensorstore/kvstore/s3/aws_credential_provider.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/tensorstore/kvstore/s3/aws_credential_provider.h b/tensorstore/kvstore/s3/aws_credential_provider.h index 499612efb..b5bf4da56 100644 --- a/tensorstore/kvstore/s3/aws_credential_provider.h +++ b/tensorstore/kvstore/s3/aws_credential_provider.h @@ -17,12 +17,9 @@ #include #include -#include #include #include -#include "absl/status/status.h" -#include "absl/synchronization/mutex.h" #include "tensorstore/kvstore/s3/aws_credentials.h" #include "tensorstore/internal/http/http_transport.h" #include "tensorstore/util/result.h" From dc2a529d801c9a8b3c69287cf01452ec0c4cc428 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Sat, 23 Sep 2023 14:16:55 +0200 Subject: [PATCH 07/63] Correct header guards --- tensorstore/kvstore/s3/aws_credential_provider.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorstore/kvstore/s3/aws_credential_provider.h b/tensorstore/kvstore/s3/aws_credential_provider.h index b5bf4da56..b7f022fd8 100644 --- a/tensorstore/kvstore/s3/aws_credential_provider.h +++ b/tensorstore/kvstore/s3/aws_credential_provider.h @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef TENSORSTORE_KVSTORE_S3_S3_CREDENTIAL_PROVIDER_H -#define TENSORSTORE_KVSTORE_S3_S3_CREDENTIAL_PROVIDER_H +#ifndef TENSORSTORE_KVSTORE_S3_AWS_CREDENTIAL_PROVIDER +#define TENSORSTORE_KVSTORE_S3_AWS_CREDENTIAL_PROVIDER #include #include @@ -50,4 +50,4 @@ Result> GetAwsCredentialProvider( } // namespace internal_kvstore_s3 } // namespace tensorstore -#endif // TENSORSTORE_KVSTORE_S3_S3_CREDENTIAL_PROVIDER_H +#endif // TENSORSTORE_KVSTORE_S3_AWS_CREDENTIAL_PROVIDER From c8975ed215d25ab23ba8dedbe350724e5376de70 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Sat, 23 Sep 2023 14:28:37 +0200 Subject: [PATCH 08/63] Add an AnonymousCredentialProvider --- tensorstore/kvstore/s3/aws_credential_provider.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tensorstore/kvstore/s3/aws_credential_provider.h b/tensorstore/kvstore/s3/aws_credential_provider.h index b7f022fd8..2dd828c9e 100644 --- a/tensorstore/kvstore/s3/aws_credential_provider.h +++ b/tensorstore/kvstore/s3/aws_credential_provider.h @@ -36,6 +36,12 @@ class AwsCredentialProvider { virtual Result GetCredentials() = 0; }; +class AnonymousCredentialProvider : public AwsCredentialProvider { + public: + Result GetCredentials() override { + return AwsCredentials{}; + } +}; using AwsCredentialProviderFn = std::function>()>; From e34cdeb5830b3d96be220e7501fc84e83eb1d11f Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Sat, 23 Sep 2023 15:14:25 +0200 Subject: [PATCH 09/63] Add a chained credential provider --- tensorstore/kvstore/s3/BUILD | 2 + .../kvstore/s3/aws_credential_provider.cc | 28 +++---------- .../kvstore/s3/aws_credential_provider.h | 1 + .../s3/aws_credential_provider_test.cc | 11 +++-- .../kvstore/s3/chained_credential_provider.cc | 35 ++++++++++++++++ .../kvstore/s3/chained_credential_provider.h | 40 +++++++++++++++++++ .../kvstore/s3/ec2_credential_provider.cc | 2 +- .../kvstore/s3/file_credential_provider.cc | 3 ++ 8 files changed, 96 insertions(+), 26 deletions(-) create mode 100644 tensorstore/kvstore/s3/chained_credential_provider.cc create mode 100644 tensorstore/kvstore/s3/chained_credential_provider.h diff --git a/tensorstore/kvstore/s3/BUILD b/tensorstore/kvstore/s3/BUILD index b024d3025..e9cbd3b15 100644 --- a/tensorstore/kvstore/s3/BUILD +++ b/tensorstore/kvstore/s3/BUILD @@ -231,12 +231,14 @@ tensorstore_cc_library( name = "aws_credential_provider", srcs = [ "aws_credential_provider.cc", + "chained_credential_provider.cc", "environment_credential_provider.cc", "file_credential_provider.cc", "ec2_credential_provider.cc"], hdrs = [ "aws_credentials.h", "aws_credential_provider.h", + "chained_credential_provider.h", "environment_credential_provider.h", "file_credential_provider.h", "ec2_credential_provider.h"], diff --git a/tensorstore/kvstore/s3/aws_credential_provider.cc b/tensorstore/kvstore/s3/aws_credential_provider.cc index 8d7ce9e43..40dbdffb0 100644 --- a/tensorstore/kvstore/s3/aws_credential_provider.cc +++ b/tensorstore/kvstore/s3/aws_credential_provider.cc @@ -16,6 +16,7 @@ #include "tensorstore/kvstore/s3/environment_credential_provider.h" #include "tensorstore/kvstore/s3/file_credential_provider.h" #include "tensorstore/kvstore/s3/ec2_credential_provider.h" +#include "tensorstore/kvstore/s3/chained_credential_provider.h" #include #include @@ -52,29 +53,12 @@ namespace { Result> GetDefaultAwsCredentialProvider( std::string_view profile, std::shared_ptr transport) { - // 1. Obtain credentials from environment variables - auto env_creds = std::make_unique(); - if(env_creds->GetCredentials().ok()) { - return env_creds; - } - - auto file_creds = std::make_unique(std::string(profile)); - if(file_creds->GetCredentials().ok()) { - ABSL_LOG(INFO) << "Using File AwsCredentialProvider with profile " - << profile; - return file_creds; - } - - auto ec2_creds = std::make_unique(transport); - if(ec2_creds->GetCredentials().ok()) { - ABSL_LOG(INFO) << "Using EC2 Metadata Service AwsCredentialProvider"; - return ec2_creds; - } - - return absl::NotFoundError( - "No credentials provided in environment variables, " - "credentials file not found and not running on AWS."); + std::vector> providers; + providers.emplace_back(std::make_unique()); + providers.emplace_back(std::make_unique(std::string(profile))); + providers.emplace_back(std::make_unique(transport)); + return std::make_unique(std::move(providers)); } struct AwsCredentialProviderRegistry { diff --git a/tensorstore/kvstore/s3/aws_credential_provider.h b/tensorstore/kvstore/s3/aws_credential_provider.h index 2dd828c9e..81c97a055 100644 --- a/tensorstore/kvstore/s3/aws_credential_provider.h +++ b/tensorstore/kvstore/s3/aws_credential_provider.h @@ -36,6 +36,7 @@ class AwsCredentialProvider { virtual Result GetCredentials() = 0; }; +/// Provides anonymous credentials class AnonymousCredentialProvider : public AwsCredentialProvider { public: Result GetCredentials() override { diff --git a/tensorstore/kvstore/s3/aws_credential_provider_test.cc b/tensorstore/kvstore/s3/aws_credential_provider_test.cc index a342d1dfd..e61207d54 100644 --- a/tensorstore/kvstore/s3/aws_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/aws_credential_provider_test.cc @@ -89,10 +89,13 @@ class AwsCredentialProviderTest : public ::testing::Test { }; TEST_F(AwsCredentialProviderTest, ProviderNoCredentials) { - ASSERT_FALSE(GetAwsCredentialProvider("", transport_).ok()); - SetEnv("AWS_ACCESS_KEY_ID", "foo"); TENSORSTORE_ASSERT_OK_AND_ASSIGN( auto provider, GetAwsCredentialProvider("", transport_)); + ASSERT_FALSE(provider->GetCredentials().ok()); + + SetEnv("AWS_ACCESS_KEY_ID", "foo"); + TENSORSTORE_ASSERT_OK_AND_ASSIGN( + provider, GetAwsCredentialProvider("", transport_)); TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, provider->GetCredentials()); ASSERT_EQ(credentials.access_key, "foo"); @@ -165,7 +168,9 @@ TEST_F(AwsCredentialProviderTest, SetEnv("AWS_SHARED_CREDENTIALS_FILE", credentials_filename.c_str()); SetEnv("AWS_PROFILE", "bob"); - ASSERT_FALSE(GetAwsCredentialProvider("", transport_).ok()); + TENSORSTORE_ASSERT_OK_AND_ASSIGN( + auto provider, GetAwsCredentialProvider("", transport_)); + ASSERT_FALSE(provider->GetCredentials().ok()); } } // namespace diff --git a/tensorstore/kvstore/s3/chained_credential_provider.cc b/tensorstore/kvstore/s3/chained_credential_provider.cc new file mode 100644 index 000000000..fb87d3ce8 --- /dev/null +++ b/tensorstore/kvstore/s3/chained_credential_provider.cc @@ -0,0 +1,35 @@ +// Copyright 2023 The TensorStore Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/status/status.h" +#include "tensorstore/kvstore/s3/chained_credential_provider.h" + +namespace tensorstore { +namespace internal_kvstore_s3 { + +Result ChainedCredentialProvider::GetCredentials() +{ + for(auto & provider: providers_) { + auto credentials = provider->GetCredentials(); + if(credentials.ok()) { + return credentials; + } + } + + return absl::NotFoundError("No suitable AwsCredentialProvider found"); +} + + +} // namespace internal_kvstore_s3 +} // namespace tensorstore diff --git a/tensorstore/kvstore/s3/chained_credential_provider.h b/tensorstore/kvstore/s3/chained_credential_provider.h new file mode 100644 index 000000000..26cd4f400 --- /dev/null +++ b/tensorstore/kvstore/s3/chained_credential_provider.h @@ -0,0 +1,40 @@ +// Copyright 2023 The TensorStore Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef TENSORSTORE_KVSTORE_S3_CHAINED_CREDENTIAL_PROVIDER +#define TENSORSTORE_KVSTORE_S3_CHAINED_CREDENTIAL_PROVIDER + +#include +#include + +#include "tensorstore/kvstore/s3/aws_credential_provider.h" + +namespace tensorstore { +namespace internal_kvstore_s3 { + +class ChainedCredentialProvider : public AwsCredentialProvider { +private: + std::vector> providers_; + +public: + ChainedCredentialProvider(std::vector> providers) + : providers_(std::move(providers)) {} + + Result GetCredentials() override; +}; + +} // namespace tensorstore +} // namespace internal_kvstore_s3 + +#endif // TENSORSTORE_KVSTORE_S3_CHAINED_CREDENTIAL_PROVIDER \ No newline at end of file diff --git a/tensorstore/kvstore/s3/ec2_credential_provider.cc b/tensorstore/kvstore/s3/ec2_credential_provider.cc index 6cb3bf8bb..6938f2af4 100644 --- a/tensorstore/kvstore/s3/ec2_credential_provider.cc +++ b/tensorstore/kvstore/s3/ec2_credential_provider.cc @@ -20,7 +20,7 @@ namespace internal_kvstore_s3 { Result EC2MetadataCredentialProvider::GetCredentials() { - return absl::UnimplementedError("EC2 Metadata Server"); + return absl::NotFoundError("EC2 Metadata Server"); } diff --git a/tensorstore/kvstore/s3/file_credential_provider.cc b/tensorstore/kvstore/s3/file_credential_provider.cc index becd67556..7451f4f22 100644 --- a/tensorstore/kvstore/s3/file_credential_provider.cc +++ b/tensorstore/kvstore/s3/file_credential_provider.cc @@ -130,6 +130,9 @@ Result FileCredentialProvider::GetCredentials() { filename, "]")); } + ABSL_LOG_FIRST_N(INFO, 1) + << "Using profile [" << profile << "] in file [" << filename << "]"; + return credentials; } From 41cca5e5061c0e63ff726ae550c5ff672db4f5b1 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Sat, 23 Sep 2023 15:27:49 +0200 Subject: [PATCH 10/63] Split file provider tests into separate test unit --- tensorstore/kvstore/s3/BUILD | 14 +++ .../s3/aws_credential_provider_test.cc | 79 ------------- .../s3/file_credential_provider_test.cc | 111 ++++++++++++++++++ 3 files changed, 125 insertions(+), 79 deletions(-) create mode 100644 tensorstore/kvstore/s3/file_credential_provider_test.cc diff --git a/tensorstore/kvstore/s3/BUILD b/tensorstore/kvstore/s3/BUILD index e9cbd3b15..77274415b 100644 --- a/tensorstore/kvstore/s3/BUILD +++ b/tensorstore/kvstore/s3/BUILD @@ -271,6 +271,20 @@ tensorstore_cc_test( ], ) +tensorstore_cc_test( + name = "file_credential_provider_test", + srcs = ["file_credential_provider_test.cc"], + deps = [ + ":aws_credential_provider", + "//tensorstore/internal:path", + "//tensorstore/internal:test_util", + #"//tensorstore/internal/http:curl_transport", + "//tensorstore/util:result", + "//tensorstore/util:status_testutil", + "@com_google_googletest//:gtest_main", + ], +) + tensorstore_cc_test( name = "localstack_test", srcs = ["localstack_test.cc"], diff --git a/tensorstore/kvstore/s3/aws_credential_provider_test.cc b/tensorstore/kvstore/s3/aws_credential_provider_test.cc index e61207d54..1b44c00bd 100644 --- a/tensorstore/kvstore/s3/aws_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/aws_credential_provider_test.cc @@ -30,7 +30,6 @@ namespace { using ::tensorstore::Future; using ::tensorstore::internal::GetEnv; -using ::tensorstore::internal::JoinPath; using ::tensorstore::internal::SetEnv; using ::tensorstore::internal::UnsetEnv; using ::tensorstore::internal_http::HttpRequest; @@ -38,27 +37,6 @@ using ::tensorstore::internal_http::HttpResponse; using ::tensorstore::internal_http::HttpTransport; using ::tensorstore::internal_kvstore_s3::GetAwsCredentialProvider; -class TestData : public tensorstore::internal::ScopedTemporaryDirectory { - public: - std::string WriteCredentialsFile() { - auto p = JoinPath(path(), "aws_config"); - std::ofstream ofs(p); - ofs << "discarded_value = 500\n" - "\n" - "[default]\n" - "aws_access_key_id =AKIAIOSFODNN7EXAMPLE\n" - "aws_secret_access_key = wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY\n" - "aws_session_token= abcdef1234567890 \n" - "\n" - "[alice]\n" - "aws_access_key_id = AKIAIOSFODNN6EXAMPLE\n" - "aws_secret_access_key = " - "wJalrXUtnFEMI/K7MDENG/bPxRfiCZEXAMPLEKEY\n" - "\n"; - ofs.close(); - return p; - } -}; /// Cause EC2Metadata to always fail class NotFoundTransport : public HttpTransport { @@ -116,61 +94,4 @@ TEST_F(AwsCredentialProviderTest, ProviderAwsCredentialsFromEnv) { ASSERT_EQ(credentials.session_token, "qux"); } -TEST_F(AwsCredentialProviderTest, ProviderAwsCredentialsFromFileDefault) { - TestData test_data; - std::string credentials_filename = test_data.WriteCredentialsFile(); - - SetEnv("AWS_SHARED_CREDENTIALS_FILE", credentials_filename.c_str()); - TENSORSTORE_ASSERT_OK_AND_ASSIGN( - auto provider, GetAwsCredentialProvider("", transport_)); - TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, - provider->GetCredentials()); - ASSERT_EQ(credentials.access_key, "AKIAIOSFODNN7EXAMPLE"); - ASSERT_EQ(credentials.secret_key, "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); - ASSERT_EQ(credentials.session_token, "abcdef1234567890"); -} - -TEST_F(AwsCredentialProviderTest, - ProviderAwsCredentialsFromFileProfileOverride) { - TestData test_data; - std::string credentials_filename = test_data.WriteCredentialsFile(); - - SetEnv("AWS_SHARED_CREDENTIALS_FILE", credentials_filename.c_str()); - TENSORSTORE_ASSERT_OK_AND_ASSIGN( - auto provider, - GetAwsCredentialProvider("alice", transport_)); - TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, - provider->GetCredentials()); - ASSERT_EQ(credentials.access_key, "AKIAIOSFODNN6EXAMPLE"); - ASSERT_EQ(credentials.secret_key, "wJalrXUtnFEMI/K7MDENG/bPxRfiCZEXAMPLEKEY"); - ASSERT_EQ(credentials.session_token, ""); -} - -TEST_F(AwsCredentialProviderTest, ProviderAwsCredentialsFromFileProfileEnv) { - TestData test_data; - std::string credentials_filename = test_data.WriteCredentialsFile(); - - SetEnv("AWS_SHARED_CREDENTIALS_FILE", credentials_filename.c_str()); - SetEnv("AWS_PROFILE", "alice"); - TENSORSTORE_ASSERT_OK_AND_ASSIGN( - auto provider, GetAwsCredentialProvider("", transport_)); - TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, - provider->GetCredentials()); - ASSERT_EQ(credentials.access_key, "AKIAIOSFODNN6EXAMPLE"); - ASSERT_EQ(credentials.secret_key, "wJalrXUtnFEMI/K7MDENG/bPxRfiCZEXAMPLEKEY"); - ASSERT_EQ(credentials.session_token, ""); -} - -TEST_F(AwsCredentialProviderTest, - ProviderAwsCredentialsFromFileInvalidProfileEnv) { - TestData test_data; - std::string credentials_filename = test_data.WriteCredentialsFile(); - - SetEnv("AWS_SHARED_CREDENTIALS_FILE", credentials_filename.c_str()); - SetEnv("AWS_PROFILE", "bob"); - TENSORSTORE_ASSERT_OK_AND_ASSIGN( - auto provider, GetAwsCredentialProvider("", transport_)); - ASSERT_FALSE(provider->GetCredentials().ok()); -} - } // namespace diff --git a/tensorstore/kvstore/s3/file_credential_provider_test.cc b/tensorstore/kvstore/s3/file_credential_provider_test.cc new file mode 100644 index 000000000..5d9aa02d6 --- /dev/null +++ b/tensorstore/kvstore/s3/file_credential_provider_test.cc @@ -0,0 +1,111 @@ +// Copyright 2023 The TensorStore Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "tensorstore/kvstore/s3/file_credential_provider.h" + +#include +#include + +#include +#include +#include "tensorstore/internal/env.h" +#include "tensorstore/internal/path.h" +#include "tensorstore/internal/test_util.h" +#include "tensorstore/util/status_testutil.h" + +namespace { + +using ::tensorstore::internal::SetEnv; +using ::tensorstore::internal::UnsetEnv; +using ::tensorstore::internal::JoinPath; +using ::tensorstore::internal_kvstore_s3::FileCredentialProvider; + +class TestData : public tensorstore::internal::ScopedTemporaryDirectory { + public: + std::string WriteCredentialsFile() { + auto p = JoinPath(path(), "aws_config"); + std::ofstream ofs(p); + ofs << "discarded_value = 500\n" + "\n" + "[default]\n" + "aws_access_key_id =AKIAIOSFODNN7EXAMPLE\n" + "aws_secret_access_key = wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY\n" + "aws_session_token= abcdef1234567890 \n" + "\n" + "[alice]\n" + "aws_access_key_id = AKIAIOSFODNN6EXAMPLE\n" + "aws_secret_access_key = " + "wJalrXUtnFEMI/K7MDENG/bPxRfiCZEXAMPLEKEY\n" + "\n"; + ofs.close(); + return p; + } +}; + +class FileCredentialProviderTest : public ::testing::Test { + protected: + void SetUp() override { UnsetEnv("AWS_SHARED_CREDENTIALS_FILE"); } +}; + + +TEST_F(FileCredentialProviderTest, ProviderAwsCredentialsFromFileDefault) { + TestData test_data; + std::string credentials_filename = test_data.WriteCredentialsFile(); + + SetEnv("AWS_SHARED_CREDENTIALS_FILE", credentials_filename.c_str()); + auto provider = FileCredentialProvider(""); + TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, provider.GetCredentials()); + ASSERT_EQ(credentials.access_key, "AKIAIOSFODNN7EXAMPLE"); + ASSERT_EQ(credentials.secret_key, "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); + ASSERT_EQ(credentials.session_token, "abcdef1234567890"); +} + +TEST_F(FileCredentialProviderTest, + ProviderAwsCredentialsFromFileProfileOverride) { + TestData test_data; + std::string credentials_filename = test_data.WriteCredentialsFile(); + + SetEnv("AWS_SHARED_CREDENTIALS_FILE", credentials_filename.c_str()); + auto provider = FileCredentialProvider("alice"); + TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, provider.GetCredentials()); + ASSERT_EQ(credentials.access_key, "AKIAIOSFODNN6EXAMPLE"); + ASSERT_EQ(credentials.secret_key, "wJalrXUtnFEMI/K7MDENG/bPxRfiCZEXAMPLEKEY"); + ASSERT_EQ(credentials.session_token, ""); +} + +TEST_F(FileCredentialProviderTest, ProviderAwsCredentialsFromFileProfileEnv) { + TestData test_data; + std::string credentials_filename = test_data.WriteCredentialsFile(); + + SetEnv("AWS_SHARED_CREDENTIALS_FILE", credentials_filename.c_str()); + SetEnv("AWS_PROFILE", "alice"); + auto provider = FileCredentialProvider(""); + TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, provider.GetCredentials()); + ASSERT_EQ(credentials.access_key, "AKIAIOSFODNN6EXAMPLE"); + ASSERT_EQ(credentials.secret_key, "wJalrXUtnFEMI/K7MDENG/bPxRfiCZEXAMPLEKEY"); + ASSERT_EQ(credentials.session_token, ""); +} + +TEST_F(FileCredentialProviderTest, + ProviderAwsCredentialsFromFileInvalidProfileEnv) { + TestData test_data; + std::string credentials_filename = test_data.WriteCredentialsFile(); + + SetEnv("AWS_SHARED_CREDENTIALS_FILE", credentials_filename.c_str()); + SetEnv("AWS_PROFILE", "bob"); + auto provider = FileCredentialProvider(""); + ASSERT_FALSE(provider.GetCredentials().ok()); +} + +} // namespace From 5b831e1aa8131cec25c3e29b9849b8e3f7641fc7 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Sat, 23 Sep 2023 15:36:16 +0200 Subject: [PATCH 11/63] Split out environment provider test cases into separate test unit --- tensorstore/kvstore/s3/BUILD | 14 +++- .../s3/aws_credential_provider_test.cc | 28 -------- .../environment_credential_provider_test.cc | 70 +++++++++++++++++++ 3 files changed, 83 insertions(+), 29 deletions(-) create mode 100644 tensorstore/kvstore/s3/environment_credential_provider_test.cc diff --git a/tensorstore/kvstore/s3/BUILD b/tensorstore/kvstore/s3/BUILD index 77274415b..79f2d362b 100644 --- a/tensorstore/kvstore/s3/BUILD +++ b/tensorstore/kvstore/s3/BUILD @@ -271,6 +271,19 @@ tensorstore_cc_test( ], ) +tensorstore_cc_test( + name = "environmental_credential_provider_test", + srcs = ["environment_credential_provider_test.cc"], + deps = [ + ":aws_credential_provider", + "//tensorstore/internal:test_util", + "//tensorstore/util:result", + "//tensorstore/util:status_testutil", + "@com_google_googletest//:gtest_main", + ], +) + + tensorstore_cc_test( name = "file_credential_provider_test", srcs = ["file_credential_provider_test.cc"], @@ -278,7 +291,6 @@ tensorstore_cc_test( ":aws_credential_provider", "//tensorstore/internal:path", "//tensorstore/internal:test_util", - #"//tensorstore/internal/http:curl_transport", "//tensorstore/util:result", "//tensorstore/util:status_testutil", "@com_google_googletest//:gtest_main", diff --git a/tensorstore/kvstore/s3/aws_credential_provider_test.cc b/tensorstore/kvstore/s3/aws_credential_provider_test.cc index 1b44c00bd..a19be1998 100644 --- a/tensorstore/kvstore/s3/aws_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/aws_credential_provider_test.cc @@ -66,32 +66,4 @@ class AwsCredentialProviderTest : public ::testing::Test { } }; -TEST_F(AwsCredentialProviderTest, ProviderNoCredentials) { - TENSORSTORE_ASSERT_OK_AND_ASSIGN( - auto provider, GetAwsCredentialProvider("", transport_)); - ASSERT_FALSE(provider->GetCredentials().ok()); - - SetEnv("AWS_ACCESS_KEY_ID", "foo"); - TENSORSTORE_ASSERT_OK_AND_ASSIGN( - provider, GetAwsCredentialProvider("", transport_)); - TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, - provider->GetCredentials()); - ASSERT_EQ(credentials.access_key, "foo"); - ASSERT_TRUE(credentials.secret_key.empty()); - ASSERT_TRUE(credentials.session_token.empty()); -} - -TEST_F(AwsCredentialProviderTest, ProviderAwsCredentialsFromEnv) { - SetEnv("AWS_ACCESS_KEY_ID", "foo"); - SetEnv("AWS_SECRET_ACCESS_KEY", "bar"); - SetEnv("AWS_SESSION_TOKEN", "qux"); - TENSORSTORE_ASSERT_OK_AND_ASSIGN( - auto provider, GetAwsCredentialProvider("", transport_)); - TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, - provider->GetCredentials()); - ASSERT_EQ(credentials.access_key, "foo"); - ASSERT_EQ(credentials.secret_key, "bar"); - ASSERT_EQ(credentials.session_token, "qux"); -} - } // namespace diff --git a/tensorstore/kvstore/s3/environment_credential_provider_test.cc b/tensorstore/kvstore/s3/environment_credential_provider_test.cc new file mode 100644 index 000000000..b45d96bd1 --- /dev/null +++ b/tensorstore/kvstore/s3/environment_credential_provider_test.cc @@ -0,0 +1,70 @@ +// Copyright 2023 The TensorStore Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "tensorstore/kvstore/s3/environment_credential_provider.h" + +#include + +#include +#include +#include "tensorstore/internal/env.h" +#include "tensorstore/internal/test_util.h" +#include "tensorstore/util/result.h" +#include "tensorstore/util/status_testutil.h" + +namespace { + +using ::tensorstore::Future; +using ::tensorstore::internal::GetEnv; +using ::tensorstore::internal::SetEnv; +using ::tensorstore::internal::UnsetEnv; +using ::tensorstore::internal_kvstore_s3::EnvironmentCredentialProvider; + + +class EnvironmentCredentialProviderTest : public ::testing::Test { + protected: + void SetUp() override { + // Make sure that env vars are not set. + for (const char* var : + {"AWS_SHARED_CREDENTIALS_FILE", "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "AWS_PROFILE"}) { + UnsetEnv(var); + } + } +}; + +TEST_F(EnvironmentCredentialProviderTest, ProviderNoCredentials) { + auto provider = EnvironmentCredentialProvider(); + ASSERT_FALSE(provider.GetCredentials().ok()); + + SetEnv("AWS_ACCESS_KEY_ID", "foo"); + TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, + provider.GetCredentials()); + ASSERT_EQ(credentials.access_key, "foo"); + ASSERT_TRUE(credentials.secret_key.empty()); + ASSERT_TRUE(credentials.session_token.empty()); +} + +TEST_F(EnvironmentCredentialProviderTest, ProviderAwsCredentialsFromEnv) { + SetEnv("AWS_ACCESS_KEY_ID", "foo"); + SetEnv("AWS_SECRET_ACCESS_KEY", "bar"); + SetEnv("AWS_SESSION_TOKEN", "qux"); + auto provider = EnvironmentCredentialProvider(); + TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, provider.GetCredentials()); + ASSERT_EQ(credentials.access_key, "foo"); + ASSERT_EQ(credentials.secret_key, "bar"); + ASSERT_EQ(credentials.session_token, "qux"); +} + +} // namespace From 2a2303913974de9e283df40e903ae9853b7e32d1 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Sat, 23 Sep 2023 16:12:37 +0200 Subject: [PATCH 12/63] Update comments --- .../kvstore/s3/aws_credential_provider.cc | 23 ++++++++----------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/tensorstore/kvstore/s3/aws_credential_provider.cc b/tensorstore/kvstore/s3/aws_credential_provider.cc index 40dbdffb0..512f7c5f3 100644 --- a/tensorstore/kvstore/s3/aws_credential_provider.cc +++ b/tensorstore/kvstore/s3/aws_credential_provider.cc @@ -46,10 +46,10 @@ namespace tensorstore { namespace internal_kvstore_s3 { namespace { -// For reference, see the latest AWS environment variables used by the cli: -// https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html - - +/// Return a ChainedCredentialProvider that attempts to retrieve credentials from +/// 1. AWS Environment Variables, e.g. AWS_ACCESS_KEY_ID +/// 2. Shared Credential File, e.g. $HOME/.aws/credentials +/// 3. EC2 Metadata Server Result> GetDefaultAwsCredentialProvider( std::string_view profile, std::shared_ptr transport) { @@ -83,22 +83,17 @@ void RegisterAwsCredentialProviderProvider(AwsCredentialProviderFn provider, [](const auto& a, const auto& b) { return a.first < b.first; }); } -/// @brief Obtain a credential provider from a series of registered and default +/// Obtain a credential provider from a series of registered and default /// providers /// /// Providers are returned in the following order: /// 1. Any registered providers that supply valid credentials /// 2. Environment variable provider if valid credential can be obtained from /// AWS_* environment variables -/// 3. File provider containing credentials from an ~/.aws/credentials file -/// 4. EC2 Metadata server -/// -/// @param profile The profile to use when retrieving credentials from a -/// credentials file. -/// @param transport Optionally specify the http transport used to retreive S3 -/// credentials -/// from the EC2 metadata server. -/// @return Provider that supplies S3 Credentials +/// 3. File provider containing credentials from the $HOME/.aws/credentials file. +/// The `profile` variable overrides the default profile in this file. +/// 4. EC2 Metadata server. The `transport` variable overrides the default +/// HttpTransport. Result> GetAwsCredentialProvider( std::string_view profile, std::shared_ptr transport) { From 786ba4d744217aa0208d42ba6fa55ac26ba946ca Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Sat, 23 Sep 2023 18:06:50 +0200 Subject: [PATCH 13/63] Add expiry concepts to AwsCredentialProvider --- .../kvstore/s3/aws_credential_provider.h | 8 ++-- .../kvstore/s3/chained_credential_provider.h | 3 ++ .../kvstore/s3/ec2_credential_provider.cc | 5 ++- .../kvstore/s3/ec2_credential_provider.h | 4 +- .../s3/environment_credential_provider.cc | 23 +++++------ .../s3/environment_credential_provider.h | 6 ++- .../kvstore/s3/file_credential_provider.cc | 41 ++++++++----------- .../kvstore/s3/file_credential_provider.h | 9 ++-- 8 files changed, 49 insertions(+), 50 deletions(-) diff --git a/tensorstore/kvstore/s3/aws_credential_provider.h b/tensorstore/kvstore/s3/aws_credential_provider.h index 81c97a055..37edfd954 100644 --- a/tensorstore/kvstore/s3/aws_credential_provider.h +++ b/tensorstore/kvstore/s3/aws_credential_provider.h @@ -16,9 +16,7 @@ #define TENSORSTORE_KVSTORE_S3_AWS_CREDENTIAL_PROVIDER #include -#include #include -#include #include "tensorstore/kvstore/s3/aws_credentials.h" #include "tensorstore/internal/http/http_transport.h" @@ -34,14 +32,14 @@ class AwsCredentialProvider { public: virtual ~AwsCredentialProvider() = default; virtual Result GetCredentials() = 0; + virtual bool IsExpired() = 0; }; /// Provides anonymous credentials class AnonymousCredentialProvider : public AwsCredentialProvider { public: - Result GetCredentials() override { - return AwsCredentials{}; - } + Result GetCredentials() override { return AwsCredentials{}; } + bool IsExpired() override { return false; } }; using AwsCredentialProviderFn = diff --git a/tensorstore/kvstore/s3/chained_credential_provider.h b/tensorstore/kvstore/s3/chained_credential_provider.h index 26cd4f400..f12480874 100644 --- a/tensorstore/kvstore/s3/chained_credential_provider.h +++ b/tensorstore/kvstore/s3/chained_credential_provider.h @@ -23,15 +23,18 @@ namespace tensorstore { namespace internal_kvstore_s3 { +// Returns credentials from a chain of providers class ChainedCredentialProvider : public AwsCredentialProvider { private: std::vector> providers_; + bool retrieved_; public: ChainedCredentialProvider(std::vector> providers) : providers_(std::move(providers)) {} Result GetCredentials() override; + bool IsExpired() override ABSL_LOCKS_EXCLUDED(mutex_) { return retrieved_; } }; } // namespace tensorstore diff --git a/tensorstore/kvstore/s3/ec2_credential_provider.cc b/tensorstore/kvstore/s3/ec2_credential_provider.cc index 6938f2af4..58385a91a 100644 --- a/tensorstore/kvstore/s3/ec2_credential_provider.cc +++ b/tensorstore/kvstore/s3/ec2_credential_provider.cc @@ -18,9 +18,10 @@ namespace tensorstore { namespace internal_kvstore_s3 { -Result EC2MetadataCredentialProvider::GetCredentials() +Result +EC2MetadataCredentialProvider::GetCredentials() { - return absl::NotFoundError("EC2 Metadata Server"); + return absl::NotFoundError("EC2 Metadata Server"); } diff --git a/tensorstore/kvstore/s3/ec2_credential_provider.h b/tensorstore/kvstore/s3/ec2_credential_provider.h index 741be26d5..ab271b85f 100644 --- a/tensorstore/kvstore/s3/ec2_credential_provider.h +++ b/tensorstore/kvstore/s3/ec2_credential_provider.h @@ -25,13 +25,15 @@ namespace internal_kvstore_s3 { class EC2MetadataCredentialProvider : public AwsCredentialProvider { private: std::shared_ptr transport_; + bool retrieved_; public: EC2MetadataCredentialProvider( std::shared_ptr transport) - : transport_(std::move(transport)) {} + : transport_(std::move(transport)), retrieved_(false) {} Result GetCredentials() override; + bool IsExpired() override { return retrieved_; } }; } // internal_kvstore_s3 diff --git a/tensorstore/kvstore/s3/environment_credential_provider.cc b/tensorstore/kvstore/s3/environment_credential_provider.cc index 2f7593fb6..22e689aee 100644 --- a/tensorstore/kvstore/s3/environment_credential_provider.cc +++ b/tensorstore/kvstore/s3/environment_credential_provider.cc @@ -23,37 +23,34 @@ namespace internal_kvstore_s3 { namespace { // AWS user identifier -constexpr char kEnvAwsAccessKeyId[] = "AWS_ACCESS_KEY_ID"; +static constexpr char kEnvAwsAccessKeyId[] = "AWS_ACCESS_KEY_ID"; // AWS user password -constexpr char kEnvAwsSecretAccessKey[] = "AWS_SECRET_ACCESS_KEY"; +static constexpr char kEnvAwsSecretAccessKey[] = "AWS_SECRET_ACCESS_KEY"; // AWS session token -constexpr char kEnvAwsSessionToken[] = "AWS_SESSION_TOKEN"; +static constexpr char kEnvAwsSessionToken[] = "AWS_SESSION_TOKEN"; } // namespace -Result EnvironmentCredentialProvider::GetCredentials() { -// 1. Obtain credentials from environment variables +Result +EnvironmentCredentialProvider::GetCredentials() { if (auto access_key = GetEnv(kEnvAwsAccessKeyId); access_key) { ABSL_LOG_FIRST_N(INFO, 1) << "Using Environment Variable " << kEnvAwsAccessKeyId; - AwsCredentials credentials; - credentials.access_key = *access_key; - auto secret_key = GetEnv(kEnvAwsSecretAccessKey); + auto credentials = AwsCredentials{*access_key}; - if (secret_key.has_value()) { + if(auto secret_key = GetEnv(kEnvAwsSecretAccessKey); secret_key) { credentials.secret_key = *secret_key; } - auto session_token = GetEnv(kEnvAwsSessionToken); - - if (session_token.has_value()) { + if(auto session_token = GetEnv(kEnvAwsSessionToken); session_token) { credentials.session_token = *session_token; } + retrieved_ = true; return credentials; } - return absl::NotFoundError("AWS Credentials not found in environment variables"); + return absl::NotFoundError(absl::StrCat(kEnvAwsAccessKeyId, " not set")); } } // namespace internal_kvstore_s3 diff --git a/tensorstore/kvstore/s3/environment_credential_provider.h b/tensorstore/kvstore/s3/environment_credential_provider.h index 211391a42..e1b44988d 100644 --- a/tensorstore/kvstore/s3/environment_credential_provider.h +++ b/tensorstore/kvstore/s3/environment_credential_provider.h @@ -15,7 +15,6 @@ #ifndef TENSORSTORE_KVSTORE_S3_ENVIRONMENT_CREDENTIAL_PROVIDER_H #define TENSORSTORE_KVSTORE_S3_ENVIRONMENT_CREDENTIAL_PROVIDER_H -#include "tensorstore/kvstore/s3/aws_credentials.h" #include "tensorstore/kvstore/s3/aws_credential_provider.h" #include "tensorstore/util/result.h" @@ -26,10 +25,13 @@ namespace internal_kvstore_s3 { /// AWS_ACCESS_KEY_ID, AWS_SECRET_KEY_ID, AWS_SESSION_TOKEN class EnvironmentCredentialProvider : public AwsCredentialProvider { private: - AwsCredentials credentials_; + bool retrieved_; public: + EnvironmentCredentialProvider() : retrieved_(false) {} Result GetCredentials() override; + // Credentials obtained from the environment never expire + bool IsExpired() override { return retrieved_; } }; } // namespace internal_kvstore_s3 diff --git a/tensorstore/kvstore/s3/file_credential_provider.cc b/tensorstore/kvstore/s3/file_credential_provider.cc index 7451f4f22..e144eb907 100644 --- a/tensorstore/kvstore/s3/file_credential_provider.cc +++ b/tensorstore/kvstore/s3/file_credential_provider.cc @@ -13,6 +13,7 @@ // limitations under the License. #include +#include #include "absl/strings/strip.h" #include "tensorstore/kvstore/s3/file_credential_provider.h" @@ -43,31 +44,25 @@ static constexpr char kEnvAwsProfile[] = "AWS_PROFILE"; // Default profile static constexpr char kDefaultProfile[] = "default"; -/// Returns whether the given path points to a readable file. -bool IsFile(const std::string& filename) { - std::ifstream fstream(filename.c_str()); - return fstream.good(); -} - Result GetAwsCredentialsFileName() { std::string result; - auto credentials_file = GetEnv(kEnvAwsCredentialsFile); - if (!credentials_file) { - auto home_dir = GetEnv("HOME"); - if (!home_dir) { + if (auto credentials_file = GetEnv(kEnvAwsCredentialsFile); credentials_file) { + result = *credentials_file; + } else { + if(auto home_dir = GetEnv("HOME"); home_dir) { + result = JoinPath(*home_dir, kDefaultAwsCredentialsFilePath); + } else { return absl::NotFoundError("Could not read $HOME"); } - result = JoinPath(*home_dir, kDefaultAwsCredentialsFilePath); - } else { - result = *credentials_file; } - if (!IsFile(result)) { + + if(auto fstream = std::ifstream(result.c_str()); !fstream.good()) { return absl::NotFoundError( absl::StrCat("Could not find the credentials file at " - "location [", - result, "]")); + "location [", result, "]")); } + return result; } @@ -75,19 +70,16 @@ Result GetAwsCredentialsFileName() { /// https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html#cli-configure-files-format Result FileCredentialProvider::GetCredentials() { - absl::ReaderMutexLock lock(&mutex_); TENSORSTORE_ASSIGN_OR_RETURN(auto filename, GetAwsCredentialsFileName()); - std::ifstream ifs(filename); - std::string profile = profile_.empty() ? - GetEnv(kEnvAwsProfile).value_or(kDefaultProfile) : - profile_; - - + std::ifstream ifs(filename.c_str()); if (!ifs) { return absl::NotFoundError( - absl::StrCat("Could not open the credentials file [", filename, "]")); + absl::StrCat("Could not open credentials file [", filename, "]")); } + std::string profile = !profile_.empty() ? std::string(profile_) : + GetEnv(kEnvAwsProfile).value_or(kDefaultProfile); + AwsCredentials credentials; std::string section_name; std::string line; @@ -133,6 +125,7 @@ Result FileCredentialProvider::GetCredentials() { ABSL_LOG_FIRST_N(INFO, 1) << "Using profile [" << profile << "] in file [" << filename << "]"; + retrieved_ = true; return credentials; } diff --git a/tensorstore/kvstore/s3/file_credential_provider.h b/tensorstore/kvstore/s3/file_credential_provider.h index e6b3780a9..5bc92f8bd 100644 --- a/tensorstore/kvstore/s3/file_credential_provider.h +++ b/tensorstore/kvstore/s3/file_credential_provider.h @@ -15,7 +15,6 @@ #ifndef TENSORSTORE_KVSTORE_S3_FILE_CREDENTIAL_PROVIDER_H #define TENSORSTORE_KVSTORE_S3_FILE_CREDENTIAL_PROVIDER_H -#include "tensorstore/kvstore/s3/aws_credentials.h" #include "tensorstore/kvstore/s3/aws_credential_provider.h" #include "tensorstore/util/result.h" @@ -31,12 +30,16 @@ namespace internal_kvstore_s3 { /// "default". class FileCredentialProvider : public AwsCredentialProvider { private: - absl::Mutex mutex_; + // desired profile std::string profile_; + bool retrieved_; public: - FileCredentialProvider(std::string profile) : profile_(std::move(profile)) {} + FileCredentialProvider(std::string profile) + : profile_(std::move(profile)), retrieved_(false) {} Result GetCredentials() override; + // Shared Credentials never expire once retrieved + bool IsExpired() override { return retrieved_; } }; } // namespace internal_kvstore_s3 From a4a6bcb36fd471f7dc82842aad49b4208f5a21c0 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Sat, 23 Sep 2023 19:58:02 +0200 Subject: [PATCH 14/63] Chained Credential Provider Test Cases --- tensorstore/kvstore/s3/BUILD | 14 +++ .../kvstore/s3/aws_credential_provider.h | 13 ++- .../kvstore/s3/chained_credential_provider.cc | 27 ++++- .../kvstore/s3/chained_credential_provider.h | 9 +- .../s3/chained_credential_provider_test.cc | 100 ++++++++++++++++++ 5 files changed, 153 insertions(+), 10 deletions(-) create mode 100644 tensorstore/kvstore/s3/chained_credential_provider_test.cc diff --git a/tensorstore/kvstore/s3/BUILD b/tensorstore/kvstore/s3/BUILD index 79f2d362b..863903f15 100644 --- a/tensorstore/kvstore/s3/BUILD +++ b/tensorstore/kvstore/s3/BUILD @@ -297,6 +297,20 @@ tensorstore_cc_test( ], ) + +tensorstore_cc_test( + name = "chained_credential_provider_test", + srcs = ["chained_credential_provider_test.cc"], + deps = [ + ":aws_credential_provider", + "//tensorstore/internal:test_util", + "//tensorstore/util:result", + "//tensorstore/util:status_testutil", + "@com_google_googletest//:gtest_main", + ], +) + + tensorstore_cc_test( name = "localstack_test", srcs = ["localstack_test.cc"], diff --git a/tensorstore/kvstore/s3/aws_credential_provider.h b/tensorstore/kvstore/s3/aws_credential_provider.h index 37edfd954..f87d70e93 100644 --- a/tensorstore/kvstore/s3/aws_credential_provider.h +++ b/tensorstore/kvstore/s3/aws_credential_provider.h @@ -36,9 +36,18 @@ class AwsCredentialProvider { }; /// Provides anonymous credentials -class AnonymousCredentialProvider : public AwsCredentialProvider { +class StaticCredentialProvider : public AwsCredentialProvider { + private: + AwsCredentials credentials_; public: - Result GetCredentials() override { return AwsCredentials{}; } + StaticCredentialProvider(std::string_view access_key="", + std::string_view secret_key="", + std::string_view session_token="") + : credentials_{std::string{access_key}, + std::string{secret_key}, + std::string{session_token}} {} + + Result GetCredentials() override { return credentials_; } bool IsExpired() override { return false; } }; diff --git a/tensorstore/kvstore/s3/chained_credential_provider.cc b/tensorstore/kvstore/s3/chained_credential_provider.cc index fb87d3ce8..cc6d5f2c3 100644 --- a/tensorstore/kvstore/s3/chained_credential_provider.cc +++ b/tensorstore/kvstore/s3/chained_credential_provider.cc @@ -13,21 +13,38 @@ // limitations under the License. #include "absl/status/status.h" +#include "absl/strings/str_join.h" #include "tensorstore/kvstore/s3/chained_credential_provider.h" namespace tensorstore { namespace internal_kvstore_s3 { -Result ChainedCredentialProvider::GetCredentials() -{ - for(auto & provider: providers_) { - auto credentials = provider->GetCredentials(); + +bool ChainedCredentialProvider::IsExpired() { + if(last_provider_ < 0 || last_provider_ >= providers_.size()) { + return true; + } + + return providers_[last_provider_]->IsExpired(); +} + +Result ChainedCredentialProvider::GetCredentials() { + std::vector errors; + last_provider_ = -1; + + for(std::size_t i=0; i < providers_.size(); ++i) { + auto credentials = providers_[i]->GetCredentials(); if(credentials.ok()) { + last_provider_ = i; return credentials; + } else { + errors.push_back(credentials.status().ToString()); } } - return absl::NotFoundError("No suitable AwsCredentialProvider found"); + return absl::NotFoundError( + absl::StrCat("No valid AwsCredentialProvider in chain:\n", + absl::StrJoin(errors, "\n"))); } diff --git a/tensorstore/kvstore/s3/chained_credential_provider.h b/tensorstore/kvstore/s3/chained_credential_provider.h index f12480874..526ab4aa4 100644 --- a/tensorstore/kvstore/s3/chained_credential_provider.h +++ b/tensorstore/kvstore/s3/chained_credential_provider.h @@ -24,17 +24,20 @@ namespace tensorstore { namespace internal_kvstore_s3 { // Returns credentials from a chain of providers +// The chain is iterated through, in order, until valid credentials are returned. +// The index of the returned provider is stored, and if valid, is used to +// proxy IsExpired calls to the relevant provider class ChainedCredentialProvider : public AwsCredentialProvider { private: std::vector> providers_; - bool retrieved_; + std::size_t last_provider_; public: ChainedCredentialProvider(std::vector> providers) - : providers_(std::move(providers)) {} + : providers_(std::move(providers)), last_provider_(-1) {} Result GetCredentials() override; - bool IsExpired() override ABSL_LOCKS_EXCLUDED(mutex_) { return retrieved_; } + bool IsExpired() override; }; } // namespace tensorstore diff --git a/tensorstore/kvstore/s3/chained_credential_provider_test.cc b/tensorstore/kvstore/s3/chained_credential_provider_test.cc new file mode 100644 index 000000000..59e9fb6ae --- /dev/null +++ b/tensorstore/kvstore/s3/chained_credential_provider_test.cc @@ -0,0 +1,100 @@ +// Copyright 2023 The TensorStore Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "tensorstore/kvstore/s3/chained_credential_provider.h" + +#include +#include +#include "tensorstore/internal/test_util.h" +#include "tensorstore/util/result.h" +#include "tensorstore/util/status_testutil.h" + +using ::tensorstore::Future; +using ::tensorstore::Result; +using ::tensorstore::internal_kvstore_s3::AwsCredentials; +using ::tensorstore::internal_kvstore_s3::AwsCredentialProvider; +using ::tensorstore::internal_kvstore_s3::ChainedCredentialProvider; + +namespace { + +class TestCredentialProvider : public AwsCredentialProvider { + public: + AwsCredentials credentials_; + bool expired_; + bool found_; + TestCredentialProvider(std::string access_key="") + : credentials_{access_key}, expired_(false), found_(true) {} + + Result GetCredentials() override { + if(found_) { expired_ = true; return credentials_; } + return absl::NotFoundError("Credentials Not Found"); + } + bool IsExpired() override { return expired_; } +}; + +TEST(ChainedCredentialProviderTest, EmptyProvider) { + auto provider = ChainedCredentialProvider({}); + auto credentials = provider.GetCredentials(); + ASSERT_FALSE(credentials.ok()); + ASSERT_TRUE(provider.IsExpired()); +} + +// Tests that Credential Retrieval results in IsExpiry +// proxying the correct encapsulated provider +TEST(ChainedCredentialProviderTest, ChainedGetAndExpiryLogic) { + auto one = std::make_unique("key1"); + auto two = std::make_unique("key2"); + auto one_ptr = one.get(); + auto two_ptr = two.get(); + + auto providers = std::vector>(); + providers.emplace_back(std::move(one)); + providers.emplace_back(std::move(two)); + auto provider = ChainedCredentialProvider(std::move(providers)); + ASSERT_EQ(provider.IsExpired(), true); + + // First call for credentials result in one's credentials + TENSORSTORE_CHECK_OK_AND_ASSIGN(auto credentials, provider.GetCredentials()); + ASSERT_EQ(credentials.access_key, "key1"); + ASSERT_EQ(provider.IsExpired(), true); + ASSERT_EQ(one_ptr->expired_, true); + ASSERT_EQ(two_ptr->expired_, false); + + // Chained Provider proxies one for IsExpired calls + one_ptr->expired_ = false; + ASSERT_EQ(provider.IsExpired(), false); + + // Disable one's credentials and get new credentials (two) + one_ptr->found_ = false; + TENSORSTORE_CHECK_OK_AND_ASSIGN(credentials, provider.GetCredentials()); + ASSERT_EQ(credentials.access_key, "key2"); + ASSERT_EQ(provider.IsExpired(), true); + ASSERT_EQ(one_ptr->expired_, false); + ASSERT_EQ(two_ptr->expired_, true); + + // Chained Provider proxies two for IsExpired calls + two_ptr->expired_ = false; + ASSERT_EQ(provider.IsExpired(), false); + + // Disable two's credentials and get new credentials + two_ptr->found_ = false; + auto result = provider.GetCredentials(); + ASSERT_FALSE(result.ok()); + ASSERT_EQ(provider.IsExpired(), true); +} + + +} // namespace From 97e68b32799a334b988594d06de0a3f49b1fb225 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Sat, 23 Sep 2023 20:18:37 +0200 Subject: [PATCH 15/63] Move s3 credentials functionality into credentials subfolder --- tensorstore/kvstore/s3/BUILD | 90 +------------------ tensorstore/kvstore/s3/credentials/BUILD | 89 ++++++++++++++++++ .../aws_credential_provider.cc | 13 ++- .../aws_credential_provider.h | 2 +- .../aws_credential_provider_test.cc | 2 +- .../s3/{ => credentials}/aws_credentials.h | 0 .../chained_credential_provider.cc | 2 +- .../chained_credential_provider.h | 2 +- .../chained_credential_provider_test.cc | 2 +- .../ec2_credential_provider.cc | 2 +- .../ec2_credential_provider.h | 2 +- .../environment_credential_provider.cc | 2 +- .../environment_credential_provider.h | 2 +- .../environment_credential_provider_test.cc | 2 +- .../file_credential_provider.cc | 2 +- .../file_credential_provider.h | 2 +- .../file_credential_provider_test.cc | 2 +- tensorstore/kvstore/s3/s3_key_value_store.cc | 2 +- tensorstore/kvstore/s3/s3_request_builder.cc | 2 +- tensorstore/kvstore/s3/s3_request_builder.h | 2 +- .../kvstore/s3/s3_request_builder_test.cc | 2 +- 21 files changed, 116 insertions(+), 110 deletions(-) create mode 100644 tensorstore/kvstore/s3/credentials/BUILD rename tensorstore/kvstore/s3/{ => credentials}/aws_credential_provider.cc (89%) rename tensorstore/kvstore/s3/{ => credentials}/aws_credential_provider.h (97%) rename tensorstore/kvstore/s3/{ => credentials}/aws_credential_provider_test.cc (96%) rename tensorstore/kvstore/s3/{ => credentials}/aws_credentials.h (100%) rename tensorstore/kvstore/s3/{ => credentials}/chained_credential_provider.cc (95%) rename tensorstore/kvstore/s3/{ => credentials}/chained_credential_provider.h (95%) rename tensorstore/kvstore/s3/{ => credentials}/chained_credential_provider_test.cc (97%) rename tensorstore/kvstore/s3/{ => credentials}/ec2_credential_provider.cc (92%) rename tensorstore/kvstore/s3/{ => credentials}/ec2_credential_provider.h (95%) rename tensorstore/kvstore/s3/{ => credentials}/environment_credential_provider.cc (95%) rename tensorstore/kvstore/s3/{ => credentials}/environment_credential_provider.h (95%) rename tensorstore/kvstore/s3/{ => credentials}/environment_credential_provider_test.cc (96%) rename tensorstore/kvstore/s3/{ => credentials}/file_credential_provider.cc (98%) rename tensorstore/kvstore/s3/{ => credentials}/file_credential_provider.h (96%) rename tensorstore/kvstore/s3/{ => credentials}/file_credential_provider_test.cc (98%) diff --git a/tensorstore/kvstore/s3/BUILD b/tensorstore/kvstore/s3/BUILD index 863903f15..f6b2efd65 100644 --- a/tensorstore/kvstore/s3/BUILD +++ b/tensorstore/kvstore/s3/BUILD @@ -44,7 +44,6 @@ tensorstore_cc_library( hdrs = ["s3_metadata.h"], local_defines = DEBUG_LOCAL_DEFINES, deps = [ - ":aws_credential_provider", ":s3_request_builder", ":s3_resource", ":s3_uri_utils", @@ -68,6 +67,7 @@ tensorstore_cc_library( "//tensorstore/kvstore:key_range", "//tensorstore/kvstore/gcs:validate", "//tensorstore/kvstore/gcs_http:rate_limiter", + "//tensorstore/kvstore/s3/credentials:aws_credential_provider", "//tensorstore/serialization", "//tensorstore/util:executor", "//tensorstore/util:future", @@ -170,11 +170,11 @@ tensorstore_cc_library( ], local_defines = DEBUG_LOCAL_DEFINES, deps = [ - ":aws_credential_provider", ":s3_uri_utils", "//tensorstore/internal:uri_utils", "//tensorstore/internal/digest:sha256", "//tensorstore/internal/http", + "//tensorstore/kvstore/s3/credentials:aws_credential_provider", "//tensorstore/kvstore:byte_range", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/log:absl_log", @@ -191,9 +191,9 @@ tensorstore_cc_test( size = "small", srcs = ["s3_request_builder_test.cc"], deps = [ - ":aws_credential_provider", ":s3_request_builder", "//tensorstore/internal/http", + "//tensorstore/kvstore/s3/credentials:aws_credential_provider", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/time", @@ -227,88 +227,6 @@ tensorstore_cc_test( ], ) -tensorstore_cc_library( - name = "aws_credential_provider", - srcs = [ - "aws_credential_provider.cc", - "chained_credential_provider.cc", - "environment_credential_provider.cc", - "file_credential_provider.cc", - "ec2_credential_provider.cc"], - hdrs = [ - "aws_credentials.h", - "aws_credential_provider.h", - "chained_credential_provider.h", - "environment_credential_provider.h", - "file_credential_provider.h", - "ec2_credential_provider.h"], - deps = [ - "//tensorstore/internal:env", - "//tensorstore/internal:no_destructor", - "//tensorstore/internal:path", - "//tensorstore/internal/http", - "//tensorstore/util:result", - "@com_google_absl//absl/log:absl_log", - "@com_google_absl//absl/status", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/synchronization", - "@com_google_absl//absl/time", - ], -) - -tensorstore_cc_test( - name = "aws_credential_provider_test", - srcs = ["aws_credential_provider_test.cc"], - deps = [ - ":aws_credential_provider", - "//tensorstore/internal:env", - "//tensorstore/internal:path", - "//tensorstore/internal:test_util", - "//tensorstore/internal/http:curl_transport", - "//tensorstore/util:result", - "//tensorstore/util:status_testutil", - "@com_google_googletest//:gtest_main", - ], -) - -tensorstore_cc_test( - name = "environmental_credential_provider_test", - srcs = ["environment_credential_provider_test.cc"], - deps = [ - ":aws_credential_provider", - "//tensorstore/internal:test_util", - "//tensorstore/util:result", - "//tensorstore/util:status_testutil", - "@com_google_googletest//:gtest_main", - ], -) - - -tensorstore_cc_test( - name = "file_credential_provider_test", - srcs = ["file_credential_provider_test.cc"], - deps = [ - ":aws_credential_provider", - "//tensorstore/internal:path", - "//tensorstore/internal:test_util", - "//tensorstore/util:result", - "//tensorstore/util:status_testutil", - "@com_google_googletest//:gtest_main", - ], -) - - -tensorstore_cc_test( - name = "chained_credential_provider_test", - srcs = ["chained_credential_provider_test.cc"], - deps = [ - ":aws_credential_provider", - "//tensorstore/internal:test_util", - "//tensorstore/util:result", - "//tensorstore/util:status_testutil", - "@com_google_googletest//:gtest_main", - ], -) tensorstore_cc_test( @@ -320,7 +238,7 @@ tensorstore_cc_test( "skip-cmake", # localstack is python, which will not work in bazel_to_cmake. ], deps = [ - ":aws_credential_provider", + "//tensorstore/kvstore/s3/aws_credential_provider", ":s3_request_builder", "//tensorstore:context", "//tensorstore:json_serialization_options_base", diff --git a/tensorstore/kvstore/s3/credentials/BUILD b/tensorstore/kvstore/s3/credentials/BUILD new file mode 100644 index 000000000..aae5b26e6 --- /dev/null +++ b/tensorstore/kvstore/s3/credentials/BUILD @@ -0,0 +1,89 @@ +load("@bazel_skylib//rules:common_settings.bzl", "bool_flag") +load("//bazel:tensorstore.bzl", "tensorstore_cc_library", "tensorstore_cc_test") + +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +tensorstore_cc_library( + name = "aws_credential_provider", + srcs = [ + "aws_credential_provider.cc", + "chained_credential_provider.cc", + "environment_credential_provider.cc", + "file_credential_provider.cc", + "ec2_credential_provider.cc"], + hdrs = [ + "aws_credentials.h", + "aws_credential_provider.h", + "chained_credential_provider.h", + "environment_credential_provider.h", + "file_credential_provider.h", + "ec2_credential_provider.h"], + deps = [ + "//tensorstore/internal:env", + "//tensorstore/internal:no_destructor", + "//tensorstore/internal:path", + "//tensorstore/internal/http", + "//tensorstore/util:result", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/synchronization", + "@com_google_absl//absl/time", + ], +) + +tensorstore_cc_test( + name = "aws_credential_provider_test", + srcs = ["aws_credential_provider_test.cc"], + deps = [ + ":aws_credential_provider", + "//tensorstore/internal:env", + "//tensorstore/internal:path", + "//tensorstore/internal:test_util", + "//tensorstore/internal/http:curl_transport", + "//tensorstore/util:result", + "//tensorstore/util:status_testutil", + "@com_google_googletest//:gtest_main", + ], +) + +tensorstore_cc_test( + name = "environmental_credential_provider_test", + srcs = ["environment_credential_provider_test.cc"], + deps = [ + ":aws_credential_provider", + "//tensorstore/internal:test_util", + "//tensorstore/util:result", + "//tensorstore/util:status_testutil", + "@com_google_googletest//:gtest_main", + ], +) + + +tensorstore_cc_test( + name = "file_credential_provider_test", + srcs = ["file_credential_provider_test.cc"], + deps = [ + ":aws_credential_provider", + "//tensorstore/internal:path", + "//tensorstore/internal:test_util", + "//tensorstore/util:result", + "//tensorstore/util:status_testutil", + "@com_google_googletest//:gtest_main", + ], +) + + +tensorstore_cc_test( + name = "chained_credential_provider_test", + srcs = ["chained_credential_provider_test.cc"], + deps = [ + ":aws_credential_provider", + "//tensorstore/internal:test_util", + "//tensorstore/util:result", + "//tensorstore/util:status_testutil", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/tensorstore/kvstore/s3/aws_credential_provider.cc b/tensorstore/kvstore/s3/credentials/aws_credential_provider.cc similarity index 89% rename from tensorstore/kvstore/s3/aws_credential_provider.cc rename to tensorstore/kvstore/s3/credentials/aws_credential_provider.cc index 512f7c5f3..2c80dc64b 100644 --- a/tensorstore/kvstore/s3/aws_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/aws_credential_provider.cc @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "tensorstore/kvstore/s3/aws_credential_provider.h" -#include "tensorstore/kvstore/s3/environment_credential_provider.h" -#include "tensorstore/kvstore/s3/file_credential_provider.h" -#include "tensorstore/kvstore/s3/ec2_credential_provider.h" -#include "tensorstore/kvstore/s3/chained_credential_provider.h" +#include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" +#include "tensorstore/kvstore/s3/credentials/environment_credential_provider.h" +#include "tensorstore/kvstore/s3/credentials/file_credential_provider.h" +#include "tensorstore/kvstore/s3/credentials/ec2_credential_provider.h" +#include "tensorstore/kvstore/s3/credentials/chained_credential_provider.h" #include #include @@ -104,8 +104,7 @@ Result> GetAwsCredentialProvider( if (credentials.ok()) return credentials; } - return internal_kvstore_s3::GetDefaultAwsCredentialProvider(profile, - transport); + return GetDefaultAwsCredentialProvider(profile, transport); } } // namespace internal_kvstore_s3 diff --git a/tensorstore/kvstore/s3/aws_credential_provider.h b/tensorstore/kvstore/s3/credentials/aws_credential_provider.h similarity index 97% rename from tensorstore/kvstore/s3/aws_credential_provider.h rename to tensorstore/kvstore/s3/credentials/aws_credential_provider.h index f87d70e93..4e0ecb3b8 100644 --- a/tensorstore/kvstore/s3/aws_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/aws_credential_provider.h @@ -18,7 +18,7 @@ #include #include -#include "tensorstore/kvstore/s3/aws_credentials.h" +#include "tensorstore/kvstore/s3/credentials/aws_credentials.h" #include "tensorstore/internal/http/http_transport.h" #include "tensorstore/util/result.h" diff --git a/tensorstore/kvstore/s3/aws_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/aws_credential_provider_test.cc similarity index 96% rename from tensorstore/kvstore/s3/aws_credential_provider_test.cc rename to tensorstore/kvstore/s3/credentials/aws_credential_provider_test.cc index a19be1998..141a5fb41 100644 --- a/tensorstore/kvstore/s3/aws_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/aws_credential_provider_test.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "tensorstore/kvstore/s3/aws_credential_provider.h" +#include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" #include #include diff --git a/tensorstore/kvstore/s3/aws_credentials.h b/tensorstore/kvstore/s3/credentials/aws_credentials.h similarity index 100% rename from tensorstore/kvstore/s3/aws_credentials.h rename to tensorstore/kvstore/s3/credentials/aws_credentials.h diff --git a/tensorstore/kvstore/s3/chained_credential_provider.cc b/tensorstore/kvstore/s3/credentials/chained_credential_provider.cc similarity index 95% rename from tensorstore/kvstore/s3/chained_credential_provider.cc rename to tensorstore/kvstore/s3/credentials/chained_credential_provider.cc index cc6d5f2c3..0a2fa9e03 100644 --- a/tensorstore/kvstore/s3/chained_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/chained_credential_provider.cc @@ -14,7 +14,7 @@ #include "absl/status/status.h" #include "absl/strings/str_join.h" -#include "tensorstore/kvstore/s3/chained_credential_provider.h" +#include "tensorstore/kvstore/s3/credentials/chained_credential_provider.h" namespace tensorstore { namespace internal_kvstore_s3 { diff --git a/tensorstore/kvstore/s3/chained_credential_provider.h b/tensorstore/kvstore/s3/credentials/chained_credential_provider.h similarity index 95% rename from tensorstore/kvstore/s3/chained_credential_provider.h rename to tensorstore/kvstore/s3/credentials/chained_credential_provider.h index 526ab4aa4..762bc71b9 100644 --- a/tensorstore/kvstore/s3/chained_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/chained_credential_provider.h @@ -18,7 +18,7 @@ #include #include -#include "tensorstore/kvstore/s3/aws_credential_provider.h" +#include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" namespace tensorstore { namespace internal_kvstore_s3 { diff --git a/tensorstore/kvstore/s3/chained_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/chained_credential_provider_test.cc similarity index 97% rename from tensorstore/kvstore/s3/chained_credential_provider_test.cc rename to tensorstore/kvstore/s3/credentials/chained_credential_provider_test.cc index 59e9fb6ae..4694b177c 100644 --- a/tensorstore/kvstore/s3/chained_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/chained_credential_provider_test.cc @@ -14,7 +14,7 @@ #include -#include "tensorstore/kvstore/s3/chained_credential_provider.h" +#include "tensorstore/kvstore/s3/credentials/chained_credential_provider.h" #include #include diff --git a/tensorstore/kvstore/s3/ec2_credential_provider.cc b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc similarity index 92% rename from tensorstore/kvstore/s3/ec2_credential_provider.cc rename to tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc index 58385a91a..3c9582e67 100644 --- a/tensorstore/kvstore/s3/ec2_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc @@ -13,7 +13,7 @@ // limitations under the License. #include "absl/status/status.h" -#include "tensorstore/kvstore/s3/ec2_credential_provider.h" +#include "tensorstore/kvstore/s3/credentials/ec2_credential_provider.h" namespace tensorstore { namespace internal_kvstore_s3 { diff --git a/tensorstore/kvstore/s3/ec2_credential_provider.h b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h similarity index 95% rename from tensorstore/kvstore/s3/ec2_credential_provider.h rename to tensorstore/kvstore/s3/credentials/ec2_credential_provider.h index ab271b85f..4e3d07090 100644 --- a/tensorstore/kvstore/s3/ec2_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h @@ -15,7 +15,7 @@ #ifndef TENSORSTORE_KVSTORE_S3_EC2_CREDENTIAL_PROVIDER_H #define TENSORSTORE_KVSTORE_S3_EC2_CREDENTIAL_PROVIDER_H -#include "tensorstore/kvstore/s3/aws_credential_provider.h" +#include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" namespace tensorstore { namespace internal_kvstore_s3 { diff --git a/tensorstore/kvstore/s3/environment_credential_provider.cc b/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc similarity index 95% rename from tensorstore/kvstore/s3/environment_credential_provider.cc rename to tensorstore/kvstore/s3/credentials/environment_credential_provider.cc index 22e689aee..90bea7022 100644 --- a/tensorstore/kvstore/s3/environment_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "tensorstore/kvstore/s3/environment_credential_provider.h" +#include "tensorstore/kvstore/s3/credentials/environment_credential_provider.h" #include "tensorstore/internal/env.h" using ::tensorstore::internal::GetEnv; diff --git a/tensorstore/kvstore/s3/environment_credential_provider.h b/tensorstore/kvstore/s3/credentials/environment_credential_provider.h similarity index 95% rename from tensorstore/kvstore/s3/environment_credential_provider.h rename to tensorstore/kvstore/s3/credentials/environment_credential_provider.h index e1b44988d..34fdcf342 100644 --- a/tensorstore/kvstore/s3/environment_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/environment_credential_provider.h @@ -15,7 +15,7 @@ #ifndef TENSORSTORE_KVSTORE_S3_ENVIRONMENT_CREDENTIAL_PROVIDER_H #define TENSORSTORE_KVSTORE_S3_ENVIRONMENT_CREDENTIAL_PROVIDER_H -#include "tensorstore/kvstore/s3/aws_credential_provider.h" +#include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" #include "tensorstore/util/result.h" namespace tensorstore { diff --git a/tensorstore/kvstore/s3/environment_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/environment_credential_provider_test.cc similarity index 96% rename from tensorstore/kvstore/s3/environment_credential_provider_test.cc rename to tensorstore/kvstore/s3/credentials/environment_credential_provider_test.cc index b45d96bd1..111d988fb 100644 --- a/tensorstore/kvstore/s3/environment_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/environment_credential_provider_test.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "tensorstore/kvstore/s3/environment_credential_provider.h" +#include "tensorstore/kvstore/s3/credentials/environment_credential_provider.h" #include diff --git a/tensorstore/kvstore/s3/file_credential_provider.cc b/tensorstore/kvstore/s3/credentials/file_credential_provider.cc similarity index 98% rename from tensorstore/kvstore/s3/file_credential_provider.cc rename to tensorstore/kvstore/s3/credentials/file_credential_provider.cc index e144eb907..604726bc6 100644 --- a/tensorstore/kvstore/s3/file_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/file_credential_provider.cc @@ -16,7 +16,7 @@ #include #include "absl/strings/strip.h" -#include "tensorstore/kvstore/s3/file_credential_provider.h" +#include "tensorstore/kvstore/s3/credentials/file_credential_provider.h" #include "tensorstore/internal/env.h" #include "tensorstore/internal/path.h" diff --git a/tensorstore/kvstore/s3/file_credential_provider.h b/tensorstore/kvstore/s3/credentials/file_credential_provider.h similarity index 96% rename from tensorstore/kvstore/s3/file_credential_provider.h rename to tensorstore/kvstore/s3/credentials/file_credential_provider.h index 5bc92f8bd..dcc3a1201 100644 --- a/tensorstore/kvstore/s3/file_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/file_credential_provider.h @@ -15,7 +15,7 @@ #ifndef TENSORSTORE_KVSTORE_S3_FILE_CREDENTIAL_PROVIDER_H #define TENSORSTORE_KVSTORE_S3_FILE_CREDENTIAL_PROVIDER_H -#include "tensorstore/kvstore/s3/aws_credential_provider.h" +#include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" #include "tensorstore/util/result.h" namespace tensorstore { diff --git a/tensorstore/kvstore/s3/file_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/file_credential_provider_test.cc similarity index 98% rename from tensorstore/kvstore/s3/file_credential_provider_test.cc rename to tensorstore/kvstore/s3/credentials/file_credential_provider_test.cc index 5d9aa02d6..48ed23118 100644 --- a/tensorstore/kvstore/s3/file_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/file_credential_provider_test.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "tensorstore/kvstore/s3/file_credential_provider.h" +#include "tensorstore/kvstore/s3/credentials/file_credential_provider.h" #include #include diff --git a/tensorstore/kvstore/s3/s3_key_value_store.cc b/tensorstore/kvstore/s3/s3_key_value_store.cc index a5596360c..400a485d6 100644 --- a/tensorstore/kvstore/s3/s3_key_value_store.cc +++ b/tensorstore/kvstore/s3/s3_key_value_store.cc @@ -59,7 +59,7 @@ #include "tensorstore/kvstore/operations.h" #include "tensorstore/kvstore/read_result.h" #include "tensorstore/kvstore/registry.h" -#include "tensorstore/kvstore/s3/aws_credential_provider.h" +#include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" #include "tensorstore/kvstore/s3/s3_metadata.h" #include "tensorstore/kvstore/s3/s3_request_builder.h" #include "tensorstore/kvstore/s3/s3_resource.h" diff --git a/tensorstore/kvstore/s3/s3_request_builder.cc b/tensorstore/kvstore/s3/s3_request_builder.cc index 0b3517888..afd2c031f 100644 --- a/tensorstore/kvstore/s3/s3_request_builder.cc +++ b/tensorstore/kvstore/s3/s3_request_builder.cc @@ -41,7 +41,7 @@ #include "tensorstore/internal/digest/sha256.h" #include "tensorstore/internal/http/http_request.h" #include "tensorstore/internal/uri_utils.h" -#include "tensorstore/kvstore/s3/aws_credential_provider.h" +#include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" #include "tensorstore/kvstore/s3/s3_uri_utils.h" #ifndef TENSORSTORE_INTERNAL_S3_LOG_AWS4 diff --git a/tensorstore/kvstore/s3/s3_request_builder.h b/tensorstore/kvstore/s3/s3_request_builder.h index f76376b6b..485a07da2 100644 --- a/tensorstore/kvstore/s3/s3_request_builder.h +++ b/tensorstore/kvstore/s3/s3_request_builder.h @@ -26,7 +26,7 @@ #include "absl/time/time.h" #include "tensorstore/internal/http/http_request.h" #include "tensorstore/kvstore/byte_range.h" -#include "tensorstore/kvstore/s3/aws_credential_provider.h" +#include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" #include "tensorstore/kvstore/s3/s3_uri_utils.h" namespace tensorstore { diff --git a/tensorstore/kvstore/s3/s3_request_builder_test.cc b/tensorstore/kvstore/s3/s3_request_builder_test.cc index 56565556b..9b8016ae7 100644 --- a/tensorstore/kvstore/s3/s3_request_builder_test.cc +++ b/tensorstore/kvstore/s3/s3_request_builder_test.cc @@ -25,7 +25,7 @@ #include "absl/time/civil_time.h" #include "absl/time/time.h" #include "tensorstore/internal/http/http_request.h" -#include "tensorstore/kvstore/s3/aws_credential_provider.h" +#include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" using ::tensorstore::internal_kvstore_s3::AwsCredentials; using ::tensorstore::internal_kvstore_s3::S3RequestBuilder; From 8ce0df862647720c356fa6854d39abca117e4be0 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Sat, 23 Sep 2023 20:43:17 +0200 Subject: [PATCH 16/63] Remove expiry date from AwsCredentials --- tensorstore/kvstore/s3/credentials/aws_credentials.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tensorstore/kvstore/s3/credentials/aws_credentials.h b/tensorstore/kvstore/s3/credentials/aws_credentials.h index 72354416b..5b1490eef 100644 --- a/tensorstore/kvstore/s3/credentials/aws_credentials.h +++ b/tensorstore/kvstore/s3/credentials/aws_credentials.h @@ -18,7 +18,6 @@ #include #include "absl/time/time.h" -#include "absl/time/clock.h" namespace tensorstore { namespace internal_kvstore_s3 { @@ -37,11 +36,8 @@ struct AwsCredentials { std::string secret_key; /// AWS_SESSION_TOKEN std::string session_token; - /// Expiry Date - absl::Time expiry_date; bool IsAnonymous() const { return access_key.empty(); } - bool isExpired() const { return expiry_date < absl::Now(); } }; } // internal_kvstore_s3 From 569a3335f129af606e63509c9e11cadf443817e5 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Sat, 23 Sep 2023 21:09:10 +0200 Subject: [PATCH 17/63] Add an ExpiresAt interface to AwsCredentialProvider --- tensorstore/kvstore/s3/credentials/BUILD | 14 +++++++++++ .../s3/credentials/aws_credential_provider.h | 7 +++++- .../chained_credential_provider.cc | 10 +++++++- .../credentials/chained_credential_provider.h | 2 ++ .../chained_credential_provider_test.cc | 25 +++++++++++++++++-- .../environment_credential_provider.cc | 3 +-- 6 files changed, 55 insertions(+), 6 deletions(-) diff --git a/tensorstore/kvstore/s3/credentials/BUILD b/tensorstore/kvstore/s3/credentials/BUILD index aae5b26e6..edc27602a 100644 --- a/tensorstore/kvstore/s3/credentials/BUILD +++ b/tensorstore/kvstore/s3/credentials/BUILD @@ -9,6 +9,7 @@ tensorstore_cc_library( name = "aws_credential_provider", srcs = [ "aws_credential_provider.cc", + "cached_credential_provider.cc", "chained_credential_provider.cc", "environment_credential_provider.cc", "file_credential_provider.cc", @@ -16,6 +17,7 @@ tensorstore_cc_library( hdrs = [ "aws_credentials.h", "aws_credential_provider.h", + "cached_credential_provider.h", "chained_credential_provider.h", "environment_credential_provider.h", "file_credential_provider.h", @@ -87,3 +89,15 @@ tensorstore_cc_test( "@com_google_googletest//:gtest_main", ], ) + +tensorstore_cc_test( + name = "cached_credential_provider_test", + srcs = ["cached_credential_provider_test.cc"], + deps = [ + ":aws_credential_provider", + "//tensorstore/internal:test_util", + "//tensorstore/util:result", + "//tensorstore/util:status_testutil", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/tensorstore/kvstore/s3/credentials/aws_credential_provider.h b/tensorstore/kvstore/s3/credentials/aws_credential_provider.h index 4e0ecb3b8..db59e7297 100644 --- a/tensorstore/kvstore/s3/credentials/aws_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/aws_credential_provider.h @@ -18,6 +18,7 @@ #include #include +#include "absl/time/time.h" #include "tensorstore/kvstore/s3/credentials/aws_credentials.h" #include "tensorstore/internal/http/http_transport.h" #include "tensorstore/util/result.h" @@ -27,11 +28,15 @@ namespace internal_kvstore_s3 { /// Base class for S3 Credential Providers /// -/// Implementers should override GetCredentials +/// Implementers should override GetCredentials, IsExpired and, +/// if the credential source supports specifying an expiry date, ExpiresAt. class AwsCredentialProvider { public: virtual ~AwsCredentialProvider() = default; virtual Result GetCredentials() = 0; + virtual Result ExpiresAt() { + return absl::UnimplementedError("AwsCredentialProvider::ExpiresAt"); + }; virtual bool IsExpired() = 0; }; diff --git a/tensorstore/kvstore/s3/credentials/chained_credential_provider.cc b/tensorstore/kvstore/s3/credentials/chained_credential_provider.cc index 0a2fa9e03..315090915 100644 --- a/tensorstore/kvstore/s3/credentials/chained_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/chained_credential_provider.cc @@ -21,13 +21,21 @@ namespace internal_kvstore_s3 { bool ChainedCredentialProvider::IsExpired() { - if(last_provider_ < 0 || last_provider_ >= providers_.size()) { + if(!LastProviderValid()) { return true; } return providers_[last_provider_]->IsExpired(); } +Result ChainedCredentialProvider::ExpiresAt() { + if(!LastProviderValid()) { + return absl::UnimplementedError("ChainedCredentialProvider::ExpiresAt"); + } + + return providers_[last_provider_]->ExpiresAt(); +} + Result ChainedCredentialProvider::GetCredentials() { std::vector errors; last_provider_ = -1; diff --git a/tensorstore/kvstore/s3/credentials/chained_credential_provider.h b/tensorstore/kvstore/s3/credentials/chained_credential_provider.h index 762bc71b9..96cca3718 100644 --- a/tensorstore/kvstore/s3/credentials/chained_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/chained_credential_provider.h @@ -32,11 +32,13 @@ class ChainedCredentialProvider : public AwsCredentialProvider { std::vector> providers_; std::size_t last_provider_; + inline bool LastProviderValid() { return last_provider_ >= 0 && last_provider_ < providers_.size(); } public: ChainedCredentialProvider(std::vector> providers) : providers_(std::move(providers)), last_provider_(-1) {} Result GetCredentials() override; + Result ExpiresAt() override; bool IsExpired() override; }; diff --git a/tensorstore/kvstore/s3/credentials/chained_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/chained_credential_provider_test.cc index 4694b177c..6f076c221 100644 --- a/tensorstore/kvstore/s3/credentials/chained_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/chained_credential_provider_test.cc @@ -23,6 +23,7 @@ #include "tensorstore/util/status_testutil.h" using ::tensorstore::Future; +using ::tensorstore::MatchesStatus; using ::tensorstore::Result; using ::tensorstore::internal_kvstore_s3::AwsCredentials; using ::tensorstore::internal_kvstore_s3::AwsCredentialProvider; @@ -35,14 +36,22 @@ class TestCredentialProvider : public AwsCredentialProvider { AwsCredentials credentials_; bool expired_; bool found_; + bool has_expired_at_; TestCredentialProvider(std::string access_key="") - : credentials_{access_key}, expired_(false), found_(true) {} + : credentials_{access_key}, expired_(false), found_(true), + has_expired_at_(true) {} Result GetCredentials() override { if(found_) { expired_ = true; return credentials_; } return absl::NotFoundError("Credentials Not Found"); } bool IsExpired() override { return expired_; } + Result ExpiresAt() override { + if(has_expired_at_) { + return absl::Now(); + } + return absl::UnimplementedError("TestCredentialProvider::ExpiresAt"); + } }; TEST(ChainedCredentialProviderTest, EmptyProvider) { @@ -50,9 +59,10 @@ TEST(ChainedCredentialProviderTest, EmptyProvider) { auto credentials = provider.GetCredentials(); ASSERT_FALSE(credentials.ok()); ASSERT_TRUE(provider.IsExpired()); + EXPECT_THAT(provider.ExpiresAt(), MatchesStatus(absl::StatusCode::kUnimplemented)); } -// Tests that Credential Retrieval results in IsExpiry +// Tests that Credential Retrieval results in IsExpiry and ExpiresAt // proxying the correct encapsulated provider TEST(ChainedCredentialProviderTest, ChainedGetAndExpiryLogic) { auto one = std::make_unique("key1"); @@ -72,10 +82,13 @@ TEST(ChainedCredentialProviderTest, ChainedGetAndExpiryLogic) { ASSERT_EQ(provider.IsExpired(), true); ASSERT_EQ(one_ptr->expired_, true); ASSERT_EQ(two_ptr->expired_, false); + EXPECT_THAT(provider.ExpiresAt(), MatchesStatus(absl::StatusCode::kOk)); // Chained Provider proxies one for IsExpired calls one_ptr->expired_ = false; + one_ptr->has_expired_at_ = false; ASSERT_EQ(provider.IsExpired(), false); + EXPECT_THAT(provider.ExpiresAt(), MatchesStatus(absl::StatusCode::kUnimplemented)); // Disable one's credentials and get new credentials (two) one_ptr->found_ = false; @@ -84,16 +97,24 @@ TEST(ChainedCredentialProviderTest, ChainedGetAndExpiryLogic) { ASSERT_EQ(provider.IsExpired(), true); ASSERT_EQ(one_ptr->expired_, false); ASSERT_EQ(two_ptr->expired_, true); + EXPECT_THAT(provider.ExpiresAt(), MatchesStatus(absl::StatusCode::kOk)); // Chained Provider proxies two for IsExpired calls two_ptr->expired_ = false; + two_ptr->has_expired_at_ = false; ASSERT_EQ(provider.IsExpired(), false); + EXPECT_THAT(provider.ExpiresAt(), MatchesStatus(absl::StatusCode::kUnimplemented)); // Disable two's credentials and get new credentials two_ptr->found_ = false; auto result = provider.GetCredentials(); ASSERT_FALSE(result.ok()); ASSERT_EQ(provider.IsExpired(), true); + + // There's no valid provider, so ExpiresAt is also unimplemented + one_ptr->has_expired_at_ = true; + two_ptr->has_expired_at_ = true; + EXPECT_THAT(provider.ExpiresAt(), MatchesStatus(absl::StatusCode::kUnimplemented)); } diff --git a/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc b/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc index 90bea7022..a655f1879 100644 --- a/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc @@ -31,8 +31,7 @@ static constexpr char kEnvAwsSessionToken[] = "AWS_SESSION_TOKEN"; } // namespace -Result -EnvironmentCredentialProvider::GetCredentials() { +Result EnvironmentCredentialProvider::GetCredentials() { if (auto access_key = GetEnv(kEnvAwsAccessKeyId); access_key) { ABSL_LOG_FIRST_N(INFO, 1) << "Using Environment Variable " << kEnvAwsAccessKeyId; From e6e5040325d177c78af3da1b91e4cbfb4b70d4f7 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Sat, 23 Sep 2023 21:42:28 +0200 Subject: [PATCH 18/63] CachedCredentialProvider --- .../s3/credentials/aws_credential_provider.cc | 4 +- .../credentials/cached_credential_provider.cc | 57 +++++++++++++++++++ .../credentials/cached_credential_provider.h | 44 ++++++++++++++ .../cached_credential_provider_test.cc | 36 ++++++++++++ 4 files changed, 140 insertions(+), 1 deletion(-) create mode 100644 tensorstore/kvstore/s3/credentials/cached_credential_provider.cc create mode 100644 tensorstore/kvstore/s3/credentials/cached_credential_provider.h create mode 100644 tensorstore/kvstore/s3/credentials/cached_credential_provider_test.cc diff --git a/tensorstore/kvstore/s3/credentials/aws_credential_provider.cc b/tensorstore/kvstore/s3/credentials/aws_credential_provider.cc index 2c80dc64b..e64dfca47 100644 --- a/tensorstore/kvstore/s3/credentials/aws_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/aws_credential_provider.cc @@ -16,6 +16,7 @@ #include "tensorstore/kvstore/s3/credentials/environment_credential_provider.h" #include "tensorstore/kvstore/s3/credentials/file_credential_provider.h" #include "tensorstore/kvstore/s3/credentials/ec2_credential_provider.h" +#include "tensorstore/kvstore/s3/credentials/cached_credential_provider.h" #include "tensorstore/kvstore/s3/credentials/chained_credential_provider.h" #include @@ -58,7 +59,8 @@ Result> GetDefaultAwsCredentialProvider( providers.emplace_back(std::make_unique()); providers.emplace_back(std::make_unique(std::string(profile))); providers.emplace_back(std::make_unique(transport)); - return std::make_unique(std::move(providers)); + return std::make_unique( + std::make_unique(std::move(providers))); } struct AwsCredentialProviderRegistry { diff --git a/tensorstore/kvstore/s3/credentials/cached_credential_provider.cc b/tensorstore/kvstore/s3/credentials/cached_credential_provider.cc new file mode 100644 index 000000000..996a1855b --- /dev/null +++ b/tensorstore/kvstore/s3/credentials/cached_credential_provider.cc @@ -0,0 +1,57 @@ +// Copyright 2023 The TensorStore Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/synchronization/mutex.h" +#include "absl/status/status.h" +#include "tensorstore/util/result.h" +#include "tensorstore/kvstore/s3/credentials/cached_credential_provider.h" + +namespace tensorstore { +namespace internal_kvstore_s3 { + +namespace { + +} // namespace + + +bool CachedCredentialProvider::IsExpired() { + absl::ReaderMutexLock lock(&mutex_); + return IsExpiredLocked(credentials_); +} + +bool CachedCredentialProvider::IsExpiredLocked(const AwsCredentials & credentials) { + return !provider_ || credentials.IsAnonymous() || provider_->IsExpired(); +} + +Result +CachedCredentialProvider::ExpiresAt() { + absl::ReaderMutexLock lock(&mutex_); + TENSORSTORE_ASSIGN_OR_RETURN(auto expires_at, provider_->ExpiresAt()); + if(credentials_.IsAnonymous()) return absl::InfiniteFuture(); + return expires_at; +} + +Result +CachedCredentialProvider::GetCredentials() { + absl::WriterMutexLock lock(&mutex_); + if(!IsExpiredLocked(credentials_)) { + return credentials_; + } + + TENSORSTORE_ASSIGN_OR_RETURN(credentials_, provider_->GetCredentials()); + return credentials_; +} + +} // namespace internal_kvstore_s3 +} // namespace tensorstore diff --git a/tensorstore/kvstore/s3/credentials/cached_credential_provider.h b/tensorstore/kvstore/s3/credentials/cached_credential_provider.h new file mode 100644 index 000000000..862135df0 --- /dev/null +++ b/tensorstore/kvstore/s3/credentials/cached_credential_provider.h @@ -0,0 +1,44 @@ +// Copyright 2023 The TensorStore Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef TENSORSTORE_KVSTORE_S3_CACHED_CREDENTIAL_PROVIDER_H +#define TENSORSTORE_KVSTORE_S3_CACHED_CREDENTIAL_PROVIDER_H + +#include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" +#include "tensorstore/util/result.h" + +namespace tensorstore { +namespace internal_kvstore_s3 { + +// +class CachedCredentialProvider : public AwsCredentialProvider { + private: + absl::Mutex mutex_; + std::unique_ptr provider_ ABSL_GUARDED_BY(mutex_); + AwsCredentials credentials_ ABSL_GUARDED_BY(mutex_); + + public: + CachedCredentialProvider(std::unique_ptr provider) + : provider_(std::move(provider)) { assert(provider_); } + + Result GetCredentials() override ABSL_LOCKS_EXCLUDED(mutex_); + bool IsExpired() override ABSL_LOCKS_EXCLUDED(mutex_); + bool IsExpiredLocked(const AwsCredentials & credentials) ABSL_SHARED_LOCKS_REQUIRED(mutex_); + Result ExpiresAt() override ABSL_LOCKS_EXCLUDED(mutex_); +}; + +} // namespace internal_kvstore_s3 +} // namespace tensorstore + +#endif // TENSORSTORE_KVSTORE_S3_CACHED_CREDENTIAL_PROVIDER_H \ No newline at end of file diff --git a/tensorstore/kvstore/s3/credentials/cached_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/cached_credential_provider_test.cc new file mode 100644 index 000000000..51c4d449f --- /dev/null +++ b/tensorstore/kvstore/s3/credentials/cached_credential_provider_test.cc @@ -0,0 +1,36 @@ +// Copyright 2023 The TensorStore Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "tensorstore/kvstore/s3/credentials/cached_credential_provider.h" + +#include + +#include +#include +#include "tensorstore/internal/test_util.h" +#include "tensorstore/util/result.h" +#include "tensorstore/util/status_testutil.h" + +namespace { + +using ::tensorstore::Future; +using ::tensorstore::internal_kvstore_s3::CachedCredentialProvider; +using ::tensorstore::MatchesStatus; + + +TEST(CachedCredentialProviderTest, Simple) { + +} + +} // namespace From dd7f691ff9066238655950dc0619d1dcd9ad1125 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Sat, 30 Sep 2023 21:56:51 +0200 Subject: [PATCH 19/63] Expiry credential provider test separate compilation unit and test cases --- tensorstore/kvstore/s3/credentials/BUILD | 13 +++ .../s3/credentials/aws_credential_provider.h | 29 ++----- .../s3/credentials/ec2_credential_provider.h | 8 +- .../credentials/expiry_credential_provider.h | 55 +++++++++++++ .../expiry_credential_provider_test.cc | 82 +++++++++++++++++++ .../credentials/file_credential_provider.cc | 5 +- 6 files changed, 163 insertions(+), 29 deletions(-) create mode 100644 tensorstore/kvstore/s3/credentials/expiry_credential_provider.h create mode 100644 tensorstore/kvstore/s3/credentials/expiry_credential_provider_test.cc diff --git a/tensorstore/kvstore/s3/credentials/BUILD b/tensorstore/kvstore/s3/credentials/BUILD index edc27602a..4df9579f6 100644 --- a/tensorstore/kvstore/s3/credentials/BUILD +++ b/tensorstore/kvstore/s3/credentials/BUILD @@ -20,6 +20,7 @@ tensorstore_cc_library( "cached_credential_provider.h", "chained_credential_provider.h", "environment_credential_provider.h", + "expiry_credential_provider.h", "file_credential_provider.h", "ec2_credential_provider.h"], deps = [ @@ -63,6 +64,18 @@ tensorstore_cc_test( ], ) +tensorstore_cc_test( + name = "expiry_credential_provider_test", + srcs = ["expiry_credential_provider_test.cc"], + deps = [ + ":aws_credential_provider", + "//tensorstore/internal:test_util", + "//tensorstore/util:result", + "//tensorstore/util:status_testutil", + "@com_google_googletest//:gtest_main", + ], +) + tensorstore_cc_test( name = "file_credential_provider_test", diff --git a/tensorstore/kvstore/s3/credentials/aws_credential_provider.h b/tensorstore/kvstore/s3/credentials/aws_credential_provider.h index db59e7297..dfe02183c 100644 --- a/tensorstore/kvstore/s3/credentials/aws_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/aws_credential_provider.h @@ -31,31 +31,16 @@ namespace internal_kvstore_s3 { /// Implementers should override GetCredentials, IsExpired and, /// if the credential source supports specifying an expiry date, ExpiresAt. class AwsCredentialProvider { - public: - virtual ~AwsCredentialProvider() = default; - virtual Result GetCredentials() = 0; - virtual Result ExpiresAt() { - return absl::UnimplementedError("AwsCredentialProvider::ExpiresAt"); - }; - virtual bool IsExpired() = 0; -}; - -/// Provides anonymous credentials -class StaticCredentialProvider : public AwsCredentialProvider { - private: - AwsCredentials credentials_; public: - StaticCredentialProvider(std::string_view access_key="", - std::string_view secret_key="", - std::string_view session_token="") - : credentials_{std::string{access_key}, - std::string{secret_key}, - std::string{session_token}} {} - - Result GetCredentials() override { return credentials_; } - bool IsExpired() override { return false; } + virtual ~AwsCredentialProvider() = default; + virtual Result GetCredentials() = 0; + virtual bool IsExpired() = 0; + virtual Result ExpiresAt() { + return absl::UnimplementedError("AwsCredentialProvider::ExpiresAt"); + }; }; + using AwsCredentialProviderFn = std::function>()>; diff --git a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h index 4e3d07090..c018cbbce 100644 --- a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h @@ -15,25 +15,23 @@ #ifndef TENSORSTORE_KVSTORE_S3_EC2_CREDENTIAL_PROVIDER_H #define TENSORSTORE_KVSTORE_S3_EC2_CREDENTIAL_PROVIDER_H -#include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" +#include "tensorstore/kvstore/s3/credentials/expiry_credential_provider.h" namespace tensorstore { namespace internal_kvstore_s3 { /// Provides S3 credentials from the EC2 Metadata server /// if running within AWS -class EC2MetadataCredentialProvider : public AwsCredentialProvider { +class EC2MetadataCredentialProvider : public ExpiryCredentialProvider { private: std::shared_ptr transport_; - bool retrieved_; public: EC2MetadataCredentialProvider( std::shared_ptr transport) - : transport_(std::move(transport)), retrieved_(false) {} + : transport_(std::move(transport)) {} Result GetCredentials() override; - bool IsExpired() override { return retrieved_; } }; } // internal_kvstore_s3 diff --git a/tensorstore/kvstore/s3/credentials/expiry_credential_provider.h b/tensorstore/kvstore/s3/credentials/expiry_credential_provider.h new file mode 100644 index 000000000..f631e6526 --- /dev/null +++ b/tensorstore/kvstore/s3/credentials/expiry_credential_provider.h @@ -0,0 +1,55 @@ +// Copyright 2023 The TensorStore Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef TENSORSTORE_KVSTORE_S3_EXPIRY_CREDENTIAL_PROVIDER_H +#define TENSORSTORE_KVSTORE_S3_EXPIRY_CREDENTIAL_PROVIDER_H + +#include "absl/functional/function_ref.h" +#include "tensorstore/util/result.h" +#include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" + +namespace tensorstore { +namespace internal_kvstore_s3 { + +/// Abstract class providing credentials with a time-based expiry. +/// Implementers overriding this class should call SetExpiration +/// when overriding the GetCredentials function. +/// In addition to an expiry date, SetExpiration can also be supplied +/// with a duration argument introduces jitter into the expiry to prevent +/// retrieval of expired credentials. +/// By default is constructed with absl::Now as a clock function, +/// but can be supplied with a custom clock for testing purposes. +class ExpiryCredentialProvider : public AwsCredentialProvider { + private: + absl::Time expiration_; + absl::FunctionRef clock_; + + public: + ExpiryCredentialProvider(const absl::FunctionRef & clock=absl::Now) + : expiration_(absl::InfinitePast()), clock_(clock) {} + void SetExpiration(const absl::Time & expiration, + const absl::Duration & window=absl::Seconds(0)) { + expiration_ = expiration; + if(window > absl::Seconds(0)) { + expiration_ -= window; + } + } + virtual bool IsExpired() override { return expiration_ < clock_(); }; + virtual Result ExpiresAt() override { return expiration_; }; +}; + +} // namespace internal_kvstore_s3 +} // namespace tenstorstore + +#endif // TENSORSTORE_KVSTORE_S3_EXPIRY_CREDENTIAL_PROVIDER_H \ No newline at end of file diff --git a/tensorstore/kvstore/s3/credentials/expiry_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/expiry_credential_provider_test.cc new file mode 100644 index 000000000..4ebea6770 --- /dev/null +++ b/tensorstore/kvstore/s3/credentials/expiry_credential_provider_test.cc @@ -0,0 +1,82 @@ +// Copyright 2023 The TensorStore Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "tensorstore/kvstore/s3/credentials/expiry_credential_provider.h" + +#include +#include +#include "tensorstore/internal/test_util.h" +#include "tensorstore/util/result.h" +#include "tensorstore/util/status_testutil.h" + +using ::tensorstore::Future; +using ::tensorstore::MatchesStatus; +using ::tensorstore::Result; +using ::tensorstore::internal_kvstore_s3::AwsCredentials; +using ::tensorstore::internal_kvstore_s3::ExpiryCredentialProvider; + +namespace { + +class TestCredentialProvider : public ExpiryCredentialProvider { + public: + TestCredentialProvider(const absl::FunctionRef & clock=absl::Now) + : ExpiryCredentialProvider(clock) {} + + Result GetCredentials() override { + return AwsCredentials{}; + } +}; + +TEST(ExpiryCredentialProviderTest, TestExpiry) { + /// Configure provider with a stuck clock + auto provider = TestCredentialProvider{[]() { + return absl::FromCivil(absl::CivilSecond(2023, 9, 6, 0, 4, 03), absl::UTCTimeZone()); + }}; + ASSERT_TRUE(provider.IsExpired()); + EXPECT_THAT(provider.ExpiresAt(), absl::InfinitePast()); + + auto utc = absl::UTCTimeZone(); + auto zero_sec = absl::Seconds(0); + auto one_sec = absl::Seconds(1); + + /// One second before clock + provider.SetExpiration(absl::FromCivil(absl::CivilSecond(2023, 9, 6, 0, 4, 02), utc), zero_sec); + ASSERT_TRUE(provider.IsExpired()); + + // Matches clock exactly + provider.SetExpiration(absl::FromCivil(absl::CivilSecond(2023, 9, 6, 0, 4, 03), utc), zero_sec); + ASSERT_FALSE(provider.IsExpired()); + + // One second after clock + provider.SetExpiration(absl::FromCivil(absl::CivilSecond(2023, 9, 6, 0, 4, 04), utc), zero_sec); + ASSERT_FALSE(provider.IsExpired()); + + /// Test expiry with duration window + // One second before clock + provider.SetExpiration(absl::FromCivil(absl::CivilSecond(2023, 9, 6, 0, 4, 03), utc), one_sec); + ASSERT_TRUE(provider.IsExpired()); + + // Matches clock exactly + provider.SetExpiration(absl::FromCivil(absl::CivilSecond(2023, 9, 6, 0, 4, 04), utc), one_sec); + ASSERT_FALSE(provider.IsExpired()); + + // One second after clock + provider.SetExpiration(absl::FromCivil(absl::CivilSecond(2023, 9, 6, 0, 4, 05), utc), one_sec); + ASSERT_FALSE(provider.IsExpired()); +} + + +} // namespace diff --git a/tensorstore/kvstore/s3/credentials/file_credential_provider.cc b/tensorstore/kvstore/s3/credentials/file_credential_provider.cc index 604726bc6..038b76dee 100644 --- a/tensorstore/kvstore/s3/credentials/file_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/file_credential_provider.cc @@ -77,8 +77,9 @@ Result FileCredentialProvider::GetCredentials() { absl::StrCat("Could not open credentials file [", filename, "]")); } - std::string profile = !profile_.empty() ? std::string(profile_) : - GetEnv(kEnvAwsProfile).value_or(kDefaultProfile); + std::string profile = !profile_.empty() ? + std::string(profile_) : + GetEnv(kEnvAwsProfile).value_or(kDefaultProfile); AwsCredentials credentials; std::string section_name; From 3eae538763c8e3fe3969a2731d8e36605d4ec3b5 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Mon, 2 Oct 2023 15:11:50 +0200 Subject: [PATCH 20/63] Update test cases --- .../credentials/cached_credential_provider.cc | 5 +- .../credentials/cached_credential_provider.h | 5 +- .../cached_credential_provider_test.cc | 83 ++++++++++++++++++- .../credentials/expiry_credential_provider.h | 4 +- .../expiry_credential_provider_test.cc | 43 ++++++---- 5 files changed, 116 insertions(+), 24 deletions(-) diff --git a/tensorstore/kvstore/s3/credentials/cached_credential_provider.cc b/tensorstore/kvstore/s3/credentials/cached_credential_provider.cc index 996a1855b..20b503194 100644 --- a/tensorstore/kvstore/s3/credentials/cached_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/cached_credential_provider.cc @@ -37,15 +37,14 @@ bool CachedCredentialProvider::IsExpiredLocked(const AwsCredentials & credential Result CachedCredentialProvider::ExpiresAt() { absl::ReaderMutexLock lock(&mutex_); - TENSORSTORE_ASSIGN_OR_RETURN(auto expires_at, provider_->ExpiresAt()); if(credentials_.IsAnonymous()) return absl::InfiniteFuture(); - return expires_at; + return provider_->ExpiresAt(); } Result CachedCredentialProvider::GetCredentials() { absl::WriterMutexLock lock(&mutex_); - if(!IsExpiredLocked(credentials_)) { + if(!IsExpiredLocked(credentials_) || !provider_) { return credentials_; } diff --git a/tensorstore/kvstore/s3/credentials/cached_credential_provider.h b/tensorstore/kvstore/s3/credentials/cached_credential_provider.h index 862135df0..917eb1e52 100644 --- a/tensorstore/kvstore/s3/credentials/cached_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/cached_credential_provider.h @@ -28,13 +28,14 @@ class CachedCredentialProvider : public AwsCredentialProvider { std::unique_ptr provider_ ABSL_GUARDED_BY(mutex_); AwsCredentials credentials_ ABSL_GUARDED_BY(mutex_); + bool IsExpiredLocked(const AwsCredentials & credentials) ABSL_SHARED_LOCKS_REQUIRED(mutex_); + public: CachedCredentialProvider(std::unique_ptr provider) - : provider_(std::move(provider)) { assert(provider_); } + : provider_(std::move(provider)) {} Result GetCredentials() override ABSL_LOCKS_EXCLUDED(mutex_); bool IsExpired() override ABSL_LOCKS_EXCLUDED(mutex_); - bool IsExpiredLocked(const AwsCredentials & credentials) ABSL_SHARED_LOCKS_REQUIRED(mutex_); Result ExpiresAt() override ABSL_LOCKS_EXCLUDED(mutex_); }; diff --git a/tensorstore/kvstore/s3/credentials/cached_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/cached_credential_provider_test.cc index 51c4d449f..81947c78f 100644 --- a/tensorstore/kvstore/s3/credentials/cached_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/cached_credential_provider_test.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "tensorstore/kvstore/s3/credentials/cached_credential_provider.h" +#include "tensorstore/kvstore/s3/credentials/expiry_credential_provider.h" #include @@ -22,15 +23,91 @@ #include "tensorstore/util/result.h" #include "tensorstore/util/status_testutil.h" -namespace { - using ::tensorstore::Future; +using ::tensorstore::Result; +using ::tensorstore::internal_kvstore_s3::AwsCredentials; using ::tensorstore::internal_kvstore_s3::CachedCredentialProvider; +using ::tensorstore::internal_kvstore_s3::ExpiryCredentialProvider; using ::tensorstore::MatchesStatus; +namespace { + +class TestCredentialProvider : public ExpiryCredentialProvider { + public: + int iteration = 0; + absl::FunctionRef clock_; + + TestCredentialProvider(absl::FunctionRef clock) : + ExpiryCredentialProvider(clock), clock_(clock) {} + + Result GetCredentials() override { + this->SetExpiration(clock_() + absl::Seconds(2)); + return AwsCredentials{"key", std::to_string(++iteration)}; + } +}; + + +TEST(CachedCredentialProviderTest, NullCase) { + auto provider = CachedCredentialProvider{nullptr}; + + // Base case + ASSERT_TRUE(provider.IsExpired()); + ASSERT_EQ(provider.ExpiresAt(), absl::InfiniteFuture()); + TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, provider.GetCredentials()); + ASSERT_TRUE(credentials.IsAnonymous()); + + // Idempotent + ASSERT_TRUE(provider.IsExpired()); + ASSERT_EQ(provider.ExpiresAt(), absl::InfiniteFuture()); + TENSORSTORE_ASSERT_OK_AND_ASSIGN(credentials, provider.GetCredentials()); + ASSERT_TRUE(credentials.IsAnonymous()); +} + +TEST(CachedCredentialProviderTest, ExpiringProvider) { + auto utc = absl::UTCTimeZone(); + auto frozen_time = absl::FromCivil(absl::CivilSecond(2023, 9, 6, 0, 4, 03), utc); + auto stuck_clock = [&frozen_time]() -> absl::Time { return frozen_time; }; + + auto test_provider = std::make_unique(stuck_clock); + auto test_prov_ptr = test_provider.get(); + auto cached_provider = CachedCredentialProvider{std::move(test_provider)}; + + // Base case, no credentials have been retrieved yet + ASSERT_TRUE(cached_provider.IsExpired()); + ASSERT_EQ(cached_provider.ExpiresAt(), absl::InfiniteFuture()); + ASSERT_EQ(test_prov_ptr->iteration, 0); + + // Retrieve some credentials + TENSORSTORE_CHECK_OK_AND_ASSIGN(auto credentials, cached_provider.GetCredentials()); + ASSERT_EQ(credentials.access_key, "key"); + ASSERT_EQ(credentials.secret_key, "1"); + ASSERT_EQ(test_prov_ptr->iteration, 1); + ASSERT_EQ(cached_provider.ExpiresAt(), frozen_time + absl::Seconds(2)); + ASSERT_FALSE(test_prov_ptr->IsExpired()); + ASSERT_FALSE(cached_provider.IsExpired()); + + // Idempotent when underlying credentials not expired + TENSORSTORE_CHECK_OK_AND_ASSIGN(credentials, cached_provider.GetCredentials()); + ASSERT_EQ(credentials.access_key, "key"); + ASSERT_EQ(credentials.secret_key, "1"); + ASSERT_EQ(test_prov_ptr->iteration, 1); + ASSERT_EQ(cached_provider.ExpiresAt(), frozen_time + absl::Seconds(2)); + ASSERT_FALSE(test_prov_ptr->IsExpired()); + ASSERT_FALSE(cached_provider.IsExpired()); -TEST(CachedCredentialProviderTest, Simple) { + // Advance clock forward + frozen_time += absl::Seconds(2.5); + ASSERT_TRUE(test_prov_ptr->IsExpired()); + ASSERT_TRUE(cached_provider.IsExpired()); + // A new set of credentials is retrieved due to expiry + TENSORSTORE_CHECK_OK_AND_ASSIGN(credentials, cached_provider.GetCredentials()); + ASSERT_EQ(credentials.access_key, "key"); + ASSERT_EQ(credentials.secret_key, "2"); + ASSERT_EQ(test_prov_ptr->iteration, 2); + ASSERT_EQ(cached_provider.ExpiresAt(), frozen_time + absl::Seconds(2)); + ASSERT_FALSE(test_prov_ptr->IsExpired()); + ASSERT_FALSE(cached_provider.IsExpired()); } } // namespace diff --git a/tensorstore/kvstore/s3/credentials/expiry_credential_provider.h b/tensorstore/kvstore/s3/credentials/expiry_credential_provider.h index f631e6526..977788074 100644 --- a/tensorstore/kvstore/s3/credentials/expiry_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/expiry_credential_provider.h @@ -28,7 +28,7 @@ namespace internal_kvstore_s3 { /// In addition to an expiry date, SetExpiration can also be supplied /// with a duration argument introduces jitter into the expiry to prevent /// retrieval of expired credentials. -/// By default is constructed with absl::Now as a clock function, +/// Constructed by default with absl::Now as a clock function, /// but can be supplied with a custom clock for testing purposes. class ExpiryCredentialProvider : public AwsCredentialProvider { private: @@ -36,7 +36,7 @@ class ExpiryCredentialProvider : public AwsCredentialProvider { absl::FunctionRef clock_; public: - ExpiryCredentialProvider(const absl::FunctionRef & clock=absl::Now) + ExpiryCredentialProvider(absl::FunctionRef clock=absl::Now) : expiration_(absl::InfinitePast()), clock_(clock) {} void SetExpiration(const absl::Time & expiration, const absl::Duration & window=absl::Seconds(0)) { diff --git a/tensorstore/kvstore/s3/credentials/expiry_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/expiry_credential_provider_test.cc index 4ebea6770..c0350d7c0 100644 --- a/tensorstore/kvstore/s3/credentials/expiry_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/expiry_credential_provider_test.cc @@ -41,41 +41,56 @@ class TestCredentialProvider : public ExpiryCredentialProvider { }; TEST(ExpiryCredentialProviderTest, TestExpiry) { + auto utc = absl::UTCTimeZone(); + + // window durations + auto window_zero = absl::Seconds(0); + auto window_one = absl::Seconds(1); + + // Timestamps + auto two_seconds = absl::FromCivil(absl::CivilSecond(2023, 9, 6, 0, 4, 02), utc); + auto three_seconds = absl::FromCivil(absl::CivilSecond(2023, 9, 6, 0, 4, 03), utc); + auto four_seconds = absl::FromCivil(absl::CivilSecond(2023, 9, 6, 0, 4, 04), utc); + auto five_seconds = absl::FromCivil(absl::CivilSecond(2023, 9, 6, 0, 4, 05), utc); + + auto frozen_time = three_seconds; + auto stuck_clock = [&frozen_time]() -> absl::Time { return frozen_time; }; + /// Configure provider with a stuck clock - auto provider = TestCredentialProvider{[]() { - return absl::FromCivil(absl::CivilSecond(2023, 9, 6, 0, 4, 03), absl::UTCTimeZone()); - }}; + auto provider = TestCredentialProvider{stuck_clock}; ASSERT_TRUE(provider.IsExpired()); - EXPECT_THAT(provider.ExpiresAt(), absl::InfinitePast()); - - auto utc = absl::UTCTimeZone(); - auto zero_sec = absl::Seconds(0); - auto one_sec = absl::Seconds(1); + ASSERT_EQ(provider.ExpiresAt(), absl::InfinitePast()); /// One second before clock - provider.SetExpiration(absl::FromCivil(absl::CivilSecond(2023, 9, 6, 0, 4, 02), utc), zero_sec); + provider.SetExpiration(two_seconds, window_zero); ASSERT_TRUE(provider.IsExpired()); + ASSERT_EQ(provider.ExpiresAt(), two_seconds); // Matches clock exactly - provider.SetExpiration(absl::FromCivil(absl::CivilSecond(2023, 9, 6, 0, 4, 03), utc), zero_sec); + provider.SetExpiration(three_seconds, window_zero); ASSERT_FALSE(provider.IsExpired()); + ASSERT_EQ(provider.ExpiresAt(), three_seconds); // One second after clock - provider.SetExpiration(absl::FromCivil(absl::CivilSecond(2023, 9, 6, 0, 4, 04), utc), zero_sec); + provider.SetExpiration(four_seconds, window_zero); ASSERT_FALSE(provider.IsExpired()); + ASSERT_EQ(provider.ExpiresAt(), four_seconds); /// Test expiry with duration window // One second before clock - provider.SetExpiration(absl::FromCivil(absl::CivilSecond(2023, 9, 6, 0, 4, 03), utc), one_sec); + provider.SetExpiration(three_seconds, window_one); ASSERT_TRUE(provider.IsExpired()); + ASSERT_EQ(provider.ExpiresAt(), two_seconds); // Matches clock exactly - provider.SetExpiration(absl::FromCivil(absl::CivilSecond(2023, 9, 6, 0, 4, 04), utc), one_sec); + provider.SetExpiration(four_seconds, window_one); ASSERT_FALSE(provider.IsExpired()); + ASSERT_EQ(provider.ExpiresAt(), three_seconds); // One second after clock - provider.SetExpiration(absl::FromCivil(absl::CivilSecond(2023, 9, 6, 0, 4, 05), utc), one_sec); + provider.SetExpiration(five_seconds, window_one); ASSERT_FALSE(provider.IsExpired()); + ASSERT_EQ(provider.ExpiresAt(), four_seconds); } From 9a813af4c922a37166c273119085efcb66c38686 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Mon, 2 Oct 2023 15:58:54 +0200 Subject: [PATCH 21/63] IWYU --- tensorstore/kvstore/s3/credentials/BUILD | 1 - .../s3/credentials/aws_credential_provider.cc | 26 +++-------- .../s3/credentials/aws_credential_provider.h | 22 ++++++++- .../kvstore/s3/credentials/aws_credentials.h | 46 ------------------- .../credentials/cached_credential_provider.h | 2 +- .../chained_credential_provider.cc | 7 +++ .../credentials/chained_credential_provider.h | 2 + .../s3/credentials/ec2_credential_provider.cc | 1 + .../environment_credential_provider.cc | 1 + .../credentials/expiry_credential_provider.h | 1 + .../credentials/file_credential_provider.cc | 4 ++ .../s3/credentials/file_credential_provider.h | 2 + 12 files changed, 46 insertions(+), 69 deletions(-) delete mode 100644 tensorstore/kvstore/s3/credentials/aws_credentials.h diff --git a/tensorstore/kvstore/s3/credentials/BUILD b/tensorstore/kvstore/s3/credentials/BUILD index 4df9579f6..2301d0d85 100644 --- a/tensorstore/kvstore/s3/credentials/BUILD +++ b/tensorstore/kvstore/s3/credentials/BUILD @@ -15,7 +15,6 @@ tensorstore_cc_library( "file_credential_provider.cc", "ec2_credential_provider.cc"], hdrs = [ - "aws_credentials.h", "aws_credential_provider.h", "cached_credential_provider.h", "chained_credential_provider.h", diff --git a/tensorstore/kvstore/s3/credentials/aws_credential_provider.cc b/tensorstore/kvstore/s3/credentials/aws_credential_provider.cc index e64dfca47..6a11ab955 100644 --- a/tensorstore/kvstore/s3/credentials/aws_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/aws_credential_provider.cc @@ -12,36 +12,24 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" -#include "tensorstore/kvstore/s3/credentials/environment_credential_provider.h" -#include "tensorstore/kvstore/s3/credentials/file_credential_provider.h" -#include "tensorstore/kvstore/s3/credentials/ec2_credential_provider.h" -#include "tensorstore/kvstore/s3/credentials/cached_credential_provider.h" -#include "tensorstore/kvstore/s3/credentials/chained_credential_provider.h" - -#include -#include #include -#include #include #include -#include #include -#include "absl/log/absl_log.h" -#include "absl/status/status.h" -#include "absl/strings/ascii.h" -#include "absl/strings/str_cat.h" #include "absl/synchronization/mutex.h" -#include "tensorstore/internal/env.h" +#include "absl/strings/str_cat.h" #include "tensorstore/internal/http/http_transport.h" #include "tensorstore/internal/no_destructor.h" -#include "tensorstore/internal/path.h" +#include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" +#include "tensorstore/kvstore/s3/credentials/environment_credential_provider.h" +#include "tensorstore/kvstore/s3/credentials/file_credential_provider.h" +#include "tensorstore/kvstore/s3/credentials/ec2_credential_provider.h" +#include "tensorstore/kvstore/s3/credentials/cached_credential_provider.h" +#include "tensorstore/kvstore/s3/credentials/chained_credential_provider.h" #include "tensorstore/util/result.h" using ::tensorstore::Result; -using ::tensorstore::internal::GetEnv; -using ::tensorstore::internal::JoinPath; namespace tensorstore { namespace internal_kvstore_s3 { diff --git a/tensorstore/kvstore/s3/credentials/aws_credential_provider.h b/tensorstore/kvstore/s3/credentials/aws_credential_provider.h index dfe02183c..a083a76be 100644 --- a/tensorstore/kvstore/s3/credentials/aws_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/aws_credential_provider.h @@ -19,17 +19,35 @@ #include #include "absl/time/time.h" -#include "tensorstore/kvstore/s3/credentials/aws_credentials.h" #include "tensorstore/internal/http/http_transport.h" #include "tensorstore/util/result.h" namespace tensorstore { namespace internal_kvstore_s3 { +/// Holds AWS credentials +/// +/// Contains the access key, secret key and session token. +/// An empty access key implies anonymous access, +/// while the presence of a session token implies the use of +/// short-lived STS credentials +/// https://docs.aws.amazon.com/STS/latest/APIReference/welcome.html +struct AwsCredentials { + /// AWS_ACCESS_KEY_ID + std::string access_key; + /// AWS_SECRET_KEY_ID + std::string secret_key; + /// AWS_SESSION_TOKEN + std::string session_token; + + bool IsAnonymous() const { return access_key.empty(); } +}; + /// Base class for S3 Credential Providers /// /// Implementers should override GetCredentials, IsExpired and, -/// if the credential source supports specifying an expiry date, ExpiresAt. +/// if the credential source supports the +/// representation of an expiry date, ExpiresAt. class AwsCredentialProvider { public: virtual ~AwsCredentialProvider() = default; diff --git a/tensorstore/kvstore/s3/credentials/aws_credentials.h b/tensorstore/kvstore/s3/credentials/aws_credentials.h deleted file mode 100644 index 5b1490eef..000000000 --- a/tensorstore/kvstore/s3/credentials/aws_credentials.h +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2023 The TensorStore Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef TENSORSTORE_KVSTORE_S3_S3_CREDENTIALS_H -#define TENSORSTORE_KVSTORE_S3_S3_CREDENTIALS_H - -#include - -#include "absl/time/time.h" - -namespace tensorstore { -namespace internal_kvstore_s3 { - -/// Holds AWS credentials -/// -/// Contains the access key, secret key and session token. -/// An empty access key implies anonymous access, -/// while the presence of a session token implies the use of -/// short-lived STS credentials -/// https://docs.aws.amazon.com/STS/latest/APIReference/welcome.html -struct AwsCredentials { - /// AWS_ACCESS_KEY_ID - std::string access_key; - /// AWS_SECRET_KEY_ID - std::string secret_key; - /// AWS_SESSION_TOKEN - std::string session_token; - - bool IsAnonymous() const { return access_key.empty(); } -}; - -} // internal_kvstore_s3 -} // tensorstore - -#endif // TENSORSTORE_KVSTORE_S3_S3_CREDENTIALS_H \ No newline at end of file diff --git a/tensorstore/kvstore/s3/credentials/cached_credential_provider.h b/tensorstore/kvstore/s3/credentials/cached_credential_provider.h index 917eb1e52..32fb0d2c5 100644 --- a/tensorstore/kvstore/s3/credentials/cached_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/cached_credential_provider.h @@ -15,8 +15,8 @@ #ifndef TENSORSTORE_KVSTORE_S3_CACHED_CREDENTIAL_PROVIDER_H #define TENSORSTORE_KVSTORE_S3_CACHED_CREDENTIAL_PROVIDER_H -#include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" #include "tensorstore/util/result.h" +#include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" namespace tensorstore { namespace internal_kvstore_s3 { diff --git a/tensorstore/kvstore/s3/credentials/chained_credential_provider.cc b/tensorstore/kvstore/s3/credentials/chained_credential_provider.cc index 315090915..122f7f46d 100644 --- a/tensorstore/kvstore/s3/credentials/chained_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/chained_credential_provider.cc @@ -12,8 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include +#include +#include + +#include "tensorstore/util/result.h" #include "absl/status/status.h" #include "absl/strings/str_join.h" +#include "absl/time/time.h" +#include "tensorstore/util/result.h" #include "tensorstore/kvstore/s3/credentials/chained_credential_provider.h" namespace tensorstore { diff --git a/tensorstore/kvstore/s3/credentials/chained_credential_provider.h b/tensorstore/kvstore/s3/credentials/chained_credential_provider.h index 96cca3718..c28e193a7 100644 --- a/tensorstore/kvstore/s3/credentials/chained_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/chained_credential_provider.h @@ -18,7 +18,9 @@ #include #include +#include "absl/time/time.h" #include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" +#include "tensorstore/util/result.h" namespace tensorstore { namespace internal_kvstore_s3 { diff --git a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc index 3c9582e67..d672ca575 100644 --- a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "absl/status/status.h" +#include "tensorstore/util/result.h" #include "tensorstore/kvstore/s3/credentials/ec2_credential_provider.h" namespace tensorstore { diff --git a/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc b/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc index a655f1879..d9c09ed25 100644 --- a/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "absl/status/status.h" #include "tensorstore/kvstore/s3/credentials/environment_credential_provider.h" #include "tensorstore/internal/env.h" diff --git a/tensorstore/kvstore/s3/credentials/expiry_credential_provider.h b/tensorstore/kvstore/s3/credentials/expiry_credential_provider.h index 977788074..9de2388b6 100644 --- a/tensorstore/kvstore/s3/credentials/expiry_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/expiry_credential_provider.h @@ -16,6 +16,7 @@ #define TENSORSTORE_KVSTORE_S3_EXPIRY_CREDENTIAL_PROVIDER_H #include "absl/functional/function_ref.h" +#include "absl/time/time.h" #include "tensorstore/util/result.h" #include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" diff --git a/tensorstore/kvstore/s3/credentials/file_credential_provider.cc b/tensorstore/kvstore/s3/credentials/file_credential_provider.cc index 038b76dee..4d65e15eb 100644 --- a/tensorstore/kvstore/s3/credentials/file_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/file_credential_provider.cc @@ -13,12 +13,16 @@ // limitations under the License. #include +#include #include #include "absl/strings/strip.h" +#include "absl/strings/str_cat.h" +#include "absl/status/status.h" #include "tensorstore/kvstore/s3/credentials/file_credential_provider.h" #include "tensorstore/internal/env.h" #include "tensorstore/internal/path.h" +#include "tensorstore/util/result.h" using ::tensorstore::internal::GetEnv; using ::tensorstore::internal::JoinPath; diff --git a/tensorstore/kvstore/s3/credentials/file_credential_provider.h b/tensorstore/kvstore/s3/credentials/file_credential_provider.h index dcc3a1201..72b7644dc 100644 --- a/tensorstore/kvstore/s3/credentials/file_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/file_credential_provider.h @@ -15,6 +15,8 @@ #ifndef TENSORSTORE_KVSTORE_S3_FILE_CREDENTIAL_PROVIDER_H #define TENSORSTORE_KVSTORE_S3_FILE_CREDENTIAL_PROVIDER_H +#include + #include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" #include "tensorstore/util/result.h" From dffe7a1348453b2b581cc71261d0b43561a7e536 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Mon, 2 Oct 2023 16:21:38 +0200 Subject: [PATCH 22/63] Remove empty test case --- tensorstore/kvstore/s3/credentials/BUILD | 15 ---- .../aws_credential_provider_test.cc | 69 ------------------- 2 files changed, 84 deletions(-) delete mode 100644 tensorstore/kvstore/s3/credentials/aws_credential_provider_test.cc diff --git a/tensorstore/kvstore/s3/credentials/BUILD b/tensorstore/kvstore/s3/credentials/BUILD index 2301d0d85..4b3cbbc80 100644 --- a/tensorstore/kvstore/s3/credentials/BUILD +++ b/tensorstore/kvstore/s3/credentials/BUILD @@ -36,21 +36,6 @@ tensorstore_cc_library( ], ) -tensorstore_cc_test( - name = "aws_credential_provider_test", - srcs = ["aws_credential_provider_test.cc"], - deps = [ - ":aws_credential_provider", - "//tensorstore/internal:env", - "//tensorstore/internal:path", - "//tensorstore/internal:test_util", - "//tensorstore/internal/http:curl_transport", - "//tensorstore/util:result", - "//tensorstore/util:status_testutil", - "@com_google_googletest//:gtest_main", - ], -) - tensorstore_cc_test( name = "environmental_credential_provider_test", srcs = ["environment_credential_provider_test.cc"], diff --git a/tensorstore/kvstore/s3/credentials/aws_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/aws_credential_provider_test.cc deleted file mode 100644 index 141a5fb41..000000000 --- a/tensorstore/kvstore/s3/credentials/aws_credential_provider_test.cc +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2023 The TensorStore Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" - -#include -#include - -#include -#include -#include "tensorstore/internal/env.h" -#include "tensorstore/internal/http/curl_transport.h" -#include "tensorstore/internal/path.h" -#include "tensorstore/internal/test_util.h" -#include "tensorstore/util/result.h" -#include "tensorstore/util/status_testutil.h" - -namespace { - -using ::tensorstore::Future; -using ::tensorstore::internal::GetEnv; -using ::tensorstore::internal::SetEnv; -using ::tensorstore::internal::UnsetEnv; -using ::tensorstore::internal_http::HttpRequest; -using ::tensorstore::internal_http::HttpResponse; -using ::tensorstore::internal_http::HttpTransport; -using ::tensorstore::internal_kvstore_s3::GetAwsCredentialProvider; - - -/// Cause EC2Metadata to always fail -class NotFoundTransport : public HttpTransport { -public: - Future IssueRequest(const HttpRequest& request, - absl::Cord payload, - absl::Duration request_timeout, - absl::Duration connect_timeout) override { - return HttpResponse{404, absl::Cord(), {}}; - } -}; - - -class AwsCredentialProviderTest : public ::testing::Test { - protected: - std::shared_ptr transport_; - - void SetUp() override { - // Make sure that env vars are not set. - for (const char* var : - {"AWS_SHARED_CREDENTIALS_FILE", "AWS_ACCESS_KEY_ID", - "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "AWS_PROFILE"}) { - UnsetEnv(var); - } - - transport_ = std::make_shared(); - } -}; - -} // namespace From 8955bf8a9024c025aa7d5dd5b996d25b39266370 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Tue, 3 Oct 2023 11:26:36 +0200 Subject: [PATCH 23/63] Forbid encapsulating nullptr's in CachedCredentialsProvider --- .../s3/credentials/cached_credential_provider.cc | 4 ++-- .../s3/credentials/cached_credential_provider.h | 2 +- .../cached_credential_provider_test.cc | 16 ---------------- 3 files changed, 3 insertions(+), 19 deletions(-) diff --git a/tensorstore/kvstore/s3/credentials/cached_credential_provider.cc b/tensorstore/kvstore/s3/credentials/cached_credential_provider.cc index 20b503194..32010cda6 100644 --- a/tensorstore/kvstore/s3/credentials/cached_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/cached_credential_provider.cc @@ -31,7 +31,7 @@ bool CachedCredentialProvider::IsExpired() { } bool CachedCredentialProvider::IsExpiredLocked(const AwsCredentials & credentials) { - return !provider_ || credentials.IsAnonymous() || provider_->IsExpired(); + return credentials.IsAnonymous() || provider_->IsExpired(); } Result @@ -44,7 +44,7 @@ CachedCredentialProvider::ExpiresAt() { Result CachedCredentialProvider::GetCredentials() { absl::WriterMutexLock lock(&mutex_); - if(!IsExpiredLocked(credentials_) || !provider_) { + if(!IsExpiredLocked(credentials_)) { return credentials_; } diff --git a/tensorstore/kvstore/s3/credentials/cached_credential_provider.h b/tensorstore/kvstore/s3/credentials/cached_credential_provider.h index 32fb0d2c5..b70ad596a 100644 --- a/tensorstore/kvstore/s3/credentials/cached_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/cached_credential_provider.h @@ -32,7 +32,7 @@ class CachedCredentialProvider : public AwsCredentialProvider { public: CachedCredentialProvider(std::unique_ptr provider) - : provider_(std::move(provider)) {} + : provider_(std::move(provider)) { assert(provider_); } Result GetCredentials() override ABSL_LOCKS_EXCLUDED(mutex_); bool IsExpired() override ABSL_LOCKS_EXCLUDED(mutex_); diff --git a/tensorstore/kvstore/s3/credentials/cached_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/cached_credential_provider_test.cc index 81947c78f..4aac59f93 100644 --- a/tensorstore/kvstore/s3/credentials/cached_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/cached_credential_provider_test.cc @@ -47,22 +47,6 @@ class TestCredentialProvider : public ExpiryCredentialProvider { }; -TEST(CachedCredentialProviderTest, NullCase) { - auto provider = CachedCredentialProvider{nullptr}; - - // Base case - ASSERT_TRUE(provider.IsExpired()); - ASSERT_EQ(provider.ExpiresAt(), absl::InfiniteFuture()); - TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, provider.GetCredentials()); - ASSERT_TRUE(credentials.IsAnonymous()); - - // Idempotent - ASSERT_TRUE(provider.IsExpired()); - ASSERT_EQ(provider.ExpiresAt(), absl::InfiniteFuture()); - TENSORSTORE_ASSERT_OK_AND_ASSIGN(credentials, provider.GetCredentials()); - ASSERT_TRUE(credentials.IsAnonymous()); -} - TEST(CachedCredentialProviderTest, ExpiringProvider) { auto utc = absl::UTCTimeZone(); auto frozen_time = absl::FromCivil(absl::CivilSecond(2023, 9, 6, 0, 4, 03), utc); From 484b12b0b1bd9828467430b26f9ec9882e2b1dcc Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Tue, 3 Oct 2023 11:28:01 +0200 Subject: [PATCH 24/63] Primary .h header include order --- .../kvstore/s3/credentials/cached_credential_provider.cc | 3 ++- .../kvstore/s3/credentials/chained_credential_provider.cc | 3 ++- tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc | 3 ++- .../kvstore/s3/credentials/environment_credential_provider.cc | 3 ++- tensorstore/kvstore/s3/credentials/file_credential_provider.cc | 3 ++- 5 files changed, 10 insertions(+), 5 deletions(-) diff --git a/tensorstore/kvstore/s3/credentials/cached_credential_provider.cc b/tensorstore/kvstore/s3/credentials/cached_credential_provider.cc index 32010cda6..a621673d6 100644 --- a/tensorstore/kvstore/s3/credentials/cached_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/cached_credential_provider.cc @@ -12,10 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "tensorstore/kvstore/s3/credentials/cached_credential_provider.h" + #include "absl/synchronization/mutex.h" #include "absl/status/status.h" #include "tensorstore/util/result.h" -#include "tensorstore/kvstore/s3/credentials/cached_credential_provider.h" namespace tensorstore { namespace internal_kvstore_s3 { diff --git a/tensorstore/kvstore/s3/credentials/chained_credential_provider.cc b/tensorstore/kvstore/s3/credentials/chained_credential_provider.cc index 122f7f46d..361d49130 100644 --- a/tensorstore/kvstore/s3/credentials/chained_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/chained_credential_provider.cc @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "tensorstore/kvstore/s3/credentials/chained_credential_provider.h" + #include #include #include @@ -21,7 +23,6 @@ #include "absl/strings/str_join.h" #include "absl/time/time.h" #include "tensorstore/util/result.h" -#include "tensorstore/kvstore/s3/credentials/chained_credential_provider.h" namespace tensorstore { namespace internal_kvstore_s3 { diff --git a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc index d672ca575..7a0a3dd25 100644 --- a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc @@ -12,9 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "tensorstore/kvstore/s3/credentials/ec2_credential_provider.h" + #include "absl/status/status.h" #include "tensorstore/util/result.h" -#include "tensorstore/kvstore/s3/credentials/ec2_credential_provider.h" namespace tensorstore { namespace internal_kvstore_s3 { diff --git a/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc b/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc index d9c09ed25..28ab071ae 100644 --- a/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc @@ -12,8 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "absl/status/status.h" #include "tensorstore/kvstore/s3/credentials/environment_credential_provider.h" + +#include "absl/status/status.h" #include "tensorstore/internal/env.h" using ::tensorstore::internal::GetEnv; diff --git a/tensorstore/kvstore/s3/credentials/file_credential_provider.cc b/tensorstore/kvstore/s3/credentials/file_credential_provider.cc index 4d65e15eb..d6bd3748f 100644 --- a/tensorstore/kvstore/s3/credentials/file_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/file_credential_provider.cc @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "tensorstore/kvstore/s3/credentials/file_credential_provider.h" + #include #include #include @@ -19,7 +21,6 @@ #include "absl/strings/strip.h" #include "absl/strings/str_cat.h" #include "absl/status/status.h" -#include "tensorstore/kvstore/s3/credentials/file_credential_provider.h" #include "tensorstore/internal/env.h" #include "tensorstore/internal/path.h" #include "tensorstore/util/result.h" From 0ae68ec8dd834fdb01e2579f613eb886b0c29684 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Tue, 3 Oct 2023 11:41:21 +0200 Subject: [PATCH 25/63] Fix CachedCredentialsProvider base case --- .../kvstore/s3/credentials/cached_credential_provider.cc | 5 +++-- .../s3/credentials/cached_credential_provider_test.cc | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tensorstore/kvstore/s3/credentials/cached_credential_provider.cc b/tensorstore/kvstore/s3/credentials/cached_credential_provider.cc index a621673d6..7b5862d6e 100644 --- a/tensorstore/kvstore/s3/credentials/cached_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/cached_credential_provider.cc @@ -38,8 +38,9 @@ bool CachedCredentialProvider::IsExpiredLocked(const AwsCredentials & credential Result CachedCredentialProvider::ExpiresAt() { absl::ReaderMutexLock lock(&mutex_); - if(credentials_.IsAnonymous()) return absl::InfiniteFuture(); - return provider_->ExpiresAt(); + TENSORSTORE_ASSIGN_OR_RETURN(auto expires, provider_->ExpiresAt()); + if(credentials_.IsAnonymous()) return absl::InfinitePast(); + return expires; } Result diff --git a/tensorstore/kvstore/s3/credentials/cached_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/cached_credential_provider_test.cc index 4aac59f93..88d1970d6 100644 --- a/tensorstore/kvstore/s3/credentials/cached_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/cached_credential_provider_test.cc @@ -58,7 +58,7 @@ TEST(CachedCredentialProviderTest, ExpiringProvider) { // Base case, no credentials have been retrieved yet ASSERT_TRUE(cached_provider.IsExpired()); - ASSERT_EQ(cached_provider.ExpiresAt(), absl::InfiniteFuture()); + ASSERT_EQ(cached_provider.ExpiresAt(), absl::InfinitePast()); ASSERT_EQ(test_prov_ptr->iteration, 0); // Retrieve some credentials From 80234650905e8146c67b96d675c637cebce4a779 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Tue, 3 Oct 2023 14:56:27 +0200 Subject: [PATCH 26/63] CachedCredentialProvider comments --- .../kvstore/s3/credentials/cached_credential_provider.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tensorstore/kvstore/s3/credentials/cached_credential_provider.h b/tensorstore/kvstore/s3/credentials/cached_credential_provider.h index b70ad596a..a9641aca1 100644 --- a/tensorstore/kvstore/s3/credentials/cached_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/cached_credential_provider.h @@ -21,7 +21,11 @@ namespace tensorstore { namespace internal_kvstore_s3 { -// +/// The Cached Credential Provider encapsulates an instance of AwsCredentialProvider +/// to provide: +/// 1. Thread-safe credential retrieval +/// 2. Caching of credentials +/// 3. Refresh of credentials upon expiry class CachedCredentialProvider : public AwsCredentialProvider { private: absl::Mutex mutex_; From 42c32a8dcde4c3de72117448738b676884dc73ca Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Fri, 13 Oct 2023 10:52:44 +0200 Subject: [PATCH 27/63] Remove caching in the EC2CredentialsProvider (as this occurs in the CachedCredentialsProvider) --- .../s3/credentials/ec2_credential_provider.cc | 19 +++++-------------- .../s3/credentials/ec2_credential_provider.h | 10 +++------- 2 files changed, 8 insertions(+), 21 deletions(-) diff --git a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc index d51ea31ec..1a788f517 100644 --- a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc @@ -140,11 +140,6 @@ bool IsEC2MetadataServiceAvailable(internal_http::HttpTransport& transport) { return GetEC2ApiToken(transport).ok(); } -bool EC2MetadataCredentialProvider::IsExpired() { - absl::MutexLock l(&mutex_); - return absl::Now() + absl::Seconds(60) < timeout_; -} - /// Obtains AWS Credentials from the EC2Metadata. /// /// https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html#instancedata-meta-data-retrieval-examples @@ -158,10 +153,6 @@ bool EC2MetadataCredentialProvider::IsExpired() { /// 3. Obtain the associated credentials from path /// "/latest/meta-data/iam/security-credentials/". Result EC2MetadataCredentialProvider::GetCredentials() { - absl::MutexLock l(&mutex_); - if (absl::Now() + absl::Seconds(60) < timeout_) { - return credentials_; - } auto default_timeout = absl::Now() + kDefaultTimeout; // Obtain an API token for communicating with the EC2 Metadata server @@ -218,11 +209,11 @@ Result EC2MetadataCredentialProvider::GetCredentials() { "] failed with ", json_sv)); } - timeout_ = iam_credentials.expiration.value_or(default_timeout); - credentials_ = AwsCredentials{iam_credentials.access_key_id.value_or(""), - iam_credentials.secret_access_key.value_or(""), - iam_credentials.token.value_or("")}; - return credentials_; + SetExpiration(iam_credentials.expiration.value_or(default_timeout)); + + return AwsCredentials{iam_credentials.access_key_id.value_or(""), + iam_credentials.secret_access_key.value_or(""), + iam_credentials.token.value_or("")}; } } // namespace internal_kvstore_s3 diff --git a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h index 0add9c47b..925cdab18 100644 --- a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h @@ -23,27 +23,23 @@ #include "absl/time/time.h" #include "tensorstore/internal/http/http_transport.h" #include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" +#include "tensorstore/kvstore/s3/credentials/expiry_credential_provider.h" #include "tensorstore/util/result.h" namespace tensorstore { namespace internal_kvstore_s3 { /// Provide S3 credentials from the EC2 Metadata server -class EC2MetadataCredentialProvider : public AwsCredentialProvider { +class EC2MetadataCredentialProvider : public ExpiryCredentialProvider { public: EC2MetadataCredentialProvider( std::shared_ptr transport) - : transport_(std::move(transport)), timeout_(absl::InfinitePast()) {} + : transport_(std::move(transport)) {} Result GetCredentials() override; - bool IsExpired() override; private: std::shared_ptr transport_; - - absl::Mutex mutex_; - absl::Time timeout_ ABSL_GUARDED_BY(mutex_); - AwsCredentials credentials_ ABSL_GUARDED_BY(mutex_); }; // Returns whether the EC2 Metadata Server is available. From 1e5174b109fcac3133c0197d9371836f992c2d91 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Fri, 13 Oct 2023 11:00:39 +0200 Subject: [PATCH 28/63] Don't query EC2 Metadata server if it doesn't respond timeously on first connect --- .../kvstore/s3/credentials/aws_credential_provider.cc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tensorstore/kvstore/s3/credentials/aws_credential_provider.cc b/tensorstore/kvstore/s3/credentials/aws_credential_provider.cc index 6a11ab955..428d52fd3 100644 --- a/tensorstore/kvstore/s3/credentials/aws_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/aws_credential_provider.cc @@ -46,7 +46,14 @@ Result> GetDefaultAwsCredentialProvider( std::vector> providers; providers.emplace_back(std::make_unique()); providers.emplace_back(std::make_unique(std::string(profile))); - providers.emplace_back(std::make_unique(transport)); + + if(IsEC2MetadataServiceAvailable(*transport)) { + providers.emplace_back(std::make_unique(transport)); + } else { + ABSL_LOG(WARNING) << "Initial connection to EC2 Metadata server timed out. " + << "Credentials from this source will be ignored."; + } + return std::make_unique( std::make_unique(std::move(providers))); } From 70850e1ff94cadda322020a12aed885e10f491dc Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Fri, 13 Oct 2023 11:11:28 +0200 Subject: [PATCH 29/63] Remove stray merge artifacts --- .../kvstore/s3/aws_credential_provider.cc | 257 ------------------ .../kvstore/s3/aws_credential_provider.h | 104 ------- 2 files changed, 361 deletions(-) delete mode 100644 tensorstore/kvstore/s3/aws_credential_provider.cc delete mode 100644 tensorstore/kvstore/s3/aws_credential_provider.h diff --git a/tensorstore/kvstore/s3/aws_credential_provider.cc b/tensorstore/kvstore/s3/aws_credential_provider.cc deleted file mode 100644 index c334ca3dc..000000000 --- a/tensorstore/kvstore/s3/aws_credential_provider.cc +++ /dev/null @@ -1,257 +0,0 @@ -// Copyright 2023 The TensorStore Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "tensorstore/kvstore/s3/aws_credential_provider.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "absl/log/absl_log.h" -#include "absl/status/status.h" -#include "absl/strings/ascii.h" -#include "absl/strings/str_cat.h" -#include "absl/synchronization/mutex.h" -#include "tensorstore/internal/env.h" -#include "tensorstore/internal/http/http_transport.h" -#include "tensorstore/internal/no_destructor.h" -#include "tensorstore/internal/path.h" -#include "tensorstore/kvstore/s3/aws_metadata_credential_provider.h" -#include "tensorstore/util/result.h" - -using ::tensorstore::Result; -using ::tensorstore::internal::GetEnv; -using ::tensorstore::internal::JoinPath; - -namespace tensorstore { -namespace internal_kvstore_s3 { -namespace { - -// For reference, see the latest AWS environment variables used by the cli: -// https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html - -// AWS user identifier -constexpr char kEnvAwsAccessKeyId[] = "AWS_ACCESS_KEY_ID"; -constexpr char kCfgAwsAccessKeyId[] = "aws_access_key_id"; - -// AWS user password -constexpr char kEnvAwsSecretAccessKey[] = "AWS_SECRET_ACCESS_KEY"; -constexpr char kCfgAwsSecretAccessKeyId[] = "aws_secret_access_key"; - -// AWS session token -constexpr char kEnvAwsSessionToken[] = "AWS_SESSION_TOKEN"; -constexpr char kCfgAwsSessionToken[] = "aws_session_token"; - -// AWS Profile environment variables -constexpr char kEnvAwsProfile[] = "AWS_PROFILE"; -constexpr char kDefaultProfile[] = "default"; - -// Credentials file environment variable -constexpr char kEnvAwsCredentialsFile[] = "AWS_SHARED_CREDENTIALS_FILE"; - -// Default path to the AWS credentials file, relative to the home folder -constexpr char kDefaultAwsCredentialsFilePath[] = ".aws/credentials"; - -/// Returns whether the given path points to a readable file. -bool IsFile(const std::string& filename) { - std::ifstream fstream(filename.c_str()); - return fstream.good(); -} - -Result GetAwsCredentialsFileName() { - std::string result; - - auto credentials_file = GetEnv(kEnvAwsCredentialsFile); - if (!credentials_file) { - auto home_dir = GetEnv("HOME"); - if (!home_dir) { - return absl::NotFoundError("Could not read $HOME"); - } - result = JoinPath(*home_dir, kDefaultAwsCredentialsFilePath); - } else { - result = *credentials_file; - } - if (!IsFile(result)) { - return absl::NotFoundError( - absl::StrCat("Could not find the credentials file at " - "location [", - result, "]")); - } - return result; -} - -Result> GetDefaultAwsCredentialProvider( - std::string_view profile, - std::shared_ptr transport) { - // 1. Obtain credentials from environment variables - if (auto access_key = GetEnv(kEnvAwsAccessKeyId); access_key) { - ABSL_LOG_FIRST_N(INFO, 1) - << "Using Environment Variable " << kEnvAwsAccessKeyId; - AwsCredentials credentials; - credentials.access_key = *access_key; - auto secret_key = GetEnv(kEnvAwsSecretAccessKey); - - if (secret_key.has_value()) { - credentials.secret_key = *secret_key; - } - - auto session_token = GetEnv(kEnvAwsSessionToken); - - if (session_token.has_value()) { - credentials.session_token = *session_token; - } - - return std::make_unique( - std::move(credentials)); - } - - // 2. Obtain credentials from AWS_SHARED_CREDENTIALS_FILE or - // ~/.aws/credentials - if (auto credentials_file = GetAwsCredentialsFileName(); - credentials_file.ok()) { - std::string env_profile; // value must not outlive view - if (profile.empty()) { - env_profile = GetEnv(kEnvAwsProfile).value_or(kDefaultProfile); - profile = std::string_view(env_profile); - } - ABSL_LOG(INFO) << "Using File AwsCredentialProvider with profile " - << profile; - return std::make_unique( - std::move(credentials_file).value(), std::string(profile)); - } - - // 3. Obtain credentials from EC2 Metadata server - if (IsEC2MetadataServiceAvailable(*transport)) { - ABSL_LOG(INFO) << "Using EC2 Metadata Service AwsCredentialProvider"; - return std::make_unique(transport); - } - - return absl::NotFoundError( - "No credentials provided in environment variables, " - "credentials file not found and not running on AWS."); -} - -struct AwsCredentialProviderRegistry { - std::vector> providers; - absl::Mutex mutex; -}; - -AwsCredentialProviderRegistry& GetAwsProviderRegistry() { - static internal::NoDestructor registry; - return *registry; -} - -} // namespace - -/// https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html#cli-configure-files-format -Result FileCredentialProvider::GetCredentials() { - absl::ReaderMutexLock lock(&mutex_); - std::ifstream ifs(filename_); - - if (!ifs) { - return absl::NotFoundError( - absl::StrCat("Could not open the credentials file [", filename_, "]")); - } - - AwsCredentials credentials; - std::string section_name; - std::string line; - bool profile_found = false; - - while (std::getline(ifs, line)) { - auto sline = absl::StripAsciiWhitespace(line); - // Ignore empty and commented out lines - if (sline.empty() || sline[0] == '#') continue; - - // A configuration section name has been encountered - if (sline[0] == '[' && sline[sline.size() - 1] == ']') { - section_name = - absl::StripAsciiWhitespace(sline.substr(1, sline.size() - 2)); - continue; - } - - // Look for key=value pairs if we're in the appropriate profile - if (section_name == profile_) { - profile_found = true; - if (auto pos = sline.find('='); pos != std::string::npos) { - auto key = absl::StripAsciiWhitespace(sline.substr(0, pos)); - auto value = absl::StripAsciiWhitespace(sline.substr(pos + 1)); - - if (key == kCfgAwsAccessKeyId) { - credentials.access_key = value; - } else if (key == kCfgAwsSecretAccessKeyId) { - credentials.secret_key = value; - } else if (key == kCfgAwsSessionToken) { - credentials.session_token = value; - } - } - } - } - - if (!profile_found) { - return absl::NotFoundError(absl::StrCat("Profile [", profile_, - "] not found " - "in credentials file [", - filename_, "]")); - } - - return credentials; -} - -void RegisterAwsCredentialProviderProvider(AwsCredentialProviderFn provider, - int priority) { - auto& registry = GetAwsProviderRegistry(); - absl::WriterMutexLock lock(®istry.mutex); - registry.providers.emplace_back(priority, std::move(provider)); - std::sort(registry.providers.begin(), registry.providers.end(), - [](const auto& a, const auto& b) { return a.first < b.first; }); -} - -/// @brief Obtain a credential provider from a series of registered and default -/// providers -/// -/// Providers are returned in the following order: -/// 1. Any registered providers that supply valid credentials -/// 2. Environment variable provider if valid credential can be obtained from -/// AWS_* environment variables -/// 3. File provider containing credentials from an ~/.aws/credentials file -/// 4. EC2 Metadata server -/// -/// @param profile The profile to use when retrieving credentials from a -/// credentials file. -/// @param transport Optionally specify the http transport used to retreive S3 -/// credentials -/// from the EC2 metadata server. -/// @return Provider that supplies S3 Credentials -Result> GetAwsCredentialProvider( - std::string_view profile, - std::shared_ptr transport) { - auto& registry = GetAwsProviderRegistry(); - absl::WriterMutexLock lock(®istry.mutex); - for (const auto& provider : registry.providers) { - auto credentials = provider.second(); - if (credentials.ok()) return credentials; - } - - return internal_kvstore_s3::GetDefaultAwsCredentialProvider(profile, - transport); -} - -} // namespace internal_kvstore_s3 -} // namespace tensorstore diff --git a/tensorstore/kvstore/s3/aws_credential_provider.h b/tensorstore/kvstore/s3/aws_credential_provider.h deleted file mode 100644 index ee946ec26..000000000 --- a/tensorstore/kvstore/s3/aws_credential_provider.h +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright 2023 The TensorStore Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef TENSORSTORE_KVSTORE_S3_AWS_CREDENTIAL_PROVIDER_H -#define TENSORSTORE_KVSTORE_S3_AWS_CREDENTIAL_PROVIDER_H - -#include -#include -#include -#include -#include - -#include "absl/synchronization/mutex.h" -#include "tensorstore/internal/http/http_transport.h" -#include "tensorstore/util/result.h" - -namespace tensorstore { -namespace internal_kvstore_s3 { - -/// Holds S3 credentials -/// -/// Contains the access key, secret key and session token. -/// An empty access key implies anonymous access, -/// while the presence of a session token implies the use of -/// short-lived STS credentials -/// https://docs.aws.amazon.com/STS/latest/APIReference/welcome.html -struct AwsCredentials { - /// AWS_ACCESS_KEY_ID - std::string access_key; - /// AWS_SECRET_KEY_ID - std::string secret_key; - /// AWS_SESSION_TOKEN - std::string session_token; - - bool IsAnonymous() const { return access_key.empty(); } -}; - -/// Base class for S3 Credential Providers -/// -/// Implementers should override GetCredentials -class AwsCredentialProvider { - public: - virtual ~AwsCredentialProvider() = default; - virtual Result GetCredentials() = 0; -}; - -/// Provides credentials from the following environment variables: -/// AWS_ACCESS_KEY_ID, AWS_SECRET_KEY_ID, AWS_SESSION_TOKEN -class EnvironmentCredentialProvider : public AwsCredentialProvider { - private: - AwsCredentials credentials_; - - public: - EnvironmentCredentialProvider(const AwsCredentials& credentials) - : credentials_(credentials) {} - - Result GetCredentials() override { return credentials_; } -}; - -/// Obtains S3 credentials from a profile in a file, usually -/// `~/.aws/credentials` or a file specified in AWS_SHARED_CREDENTIALS_FILE. A -/// desired profile may be specified in the constructor: This value should be -/// derived from the s3 json spec. -/// However, if profile is passed as an empty string, the profile is obtained -/// from AWS_DEFAULT_PROFILE, AWS_PROFILE before finally defaulting to -/// "default". -class FileCredentialProvider : public AwsCredentialProvider { - private: - absl::Mutex mutex_; - std::string filename_; - std::string profile_; - - public: - FileCredentialProvider(std::string filename, std::string profile) - : filename_(std::move(filename)), profile_(std::move(profile)) {} - - Result GetCredentials() override; -}; - -using AwsCredentialProviderFn = - std::function>()>; - -void RegisterAwsCredentialProviderProvider(AwsCredentialProviderFn provider, - int priority); - -Result> GetAwsCredentialProvider( - std::string_view profile, - std::shared_ptr transport); - -} // namespace internal_kvstore_s3 -} // namespace tensorstore - -#endif // TENSORSTORE_KVSTORE_S3_AWS_CREDENTIAL_PROVIDER_H From 948f5ffb5148c758cd002a80da1e0b3895959598 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Fri, 13 Oct 2023 14:48:10 +0200 Subject: [PATCH 30/63] Fix IsExpired bug and implement ExpiresAt for Environment and File Providers --- .../environment_credential_provider.h | 5 ++++- .../environment_credential_provider_test.cc | 10 +++++++++- .../s3/credentials/file_credential_provider.h | 5 ++++- .../credentials/file_credential_provider_test.cc | 16 ++++++++++++++++ 4 files changed, 33 insertions(+), 3 deletions(-) diff --git a/tensorstore/kvstore/s3/credentials/environment_credential_provider.h b/tensorstore/kvstore/s3/credentials/environment_credential_provider.h index 34fdcf342..b4c00128f 100644 --- a/tensorstore/kvstore/s3/credentials/environment_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/environment_credential_provider.h @@ -31,7 +31,10 @@ class EnvironmentCredentialProvider : public AwsCredentialProvider { EnvironmentCredentialProvider() : retrieved_(false) {} Result GetCredentials() override; // Credentials obtained from the environment never expire - bool IsExpired() override { return retrieved_; } + bool IsExpired() override { return !retrieved_; } + Result ExpiresAt() override { + return retrieved_ ? absl::InfiniteFuture() : absl::InfinitePast(); + } }; } // namespace internal_kvstore_s3 diff --git a/tensorstore/kvstore/s3/credentials/environment_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/environment_credential_provider_test.cc index 111d988fb..d0f0bbf86 100644 --- a/tensorstore/kvstore/s3/credentials/environment_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/environment_credential_provider_test.cc @@ -46,14 +46,18 @@ class EnvironmentCredentialProviderTest : public ::testing::Test { TEST_F(EnvironmentCredentialProviderTest, ProviderNoCredentials) { auto provider = EnvironmentCredentialProvider(); + ASSERT_TRUE(provider.IsExpired()); + ASSERT_EQ(provider.ExpiresAt(), absl::InfinitePast()); ASSERT_FALSE(provider.GetCredentials().ok()); - SetEnv("AWS_ACCESS_KEY_ID", "foo"); TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, provider.GetCredentials()); ASSERT_EQ(credentials.access_key, "foo"); ASSERT_TRUE(credentials.secret_key.empty()); ASSERT_TRUE(credentials.session_token.empty()); + ASSERT_FALSE(provider.IsExpired()); + ASSERT_EQ(provider.ExpiresAt(), absl::InfiniteFuture()); + } TEST_F(EnvironmentCredentialProviderTest, ProviderAwsCredentialsFromEnv) { @@ -61,10 +65,14 @@ TEST_F(EnvironmentCredentialProviderTest, ProviderAwsCredentialsFromEnv) { SetEnv("AWS_SECRET_ACCESS_KEY", "bar"); SetEnv("AWS_SESSION_TOKEN", "qux"); auto provider = EnvironmentCredentialProvider(); + ASSERT_TRUE(provider.IsExpired()); + ASSERT_EQ(provider.ExpiresAt(), absl::InfinitePast()); TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, provider.GetCredentials()); ASSERT_EQ(credentials.access_key, "foo"); ASSERT_EQ(credentials.secret_key, "bar"); ASSERT_EQ(credentials.session_token, "qux"); + ASSERT_FALSE(provider.IsExpired()); + ASSERT_EQ(provider.ExpiresAt(), absl::InfiniteFuture()); } } // namespace diff --git a/tensorstore/kvstore/s3/credentials/file_credential_provider.h b/tensorstore/kvstore/s3/credentials/file_credential_provider.h index 72b7644dc..8c7e014aa 100644 --- a/tensorstore/kvstore/s3/credentials/file_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/file_credential_provider.h @@ -41,7 +41,10 @@ class FileCredentialProvider : public AwsCredentialProvider { : profile_(std::move(profile)), retrieved_(false) {} Result GetCredentials() override; // Shared Credentials never expire once retrieved - bool IsExpired() override { return retrieved_; } + bool IsExpired() override { return !retrieved_; } + Result ExpiresAt() override { + return retrieved_ ? absl::InfiniteFuture() : absl::InfinitePast(); + } }; } // namespace internal_kvstore_s3 diff --git a/tensorstore/kvstore/s3/credentials/file_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/file_credential_provider_test.cc index 48ed23118..1ae16ab6d 100644 --- a/tensorstore/kvstore/s3/credentials/file_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/file_credential_provider_test.cc @@ -65,10 +65,14 @@ TEST_F(FileCredentialProviderTest, ProviderAwsCredentialsFromFileDefault) { SetEnv("AWS_SHARED_CREDENTIALS_FILE", credentials_filename.c_str()); auto provider = FileCredentialProvider(""); + ASSERT_TRUE(provider.IsExpired()); + ASSERT_EQ(provider.ExpiresAt(), absl::InfinitePast()); TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, provider.GetCredentials()); ASSERT_EQ(credentials.access_key, "AKIAIOSFODNN7EXAMPLE"); ASSERT_EQ(credentials.secret_key, "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); ASSERT_EQ(credentials.session_token, "abcdef1234567890"); + ASSERT_FALSE(provider.IsExpired()); + ASSERT_EQ(provider.ExpiresAt(), absl::InfiniteFuture()); } TEST_F(FileCredentialProviderTest, @@ -78,10 +82,14 @@ TEST_F(FileCredentialProviderTest, SetEnv("AWS_SHARED_CREDENTIALS_FILE", credentials_filename.c_str()); auto provider = FileCredentialProvider("alice"); + ASSERT_TRUE(provider.IsExpired()); + ASSERT_EQ(provider.ExpiresAt(), absl::InfinitePast()); TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, provider.GetCredentials()); ASSERT_EQ(credentials.access_key, "AKIAIOSFODNN6EXAMPLE"); ASSERT_EQ(credentials.secret_key, "wJalrXUtnFEMI/K7MDENG/bPxRfiCZEXAMPLEKEY"); ASSERT_EQ(credentials.session_token, ""); + ASSERT_FALSE(provider.IsExpired()); + ASSERT_EQ(provider.ExpiresAt(), absl::InfiniteFuture()); } TEST_F(FileCredentialProviderTest, ProviderAwsCredentialsFromFileProfileEnv) { @@ -91,10 +99,14 @@ TEST_F(FileCredentialProviderTest, ProviderAwsCredentialsFromFileProfileEnv) { SetEnv("AWS_SHARED_CREDENTIALS_FILE", credentials_filename.c_str()); SetEnv("AWS_PROFILE", "alice"); auto provider = FileCredentialProvider(""); + ASSERT_TRUE(provider.IsExpired()); + ASSERT_EQ(provider.ExpiresAt(), absl::InfinitePast()); TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, provider.GetCredentials()); ASSERT_EQ(credentials.access_key, "AKIAIOSFODNN6EXAMPLE"); ASSERT_EQ(credentials.secret_key, "wJalrXUtnFEMI/K7MDENG/bPxRfiCZEXAMPLEKEY"); ASSERT_EQ(credentials.session_token, ""); + ASSERT_FALSE(provider.IsExpired()); + ASSERT_EQ(provider.ExpiresAt(), absl::InfiniteFuture()); } TEST_F(FileCredentialProviderTest, @@ -105,7 +117,11 @@ TEST_F(FileCredentialProviderTest, SetEnv("AWS_SHARED_CREDENTIALS_FILE", credentials_filename.c_str()); SetEnv("AWS_PROFILE", "bob"); auto provider = FileCredentialProvider(""); + ASSERT_TRUE(provider.IsExpired()); + ASSERT_EQ(provider.ExpiresAt(), absl::InfinitePast()); ASSERT_FALSE(provider.GetCredentials().ok()); + ASSERT_TRUE(provider.IsExpired()); + ASSERT_EQ(provider.ExpiresAt(), absl::InfinitePast()); } } // namespace From 407923b2015a45f71b60fec9d988cb68bc0f063a Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Fri, 13 Oct 2023 15:09:40 +0200 Subject: [PATCH 31/63] Add some leeway to EC2 timeout and check ExpiresAt in EC2 test case --- .../s3/credentials/ec2_credential_provider.cc | 3 +- .../ec2_credential_provider_test.cc | 28 ++++++++++++------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc index 1a788f517..1abe0cfaa 100644 --- a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc @@ -209,7 +209,8 @@ Result EC2MetadataCredentialProvider::GetCredentials() { "] failed with ", json_sv)); } - SetExpiration(iam_credentials.expiration.value_or(default_timeout)); + SetExpiration(iam_credentials.expiration.value_or(default_timeout), + absl::Seconds(60)); return AwsCredentials{iam_credentials.access_key_id.value_or(""), iam_credentials.secret_access_key.value_or(""), diff --git a/tensorstore/kvstore/s3/credentials/ec2_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/ec2_credential_provider_test.cc index 8560c94d0..30e616853 100644 --- a/tensorstore/kvstore/s3/credentials/ec2_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/ec2_credential_provider_test.cc @@ -63,6 +63,9 @@ class EC2MetadataMockTransport : public HttpTransport { }; TEST(EC2MetadataCredentialProviderTest, CredentialRetrievalFlow) { + auto expiry = absl::Now() + absl::Seconds(240); + auto utc = absl::UTCTimeZone(); + auto url_to_response = absl::flat_hash_map{ {"POST http://169.254.169.254/latest/api/token", HttpResponse{200, absl::Cord{"1234567890"}}}, @@ -77,26 +80,31 @@ TEST(EC2MetadataCredentialProviderTest, CredentialRetrievalFlow) { {"GET " "http://169.254.169.254/latest/meta-data/iam/security-credentials/" "mock-iam-role", - HttpResponse{200, - absl::Cord(R"({ - "Code": "Success", - "LastUpdated": "2023-09-21T12:42:12Z", - "Type": "AWS-HMAC", - "AccessKeyId": "ASIA1234567890", - "SecretAccessKey": "1234567890abcdef", - "Token": "abcdef123456790", - "Expiration": "2023-09-21T12:42:12Z" - })"), + HttpResponse{200, absl::Cord( + absl::FormatTime(R"({ + "Code": "Success", + "LastUpdated": "2023-09-21T12:42:12Z", + "Type": "AWS-HMAC", + "AccessKeyId": "ASIA1234567890", + "SecretAccessKey": "1234567890abcdef", + "Token": "abcdef123456790", + "Expiration": "%Y-%m-%d%ET%H:%M:%E*S%Ez" + })", expiry, utc)), {{"x-aws-ec2-metadata-token", "1234567890"}}}}}; auto mock_transport = std::make_shared(url_to_response); auto provider = std::make_shared(mock_transport); + ASSERT_TRUE(provider->IsExpired()); + ASSERT_EQ(provider->ExpiresAt(), absl::InfinitePast()); TENSORSTORE_CHECK_OK_AND_ASSIGN(auto credentials, provider->GetCredentials()); ASSERT_EQ(credentials.access_key, "ASIA1234567890"); ASSERT_EQ(credentials.secret_key, "1234567890abcdef"); ASSERT_EQ(credentials.session_token, "abcdef123456790"); + // expiry less the 60s leeway + ASSERT_EQ(provider->ExpiresAt(), expiry - absl::Seconds(60)); + ASSERT_FALSE(provider->IsExpired()); } TEST(EC2MetadataCredentialProviderTest, NoIamRolesInSecurityCredentials) { From 3b49ee3a4f29bc09bcfe330878e972e3341cf565 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Fri, 13 Oct 2023 15:33:50 +0200 Subject: [PATCH 32/63] Remove some unneeded #includes --- tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc | 2 -- tensorstore/kvstore/s3/credentials/ec2_credential_provider.h | 2 -- 2 files changed, 4 deletions(-) diff --git a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc index 1abe0cfaa..1c76dc77e 100644 --- a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc @@ -25,8 +25,6 @@ #include "absl/strings/cord.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_split.h" -#include "absl/synchronization/mutex.h" -#include "absl/time/clock.h" #include "absl/time/time.h" #include "tensorstore/internal/env.h" #include "tensorstore/internal/http/http_request.h" diff --git a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h index 925cdab18..9df387905 100644 --- a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h @@ -19,8 +19,6 @@ #include #include "absl/base/thread_annotations.h" -#include "absl/synchronization/mutex.h" -#include "absl/time/time.h" #include "tensorstore/internal/http/http_transport.h" #include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" #include "tensorstore/kvstore/s3/credentials/expiry_credential_provider.h" From 47ef62375d531e826055dc8135ce57beee2a3ba6 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Mon, 23 Oct 2023 11:09:01 +0200 Subject: [PATCH 33/63] clang-format --- .../s3/credentials/aws_credential_provider.cc | 29 +++-- .../credentials/cached_credential_provider.cc | 46 +++---- .../cached_credential_provider_test.cc | 122 +++++++++--------- .../chained_credential_provider.cc | 51 ++++---- .../chained_credential_provider_test.cc | 35 +++-- .../ec2_credential_provider_test.cc | 12 +- .../environment_credential_provider.cc | 10 +- .../environment_credential_provider_test.cc | 10 +- .../expiry_credential_provider_test.cc | 27 ++-- .../credentials/file_credential_provider.cc | 27 ++-- .../file_credential_provider_test.cc | 8 +- 11 files changed, 195 insertions(+), 182 deletions(-) diff --git a/tensorstore/kvstore/s3/credentials/aws_credential_provider.cc b/tensorstore/kvstore/s3/credentials/aws_credential_provider.cc index 428d52fd3..96a1aeb84 100644 --- a/tensorstore/kvstore/s3/credentials/aws_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/aws_credential_provider.cc @@ -12,21 +12,22 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" + #include #include #include #include -#include "absl/synchronization/mutex.h" #include "absl/strings/str_cat.h" +#include "absl/synchronization/mutex.h" #include "tensorstore/internal/http/http_transport.h" #include "tensorstore/internal/no_destructor.h" -#include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" -#include "tensorstore/kvstore/s3/credentials/environment_credential_provider.h" -#include "tensorstore/kvstore/s3/credentials/file_credential_provider.h" -#include "tensorstore/kvstore/s3/credentials/ec2_credential_provider.h" #include "tensorstore/kvstore/s3/credentials/cached_credential_provider.h" #include "tensorstore/kvstore/s3/credentials/chained_credential_provider.h" +#include "tensorstore/kvstore/s3/credentials/ec2_credential_provider.h" +#include "tensorstore/kvstore/s3/credentials/environment_credential_provider.h" +#include "tensorstore/kvstore/s3/credentials/file_credential_provider.h" #include "tensorstore/util/result.h" using ::tensorstore::Result; @@ -35,27 +36,29 @@ namespace tensorstore { namespace internal_kvstore_s3 { namespace { -/// Return a ChainedCredentialProvider that attempts to retrieve credentials from +/// Return a ChainedCredentialProvider that attempts to retrieve credentials +/// from /// 1. AWS Environment Variables, e.g. AWS_ACCESS_KEY_ID /// 2. Shared Credential File, e.g. $HOME/.aws/credentials /// 3. EC2 Metadata Server Result> GetDefaultAwsCredentialProvider( std::string_view profile, std::shared_ptr transport) { - std::vector> providers; providers.emplace_back(std::make_unique()); - providers.emplace_back(std::make_unique(std::string(profile))); + providers.emplace_back( + std::make_unique(std::string(profile))); - if(IsEC2MetadataServiceAvailable(*transport)) { - providers.emplace_back(std::make_unique(transport)); + if (IsEC2MetadataServiceAvailable(*transport)) { + providers.emplace_back( + std::make_unique(transport)); } else { ABSL_LOG(WARNING) << "Initial connection to EC2 Metadata server timed out. " << "Credentials from this source will be ignored."; } return std::make_unique( - std::make_unique(std::move(providers))); + std::make_unique(std::move(providers))); } struct AwsCredentialProviderRegistry { @@ -70,7 +73,6 @@ AwsCredentialProviderRegistry& GetAwsProviderRegistry() { } // namespace - void RegisterAwsCredentialProviderProvider(AwsCredentialProviderFn provider, int priority) { auto& registry = GetAwsProviderRegistry(); @@ -87,7 +89,8 @@ void RegisterAwsCredentialProviderProvider(AwsCredentialProviderFn provider, /// 1. Any registered providers that supply valid credentials /// 2. Environment variable provider if valid credential can be obtained from /// AWS_* environment variables -/// 3. File provider containing credentials from the $HOME/.aws/credentials file. +/// 3. File provider containing credentials from the $HOME/.aws/credentials +/// file. /// The `profile` variable overrides the default profile in this file. /// 4. EC2 Metadata server. The `transport` variable overrides the default /// HttpTransport. diff --git a/tensorstore/kvstore/s3/credentials/cached_credential_provider.cc b/tensorstore/kvstore/s3/credentials/cached_credential_provider.cc index 7b5862d6e..3fef23749 100644 --- a/tensorstore/kvstore/s3/credentials/cached_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/cached_credential_provider.cc @@ -14,45 +14,41 @@ #include "tensorstore/kvstore/s3/credentials/cached_credential_provider.h" -#include "absl/synchronization/mutex.h" #include "absl/status/status.h" +#include "absl/synchronization/mutex.h" #include "tensorstore/util/result.h" namespace tensorstore { namespace internal_kvstore_s3 { -namespace { - -} // namespace - +namespace {} // namespace bool CachedCredentialProvider::IsExpired() { - absl::ReaderMutexLock lock(&mutex_); - return IsExpiredLocked(credentials_); + absl::ReaderMutexLock lock(&mutex_); + return IsExpiredLocked(credentials_); } -bool CachedCredentialProvider::IsExpiredLocked(const AwsCredentials & credentials) { - return credentials.IsAnonymous() || provider_->IsExpired(); +bool CachedCredentialProvider::IsExpiredLocked( + const AwsCredentials& credentials) { + return credentials.IsAnonymous() || provider_->IsExpired(); } -Result -CachedCredentialProvider::ExpiresAt() { - absl::ReaderMutexLock lock(&mutex_); - TENSORSTORE_ASSIGN_OR_RETURN(auto expires, provider_->ExpiresAt()); - if(credentials_.IsAnonymous()) return absl::InfinitePast(); - return expires; +Result CachedCredentialProvider::ExpiresAt() { + absl::ReaderMutexLock lock(&mutex_); + TENSORSTORE_ASSIGN_OR_RETURN(auto expires, provider_->ExpiresAt()); + if (credentials_.IsAnonymous()) return absl::InfinitePast(); + return expires; } -Result -CachedCredentialProvider::GetCredentials() { - absl::WriterMutexLock lock(&mutex_); - if(!IsExpiredLocked(credentials_)) { - return credentials_; - } - - TENSORSTORE_ASSIGN_OR_RETURN(credentials_, provider_->GetCredentials()); +Result CachedCredentialProvider::GetCredentials() { + absl::WriterMutexLock lock(&mutex_); + if (!IsExpiredLocked(credentials_)) { return credentials_; + } + + TENSORSTORE_ASSIGN_OR_RETURN(credentials_, provider_->GetCredentials()); + return credentials_; } -} // namespace internal_kvstore_s3 -} // namespace tensorstore +} // namespace internal_kvstore_s3 +} // namespace tensorstore diff --git a/tensorstore/kvstore/s3/credentials/cached_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/cached_credential_provider_test.cc index 3b4b0cbd8..329298eb2 100644 --- a/tensorstore/kvstore/s3/credentials/cached_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/cached_credential_provider_test.cc @@ -13,87 +13,91 @@ // limitations under the License. #include "tensorstore/kvstore/s3/credentials/cached_credential_provider.h" -#include "tensorstore/kvstore/s3/credentials/expiry_credential_provider.h" - -#include #include #include -#include "absl/log/absl_log.h" // remove + +#include + +#include "absl/log/absl_log.h" // remove #include "tensorstore/internal/test_util.h" +#include "tensorstore/kvstore/s3/credentials/expiry_credential_provider.h" #include "tensorstore/util/result.h" #include "tensorstore/util/status_testutil.h" using ::tensorstore::Future; +using ::tensorstore::MatchesStatus; using ::tensorstore::Result; using ::tensorstore::internal_kvstore_s3::AwsCredentials; using ::tensorstore::internal_kvstore_s3::CachedCredentialProvider; using ::tensorstore::internal_kvstore_s3::ExpiryCredentialProvider; -using ::tensorstore::MatchesStatus; namespace { class TestCredentialProvider : public ExpiryCredentialProvider { public: - int iteration = 0; + int iteration = 0; - TestCredentialProvider(absl::FunctionRef clock) : - ExpiryCredentialProvider(std::move(clock)) {} + TestCredentialProvider(absl::FunctionRef clock) + : ExpiryCredentialProvider(std::move(clock)) {} - Result GetCredentials() override { - ABSL_LOG(INFO) << "GetCredentials() == " << clock(); - this->SetExpiration(clock() + absl::Seconds(2)); - return AwsCredentials{"key", std::to_string(++iteration)}; - } + Result GetCredentials() override { + ABSL_LOG(INFO) << "GetCredentials() == " << clock(); + this->SetExpiration(clock() + absl::Seconds(2)); + return AwsCredentials{"key", std::to_string(++iteration)}; + } }; - TEST(CachedCredentialProviderTest, ExpiringProvider) { - auto utc = absl::UTCTimeZone(); - auto frozen_time = absl::FromCivil(absl::CivilSecond(2023, 9, 6, 0, 4, 03), utc); - auto stuck_clock = [&frozen_time]() -> absl::Time { return frozen_time; }; - ASSERT_EQ(stuck_clock(), frozen_time); - - auto test_provider = std::make_unique(stuck_clock); - auto test_prov_ptr = test_provider.get(); - auto cached_provider = CachedCredentialProvider{std::move(test_provider)}; - - // Base case, no credentials have been retrieved yet - ASSERT_TRUE(cached_provider.IsExpired()); - ASSERT_EQ(cached_provider.ExpiresAt(), absl::InfinitePast()); - ASSERT_EQ(test_prov_ptr->iteration, 0); - - // Retrieve some credentials - TENSORSTORE_CHECK_OK_AND_ASSIGN(auto credentials, cached_provider.GetCredentials()); - ASSERT_EQ(credentials.access_key, "key"); - ASSERT_EQ(credentials.secret_key, "1"); - ASSERT_EQ(test_prov_ptr->iteration, 1); - ASSERT_EQ(cached_provider.ExpiresAt(), frozen_time + absl::Seconds(2)); - ASSERT_FALSE(test_prov_ptr->IsExpired()); - ASSERT_FALSE(cached_provider.IsExpired()); - - // Idempotent when underlying credentials not expired - TENSORSTORE_CHECK_OK_AND_ASSIGN(credentials, cached_provider.GetCredentials()); - ASSERT_EQ(credentials.access_key, "key"); - ASSERT_EQ(credentials.secret_key, "1"); - ASSERT_EQ(test_prov_ptr->iteration, 1); - ASSERT_EQ(cached_provider.ExpiresAt(), frozen_time + absl::Seconds(2)); - ASSERT_FALSE(test_prov_ptr->IsExpired()); - ASSERT_FALSE(cached_provider.IsExpired()); - - // Advance clock forward - frozen_time += absl::Seconds(2.5); - ASSERT_TRUE(test_prov_ptr->IsExpired()); - ASSERT_TRUE(cached_provider.IsExpired()); - - // A new set of credentials is retrieved due to expiry - TENSORSTORE_CHECK_OK_AND_ASSIGN(credentials, cached_provider.GetCredentials()); - ASSERT_EQ(credentials.access_key, "key"); - ASSERT_EQ(credentials.secret_key, "2"); - ASSERT_EQ(test_prov_ptr->iteration, 2); - ASSERT_EQ(cached_provider.ExpiresAt(), frozen_time + absl::Seconds(2)); - ASSERT_FALSE(test_prov_ptr->IsExpired()); - ASSERT_FALSE(cached_provider.IsExpired()); + auto utc = absl::UTCTimeZone(); + auto frozen_time = + absl::FromCivil(absl::CivilSecond(2023, 9, 6, 0, 4, 03), utc); + auto stuck_clock = [&frozen_time]() -> absl::Time { return frozen_time; }; + ASSERT_EQ(stuck_clock(), frozen_time); + + auto test_provider = std::make_unique(stuck_clock); + auto test_prov_ptr = test_provider.get(); + auto cached_provider = CachedCredentialProvider{std::move(test_provider)}; + + // Base case, no credentials have been retrieved yet + ASSERT_TRUE(cached_provider.IsExpired()); + ASSERT_EQ(cached_provider.ExpiresAt(), absl::InfinitePast()); + ASSERT_EQ(test_prov_ptr->iteration, 0); + + // Retrieve some credentials + TENSORSTORE_CHECK_OK_AND_ASSIGN(auto credentials, + cached_provider.GetCredentials()); + ASSERT_EQ(credentials.access_key, "key"); + ASSERT_EQ(credentials.secret_key, "1"); + ASSERT_EQ(test_prov_ptr->iteration, 1); + ASSERT_EQ(cached_provider.ExpiresAt(), frozen_time + absl::Seconds(2)); + ASSERT_FALSE(test_prov_ptr->IsExpired()); + ASSERT_FALSE(cached_provider.IsExpired()); + + // Idempotent when underlying credentials not expired + TENSORSTORE_CHECK_OK_AND_ASSIGN(credentials, + cached_provider.GetCredentials()); + ASSERT_EQ(credentials.access_key, "key"); + ASSERT_EQ(credentials.secret_key, "1"); + ASSERT_EQ(test_prov_ptr->iteration, 1); + ASSERT_EQ(cached_provider.ExpiresAt(), frozen_time + absl::Seconds(2)); + ASSERT_FALSE(test_prov_ptr->IsExpired()); + ASSERT_FALSE(cached_provider.IsExpired()); + + // Advance clock forward + frozen_time += absl::Seconds(2.5); + ASSERT_TRUE(test_prov_ptr->IsExpired()); + ASSERT_TRUE(cached_provider.IsExpired()); + + // A new set of credentials is retrieved due to expiry + TENSORSTORE_CHECK_OK_AND_ASSIGN(credentials, + cached_provider.GetCredentials()); + ASSERT_EQ(credentials.access_key, "key"); + ASSERT_EQ(credentials.secret_key, "2"); + ASSERT_EQ(test_prov_ptr->iteration, 2); + ASSERT_EQ(cached_provider.ExpiresAt(), frozen_time + absl::Seconds(2)); + ASSERT_FALSE(test_prov_ptr->IsExpired()); + ASSERT_FALSE(cached_provider.IsExpired()); } } // namespace diff --git a/tensorstore/kvstore/s3/credentials/chained_credential_provider.cc b/tensorstore/kvstore/s3/credentials/chained_credential_provider.cc index 361d49130..5408faa0b 100644 --- a/tensorstore/kvstore/s3/credentials/chained_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/chained_credential_provider.cc @@ -18,7 +18,6 @@ #include #include -#include "tensorstore/util/result.h" #include "absl/status/status.h" #include "absl/strings/str_join.h" #include "absl/time/time.h" @@ -27,42 +26,40 @@ namespace tensorstore { namespace internal_kvstore_s3 { +bool ChainedCredentialProvider::IsExpired() { + if (!LastProviderValid()) { + return true; + } -bool ChainedCredentialProvider::IsExpired() { - if(!LastProviderValid()) { - return true; - } - - return providers_[last_provider_]->IsExpired(); + return providers_[last_provider_]->IsExpired(); } Result ChainedCredentialProvider::ExpiresAt() { - if(!LastProviderValid()) { - return absl::UnimplementedError("ChainedCredentialProvider::ExpiresAt"); - } + if (!LastProviderValid()) { + return absl::UnimplementedError("ChainedCredentialProvider::ExpiresAt"); + } - return providers_[last_provider_]->ExpiresAt(); + return providers_[last_provider_]->ExpiresAt(); } Result ChainedCredentialProvider::GetCredentials() { - std::vector errors; - last_provider_ = -1; + std::vector errors; + last_provider_ = -1; - for(std::size_t i=0; i < providers_.size(); ++i) { - auto credentials = providers_[i]->GetCredentials(); - if(credentials.ok()) { - last_provider_ = i; - return credentials; - } else { - errors.push_back(credentials.status().ToString()); - } + for (std::size_t i = 0; i < providers_.size(); ++i) { + auto credentials = providers_[i]->GetCredentials(); + if (credentials.ok()) { + last_provider_ = i; + return credentials; + } else { + errors.push_back(credentials.status().ToString()); } + } - return absl::NotFoundError( - absl::StrCat("No valid AwsCredentialProvider in chain:\n", - absl::StrJoin(errors, "\n"))); + return absl::NotFoundError( + absl::StrCat("No valid AwsCredentialProvider in chain:\n", + absl::StrJoin(errors, "\n"))); } - -} // namespace internal_kvstore_s3 -} // namespace tensorstore +} // namespace internal_kvstore_s3 +} // namespace tensorstore diff --git a/tensorstore/kvstore/s3/credentials/chained_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/chained_credential_provider_test.cc index 6f076c221..2904f9456 100644 --- a/tensorstore/kvstore/s3/credentials/chained_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/chained_credential_provider_test.cc @@ -12,12 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include - #include "tensorstore/kvstore/s3/credentials/chained_credential_provider.h" #include #include + +#include + #include "tensorstore/internal/test_util.h" #include "tensorstore/util/result.h" #include "tensorstore/util/status_testutil.h" @@ -25,8 +26,8 @@ using ::tensorstore::Future; using ::tensorstore::MatchesStatus; using ::tensorstore::Result; -using ::tensorstore::internal_kvstore_s3::AwsCredentials; using ::tensorstore::internal_kvstore_s3::AwsCredentialProvider; +using ::tensorstore::internal_kvstore_s3::AwsCredentials; using ::tensorstore::internal_kvstore_s3::ChainedCredentialProvider; namespace { @@ -37,17 +38,22 @@ class TestCredentialProvider : public AwsCredentialProvider { bool expired_; bool found_; bool has_expired_at_; - TestCredentialProvider(std::string access_key="") - : credentials_{access_key}, expired_(false), found_(true), - has_expired_at_(true) {} + TestCredentialProvider(std::string access_key = "") + : credentials_{access_key}, + expired_(false), + found_(true), + has_expired_at_(true) {} Result GetCredentials() override { - if(found_) { expired_ = true; return credentials_; } + if (found_) { + expired_ = true; + return credentials_; + } return absl::NotFoundError("Credentials Not Found"); } bool IsExpired() override { return expired_; } Result ExpiresAt() override { - if(has_expired_at_) { + if (has_expired_at_) { return absl::Now(); } return absl::UnimplementedError("TestCredentialProvider::ExpiresAt"); @@ -59,7 +65,8 @@ TEST(ChainedCredentialProviderTest, EmptyProvider) { auto credentials = provider.GetCredentials(); ASSERT_FALSE(credentials.ok()); ASSERT_TRUE(provider.IsExpired()); - EXPECT_THAT(provider.ExpiresAt(), MatchesStatus(absl::StatusCode::kUnimplemented)); + EXPECT_THAT(provider.ExpiresAt(), + MatchesStatus(absl::StatusCode::kUnimplemented)); } // Tests that Credential Retrieval results in IsExpiry and ExpiresAt @@ -88,7 +95,8 @@ TEST(ChainedCredentialProviderTest, ChainedGetAndExpiryLogic) { one_ptr->expired_ = false; one_ptr->has_expired_at_ = false; ASSERT_EQ(provider.IsExpired(), false); - EXPECT_THAT(provider.ExpiresAt(), MatchesStatus(absl::StatusCode::kUnimplemented)); + EXPECT_THAT(provider.ExpiresAt(), + MatchesStatus(absl::StatusCode::kUnimplemented)); // Disable one's credentials and get new credentials (two) one_ptr->found_ = false; @@ -103,7 +111,8 @@ TEST(ChainedCredentialProviderTest, ChainedGetAndExpiryLogic) { two_ptr->expired_ = false; two_ptr->has_expired_at_ = false; ASSERT_EQ(provider.IsExpired(), false); - EXPECT_THAT(provider.ExpiresAt(), MatchesStatus(absl::StatusCode::kUnimplemented)); + EXPECT_THAT(provider.ExpiresAt(), + MatchesStatus(absl::StatusCode::kUnimplemented)); // Disable two's credentials and get new credentials two_ptr->found_ = false; @@ -114,8 +123,8 @@ TEST(ChainedCredentialProviderTest, ChainedGetAndExpiryLogic) { // There's no valid provider, so ExpiresAt is also unimplemented one_ptr->has_expired_at_ = true; two_ptr->has_expired_at_ = true; - EXPECT_THAT(provider.ExpiresAt(), MatchesStatus(absl::StatusCode::kUnimplemented)); + EXPECT_THAT(provider.ExpiresAt(), + MatchesStatus(absl::StatusCode::kUnimplemented)); } - } // namespace diff --git a/tensorstore/kvstore/s3/credentials/ec2_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/ec2_credential_provider_test.cc index 30e616853..6ad82b69e 100644 --- a/tensorstore/kvstore/s3/credentials/ec2_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/ec2_credential_provider_test.cc @@ -14,11 +14,12 @@ #include "tensorstore/kvstore/s3/credentials/ec2_credential_provider.h" +#include +#include + #include #include -#include -#include #include "absl/container/flat_hash_map.h" #include "absl/log/absl_log.h" #include "absl/status/status.h" @@ -80,8 +81,8 @@ TEST(EC2MetadataCredentialProviderTest, CredentialRetrievalFlow) { {"GET " "http://169.254.169.254/latest/meta-data/iam/security-credentials/" "mock-iam-role", - HttpResponse{200, absl::Cord( - absl::FormatTime(R"({ + HttpResponse{200, + absl::Cord(absl::FormatTime(R"({ "Code": "Success", "LastUpdated": "2023-09-21T12:42:12Z", "Type": "AWS-HMAC", @@ -89,7 +90,8 @@ TEST(EC2MetadataCredentialProviderTest, CredentialRetrievalFlow) { "SecretAccessKey": "1234567890abcdef", "Token": "abcdef123456790", "Expiration": "%Y-%m-%d%ET%H:%M:%E*S%Ez" - })", expiry, utc)), + })", + expiry, utc)), {{"x-aws-ec2-metadata-token", "1234567890"}}}}}; auto mock_transport = diff --git a/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc b/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc index 28ab071ae..5ef44dd92 100644 --- a/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc @@ -31,7 +31,7 @@ static constexpr char kEnvAwsSecretAccessKey[] = "AWS_SECRET_ACCESS_KEY"; // AWS session token static constexpr char kEnvAwsSessionToken[] = "AWS_SESSION_TOKEN"; -} // namespace +} // namespace Result EnvironmentCredentialProvider::GetCredentials() { if (auto access_key = GetEnv(kEnvAwsAccessKeyId); access_key) { @@ -39,11 +39,11 @@ Result EnvironmentCredentialProvider::GetCredentials() { << "Using Environment Variable " << kEnvAwsAccessKeyId; auto credentials = AwsCredentials{*access_key}; - if(auto secret_key = GetEnv(kEnvAwsSecretAccessKey); secret_key) { + if (auto secret_key = GetEnv(kEnvAwsSecretAccessKey); secret_key) { credentials.secret_key = *secret_key; } - if(auto session_token = GetEnv(kEnvAwsSessionToken); session_token) { + if (auto session_token = GetEnv(kEnvAwsSessionToken); session_token) { credentials.session_token = *session_token; } @@ -54,5 +54,5 @@ Result EnvironmentCredentialProvider::GetCredentials() { return absl::NotFoundError(absl::StrCat(kEnvAwsAccessKeyId, " not set")); } -} // namespace internal_kvstore_s3 -} // namespace tensorstore +} // namespace internal_kvstore_s3 +} // namespace tensorstore diff --git a/tensorstore/kvstore/s3/credentials/environment_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/environment_credential_provider_test.cc index d0f0bbf86..8666776f5 100644 --- a/tensorstore/kvstore/s3/credentials/environment_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/environment_credential_provider_test.cc @@ -14,10 +14,11 @@ #include "tensorstore/kvstore/s3/credentials/environment_credential_provider.h" -#include - #include #include + +#include + #include "tensorstore/internal/env.h" #include "tensorstore/internal/test_util.h" #include "tensorstore/util/result.h" @@ -31,7 +32,6 @@ using ::tensorstore::internal::SetEnv; using ::tensorstore::internal::UnsetEnv; using ::tensorstore::internal_kvstore_s3::EnvironmentCredentialProvider; - class EnvironmentCredentialProviderTest : public ::testing::Test { protected: void SetUp() override { @@ -50,14 +50,12 @@ TEST_F(EnvironmentCredentialProviderTest, ProviderNoCredentials) { ASSERT_EQ(provider.ExpiresAt(), absl::InfinitePast()); ASSERT_FALSE(provider.GetCredentials().ok()); SetEnv("AWS_ACCESS_KEY_ID", "foo"); - TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, - provider.GetCredentials()); + TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, provider.GetCredentials()); ASSERT_EQ(credentials.access_key, "foo"); ASSERT_TRUE(credentials.secret_key.empty()); ASSERT_TRUE(credentials.session_token.empty()); ASSERT_FALSE(provider.IsExpired()); ASSERT_EQ(provider.ExpiresAt(), absl::InfiniteFuture()); - } TEST_F(EnvironmentCredentialProviderTest, ProviderAwsCredentialsFromEnv) { diff --git a/tensorstore/kvstore/s3/credentials/expiry_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/expiry_credential_provider_test.cc index c0350d7c0..1b31157dc 100644 --- a/tensorstore/kvstore/s3/credentials/expiry_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/expiry_credential_provider_test.cc @@ -12,12 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include - #include "tensorstore/kvstore/s3/credentials/expiry_credential_provider.h" #include #include + +#include + #include "tensorstore/internal/test_util.h" #include "tensorstore/util/result.h" #include "tensorstore/util/status_testutil.h" @@ -32,12 +33,11 @@ namespace { class TestCredentialProvider : public ExpiryCredentialProvider { public: - TestCredentialProvider(const absl::FunctionRef & clock=absl::Now) - : ExpiryCredentialProvider(clock) {} + TestCredentialProvider( + const absl::FunctionRef& clock = absl::Now) + : ExpiryCredentialProvider(clock) {} - Result GetCredentials() override { - return AwsCredentials{}; - } + Result GetCredentials() override { return AwsCredentials{}; } }; TEST(ExpiryCredentialProviderTest, TestExpiry) { @@ -48,10 +48,14 @@ TEST(ExpiryCredentialProviderTest, TestExpiry) { auto window_one = absl::Seconds(1); // Timestamps - auto two_seconds = absl::FromCivil(absl::CivilSecond(2023, 9, 6, 0, 4, 02), utc); - auto three_seconds = absl::FromCivil(absl::CivilSecond(2023, 9, 6, 0, 4, 03), utc); - auto four_seconds = absl::FromCivil(absl::CivilSecond(2023, 9, 6, 0, 4, 04), utc); - auto five_seconds = absl::FromCivil(absl::CivilSecond(2023, 9, 6, 0, 4, 05), utc); + auto two_seconds = + absl::FromCivil(absl::CivilSecond(2023, 9, 6, 0, 4, 02), utc); + auto three_seconds = + absl::FromCivil(absl::CivilSecond(2023, 9, 6, 0, 4, 03), utc); + auto four_seconds = + absl::FromCivil(absl::CivilSecond(2023, 9, 6, 0, 4, 04), utc); + auto five_seconds = + absl::FromCivil(absl::CivilSecond(2023, 9, 6, 0, 4, 05), utc); auto frozen_time = three_seconds; auto stuck_clock = [&frozen_time]() -> absl::Time { return frozen_time; }; @@ -93,5 +97,4 @@ TEST(ExpiryCredentialProviderTest, TestExpiry) { ASSERT_EQ(provider.ExpiresAt(), four_seconds); } - } // namespace diff --git a/tensorstore/kvstore/s3/credentials/file_credential_provider.cc b/tensorstore/kvstore/s3/credentials/file_credential_provider.cc index d6bd3748f..4970fb5dc 100644 --- a/tensorstore/kvstore/s3/credentials/file_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/file_credential_provider.cc @@ -18,9 +18,9 @@ #include #include -#include "absl/strings/strip.h" -#include "absl/strings/str_cat.h" #include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/strip.h" #include "tensorstore/internal/env.h" #include "tensorstore/internal/path.h" #include "tensorstore/util/result.h" @@ -28,7 +28,6 @@ using ::tensorstore::internal::GetEnv; using ::tensorstore::internal::JoinPath; - namespace tensorstore { namespace internal_kvstore_s3 { @@ -52,26 +51,28 @@ static constexpr char kDefaultProfile[] = "default"; Result GetAwsCredentialsFileName() { std::string result; - if (auto credentials_file = GetEnv(kEnvAwsCredentialsFile); credentials_file) { + if (auto credentials_file = GetEnv(kEnvAwsCredentialsFile); + credentials_file) { result = *credentials_file; } else { - if(auto home_dir = GetEnv("HOME"); home_dir) { + if (auto home_dir = GetEnv("HOME"); home_dir) { result = JoinPath(*home_dir, kDefaultAwsCredentialsFilePath); } else { return absl::NotFoundError("Could not read $HOME"); } } - if(auto fstream = std::ifstream(result.c_str()); !fstream.good()) { + if (auto fstream = std::ifstream(result.c_str()); !fstream.good()) { return absl::NotFoundError( absl::StrCat("Could not find the credentials file at " - "location [", result, "]")); + "location [", + result, "]")); } return result; } -} // namespace +} // namespace /// https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html#cli-configure-files-format Result FileCredentialProvider::GetCredentials() { @@ -82,9 +83,9 @@ Result FileCredentialProvider::GetCredentials() { absl::StrCat("Could not open credentials file [", filename, "]")); } - std::string profile = !profile_.empty() ? - std::string(profile_) : - GetEnv(kEnvAwsProfile).value_or(kDefaultProfile); + std::string profile = !profile_.empty() + ? std::string(profile_) + : GetEnv(kEnvAwsProfile).value_or(kDefaultProfile); AwsCredentials credentials; std::string section_name; @@ -135,5 +136,5 @@ Result FileCredentialProvider::GetCredentials() { return credentials; } -} // namespace internal_kvstore_s3 -} // namespace tensorstore +} // namespace internal_kvstore_s3 +} // namespace tensorstore diff --git a/tensorstore/kvstore/s3/credentials/file_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/file_credential_provider_test.cc index 1ae16ab6d..f9b64d8e5 100644 --- a/tensorstore/kvstore/s3/credentials/file_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/file_credential_provider_test.cc @@ -14,11 +14,12 @@ #include "tensorstore/kvstore/s3/credentials/file_credential_provider.h" +#include +#include + #include #include -#include -#include #include "tensorstore/internal/env.h" #include "tensorstore/internal/path.h" #include "tensorstore/internal/test_util.h" @@ -26,9 +27,9 @@ namespace { +using ::tensorstore::internal::JoinPath; using ::tensorstore::internal::SetEnv; using ::tensorstore::internal::UnsetEnv; -using ::tensorstore::internal::JoinPath; using ::tensorstore::internal_kvstore_s3::FileCredentialProvider; class TestData : public tensorstore::internal::ScopedTemporaryDirectory { @@ -58,7 +59,6 @@ class FileCredentialProviderTest : public ::testing::Test { void SetUp() override { UnsetEnv("AWS_SHARED_CREDENTIALS_FILE"); } }; - TEST_F(FileCredentialProviderTest, ProviderAwsCredentialsFromFileDefault) { TestData test_data; std::string credentials_filename = test_data.WriteCredentialsFile(); From 45d52965a590e8bc1cfda60ba8533648cafed53e Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Fri, 10 Nov 2023 18:05:06 +0200 Subject: [PATCH 34/63] Include CREDENTIALs in #includes --- .../kvstore/s3/credentials/aws_credential_provider.h | 6 +++--- .../kvstore/s3/credentials/ec2_credential_provider.h | 6 +++--- .../s3/credentials/environment_credential_provider.h | 6 +++--- .../kvstore/s3/credentials/file_credential_provider.h | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tensorstore/kvstore/s3/credentials/aws_credential_provider.h b/tensorstore/kvstore/s3/credentials/aws_credential_provider.h index a083a76be..6a2446af9 100644 --- a/tensorstore/kvstore/s3/credentials/aws_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/aws_credential_provider.h @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef TENSORSTORE_KVSTORE_S3_AWS_CREDENTIAL_PROVIDER -#define TENSORSTORE_KVSTORE_S3_AWS_CREDENTIAL_PROVIDER +#ifndef TENSORSTORE_KVSTORE_S3_CREDENTIALS_AWS_CREDENTIAL_PROVIDER +#define TENSORSTORE_KVSTORE_S3_CREDENTIALS_AWS_CREDENTIAL_PROVIDER #include #include @@ -72,4 +72,4 @@ Result> GetAwsCredentialProvider( } // namespace internal_kvstore_s3 } // namespace tensorstore -#endif // TENSORSTORE_KVSTORE_S3_AWS_CREDENTIAL_PROVIDER +#endif // TENSORSTORE_KVSTORE_S3_CREDENTIALS_AWS_CREDENTIAL_PROVIDER diff --git a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h index 9df387905..8b0e2db8d 100644 --- a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef TENSORSTORE_KVSTORE_S3_AWS_METADATA_CREDENTIAL_PROVIDER_H -#define TENSORSTORE_KVSTORE_S3_AWS_METADATA_CREDENTIAL_PROVIDER_H +#ifndef TENSORSTORE_KVSTORE_S3_CREDENTIALS_EC2_METADATA_CREDENTIAL_PROVIDER_H +#define TENSORSTORE_KVSTORE_S3_CREDENTIALS_EC2_METADATA_CREDENTIAL_PROVIDER_H #include #include @@ -46,4 +46,4 @@ bool IsEC2MetadataServiceAvailable(internal_http::HttpTransport& transport); } // namespace internal_kvstore_s3 } // namespace tensorstore -#endif // TENSORSTORE_KVSTORE_S3_AWS_METADATA_CREDENTIAL_PROVIDER_H +#endif // TENSORSTORE_KVSTORE_S3_CREDENTIALS_EC2_METADATA_CREDENTIAL_PROVIDER_H diff --git a/tensorstore/kvstore/s3/credentials/environment_credential_provider.h b/tensorstore/kvstore/s3/credentials/environment_credential_provider.h index b4c00128f..7744521cf 100644 --- a/tensorstore/kvstore/s3/credentials/environment_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/environment_credential_provider.h @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef TENSORSTORE_KVSTORE_S3_ENVIRONMENT_CREDENTIAL_PROVIDER_H -#define TENSORSTORE_KVSTORE_S3_ENVIRONMENT_CREDENTIAL_PROVIDER_H +#ifndef TENSORSTORE_KVSTORE_S3_CREDENTIALS_ENVIRONMENT_CREDENTIAL_PROVIDER_H +#define TENSORSTORE_KVSTORE_S3_CREDENTIALS_ENVIRONMENT_CREDENTIAL_PROVIDER_H #include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" #include "tensorstore/util/result.h" @@ -40,4 +40,4 @@ class EnvironmentCredentialProvider : public AwsCredentialProvider { } // namespace internal_kvstore_s3 } // namespace tensorstore -#endif // TENSORSTORE_KVSTORE_S3_ENVIRONMENT_CREDENTIAL_PROVIDER_H \ No newline at end of file +#endif // TENSORSTORE_KVSTORE_S3_CREDENTIALS_ENVIRONMENT_CREDENTIAL_PROVIDER_H \ No newline at end of file diff --git a/tensorstore/kvstore/s3/credentials/file_credential_provider.h b/tensorstore/kvstore/s3/credentials/file_credential_provider.h index 8c7e014aa..df91589b3 100644 --- a/tensorstore/kvstore/s3/credentials/file_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/file_credential_provider.h @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef TENSORSTORE_KVSTORE_S3_FILE_CREDENTIAL_PROVIDER_H -#define TENSORSTORE_KVSTORE_S3_FILE_CREDENTIAL_PROVIDER_H +#ifndef TENSORSTORE_KVSTORE_S3_CREDENTIALS_FILE_CREDENTIAL_PROVIDER_H +#define TENSORSTORE_KVSTORE_S3_CREDENTIALS_FILE_CREDENTIAL_PROVIDER_H #include @@ -50,4 +50,4 @@ class FileCredentialProvider : public AwsCredentialProvider { } // namespace internal_kvstore_s3 } // namespace tensorstore -#endif // TENSORSTORE_KVSTORE_S3_FILE_CREDENTIAL_PROVIDER_H \ No newline at end of file +#endif // TENSORSTORE_KVSTORE_S3_CREDENTIALS_FILE_CREDENTIAL_PROVIDER_H \ No newline at end of file From a5f421fb57e15aa392d164b25add7465ec4bb1d4 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Fri, 10 Nov 2023 19:26:49 +0200 Subject: [PATCH 35/63] Removed expired, chained and cached credential providers, replace with default credential provider --- tensorstore/kvstore/s3/credentials/BUILD | 46 +------ .../s3/credentials/aws_credential_provider.cc | 22 +-- .../s3/credentials/aws_credential_provider.h | 10 +- .../credentials/cached_credential_provider.cc | 54 -------- .../credentials/cached_credential_provider.h | 49 ------- .../cached_credential_provider_test.cc | 103 -------------- .../chained_credential_provider.cc | 65 --------- .../credentials/chained_credential_provider.h | 50 ------- .../chained_credential_provider_test.cc | 130 ------------------ .../default_credential_provider.cc | 86 ++++++++++++ .../credentials/default_credential_provider.h | 52 +++++++ .../s3/credentials/ec2_credential_provider.cc | 7 +- .../s3/credentials/ec2_credential_provider.h | 3 +- .../ec2_credential_provider_test.cc | 7 +- .../environment_credential_provider.h | 5 - .../environment_credential_provider_test.cc | 8 -- .../credentials/expiry_credential_provider.h | 59 -------- .../expiry_credential_provider_test.cc | 100 -------------- .../s3/credentials/file_credential_provider.h | 5 - .../file_credential_provider_test.cc | 16 --- 20 files changed, 157 insertions(+), 720 deletions(-) delete mode 100644 tensorstore/kvstore/s3/credentials/cached_credential_provider.cc delete mode 100644 tensorstore/kvstore/s3/credentials/cached_credential_provider.h delete mode 100644 tensorstore/kvstore/s3/credentials/cached_credential_provider_test.cc delete mode 100644 tensorstore/kvstore/s3/credentials/chained_credential_provider.cc delete mode 100644 tensorstore/kvstore/s3/credentials/chained_credential_provider.h delete mode 100644 tensorstore/kvstore/s3/credentials/chained_credential_provider_test.cc create mode 100644 tensorstore/kvstore/s3/credentials/default_credential_provider.cc create mode 100644 tensorstore/kvstore/s3/credentials/default_credential_provider.h delete mode 100644 tensorstore/kvstore/s3/credentials/expiry_credential_provider.h delete mode 100644 tensorstore/kvstore/s3/credentials/expiry_credential_provider_test.cc diff --git a/tensorstore/kvstore/s3/credentials/BUILD b/tensorstore/kvstore/s3/credentials/BUILD index e268f4cb4..7a18c8717 100644 --- a/tensorstore/kvstore/s3/credentials/BUILD +++ b/tensorstore/kvstore/s3/credentials/BUILD @@ -9,17 +9,14 @@ tensorstore_cc_library( name = "aws_credential_provider", srcs = [ "aws_credential_provider.cc", - "cached_credential_provider.cc", - "chained_credential_provider.cc", + "default_credential_provider.cc", "environment_credential_provider.cc", "file_credential_provider.cc", "ec2_credential_provider.cc"], hdrs = [ "aws_credential_provider.h", - "cached_credential_provider.h", - "chained_credential_provider.h", + "default_credential_provider.h", "environment_credential_provider.h", - "expiry_credential_provider.h", "file_credential_provider.h", "ec2_credential_provider.h"], deps = [ @@ -52,18 +49,6 @@ tensorstore_cc_test( ], ) -tensorstore_cc_test( - name = "expiry_credential_provider_test", - srcs = ["expiry_credential_provider_test.cc"], - deps = [ - ":aws_credential_provider", - "//tensorstore/internal:test_util", - "//tensorstore/util:result", - "//tensorstore/util:status_testutil", - "@com_google_googletest//:gtest_main", - ], -) - tensorstore_cc_test( name = "file_credential_provider_test", @@ -95,29 +80,4 @@ tensorstore_cc_test( "@com_google_absl//absl/time", "@com_google_googletest//:gtest_main", ], -) - - -tensorstore_cc_test( - name = "chained_credential_provider_test", - srcs = ["chained_credential_provider_test.cc"], - deps = [ - ":aws_credential_provider", - "//tensorstore/internal:test_util", - "//tensorstore/util:result", - "//tensorstore/util:status_testutil", - "@com_google_googletest//:gtest_main", - ], -) - -tensorstore_cc_test( - name = "cached_credential_provider_test", - srcs = ["cached_credential_provider_test.cc"], - deps = [ - ":aws_credential_provider", - "//tensorstore/internal:test_util", - "//tensorstore/util:result", - "//tensorstore/util:status_testutil", - "@com_google_googletest//:gtest_main", - ], -) +) \ No newline at end of file diff --git a/tensorstore/kvstore/s3/credentials/aws_credential_provider.cc b/tensorstore/kvstore/s3/credentials/aws_credential_provider.cc index 96a1aeb84..7eb55338e 100644 --- a/tensorstore/kvstore/s3/credentials/aws_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/aws_credential_provider.cc @@ -23,11 +23,7 @@ #include "absl/synchronization/mutex.h" #include "tensorstore/internal/http/http_transport.h" #include "tensorstore/internal/no_destructor.h" -#include "tensorstore/kvstore/s3/credentials/cached_credential_provider.h" -#include "tensorstore/kvstore/s3/credentials/chained_credential_provider.h" -#include "tensorstore/kvstore/s3/credentials/ec2_credential_provider.h" -#include "tensorstore/kvstore/s3/credentials/environment_credential_provider.h" -#include "tensorstore/kvstore/s3/credentials/file_credential_provider.h" +#include "tensorstore/kvstore/s3/credentials/default_credential_provider.h" #include "tensorstore/util/result.h" using ::tensorstore::Result; @@ -44,21 +40,9 @@ namespace { Result> GetDefaultAwsCredentialProvider( std::string_view profile, std::shared_ptr transport) { - std::vector> providers; - providers.emplace_back(std::make_unique()); - providers.emplace_back( - std::make_unique(std::string(profile))); - if (IsEC2MetadataServiceAvailable(*transport)) { - providers.emplace_back( - std::make_unique(transport)); - } else { - ABSL_LOG(WARNING) << "Initial connection to EC2 Metadata server timed out. " - << "Credentials from this source will be ignored."; - } - - return std::make_unique( - std::make_unique(std::move(providers))); + return std::make_unique( + DefaultAwsCredentialsProvider::Options{{}, std::string{profile}, transport}); } struct AwsCredentialProviderRegistry { diff --git a/tensorstore/kvstore/s3/credentials/aws_credential_provider.h b/tensorstore/kvstore/s3/credentials/aws_credential_provider.h index 6a2446af9..d95223ee8 100644 --- a/tensorstore/kvstore/s3/credentials/aws_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/aws_credential_provider.h @@ -39,6 +39,12 @@ struct AwsCredentials { std::string secret_key; /// AWS_SESSION_TOKEN std::string session_token; + /// Credentials expiry + absl::Time expires_at; + + static AwsCredentials Anonymous() { + return AwsCredentials{{}, {}, {}, absl::InfiniteFuture()}; + } bool IsAnonymous() const { return access_key.empty(); } }; @@ -52,10 +58,6 @@ class AwsCredentialProvider { public: virtual ~AwsCredentialProvider() = default; virtual Result GetCredentials() = 0; - virtual bool IsExpired() = 0; - virtual Result ExpiresAt() { - return absl::UnimplementedError("AwsCredentialProvider::ExpiresAt"); - }; }; diff --git a/tensorstore/kvstore/s3/credentials/cached_credential_provider.cc b/tensorstore/kvstore/s3/credentials/cached_credential_provider.cc deleted file mode 100644 index 3fef23749..000000000 --- a/tensorstore/kvstore/s3/credentials/cached_credential_provider.cc +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2023 The TensorStore Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "tensorstore/kvstore/s3/credentials/cached_credential_provider.h" - -#include "absl/status/status.h" -#include "absl/synchronization/mutex.h" -#include "tensorstore/util/result.h" - -namespace tensorstore { -namespace internal_kvstore_s3 { - -namespace {} // namespace - -bool CachedCredentialProvider::IsExpired() { - absl::ReaderMutexLock lock(&mutex_); - return IsExpiredLocked(credentials_); -} - -bool CachedCredentialProvider::IsExpiredLocked( - const AwsCredentials& credentials) { - return credentials.IsAnonymous() || provider_->IsExpired(); -} - -Result CachedCredentialProvider::ExpiresAt() { - absl::ReaderMutexLock lock(&mutex_); - TENSORSTORE_ASSIGN_OR_RETURN(auto expires, provider_->ExpiresAt()); - if (credentials_.IsAnonymous()) return absl::InfinitePast(); - return expires; -} - -Result CachedCredentialProvider::GetCredentials() { - absl::WriterMutexLock lock(&mutex_); - if (!IsExpiredLocked(credentials_)) { - return credentials_; - } - - TENSORSTORE_ASSIGN_OR_RETURN(credentials_, provider_->GetCredentials()); - return credentials_; -} - -} // namespace internal_kvstore_s3 -} // namespace tensorstore diff --git a/tensorstore/kvstore/s3/credentials/cached_credential_provider.h b/tensorstore/kvstore/s3/credentials/cached_credential_provider.h deleted file mode 100644 index a9641aca1..000000000 --- a/tensorstore/kvstore/s3/credentials/cached_credential_provider.h +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2023 The TensorStore Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef TENSORSTORE_KVSTORE_S3_CACHED_CREDENTIAL_PROVIDER_H -#define TENSORSTORE_KVSTORE_S3_CACHED_CREDENTIAL_PROVIDER_H - -#include "tensorstore/util/result.h" -#include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" - -namespace tensorstore { -namespace internal_kvstore_s3 { - -/// The Cached Credential Provider encapsulates an instance of AwsCredentialProvider -/// to provide: -/// 1. Thread-safe credential retrieval -/// 2. Caching of credentials -/// 3. Refresh of credentials upon expiry -class CachedCredentialProvider : public AwsCredentialProvider { - private: - absl::Mutex mutex_; - std::unique_ptr provider_ ABSL_GUARDED_BY(mutex_); - AwsCredentials credentials_ ABSL_GUARDED_BY(mutex_); - - bool IsExpiredLocked(const AwsCredentials & credentials) ABSL_SHARED_LOCKS_REQUIRED(mutex_); - - public: - CachedCredentialProvider(std::unique_ptr provider) - : provider_(std::move(provider)) { assert(provider_); } - - Result GetCredentials() override ABSL_LOCKS_EXCLUDED(mutex_); - bool IsExpired() override ABSL_LOCKS_EXCLUDED(mutex_); - Result ExpiresAt() override ABSL_LOCKS_EXCLUDED(mutex_); -}; - -} // namespace internal_kvstore_s3 -} // namespace tensorstore - -#endif // TENSORSTORE_KVSTORE_S3_CACHED_CREDENTIAL_PROVIDER_H \ No newline at end of file diff --git a/tensorstore/kvstore/s3/credentials/cached_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/cached_credential_provider_test.cc deleted file mode 100644 index 329298eb2..000000000 --- a/tensorstore/kvstore/s3/credentials/cached_credential_provider_test.cc +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright 2023 The TensorStore Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "tensorstore/kvstore/s3/credentials/cached_credential_provider.h" - -#include -#include - -#include - -#include "absl/log/absl_log.h" // remove -#include "tensorstore/internal/test_util.h" -#include "tensorstore/kvstore/s3/credentials/expiry_credential_provider.h" -#include "tensorstore/util/result.h" -#include "tensorstore/util/status_testutil.h" - -using ::tensorstore::Future; -using ::tensorstore::MatchesStatus; -using ::tensorstore::Result; -using ::tensorstore::internal_kvstore_s3::AwsCredentials; -using ::tensorstore::internal_kvstore_s3::CachedCredentialProvider; -using ::tensorstore::internal_kvstore_s3::ExpiryCredentialProvider; - -namespace { - -class TestCredentialProvider : public ExpiryCredentialProvider { - public: - int iteration = 0; - - TestCredentialProvider(absl::FunctionRef clock) - : ExpiryCredentialProvider(std::move(clock)) {} - - Result GetCredentials() override { - ABSL_LOG(INFO) << "GetCredentials() == " << clock(); - this->SetExpiration(clock() + absl::Seconds(2)); - return AwsCredentials{"key", std::to_string(++iteration)}; - } -}; - -TEST(CachedCredentialProviderTest, ExpiringProvider) { - auto utc = absl::UTCTimeZone(); - auto frozen_time = - absl::FromCivil(absl::CivilSecond(2023, 9, 6, 0, 4, 03), utc); - auto stuck_clock = [&frozen_time]() -> absl::Time { return frozen_time; }; - ASSERT_EQ(stuck_clock(), frozen_time); - - auto test_provider = std::make_unique(stuck_clock); - auto test_prov_ptr = test_provider.get(); - auto cached_provider = CachedCredentialProvider{std::move(test_provider)}; - - // Base case, no credentials have been retrieved yet - ASSERT_TRUE(cached_provider.IsExpired()); - ASSERT_EQ(cached_provider.ExpiresAt(), absl::InfinitePast()); - ASSERT_EQ(test_prov_ptr->iteration, 0); - - // Retrieve some credentials - TENSORSTORE_CHECK_OK_AND_ASSIGN(auto credentials, - cached_provider.GetCredentials()); - ASSERT_EQ(credentials.access_key, "key"); - ASSERT_EQ(credentials.secret_key, "1"); - ASSERT_EQ(test_prov_ptr->iteration, 1); - ASSERT_EQ(cached_provider.ExpiresAt(), frozen_time + absl::Seconds(2)); - ASSERT_FALSE(test_prov_ptr->IsExpired()); - ASSERT_FALSE(cached_provider.IsExpired()); - - // Idempotent when underlying credentials not expired - TENSORSTORE_CHECK_OK_AND_ASSIGN(credentials, - cached_provider.GetCredentials()); - ASSERT_EQ(credentials.access_key, "key"); - ASSERT_EQ(credentials.secret_key, "1"); - ASSERT_EQ(test_prov_ptr->iteration, 1); - ASSERT_EQ(cached_provider.ExpiresAt(), frozen_time + absl::Seconds(2)); - ASSERT_FALSE(test_prov_ptr->IsExpired()); - ASSERT_FALSE(cached_provider.IsExpired()); - - // Advance clock forward - frozen_time += absl::Seconds(2.5); - ASSERT_TRUE(test_prov_ptr->IsExpired()); - ASSERT_TRUE(cached_provider.IsExpired()); - - // A new set of credentials is retrieved due to expiry - TENSORSTORE_CHECK_OK_AND_ASSIGN(credentials, - cached_provider.GetCredentials()); - ASSERT_EQ(credentials.access_key, "key"); - ASSERT_EQ(credentials.secret_key, "2"); - ASSERT_EQ(test_prov_ptr->iteration, 2); - ASSERT_EQ(cached_provider.ExpiresAt(), frozen_time + absl::Seconds(2)); - ASSERT_FALSE(test_prov_ptr->IsExpired()); - ASSERT_FALSE(cached_provider.IsExpired()); -} - -} // namespace diff --git a/tensorstore/kvstore/s3/credentials/chained_credential_provider.cc b/tensorstore/kvstore/s3/credentials/chained_credential_provider.cc deleted file mode 100644 index 5408faa0b..000000000 --- a/tensorstore/kvstore/s3/credentials/chained_credential_provider.cc +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2023 The TensorStore Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "tensorstore/kvstore/s3/credentials/chained_credential_provider.h" - -#include -#include -#include - -#include "absl/status/status.h" -#include "absl/strings/str_join.h" -#include "absl/time/time.h" -#include "tensorstore/util/result.h" - -namespace tensorstore { -namespace internal_kvstore_s3 { - -bool ChainedCredentialProvider::IsExpired() { - if (!LastProviderValid()) { - return true; - } - - return providers_[last_provider_]->IsExpired(); -} - -Result ChainedCredentialProvider::ExpiresAt() { - if (!LastProviderValid()) { - return absl::UnimplementedError("ChainedCredentialProvider::ExpiresAt"); - } - - return providers_[last_provider_]->ExpiresAt(); -} - -Result ChainedCredentialProvider::GetCredentials() { - std::vector errors; - last_provider_ = -1; - - for (std::size_t i = 0; i < providers_.size(); ++i) { - auto credentials = providers_[i]->GetCredentials(); - if (credentials.ok()) { - last_provider_ = i; - return credentials; - } else { - errors.push_back(credentials.status().ToString()); - } - } - - return absl::NotFoundError( - absl::StrCat("No valid AwsCredentialProvider in chain:\n", - absl::StrJoin(errors, "\n"))); -} - -} // namespace internal_kvstore_s3 -} // namespace tensorstore diff --git a/tensorstore/kvstore/s3/credentials/chained_credential_provider.h b/tensorstore/kvstore/s3/credentials/chained_credential_provider.h deleted file mode 100644 index c28e193a7..000000000 --- a/tensorstore/kvstore/s3/credentials/chained_credential_provider.h +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2023 The TensorStore Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef TENSORSTORE_KVSTORE_S3_CHAINED_CREDENTIAL_PROVIDER -#define TENSORSTORE_KVSTORE_S3_CHAINED_CREDENTIAL_PROVIDER - -#include -#include - -#include "absl/time/time.h" -#include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" -#include "tensorstore/util/result.h" - -namespace tensorstore { -namespace internal_kvstore_s3 { - -// Returns credentials from a chain of providers -// The chain is iterated through, in order, until valid credentials are returned. -// The index of the returned provider is stored, and if valid, is used to -// proxy IsExpired calls to the relevant provider -class ChainedCredentialProvider : public AwsCredentialProvider { -private: - std::vector> providers_; - std::size_t last_provider_; - - inline bool LastProviderValid() { return last_provider_ >= 0 && last_provider_ < providers_.size(); } -public: - ChainedCredentialProvider(std::vector> providers) - : providers_(std::move(providers)), last_provider_(-1) {} - - Result GetCredentials() override; - Result ExpiresAt() override; - bool IsExpired() override; -}; - -} // namespace tensorstore -} // namespace internal_kvstore_s3 - -#endif // TENSORSTORE_KVSTORE_S3_CHAINED_CREDENTIAL_PROVIDER \ No newline at end of file diff --git a/tensorstore/kvstore/s3/credentials/chained_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/chained_credential_provider_test.cc deleted file mode 100644 index 2904f9456..000000000 --- a/tensorstore/kvstore/s3/credentials/chained_credential_provider_test.cc +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright 2023 The TensorStore Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "tensorstore/kvstore/s3/credentials/chained_credential_provider.h" - -#include -#include - -#include - -#include "tensorstore/internal/test_util.h" -#include "tensorstore/util/result.h" -#include "tensorstore/util/status_testutil.h" - -using ::tensorstore::Future; -using ::tensorstore::MatchesStatus; -using ::tensorstore::Result; -using ::tensorstore::internal_kvstore_s3::AwsCredentialProvider; -using ::tensorstore::internal_kvstore_s3::AwsCredentials; -using ::tensorstore::internal_kvstore_s3::ChainedCredentialProvider; - -namespace { - -class TestCredentialProvider : public AwsCredentialProvider { - public: - AwsCredentials credentials_; - bool expired_; - bool found_; - bool has_expired_at_; - TestCredentialProvider(std::string access_key = "") - : credentials_{access_key}, - expired_(false), - found_(true), - has_expired_at_(true) {} - - Result GetCredentials() override { - if (found_) { - expired_ = true; - return credentials_; - } - return absl::NotFoundError("Credentials Not Found"); - } - bool IsExpired() override { return expired_; } - Result ExpiresAt() override { - if (has_expired_at_) { - return absl::Now(); - } - return absl::UnimplementedError("TestCredentialProvider::ExpiresAt"); - } -}; - -TEST(ChainedCredentialProviderTest, EmptyProvider) { - auto provider = ChainedCredentialProvider({}); - auto credentials = provider.GetCredentials(); - ASSERT_FALSE(credentials.ok()); - ASSERT_TRUE(provider.IsExpired()); - EXPECT_THAT(provider.ExpiresAt(), - MatchesStatus(absl::StatusCode::kUnimplemented)); -} - -// Tests that Credential Retrieval results in IsExpiry and ExpiresAt -// proxying the correct encapsulated provider -TEST(ChainedCredentialProviderTest, ChainedGetAndExpiryLogic) { - auto one = std::make_unique("key1"); - auto two = std::make_unique("key2"); - auto one_ptr = one.get(); - auto two_ptr = two.get(); - - auto providers = std::vector>(); - providers.emplace_back(std::move(one)); - providers.emplace_back(std::move(two)); - auto provider = ChainedCredentialProvider(std::move(providers)); - ASSERT_EQ(provider.IsExpired(), true); - - // First call for credentials result in one's credentials - TENSORSTORE_CHECK_OK_AND_ASSIGN(auto credentials, provider.GetCredentials()); - ASSERT_EQ(credentials.access_key, "key1"); - ASSERT_EQ(provider.IsExpired(), true); - ASSERT_EQ(one_ptr->expired_, true); - ASSERT_EQ(two_ptr->expired_, false); - EXPECT_THAT(provider.ExpiresAt(), MatchesStatus(absl::StatusCode::kOk)); - - // Chained Provider proxies one for IsExpired calls - one_ptr->expired_ = false; - one_ptr->has_expired_at_ = false; - ASSERT_EQ(provider.IsExpired(), false); - EXPECT_THAT(provider.ExpiresAt(), - MatchesStatus(absl::StatusCode::kUnimplemented)); - - // Disable one's credentials and get new credentials (two) - one_ptr->found_ = false; - TENSORSTORE_CHECK_OK_AND_ASSIGN(credentials, provider.GetCredentials()); - ASSERT_EQ(credentials.access_key, "key2"); - ASSERT_EQ(provider.IsExpired(), true); - ASSERT_EQ(one_ptr->expired_, false); - ASSERT_EQ(two_ptr->expired_, true); - EXPECT_THAT(provider.ExpiresAt(), MatchesStatus(absl::StatusCode::kOk)); - - // Chained Provider proxies two for IsExpired calls - two_ptr->expired_ = false; - two_ptr->has_expired_at_ = false; - ASSERT_EQ(provider.IsExpired(), false); - EXPECT_THAT(provider.ExpiresAt(), - MatchesStatus(absl::StatusCode::kUnimplemented)); - - // Disable two's credentials and get new credentials - two_ptr->found_ = false; - auto result = provider.GetCredentials(); - ASSERT_FALSE(result.ok()); - ASSERT_EQ(provider.IsExpired(), true); - - // There's no valid provider, so ExpiresAt is also unimplemented - one_ptr->has_expired_at_ = true; - two_ptr->has_expired_at_ = true; - EXPECT_THAT(provider.ExpiresAt(), - MatchesStatus(absl::StatusCode::kUnimplemented)); -} - -} // namespace diff --git a/tensorstore/kvstore/s3/credentials/default_credential_provider.cc b/tensorstore/kvstore/s3/credentials/default_credential_provider.cc new file mode 100644 index 000000000..c95444fe5 --- /dev/null +++ b/tensorstore/kvstore/s3/credentials/default_credential_provider.cc @@ -0,0 +1,86 @@ +// Copyright 2023 The TensorStore Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "tensorstore/kvstore/s3/credentials/default_credential_provider.h" + +#include +#include + +#include "absl/time/time.h" +#include "absl/synchronization/mutex.h" +#include "tensorstore/util/result.h" +#include "tensorstore/kvstore/s3/credentials/environment_credential_provider.h" +#include "tensorstore/kvstore/s3/credentials/file_credential_provider.h" +#include "tensorstore/kvstore/s3/credentials/ec2_credential_provider.h" + +namespace tensorstore { +namespace internal_kvstore_s3 { + +DefaultAwsCredentialsProvider::DefaultAwsCredentialsProvider() { + credentials_.expires_at = absl::InfinitePast(); +} + +DefaultAwsCredentialsProvider::DefaultAwsCredentialsProvider(Options options) : options_(options) { + credentials_.expires_at = absl::InfinitePast(); +} + +Result DefaultAwsCredentialsProvider::GetCredentials() { + { + absl::ReaderMutexLock lock(&mutex_); + if(credentials_.expires_at > absl::Now()) { + return credentials_; + } + } + + absl::WriterMutexLock lock(&mutex_); + // Refresh existing credentials + if(provider_) { + TENSORSTORE_ASSIGN_OR_RETURN(credentials_, provider_->GetCredentials()); + return credentials_; + } + + // Return credentials in this order: + // 1. AWS Environment Variables, e.g. AWS_ACCESS_KEY_ID + provider_ = std::make_unique(); + auto credentials_result = provider_->GetCredentials(); + if (credentials_result.ok()) { + credentials_ = credentials_result.value(); + return credentials_; + } + + // 2. Shared Credential File, e.g. $HOME/.aws/credentials + provider_ = std::make_unique(options_.profile); + credentials_result = provider_->GetCredentials(); + if (credentials_result.ok()) { + credentials_ = credentials_result.value(); + return credentials_; + } + + // 3. EC2 Metadata Server + provider_ = std::make_unique(options_.transport); + credentials_result = provider_->GetCredentials(); + if (credentials_result.ok()) { + credentials_ = credentials_result.value(); + return credentials_; + } + + // 4. Anonymous credentials + provider_ = nullptr; + credentials_ = AwsCredentials::Anonymous(); + return credentials_; +} + + +} // namespace internal_kvstore_s3 +} // namespace tensorstore diff --git a/tensorstore/kvstore/s3/credentials/default_credential_provider.h b/tensorstore/kvstore/s3/credentials/default_credential_provider.h new file mode 100644 index 000000000..203bcf967 --- /dev/null +++ b/tensorstore/kvstore/s3/credentials/default_credential_provider.h @@ -0,0 +1,52 @@ +// Copyright 2023 The TensorStore Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef TENSORSTORE_KVSTORE_S3_CREDENTIALS_DEFAULT_CREDENTIAL_PROVIDER +#define TENSORSTORE_KVSTORE_S3_CREDENTIALS_DEFAULT_CREDENTIAL_PROVIDER + +#include +#include + +#include "absl/synchronization/mutex.h" +#include "absl/time/time.h" +#include "tensorstore/util/result.h" +#include "tensorstore/internal/http/http_transport.h" +#include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" + +namespace tensorstore { +namespace internal_kvstore_s3 { + +class DefaultAwsCredentialsProvider : public AwsCredentialProvider { + public: + struct Options { + std::string filename = {}; + std::string profile = {}; + std::shared_ptr transport; + }; + + DefaultAwsCredentialsProvider(); + DefaultAwsCredentialsProvider(Options options); + Result GetCredentials() override; + + private: + Options options_; + absl::Mutex mutex_; + std::unique_ptr provider_ ABSL_GUARDED_BY(mutex_); + AwsCredentials credentials_ ABSL_GUARDED_BY(mutex_); +}; + +} // namespace internal_kvstore_s3 +} // namespace tensorstore + +#endif // TENSORSTORE_KVSTORE_S3_CREDENTIALS_DEFAULT_CREDENTIAL_PROVIDER \ No newline at end of file diff --git a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc index 1c76dc77e..fa4c3c1d9 100644 --- a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc @@ -207,12 +207,13 @@ Result EC2MetadataCredentialProvider::GetCredentials() { "] failed with ", json_sv)); } - SetExpiration(iam_credentials.expiration.value_or(default_timeout), - absl::Seconds(60)); + // Introduce a leeway of 60 seconds to avoid credential expiry conditions + auto expires_at = iam_credentials.expiration.value_or(default_timeout) - absl::Seconds(60); return AwsCredentials{iam_credentials.access_key_id.value_or(""), iam_credentials.secret_access_key.value_or(""), - iam_credentials.token.value_or("")}; + iam_credentials.token.value_or(""), + expires_at}; } } // namespace internal_kvstore_s3 diff --git a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h index 8b0e2db8d..69485cfb9 100644 --- a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h @@ -21,14 +21,13 @@ #include "absl/base/thread_annotations.h" #include "tensorstore/internal/http/http_transport.h" #include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" -#include "tensorstore/kvstore/s3/credentials/expiry_credential_provider.h" #include "tensorstore/util/result.h" namespace tensorstore { namespace internal_kvstore_s3 { /// Provide S3 credentials from the EC2 Metadata server -class EC2MetadataCredentialProvider : public ExpiryCredentialProvider { +class EC2MetadataCredentialProvider : public AwsCredentialProvider { public: EC2MetadataCredentialProvider( std::shared_ptr transport) diff --git a/tensorstore/kvstore/s3/credentials/ec2_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/ec2_credential_provider_test.cc index 6ad82b69e..6868a8d74 100644 --- a/tensorstore/kvstore/s3/credentials/ec2_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/ec2_credential_provider_test.cc @@ -64,7 +64,7 @@ class EC2MetadataMockTransport : public HttpTransport { }; TEST(EC2MetadataCredentialProviderTest, CredentialRetrievalFlow) { - auto expiry = absl::Now() + absl::Seconds(240); + auto expiry = absl::Now() + absl::Seconds(200); auto utc = absl::UTCTimeZone(); auto url_to_response = absl::flat_hash_map{ @@ -98,15 +98,12 @@ TEST(EC2MetadataCredentialProviderTest, CredentialRetrievalFlow) { std::make_shared(url_to_response); auto provider = std::make_shared(mock_transport); - ASSERT_TRUE(provider->IsExpired()); - ASSERT_EQ(provider->ExpiresAt(), absl::InfinitePast()); TENSORSTORE_CHECK_OK_AND_ASSIGN(auto credentials, provider->GetCredentials()); ASSERT_EQ(credentials.access_key, "ASIA1234567890"); ASSERT_EQ(credentials.secret_key, "1234567890abcdef"); ASSERT_EQ(credentials.session_token, "abcdef123456790"); // expiry less the 60s leeway - ASSERT_EQ(provider->ExpiresAt(), expiry - absl::Seconds(60)); - ASSERT_FALSE(provider->IsExpired()); + ASSERT_EQ(credentials.expires_at, expiry - absl::Seconds(60)); } TEST(EC2MetadataCredentialProviderTest, NoIamRolesInSecurityCredentials) { diff --git a/tensorstore/kvstore/s3/credentials/environment_credential_provider.h b/tensorstore/kvstore/s3/credentials/environment_credential_provider.h index 7744521cf..f105fa081 100644 --- a/tensorstore/kvstore/s3/credentials/environment_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/environment_credential_provider.h @@ -30,11 +30,6 @@ class EnvironmentCredentialProvider : public AwsCredentialProvider { public: EnvironmentCredentialProvider() : retrieved_(false) {} Result GetCredentials() override; - // Credentials obtained from the environment never expire - bool IsExpired() override { return !retrieved_; } - Result ExpiresAt() override { - return retrieved_ ? absl::InfiniteFuture() : absl::InfinitePast(); - } }; } // namespace internal_kvstore_s3 diff --git a/tensorstore/kvstore/s3/credentials/environment_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/environment_credential_provider_test.cc index 8666776f5..e81888f50 100644 --- a/tensorstore/kvstore/s3/credentials/environment_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/environment_credential_provider_test.cc @@ -46,16 +46,12 @@ class EnvironmentCredentialProviderTest : public ::testing::Test { TEST_F(EnvironmentCredentialProviderTest, ProviderNoCredentials) { auto provider = EnvironmentCredentialProvider(); - ASSERT_TRUE(provider.IsExpired()); - ASSERT_EQ(provider.ExpiresAt(), absl::InfinitePast()); ASSERT_FALSE(provider.GetCredentials().ok()); SetEnv("AWS_ACCESS_KEY_ID", "foo"); TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, provider.GetCredentials()); ASSERT_EQ(credentials.access_key, "foo"); ASSERT_TRUE(credentials.secret_key.empty()); ASSERT_TRUE(credentials.session_token.empty()); - ASSERT_FALSE(provider.IsExpired()); - ASSERT_EQ(provider.ExpiresAt(), absl::InfiniteFuture()); } TEST_F(EnvironmentCredentialProviderTest, ProviderAwsCredentialsFromEnv) { @@ -63,14 +59,10 @@ TEST_F(EnvironmentCredentialProviderTest, ProviderAwsCredentialsFromEnv) { SetEnv("AWS_SECRET_ACCESS_KEY", "bar"); SetEnv("AWS_SESSION_TOKEN", "qux"); auto provider = EnvironmentCredentialProvider(); - ASSERT_TRUE(provider.IsExpired()); - ASSERT_EQ(provider.ExpiresAt(), absl::InfinitePast()); TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, provider.GetCredentials()); ASSERT_EQ(credentials.access_key, "foo"); ASSERT_EQ(credentials.secret_key, "bar"); ASSERT_EQ(credentials.session_token, "qux"); - ASSERT_FALSE(provider.IsExpired()); - ASSERT_EQ(provider.ExpiresAt(), absl::InfiniteFuture()); } } // namespace diff --git a/tensorstore/kvstore/s3/credentials/expiry_credential_provider.h b/tensorstore/kvstore/s3/credentials/expiry_credential_provider.h deleted file mode 100644 index f961f018c..000000000 --- a/tensorstore/kvstore/s3/credentials/expiry_credential_provider.h +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2023 The TensorStore Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef TENSORSTORE_KVSTORE_S3_EXPIRY_CREDENTIAL_PROVIDER_H -#define TENSORSTORE_KVSTORE_S3_EXPIRY_CREDENTIAL_PROVIDER_H - -#include "absl/functional/function_ref.h" -#include "absl/time/time.h" -#include "tensorstore/util/result.h" -#include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" - -namespace tensorstore { -namespace internal_kvstore_s3 { - -/// Abstract class providing credentials with a time-based expiry. -/// Implementers overriding this class should call SetExpiration -/// when overriding the GetCredentials function. -/// In addition to an expiry date, SetExpiration can also be supplied -/// with a duration argument introduces jitter into the expiry to prevent -/// retrieval of expired credentials. -/// Constructed by default with absl::Now as a clock function, -/// but can be supplied with a custom clock for testing purposes. -class ExpiryCredentialProvider : public AwsCredentialProvider { - private: - absl::Time expiration_; - absl::FunctionRef clock_; - - protected: - absl::Time clock() { return clock_(); } - - public: - ExpiryCredentialProvider(absl::FunctionRef clock=absl::Now) - : expiration_(absl::InfinitePast()), clock_(std::move(clock)) {} - void SetExpiration(const absl::Time & expiration, - const absl::Duration & window=absl::Seconds(0)) { - expiration_ = expiration; - if(window > absl::Seconds(0)) { - expiration_ -= window; - } - } - virtual bool IsExpired() override { return expiration_ < clock_(); }; - virtual Result ExpiresAt() override { return expiration_; }; -}; - -} // namespace internal_kvstore_s3 -} // namespace tenstorstore - -#endif // TENSORSTORE_KVSTORE_S3_EXPIRY_CREDENTIAL_PROVIDER_H \ No newline at end of file diff --git a/tensorstore/kvstore/s3/credentials/expiry_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/expiry_credential_provider_test.cc deleted file mode 100644 index 1b31157dc..000000000 --- a/tensorstore/kvstore/s3/credentials/expiry_credential_provider_test.cc +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2023 The TensorStore Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "tensorstore/kvstore/s3/credentials/expiry_credential_provider.h" - -#include -#include - -#include - -#include "tensorstore/internal/test_util.h" -#include "tensorstore/util/result.h" -#include "tensorstore/util/status_testutil.h" - -using ::tensorstore::Future; -using ::tensorstore::MatchesStatus; -using ::tensorstore::Result; -using ::tensorstore::internal_kvstore_s3::AwsCredentials; -using ::tensorstore::internal_kvstore_s3::ExpiryCredentialProvider; - -namespace { - -class TestCredentialProvider : public ExpiryCredentialProvider { - public: - TestCredentialProvider( - const absl::FunctionRef& clock = absl::Now) - : ExpiryCredentialProvider(clock) {} - - Result GetCredentials() override { return AwsCredentials{}; } -}; - -TEST(ExpiryCredentialProviderTest, TestExpiry) { - auto utc = absl::UTCTimeZone(); - - // window durations - auto window_zero = absl::Seconds(0); - auto window_one = absl::Seconds(1); - - // Timestamps - auto two_seconds = - absl::FromCivil(absl::CivilSecond(2023, 9, 6, 0, 4, 02), utc); - auto three_seconds = - absl::FromCivil(absl::CivilSecond(2023, 9, 6, 0, 4, 03), utc); - auto four_seconds = - absl::FromCivil(absl::CivilSecond(2023, 9, 6, 0, 4, 04), utc); - auto five_seconds = - absl::FromCivil(absl::CivilSecond(2023, 9, 6, 0, 4, 05), utc); - - auto frozen_time = three_seconds; - auto stuck_clock = [&frozen_time]() -> absl::Time { return frozen_time; }; - - /// Configure provider with a stuck clock - auto provider = TestCredentialProvider{stuck_clock}; - ASSERT_TRUE(provider.IsExpired()); - ASSERT_EQ(provider.ExpiresAt(), absl::InfinitePast()); - - /// One second before clock - provider.SetExpiration(two_seconds, window_zero); - ASSERT_TRUE(provider.IsExpired()); - ASSERT_EQ(provider.ExpiresAt(), two_seconds); - - // Matches clock exactly - provider.SetExpiration(three_seconds, window_zero); - ASSERT_FALSE(provider.IsExpired()); - ASSERT_EQ(provider.ExpiresAt(), three_seconds); - - // One second after clock - provider.SetExpiration(four_seconds, window_zero); - ASSERT_FALSE(provider.IsExpired()); - ASSERT_EQ(provider.ExpiresAt(), four_seconds); - - /// Test expiry with duration window - // One second before clock - provider.SetExpiration(three_seconds, window_one); - ASSERT_TRUE(provider.IsExpired()); - ASSERT_EQ(provider.ExpiresAt(), two_seconds); - - // Matches clock exactly - provider.SetExpiration(four_seconds, window_one); - ASSERT_FALSE(provider.IsExpired()); - ASSERT_EQ(provider.ExpiresAt(), three_seconds); - - // One second after clock - provider.SetExpiration(five_seconds, window_one); - ASSERT_FALSE(provider.IsExpired()); - ASSERT_EQ(provider.ExpiresAt(), four_seconds); -} - -} // namespace diff --git a/tensorstore/kvstore/s3/credentials/file_credential_provider.h b/tensorstore/kvstore/s3/credentials/file_credential_provider.h index df91589b3..f393af2a7 100644 --- a/tensorstore/kvstore/s3/credentials/file_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/file_credential_provider.h @@ -40,11 +40,6 @@ class FileCredentialProvider : public AwsCredentialProvider { FileCredentialProvider(std::string profile) : profile_(std::move(profile)), retrieved_(false) {} Result GetCredentials() override; - // Shared Credentials never expire once retrieved - bool IsExpired() override { return !retrieved_; } - Result ExpiresAt() override { - return retrieved_ ? absl::InfiniteFuture() : absl::InfinitePast(); - } }; } // namespace internal_kvstore_s3 diff --git a/tensorstore/kvstore/s3/credentials/file_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/file_credential_provider_test.cc index f9b64d8e5..b7480eac7 100644 --- a/tensorstore/kvstore/s3/credentials/file_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/file_credential_provider_test.cc @@ -65,14 +65,10 @@ TEST_F(FileCredentialProviderTest, ProviderAwsCredentialsFromFileDefault) { SetEnv("AWS_SHARED_CREDENTIALS_FILE", credentials_filename.c_str()); auto provider = FileCredentialProvider(""); - ASSERT_TRUE(provider.IsExpired()); - ASSERT_EQ(provider.ExpiresAt(), absl::InfinitePast()); TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, provider.GetCredentials()); ASSERT_EQ(credentials.access_key, "AKIAIOSFODNN7EXAMPLE"); ASSERT_EQ(credentials.secret_key, "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); ASSERT_EQ(credentials.session_token, "abcdef1234567890"); - ASSERT_FALSE(provider.IsExpired()); - ASSERT_EQ(provider.ExpiresAt(), absl::InfiniteFuture()); } TEST_F(FileCredentialProviderTest, @@ -82,14 +78,10 @@ TEST_F(FileCredentialProviderTest, SetEnv("AWS_SHARED_CREDENTIALS_FILE", credentials_filename.c_str()); auto provider = FileCredentialProvider("alice"); - ASSERT_TRUE(provider.IsExpired()); - ASSERT_EQ(provider.ExpiresAt(), absl::InfinitePast()); TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, provider.GetCredentials()); ASSERT_EQ(credentials.access_key, "AKIAIOSFODNN6EXAMPLE"); ASSERT_EQ(credentials.secret_key, "wJalrXUtnFEMI/K7MDENG/bPxRfiCZEXAMPLEKEY"); ASSERT_EQ(credentials.session_token, ""); - ASSERT_FALSE(provider.IsExpired()); - ASSERT_EQ(provider.ExpiresAt(), absl::InfiniteFuture()); } TEST_F(FileCredentialProviderTest, ProviderAwsCredentialsFromFileProfileEnv) { @@ -99,14 +91,10 @@ TEST_F(FileCredentialProviderTest, ProviderAwsCredentialsFromFileProfileEnv) { SetEnv("AWS_SHARED_CREDENTIALS_FILE", credentials_filename.c_str()); SetEnv("AWS_PROFILE", "alice"); auto provider = FileCredentialProvider(""); - ASSERT_TRUE(provider.IsExpired()); - ASSERT_EQ(provider.ExpiresAt(), absl::InfinitePast()); TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, provider.GetCredentials()); ASSERT_EQ(credentials.access_key, "AKIAIOSFODNN6EXAMPLE"); ASSERT_EQ(credentials.secret_key, "wJalrXUtnFEMI/K7MDENG/bPxRfiCZEXAMPLEKEY"); ASSERT_EQ(credentials.session_token, ""); - ASSERT_FALSE(provider.IsExpired()); - ASSERT_EQ(provider.ExpiresAt(), absl::InfiniteFuture()); } TEST_F(FileCredentialProviderTest, @@ -117,11 +105,7 @@ TEST_F(FileCredentialProviderTest, SetEnv("AWS_SHARED_CREDENTIALS_FILE", credentials_filename.c_str()); SetEnv("AWS_PROFILE", "bob"); auto provider = FileCredentialProvider(""); - ASSERT_TRUE(provider.IsExpired()); - ASSERT_EQ(provider.ExpiresAt(), absl::InfinitePast()); ASSERT_FALSE(provider.GetCredentials().ok()); - ASSERT_TRUE(provider.IsExpired()); - ASSERT_EQ(provider.ExpiresAt(), absl::InfinitePast()); } } // namespace From 16f31e1ac69b497edc98f84154c8dc64ac0ac473 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Mon, 13 Nov 2023 17:53:16 +0200 Subject: [PATCH 36/63] Default Credential Provider tests --- tensorstore/kvstore/s3/credentials/BUILD | 31 +++- .../s3/credentials/aws_credential_provider.h | 23 +-- .../default_credential_provider.cc | 27 ++-- .../credentials/default_credential_provider.h | 24 +++- .../default_credential_provider_test.cc | 133 ++++++++++++++++++ .../s3/credentials/ec2_credential_provider.h | 2 - .../ec2_credential_provider_test.cc | 58 +------- .../environment_credential_provider.cc | 3 +- .../environment_credential_provider.h | 4 - .../credentials/file_credential_provider.cc | 23 +-- .../s3/credentials/file_credential_provider.h | 22 +-- .../file_credential_provider_test.cc | 41 +++++- .../kvstore/s3/credentials/test_utils.cc | 78 ++++++++++ .../kvstore/s3/credentials/test_utils.h | 54 +++++++ tensorstore/kvstore/s3/s3_key_value_store.cc | 6 +- 15 files changed, 412 insertions(+), 117 deletions(-) create mode 100644 tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc create mode 100644 tensorstore/kvstore/s3/credentials/test_utils.cc create mode 100644 tensorstore/kvstore/s3/credentials/test_utils.h diff --git a/tensorstore/kvstore/s3/credentials/BUILD b/tensorstore/kvstore/s3/credentials/BUILD index 7a18c8717..3903a7558 100644 --- a/tensorstore/kvstore/s3/credentials/BUILD +++ b/tensorstore/kvstore/s3/credentials/BUILD @@ -24,6 +24,7 @@ tensorstore_cc_library( "//tensorstore/internal:no_destructor", "//tensorstore/internal:path", "//tensorstore/internal/http", + "//tensorstore/internal/http:curl_transport", "//tensorstore/internal/json", "//tensorstore/internal/json_binding", "//tensorstore/internal/json_binding:absl_time", @@ -37,6 +38,34 @@ tensorstore_cc_library( ], ) + +tensorstore_cc_library( + name = "test_utils", + srcs = ["test_utils.cc"], + hdrs = ["test_utils.h"], + deps = [ + "//tensorstore/internal/http", + "//tensorstore/util:str_cat", + "//tensorstore/util:result", + "@com_google_absl//absl/container:flat_hash_map", + ] +) + +tensorstore_cc_test( + name = "default_credential_provider_test", + size = "small", + srcs = ["default_credential_provider_test.cc"], + deps = [ + ":aws_credential_provider", + ":test_utils", + "//tensorstore/internal:test_util", + "//tensorstore/util:result", + "//tensorstore/util:status_testutil", + "@com_google_googletest//:gtest_main", + ], +) + + tensorstore_cc_test( name = "environmental_credential_provider_test", srcs = ["environment_credential_provider_test.cc"], @@ -69,11 +98,11 @@ tensorstore_cc_test( srcs = ["ec2_credential_provider_test.cc"], deps = [ ":aws_credential_provider", + ":test_utils", "//tensorstore/internal/http", "//tensorstore/util:result", "//tensorstore/util:status_testutil", "//tensorstore/util:str_cat", - "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", "@com_google_absl//absl/strings:cord", diff --git a/tensorstore/kvstore/s3/credentials/aws_credential_provider.h b/tensorstore/kvstore/s3/credentials/aws_credential_provider.h index d95223ee8..384d7963e 100644 --- a/tensorstore/kvstore/s3/credentials/aws_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/aws_credential_provider.h @@ -16,6 +16,8 @@ #define TENSORSTORE_KVSTORE_S3_CREDENTIALS_AWS_CREDENTIAL_PROVIDER #include +#include +#include #include #include "absl/time/time.h" @@ -27,23 +29,24 @@ namespace internal_kvstore_s3 { /// Holds AWS credentials /// -/// Contains the access key, secret key and session token. +/// Contains the access key, secret key, session token and expiry /// An empty access key implies anonymous access, /// while the presence of a session token implies the use of /// short-lived STS credentials /// https://docs.aws.amazon.com/STS/latest/APIReference/welcome.html struct AwsCredentials { /// AWS_ACCESS_KEY_ID - std::string access_key; + std::string access_key = {}; /// AWS_SECRET_KEY_ID - std::string secret_key; + std::string secret_key = {}; /// AWS_SESSION_TOKEN - std::string session_token; - /// Credentials expiry - absl::Time expires_at; + std::string session_token = {}; + /// Expiration date + absl::Time expires_at = absl::InfinitePast(); - static AwsCredentials Anonymous() { - return AwsCredentials{{}, {}, {}, absl::InfiniteFuture()}; + /// Anonymous credentials expiring duration seconds from now + static AwsCredentials Anonymous(const absl::Duration & duration=absl::InfiniteDuration()) { + return AwsCredentials{{}, {}, {}, absl::Now() + duration}; } bool IsAnonymous() const { return access_key.empty(); } @@ -51,9 +54,7 @@ struct AwsCredentials { /// Base class for S3 Credential Providers /// -/// Implementers should override GetCredentials, IsExpired and, -/// if the credential source supports the -/// representation of an expiry date, ExpiresAt. +/// Implementers should override GetCredentials. class AwsCredentialProvider { public: virtual ~AwsCredentialProvider() = default; diff --git a/tensorstore/kvstore/s3/credentials/default_credential_provider.cc b/tensorstore/kvstore/s3/credentials/default_credential_provider.cc index c95444fe5..4da9595fe 100644 --- a/tensorstore/kvstore/s3/credentials/default_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/default_credential_provider.cc @@ -17,37 +17,38 @@ #include #include -#include "absl/time/time.h" #include "absl/synchronization/mutex.h" +#include "absl/time/time.h" #include "tensorstore/util/result.h" +#include "tensorstore/internal/http/http_transport.h" +#include "tensorstore/kvstore/s3/credentials/ec2_credential_provider.h" #include "tensorstore/kvstore/s3/credentials/environment_credential_provider.h" #include "tensorstore/kvstore/s3/credentials/file_credential_provider.h" -#include "tensorstore/kvstore/s3/credentials/ec2_credential_provider.h" namespace tensorstore { namespace internal_kvstore_s3 { -DefaultAwsCredentialsProvider::DefaultAwsCredentialsProvider() { - credentials_.expires_at = absl::InfinitePast(); -} -DefaultAwsCredentialsProvider::DefaultAwsCredentialsProvider(Options options) : options_(options) { - credentials_.expires_at = absl::InfinitePast(); -} +DefaultAwsCredentialsProvider::DefaultAwsCredentialsProvider(Options options, absl::FunctionRef clock) + : options_(std::move(options)), clock_(clock), credentials_{{}, {}, {}, absl::InfinitePast()} {} Result DefaultAwsCredentialsProvider::GetCredentials() { { absl::ReaderMutexLock lock(&mutex_); - if(credentials_.expires_at > absl::Now()) { + if (credentials_.expires_at > clock_()) { return credentials_; } } absl::WriterMutexLock lock(&mutex_); + // Refresh existing credentials - if(provider_) { - TENSORSTORE_ASSIGN_OR_RETURN(credentials_, provider_->GetCredentials()); - return credentials_; + if (provider_) { + auto credentials_result = provider_->GetCredentials(); + if (credentials_result.ok()) { + credentials_ = credentials_result.value(); + return credentials_; + } } // Return credentials in this order: @@ -60,7 +61,7 @@ Result DefaultAwsCredentialsProvider::GetCredentials() { } // 2. Shared Credential File, e.g. $HOME/.aws/credentials - provider_ = std::make_unique(options_.profile); + provider_ = std::make_unique(options_.filename, options_.profile); credentials_result = provider_->GetCredentials(); if (credentials_result.ok()) { credentials_ = credentials_result.value(); diff --git a/tensorstore/kvstore/s3/credentials/default_credential_provider.h b/tensorstore/kvstore/s3/credentials/default_credential_provider.h index 203bcf967..bb7874858 100644 --- a/tensorstore/kvstore/s3/credentials/default_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/default_credential_provider.h @@ -18,29 +18,47 @@ #include #include +#include "absl/base/thread_annotations.h" +#include "absl/functional/function_ref.h" #include "absl/synchronization/mutex.h" -#include "absl/time/time.h" #include "tensorstore/util/result.h" +#include "tensorstore/internal/http/curl_transport.h" #include "tensorstore/internal/http/http_transport.h" #include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" namespace tensorstore { namespace internal_kvstore_s3 { +/// A provider that implements a Default strategy for retrieving +/// and caching a set of AWS credentials from the following sources: +/// +/// 1. Environment variables such as AWS_ACCESS_KEY_ID +/// 2. Shared Credential Files such as ~/.aws/credentials +/// 3. The EC2 Metadata server +/// +/// The cached credentials are returned until they expire, +/// at which point the original source is queried again to +/// obtain fresher credentials class DefaultAwsCredentialsProvider : public AwsCredentialProvider { public: + /// Options to configure the provider. These include the: + /// + /// 1. Shared Credential Filename + /// 2. Shared Credential File Profile + /// 3. Http Transport for querying the EC2 Metadata Server struct Options { std::string filename = {}; std::string profile = {}; std::shared_ptr transport; }; - DefaultAwsCredentialsProvider(); - DefaultAwsCredentialsProvider(Options options); + DefaultAwsCredentialsProvider(Options options={{}, {}, internal_http::GetDefaultHttpTransport()}, + absl::FunctionRef clock=absl::Now); Result GetCredentials() override; private: Options options_; + absl::FunctionRef clock_; absl::Mutex mutex_; std::unique_ptr provider_ ABSL_GUARDED_BY(mutex_); AwsCredentials credentials_ ABSL_GUARDED_BY(mutex_); diff --git a/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc new file mode 100644 index 000000000..9fa51e7ef --- /dev/null +++ b/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc @@ -0,0 +1,133 @@ +// Copyright 2023 The TensorStore Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "tensorstore/kvstore/s3/credentials/default_credential_provider.h" + +#include +#include + +#include +#include + +#include "absl/strings/cord.h" +#include "absl/container/flat_hash_map.h" +#include "tensorstore/kvstore/s3/credentials/test_utils.h" +#include "tensorstore/internal/http/http_response.h" +#include "tensorstore/internal/env.h" +#include "tensorstore/internal/path.h" +#include "tensorstore/internal/test_util.h" +#include "tensorstore/util/status_testutil.h" +#include "tensorstore/util/str_cat.h" + +namespace { + +using ::tensorstore::Future; +using ::tensorstore::internal::JoinPath; +using ::tensorstore::internal::SetEnv; +using ::tensorstore::internal::UnsetEnv; +using ::tensorstore::internal_http::HttpResponse; +using ::tensorstore::internal_kvstore_s3::DefaultAwsCredentialsProvider; +using ::tensorstore::internal_kvstore_s3::EC2MetadataMockTransport; +using ::tensorstore::internal_kvstore_s3::DefaultEC2MetadataFlow; +using Options = ::tensorstore::internal_kvstore_s3::DefaultAwsCredentialsProvider::Options; + + +class CredentialFileFactory : public tensorstore::internal::ScopedTemporaryDirectory { + public: + std::string WriteCredentialsFile() { + auto p = JoinPath(path(), "aws_config"); + std::ofstream ofs(p); + ofs << "[alice]\n" + "aws_access_key_id = AKIAIOSFODNN6EXAMPLE\n" + "aws_secret_access_key = " + "wJalrXUtnFEMI/K7MDENG/bPxRfiCZEXAMPLEKEY\n" + "aws_session_token = abcdef1234567890\n" + "\n"; + ofs.close(); + return p; + } +}; + +/// Test configuration of FileCredentialProvider from DefaultAwsCredentialsProvider::Options +TEST(DefaultCredentialProviderTest, ConfigureFileProviderFromOptions) { + auto factory = CredentialFileFactory{}; + auto credentials_file = factory.WriteCredentialsFile(); + + auto provider = std::make_unique(Options{credentials_file, "alice"}); + TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, provider->GetCredentials()); + EXPECT_EQ(credentials.access_key, "AKIAIOSFODNN6EXAMPLE"); + EXPECT_EQ(credentials.secret_key, "wJalrXUtnFEMI/K7MDENG/bPxRfiCZEXAMPLEKEY"); + EXPECT_EQ(credentials.session_token, "abcdef1234567890"); +} + +/// Test configuration of EC2MetaDataProvider from DefaultAwsCredentialsProvider::Options +TEST(DefaultCredentialProviderTest, ConfigureEC2ProviderFromOptions) { + auto now = absl::Now(); + auto stuck_clock = [&]() -> absl::Time { return now; }; + auto expiry = now + absl::Seconds(200); + auto url_to_response = DefaultEC2MetadataFlow("1234", "ASIA1234567890", "1234567890abcdef", "token", expiry); + + auto mock_transport = + std::make_shared(url_to_response); + + auto provider = std::make_unique(Options{{}, {}, mock_transport}, stuck_clock); + TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, provider->GetCredentials()); + EXPECT_EQ(credentials.access_key, "ASIA1234567890"); + EXPECT_EQ(credentials.secret_key, "1234567890abcdef"); + EXPECT_EQ(credentials.session_token, "token"); + ASSERT_EQ(credentials.expires_at, expiry - absl::Seconds(60)); + + /// Force failure on credential retrieval + url_to_response = absl::flat_hash_map{ + {"POST http://169.254.169.254/latest/api/token", + HttpResponse{404, absl::Cord{""}}}, + }; + + // But we're not expired, so we get the original credentials + TENSORSTORE_ASSERT_OK_AND_ASSIGN(credentials, provider->GetCredentials()); + EXPECT_EQ(credentials.access_key, "ASIA1234567890"); + EXPECT_EQ(credentials.secret_key, "1234567890abcdef"); + EXPECT_EQ(credentials.session_token, "token"); + ASSERT_EQ(credentials.expires_at, expiry - absl::Seconds(60)); + + // Force expiry and retrieve new credentials + now += absl::Seconds(300); + url_to_response = DefaultEC2MetadataFlow("1234", "ASIA1234567890", "1234567890abcdef", "TOKEN", expiry); + + // A new set of credentials is returned + TENSORSTORE_ASSERT_OK_AND_ASSIGN(credentials, provider->GetCredentials()); + EXPECT_EQ(credentials.access_key, "ASIA1234567890"); + EXPECT_EQ(credentials.secret_key, "1234567890abcdef"); + EXPECT_EQ(credentials.session_token, "TOKEN"); + ASSERT_EQ(credentials.expires_at, expiry - absl::Seconds(60)); + + /// Force failure on credential retrieval + url_to_response = absl::flat_hash_map{ + {"POST http://169.254.169.254/latest/api/token", + HttpResponse{404, absl::Cord{""}}}, + }; + + // Anonymous credentials + TENSORSTORE_ASSERT_OK_AND_ASSIGN(credentials, provider->GetCredentials()); + EXPECT_EQ(credentials.access_key, ""); + EXPECT_EQ(credentials.secret_key, ""); + EXPECT_EQ(credentials.session_token, ""); + ASSERT_EQ(credentials.expires_at, absl::InfiniteFuture()); + + +} + + + +} // namespace \ No newline at end of file diff --git a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h index 69485cfb9..43e935cd8 100644 --- a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h @@ -16,9 +16,7 @@ #define TENSORSTORE_KVSTORE_S3_CREDENTIALS_EC2_METADATA_CREDENTIAL_PROVIDER_H #include -#include -#include "absl/base/thread_annotations.h" #include "tensorstore/internal/http/http_transport.h" #include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" #include "tensorstore/util/result.h" diff --git a/tensorstore/kvstore/s3/credentials/ec2_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/ec2_credential_provider_test.cc index 6868a8d74..4c230c9b1 100644 --- a/tensorstore/kvstore/s3/credentials/ec2_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/ec2_credential_provider_test.cc @@ -25,74 +25,24 @@ #include "absl/status/status.h" #include "absl/strings/cord.h" #include "absl/time/time.h" -#include "tensorstore/internal/http/http_request.h" +#include "tensorstore/kvstore/s3/credentials/test_utils.h" #include "tensorstore/internal/http/http_response.h" -#include "tensorstore/internal/http/http_transport.h" #include "tensorstore/util/result.h" #include "tensorstore/util/status_testutil.h" #include "tensorstore/util/str_cat.h" namespace { -using ::tensorstore::Future; using ::tensorstore::MatchesStatus; -using ::tensorstore::internal_http::HttpRequest; using ::tensorstore::internal_http::HttpResponse; -using ::tensorstore::internal_http::HttpTransport; +using ::tensorstore::internal_kvstore_s3::DefaultEC2MetadataFlow; using ::tensorstore::internal_kvstore_s3::EC2MetadataCredentialProvider; +using ::tensorstore::internal_kvstore_s3::EC2MetadataMockTransport; -class EC2MetadataMockTransport : public HttpTransport { - public: - EC2MetadataMockTransport( - const absl::flat_hash_map& url_to_response) - : url_to_response_(url_to_response) {} - - Future IssueRequest(const HttpRequest& request, - absl::Cord payload, - absl::Duration request_timeout, - absl::Duration connect_timeout) override { - ABSL_LOG(INFO) << request; - auto it = url_to_response_.find( - tensorstore::StrCat(request.method, " ", request.url)); - if (it != url_to_response_.end()) { - return it->second; - } - return HttpResponse{404, absl::Cord(), {}}; - } - - const absl::flat_hash_map& url_to_response_; -}; TEST(EC2MetadataCredentialProviderTest, CredentialRetrievalFlow) { auto expiry = absl::Now() + absl::Seconds(200); - auto utc = absl::UTCTimeZone(); - - auto url_to_response = absl::flat_hash_map{ - {"POST http://169.254.169.254/latest/api/token", - HttpResponse{200, absl::Cord{"1234567890"}}}, - {"GET http://169.254.169.254/latest/meta-data/iam/", - HttpResponse{200, - absl::Cord{"info"}, - {{"x-aws-ec2-metadata-token", "1234567890"}}}}, - {"GET http://169.254.169.254/latest/meta-data/iam/security-credentials/", - HttpResponse{200, - absl::Cord{"mock-iam-role\nmock-iam-role2"}, - {{"x-aws-ec2-metadata-token", "1234567890"}}}}, - {"GET " - "http://169.254.169.254/latest/meta-data/iam/security-credentials/" - "mock-iam-role", - HttpResponse{200, - absl::Cord(absl::FormatTime(R"({ - "Code": "Success", - "LastUpdated": "2023-09-21T12:42:12Z", - "Type": "AWS-HMAC", - "AccessKeyId": "ASIA1234567890", - "SecretAccessKey": "1234567890abcdef", - "Token": "abcdef123456790", - "Expiration": "%Y-%m-%d%ET%H:%M:%E*S%Ez" - })", - expiry, utc)), - {{"x-aws-ec2-metadata-token", "1234567890"}}}}}; + auto url_to_response = DefaultEC2MetadataFlow("1234567890", "ASIA1234567890", "1234567890abcdef", "abcdef123456790", expiry); auto mock_transport = std::make_shared(url_to_response); diff --git a/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc b/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc index 5ef44dd92..806743777 100644 --- a/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc @@ -15,6 +15,7 @@ #include "tensorstore/kvstore/s3/credentials/environment_credential_provider.h" #include "absl/status/status.h" +#include "absl/time/time.h" #include "tensorstore/internal/env.h" using ::tensorstore::internal::GetEnv; @@ -47,7 +48,7 @@ Result EnvironmentCredentialProvider::GetCredentials() { credentials.session_token = *session_token; } - retrieved_ = true; + credentials.expires_at = absl::InfiniteFuture(); return credentials; } diff --git a/tensorstore/kvstore/s3/credentials/environment_credential_provider.h b/tensorstore/kvstore/s3/credentials/environment_credential_provider.h index f105fa081..701d08b94 100644 --- a/tensorstore/kvstore/s3/credentials/environment_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/environment_credential_provider.h @@ -24,11 +24,7 @@ namespace internal_kvstore_s3 { /// Provides credentials from the following environment variables: /// AWS_ACCESS_KEY_ID, AWS_SECRET_KEY_ID, AWS_SESSION_TOKEN class EnvironmentCredentialProvider : public AwsCredentialProvider { - private: - bool retrieved_; - public: - EnvironmentCredentialProvider() : retrieved_(false) {} Result GetCredentials() override; }; diff --git a/tensorstore/kvstore/s3/credentials/file_credential_provider.cc b/tensorstore/kvstore/s3/credentials/file_credential_provider.cc index 4970fb5dc..7f5fd0cc9 100644 --- a/tensorstore/kvstore/s3/credentials/file_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/file_credential_provider.cc @@ -21,6 +21,7 @@ #include "absl/status/status.h" #include "absl/strings/str_cat.h" #include "absl/strings/strip.h" +#include "absl/time/time.h" #include "tensorstore/internal/env.h" #include "tensorstore/internal/path.h" #include "tensorstore/util/result.h" @@ -76,21 +77,25 @@ Result GetAwsCredentialsFileName() { /// https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html#cli-configure-files-format Result FileCredentialProvider::GetCredentials() { - TENSORSTORE_ASSIGN_OR_RETURN(auto filename, GetAwsCredentialsFileName()); + TENSORSTORE_ASSIGN_OR_RETURN(auto filename, ([&]() -> Result { + if(!filename_.empty()) return filename_; + return GetAwsCredentialsFileName(); + }())); + std::ifstream ifs(filename.c_str()); if (!ifs) { return absl::NotFoundError( absl::StrCat("Could not open credentials file [", filename, "]")); } - std::string profile = !profile_.empty() - ? std::string(profile_) - : GetEnv(kEnvAwsProfile).value_or(kDefaultProfile); + auto profile = !profile_.empty() + ? std::string(profile_) + : GetEnv(kEnvAwsProfile).value_or(kDefaultProfile); - AwsCredentials credentials; - std::string section_name; - std::string line; - bool profile_found = false; + auto credentials = AwsCredentials{}; + auto section_name = std::string{}; + auto line = std::string{}; + auto profile_found = false; while (std::getline(ifs, line)) { auto sline = absl::StripAsciiWhitespace(line); @@ -132,7 +137,7 @@ Result FileCredentialProvider::GetCredentials() { ABSL_LOG_FIRST_N(INFO, 1) << "Using profile [" << profile << "] in file [" << filename << "]"; - retrieved_ = true; + credentials.expires_at = absl::InfiniteFuture(); return credentials; } diff --git a/tensorstore/kvstore/s3/credentials/file_credential_provider.h b/tensorstore/kvstore/s3/credentials/file_credential_provider.h index f393af2a7..323cbbdfe 100644 --- a/tensorstore/kvstore/s3/credentials/file_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/file_credential_provider.h @@ -24,21 +24,23 @@ namespace tensorstore { namespace internal_kvstore_s3 { /// Obtains S3 credentials from a profile in a file, usually -/// `~/.aws/credentials` or a file specified in AWS_SHARED_CREDENTIALS_FILE. A -/// desired profile may be specified in the constructor: This value should be -/// derived from the s3 json spec. -/// However, if profile is passed as an empty string, the profile is obtained -/// from AWS_DEFAULT_PROFILE, AWS_PROFILE before finally defaulting to -/// "default". +/// `~/.aws/credentials` or a file specified in AWS_SHARED_CREDENTIALS_FILE. +/// A filename or desired profile may be specified in the constructor: +/// These values should be derived from the s3 json spec. +/// However, if filename is passed as an empty string, the filename is +/// obtained from AWS_SHARED_CREDENTIAL_FILE before defaulting to +/// `.aws/credentials`. +/// Likewise, if profile is passed as an empty string, +/// the profile is obtained from AWS_DEFAULT_PROFILE, AWS_PROFILE before +/// finally defaulting to "default". class FileCredentialProvider : public AwsCredentialProvider { private: - // desired profile + std::string filename_; std::string profile_; - bool retrieved_; public: - FileCredentialProvider(std::string profile) - : profile_(std::move(profile)), retrieved_(false) {} + FileCredentialProvider(std::string filename, std::string profile) + : filename_(std::move(filename)), profile_(std::move(profile)) {} Result GetCredentials() override; }; diff --git a/tensorstore/kvstore/s3/credentials/file_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/file_credential_provider_test.cc index b7480eac7..5989c4dc0 100644 --- a/tensorstore/kvstore/s3/credentials/file_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/file_credential_provider_test.cc @@ -56,7 +56,10 @@ class TestData : public tensorstore::internal::ScopedTemporaryDirectory { class FileCredentialProviderTest : public ::testing::Test { protected: - void SetUp() override { UnsetEnv("AWS_SHARED_CREDENTIALS_FILE"); } + void SetUp() override { + UnsetEnv("AWS_SHARED_CREDENTIALS_FILE"); + UnsetEnv("AWS_PROFILE"); + } }; TEST_F(FileCredentialProviderTest, ProviderAwsCredentialsFromFileDefault) { @@ -64,11 +67,12 @@ TEST_F(FileCredentialProviderTest, ProviderAwsCredentialsFromFileDefault) { std::string credentials_filename = test_data.WriteCredentialsFile(); SetEnv("AWS_SHARED_CREDENTIALS_FILE", credentials_filename.c_str()); - auto provider = FileCredentialProvider(""); + auto provider = FileCredentialProvider("", ""); TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, provider.GetCredentials()); ASSERT_EQ(credentials.access_key, "AKIAIOSFODNN7EXAMPLE"); ASSERT_EQ(credentials.secret_key, "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); ASSERT_EQ(credentials.session_token, "abcdef1234567890"); + ASSERT_EQ(credentials.expires_at, absl::InfiniteFuture()); } TEST_F(FileCredentialProviderTest, @@ -77,35 +81,58 @@ TEST_F(FileCredentialProviderTest, std::string credentials_filename = test_data.WriteCredentialsFile(); SetEnv("AWS_SHARED_CREDENTIALS_FILE", credentials_filename.c_str()); - auto provider = FileCredentialProvider("alice"); + auto provider = FileCredentialProvider("", "alice"); TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, provider.GetCredentials()); ASSERT_EQ(credentials.access_key, "AKIAIOSFODNN6EXAMPLE"); ASSERT_EQ(credentials.secret_key, "wJalrXUtnFEMI/K7MDENG/bPxRfiCZEXAMPLEKEY"); ASSERT_EQ(credentials.session_token, ""); + ASSERT_EQ(credentials.expires_at, absl::InfiniteFuture()); } TEST_F(FileCredentialProviderTest, ProviderAwsCredentialsFromFileProfileEnv) { TestData test_data; - std::string credentials_filename = test_data.WriteCredentialsFile(); + auto credentials_filename = test_data.WriteCredentialsFile(); SetEnv("AWS_SHARED_CREDENTIALS_FILE", credentials_filename.c_str()); SetEnv("AWS_PROFILE", "alice"); - auto provider = FileCredentialProvider(""); + auto provider = FileCredentialProvider("", ""); TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, provider.GetCredentials()); ASSERT_EQ(credentials.access_key, "AKIAIOSFODNN6EXAMPLE"); ASSERT_EQ(credentials.secret_key, "wJalrXUtnFEMI/K7MDENG/bPxRfiCZEXAMPLEKEY"); ASSERT_EQ(credentials.session_token, ""); + ASSERT_EQ(credentials.expires_at, absl::InfiniteFuture()); } TEST_F(FileCredentialProviderTest, ProviderAwsCredentialsFromFileInvalidProfileEnv) { TestData test_data; - std::string credentials_filename = test_data.WriteCredentialsFile(); + auto credentials_filename = test_data.WriteCredentialsFile(); SetEnv("AWS_SHARED_CREDENTIALS_FILE", credentials_filename.c_str()); SetEnv("AWS_PROFILE", "bob"); - auto provider = FileCredentialProvider(""); + auto provider = FileCredentialProvider("", ""); ASSERT_FALSE(provider.GetCredentials().ok()); } +TEST_F(FileCredentialProviderTest, + ProviderAwsCredentialsFromFileOverride) { + TestData test_data; + auto credentials_filename = test_data.WriteCredentialsFile(); + auto provider = std::make_unique(credentials_filename, ""); + TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, provider->GetCredentials()); + ASSERT_EQ(credentials.access_key, "AKIAIOSFODNN7EXAMPLE"); + ASSERT_EQ(credentials.secret_key, "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); + ASSERT_EQ(credentials.session_token, "abcdef1234567890"); + ASSERT_EQ(credentials.expires_at, absl::InfiniteFuture()); + + provider = std::make_unique(credentials_filename, "alice"); + TENSORSTORE_ASSERT_OK_AND_ASSIGN(credentials, provider->GetCredentials()); + ASSERT_EQ(credentials.access_key, "AKIAIOSFODNN6EXAMPLE"); + ASSERT_EQ(credentials.secret_key, "wJalrXUtnFEMI/K7MDENG/bPxRfiCZEXAMPLEKEY"); + ASSERT_EQ(credentials.session_token, ""); + ASSERT_EQ(credentials.expires_at, absl::InfiniteFuture()); + +} + + } // namespace diff --git a/tensorstore/kvstore/s3/credentials/test_utils.cc b/tensorstore/kvstore/s3/credentials/test_utils.cc new file mode 100644 index 000000000..c27da7021 --- /dev/null +++ b/tensorstore/kvstore/s3/credentials/test_utils.cc @@ -0,0 +1,78 @@ +// Copyright 2023 The TensorStore Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "tensorstore/kvstore/s3/credentials/test_utils.h" + +#include + +#include "absl/strings/cord.h" +#include "absl/time/time.h" +#include "tensorstore/internal/http/http_request.h" +#include "tensorstore/internal/http/http_response.h" +#include "tensorstore/util/str_cat.h" + +namespace tensorstore { +namespace internal_kvstore_s3 { + +Future EC2MetadataMockTransport::IssueRequest( + const internal_http::HttpRequest& request, + absl::Cord payload, + absl::Duration request_timeout, + absl::Duration connect_timeout) { + + ABSL_LOG(INFO) << request; + if(auto it = url_to_response_.find(StrCat(request.method, " ", request.url)); + it != url_to_response_.end()) { + return it->second; + } + + return internal_http::HttpResponse{404, absl::Cord(), {}}; +} + +absl::flat_hash_map DefaultEC2MetadataFlow( + const std::string & api_token, + const std::string & access_key, + const std::string & secret_key, + const std::string & session_token, + const absl::Time & expires_at) { + + return absl::flat_hash_map{ + {"POST http://169.254.169.254/latest/api/token", + internal_http::HttpResponse{200, absl::Cord{api_token}}}, + {"GET http://169.254.169.254/latest/meta-data/iam/", + internal_http::HttpResponse{200, absl::Cord{"info"}, + {{"x-aws-ec2-metadata-token", api_token}}}}, + {"GET http://169.254.169.254/latest/meta-data/iam/security-credentials/", + internal_http::HttpResponse{200, absl::Cord{"mock-iam-role"}, + {{"x-aws-ec2-metadata-token", api_token}}}}, + {"GET " + "http://169.254.169.254/latest/meta-data/iam/security-credentials/" + "mock-iam-role", + internal_http::HttpResponse{200, absl::Cord( + absl::StrFormat(R"({ + "Code": "Success", + "AccessKeyId": "%s", + "SecretAccessKey": "%s", + "Token": "%s", + "Expiration": "%s" + })", access_key, secret_key, session_token, + absl::FormatTime("%Y-%m-%d%ET%H:%M:%E*S%Ez", + expires_at, absl::UTCTimeZone()))), + {{"x-aws-ec2-metadata-token", api_token}}}}}; +} + + + +} // namespace internal_kvstore_s3 +} // namespace tensorstore \ No newline at end of file diff --git a/tensorstore/kvstore/s3/credentials/test_utils.h b/tensorstore/kvstore/s3/credentials/test_utils.h new file mode 100644 index 000000000..f9db84633 --- /dev/null +++ b/tensorstore/kvstore/s3/credentials/test_utils.h @@ -0,0 +1,54 @@ +// Copyright 2023 The TensorStore Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "absl/log/absl_log.h" +#include "absl/strings/cord.h" +#include "absl/container/flat_hash_map.h" +#include "absl/time/time.h" +#include "tensorstore/internal/http/http_transport.h" +#include "tensorstore/util/str_cat.h" + +namespace tensorstore { +namespace internal_kvstore_s3 { + +/// Mocks an HttpTransport by overriding the IssueRequest method to +/// respond with a predefined set of request-response pairs supplied +/// to the constructor +class EC2MetadataMockTransport : public internal_http::HttpTransport { + public: + EC2MetadataMockTransport( + const absl::flat_hash_map& url_to_response) + : url_to_response_(url_to_response) {} + + Future IssueRequest(const internal_http::HttpRequest& request, + absl::Cord payload, + absl::Duration request_timeout, + absl::Duration connect_timeout) override; + + const absl::flat_hash_map& url_to_response_; +}; + +/// Return a Default EC2 Metadata Credential Retrieval Flow, suitable +/// for passing to EC2MetadataMockTransport +absl::flat_hash_map DefaultEC2MetadataFlow( + const std::string & api_token, + const std::string & access_key, + const std::string & secret_key, + const std::string & session_token, + const absl::Time & expires_at); + +} // namespace internal_kvstore_s3 +} // namespace tensorstore diff --git a/tensorstore/kvstore/s3/s3_key_value_store.cc b/tensorstore/kvstore/s3/s3_key_value_store.cc index cc62b6d6c..73691012c 100644 --- a/tensorstore/kvstore/s3/s3_key_value_store.cc +++ b/tensorstore/kvstore/s3/s3_key_value_store.cc @@ -211,8 +211,9 @@ struct AwsCredentialsResource struct Spec { std::string profile; + std::string filename; constexpr static auto ApplyMembers = [](auto&& x, auto f) { - return f(x.profile); + return f(x.profile, x.filename); }; }; @@ -227,7 +228,8 @@ struct AwsCredentialsResource static constexpr auto JsonBinder() { return jb::Object( - jb::Member("profile", jb::Projection<&Spec::profile>()) /**/ + jb::Member("profile", jb::Projection<&Spec::profile>()), + jb::Member("filename", jb::Projection<&Spec::filename>()) ); } From 6bb81eb3bc082dc7cdb8cff248accfb4be05af62 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Mon, 13 Nov 2023 17:54:13 +0200 Subject: [PATCH 37/63] Mark some tests as small --- tensorstore/kvstore/s3/BUILD | 2 ++ tensorstore/kvstore/s3/credentials/BUILD | 3 +++ 2 files changed, 5 insertions(+) diff --git a/tensorstore/kvstore/s3/BUILD b/tensorstore/kvstore/s3/BUILD index 99bfb0547..456a402e8 100644 --- a/tensorstore/kvstore/s3/BUILD +++ b/tensorstore/kvstore/s3/BUILD @@ -271,6 +271,7 @@ tensorstore_cc_test( "@com_google_absl//absl/time", "@com_google_googletest//:gtest_main", ], + size = "small" ) tensorstore_cc_library( @@ -293,6 +294,7 @@ tensorstore_cc_library( tensorstore_cc_test( name = "s3_endpoint_test", + size = "small", srcs = ["s3_endpoint_test.cc"], deps = [ ":s3_endpoint", diff --git a/tensorstore/kvstore/s3/credentials/BUILD b/tensorstore/kvstore/s3/credentials/BUILD index 3903a7558..99b810678 100644 --- a/tensorstore/kvstore/s3/credentials/BUILD +++ b/tensorstore/kvstore/s3/credentials/BUILD @@ -68,6 +68,7 @@ tensorstore_cc_test( tensorstore_cc_test( name = "environmental_credential_provider_test", + size = "small", srcs = ["environment_credential_provider_test.cc"], deps = [ ":aws_credential_provider", @@ -81,6 +82,7 @@ tensorstore_cc_test( tensorstore_cc_test( name = "file_credential_provider_test", + size = "small", srcs = ["file_credential_provider_test.cc"], deps = [ ":aws_credential_provider", @@ -95,6 +97,7 @@ tensorstore_cc_test( tensorstore_cc_test( name = "ec2_credential_provider_test", + size = "small", srcs = ["ec2_credential_provider_test.cc"], deps = [ ":aws_credential_provider", From 5cd9f60d51c4ba04c64103282d3dfa58c44f8222 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Mon, 13 Nov 2023 17:57:01 +0200 Subject: [PATCH 38/63] clang-format --- .../s3/credentials/aws_credential_provider.cc | 6 +- .../s3/credentials/aws_credential_provider.h | 10 +-- .../default_credential_provider.cc | 21 +++--- .../credentials/default_credential_provider.h | 49 +++++++------- .../default_credential_provider_test.cc | 45 +++++++------ .../s3/credentials/ec2_credential_provider.cc | 6 +- .../ec2_credential_provider_test.cc | 7 +- .../environment_credential_provider.h | 6 +- .../credentials/file_credential_provider.cc | 10 +-- .../s3/credentials/file_credential_provider.h | 8 +-- .../file_credential_provider_test.cc | 14 ++-- .../kvstore/s3/credentials/test_utils.cc | 66 +++++++++---------- .../kvstore/s3/credentials/test_utils.h | 33 +++++----- 13 files changed, 145 insertions(+), 136 deletions(-) diff --git a/tensorstore/kvstore/s3/credentials/aws_credential_provider.cc b/tensorstore/kvstore/s3/credentials/aws_credential_provider.cc index 7eb55338e..c5a15b098 100644 --- a/tensorstore/kvstore/s3/credentials/aws_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/aws_credential_provider.cc @@ -40,9 +40,9 @@ namespace { Result> GetDefaultAwsCredentialProvider( std::string_view profile, std::shared_ptr transport) { - - return std::make_unique( - DefaultAwsCredentialsProvider::Options{{}, std::string{profile}, transport}); + return std::make_unique( + DefaultAwsCredentialsProvider::Options{ + {}, std::string{profile}, transport}); } struct AwsCredentialProviderRegistry { diff --git a/tensorstore/kvstore/s3/credentials/aws_credential_provider.h b/tensorstore/kvstore/s3/credentials/aws_credential_provider.h index 384d7963e..1fab927f2 100644 --- a/tensorstore/kvstore/s3/credentials/aws_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/aws_credential_provider.h @@ -45,7 +45,8 @@ struct AwsCredentials { absl::Time expires_at = absl::InfinitePast(); /// Anonymous credentials expiring duration seconds from now - static AwsCredentials Anonymous(const absl::Duration & duration=absl::InfiniteDuration()) { + static AwsCredentials Anonymous( + const absl::Duration& duration = absl::InfiniteDuration()) { return AwsCredentials{{}, {}, {}, absl::Now() + duration}; } @@ -56,12 +57,11 @@ struct AwsCredentials { /// /// Implementers should override GetCredentials. class AwsCredentialProvider { - public: - virtual ~AwsCredentialProvider() = default; - virtual Result GetCredentials() = 0; + public: + virtual ~AwsCredentialProvider() = default; + virtual Result GetCredentials() = 0; }; - using AwsCredentialProviderFn = std::function>()>; diff --git a/tensorstore/kvstore/s3/credentials/default_credential_provider.cc b/tensorstore/kvstore/s3/credentials/default_credential_provider.cc index 4da9595fe..2f2dfbc89 100644 --- a/tensorstore/kvstore/s3/credentials/default_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/default_credential_provider.cc @@ -19,18 +19,20 @@ #include "absl/synchronization/mutex.h" #include "absl/time/time.h" -#include "tensorstore/util/result.h" #include "tensorstore/internal/http/http_transport.h" #include "tensorstore/kvstore/s3/credentials/ec2_credential_provider.h" #include "tensorstore/kvstore/s3/credentials/environment_credential_provider.h" #include "tensorstore/kvstore/s3/credentials/file_credential_provider.h" +#include "tensorstore/util/result.h" namespace tensorstore { namespace internal_kvstore_s3 { - -DefaultAwsCredentialsProvider::DefaultAwsCredentialsProvider(Options options, absl::FunctionRef clock) - : options_(std::move(options)), clock_(clock), credentials_{{}, {}, {}, absl::InfinitePast()} {} +DefaultAwsCredentialsProvider::DefaultAwsCredentialsProvider( + Options options, absl::FunctionRef clock) + : options_(std::move(options)), + clock_(clock), + credentials_{{}, {}, {}, absl::InfinitePast()} {} Result DefaultAwsCredentialsProvider::GetCredentials() { { @@ -61,7 +63,8 @@ Result DefaultAwsCredentialsProvider::GetCredentials() { } // 2. Shared Credential File, e.g. $HOME/.aws/credentials - provider_ = std::make_unique(options_.filename, options_.profile); + provider_ = std::make_unique(options_.filename, + options_.profile); credentials_result = provider_->GetCredentials(); if (credentials_result.ok()) { credentials_ = credentials_result.value(); @@ -69,7 +72,8 @@ Result DefaultAwsCredentialsProvider::GetCredentials() { } // 3. EC2 Metadata Server - provider_ = std::make_unique(options_.transport); + provider_ = + std::make_unique(options_.transport); credentials_result = provider_->GetCredentials(); if (credentials_result.ok()) { credentials_ = credentials_result.value(); @@ -82,6 +86,5 @@ Result DefaultAwsCredentialsProvider::GetCredentials() { return credentials_; } - -} // namespace internal_kvstore_s3 -} // namespace tensorstore +} // namespace internal_kvstore_s3 +} // namespace tensorstore diff --git a/tensorstore/kvstore/s3/credentials/default_credential_provider.h b/tensorstore/kvstore/s3/credentials/default_credential_provider.h index bb7874858..3e323af2d 100644 --- a/tensorstore/kvstore/s3/credentials/default_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/default_credential_provider.h @@ -21,10 +21,10 @@ #include "absl/base/thread_annotations.h" #include "absl/functional/function_ref.h" #include "absl/synchronization/mutex.h" -#include "tensorstore/util/result.h" #include "tensorstore/internal/http/curl_transport.h" #include "tensorstore/internal/http/http_transport.h" #include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" +#include "tensorstore/util/result.h" namespace tensorstore { namespace internal_kvstore_s3 { @@ -40,31 +40,32 @@ namespace internal_kvstore_s3 { /// at which point the original source is queried again to /// obtain fresher credentials class DefaultAwsCredentialsProvider : public AwsCredentialProvider { - public: - /// Options to configure the provider. These include the: - /// - /// 1. Shared Credential Filename - /// 2. Shared Credential File Profile - /// 3. Http Transport for querying the EC2 Metadata Server - struct Options { - std::string filename = {}; - std::string profile = {}; - std::shared_ptr transport; - }; + public: + /// Options to configure the provider. These include the: + /// + /// 1. Shared Credential Filename + /// 2. Shared Credential File Profile + /// 3. Http Transport for querying the EC2 Metadata Server + struct Options { + std::string filename = {}; + std::string profile = {}; + std::shared_ptr transport; + }; - DefaultAwsCredentialsProvider(Options options={{}, {}, internal_http::GetDefaultHttpTransport()}, - absl::FunctionRef clock=absl::Now); - Result GetCredentials() override; + DefaultAwsCredentialsProvider( + Options options = {{}, {}, internal_http::GetDefaultHttpTransport()}, + absl::FunctionRef clock = absl::Now); + Result GetCredentials() override; - private: - Options options_; - absl::FunctionRef clock_; - absl::Mutex mutex_; - std::unique_ptr provider_ ABSL_GUARDED_BY(mutex_); - AwsCredentials credentials_ ABSL_GUARDED_BY(mutex_); + private: + Options options_; + absl::FunctionRef clock_; + absl::Mutex mutex_; + std::unique_ptr provider_ ABSL_GUARDED_BY(mutex_); + AwsCredentials credentials_ ABSL_GUARDED_BY(mutex_); }; -} // namespace internal_kvstore_s3 -} // namespace tensorstore +} // namespace internal_kvstore_s3 +} // namespace tensorstore -#endif // TENSORSTORE_KVSTORE_S3_CREDENTIALS_DEFAULT_CREDENTIAL_PROVIDER \ No newline at end of file +#endif // TENSORSTORE_KVSTORE_S3_CREDENTIALS_DEFAULT_CREDENTIAL_PROVIDER \ No newline at end of file diff --git a/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc index 9fa51e7ef..dc675ce64 100644 --- a/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc @@ -20,13 +20,13 @@ #include #include -#include "absl/strings/cord.h" #include "absl/container/flat_hash_map.h" -#include "tensorstore/kvstore/s3/credentials/test_utils.h" -#include "tensorstore/internal/http/http_response.h" +#include "absl/strings/cord.h" #include "tensorstore/internal/env.h" +#include "tensorstore/internal/http/http_response.h" #include "tensorstore/internal/path.h" #include "tensorstore/internal/test_util.h" +#include "tensorstore/kvstore/s3/credentials/test_utils.h" #include "tensorstore/util/status_testutil.h" #include "tensorstore/util/str_cat.h" @@ -38,12 +38,13 @@ using ::tensorstore::internal::SetEnv; using ::tensorstore::internal::UnsetEnv; using ::tensorstore::internal_http::HttpResponse; using ::tensorstore::internal_kvstore_s3::DefaultAwsCredentialsProvider; -using ::tensorstore::internal_kvstore_s3::EC2MetadataMockTransport; using ::tensorstore::internal_kvstore_s3::DefaultEC2MetadataFlow; -using Options = ::tensorstore::internal_kvstore_s3::DefaultAwsCredentialsProvider::Options; - +using ::tensorstore::internal_kvstore_s3::EC2MetadataMockTransport; +using Options = + ::tensorstore::internal_kvstore_s3::DefaultAwsCredentialsProvider::Options; -class CredentialFileFactory : public tensorstore::internal::ScopedTemporaryDirectory { +class CredentialFileFactory + : public tensorstore::internal::ScopedTemporaryDirectory { public: std::string WriteCredentialsFile() { auto p = JoinPath(path(), "aws_config"); @@ -59,30 +60,37 @@ class CredentialFileFactory : public tensorstore::internal::ScopedTemporaryDirec } }; -/// Test configuration of FileCredentialProvider from DefaultAwsCredentialsProvider::Options +/// Test configuration of FileCredentialProvider from +/// DefaultAwsCredentialsProvider::Options TEST(DefaultCredentialProviderTest, ConfigureFileProviderFromOptions) { auto factory = CredentialFileFactory{}; auto credentials_file = factory.WriteCredentialsFile(); - auto provider = std::make_unique(Options{credentials_file, "alice"}); - TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, provider->GetCredentials()); + auto provider = std::make_unique( + Options{credentials_file, "alice"}); + TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, + provider->GetCredentials()); EXPECT_EQ(credentials.access_key, "AKIAIOSFODNN6EXAMPLE"); EXPECT_EQ(credentials.secret_key, "wJalrXUtnFEMI/K7MDENG/bPxRfiCZEXAMPLEKEY"); EXPECT_EQ(credentials.session_token, "abcdef1234567890"); } -/// Test configuration of EC2MetaDataProvider from DefaultAwsCredentialsProvider::Options +/// Test configuration of EC2MetaDataProvider from +/// DefaultAwsCredentialsProvider::Options TEST(DefaultCredentialProviderTest, ConfigureEC2ProviderFromOptions) { auto now = absl::Now(); auto stuck_clock = [&]() -> absl::Time { return now; }; auto expiry = now + absl::Seconds(200); - auto url_to_response = DefaultEC2MetadataFlow("1234", "ASIA1234567890", "1234567890abcdef", "token", expiry); + auto url_to_response = DefaultEC2MetadataFlow( + "1234", "ASIA1234567890", "1234567890abcdef", "token", expiry); auto mock_transport = std::make_shared(url_to_response); - auto provider = std::make_unique(Options{{}, {}, mock_transport}, stuck_clock); - TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, provider->GetCredentials()); + auto provider = std::make_unique( + Options{{}, {}, mock_transport}, stuck_clock); + TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, + provider->GetCredentials()); EXPECT_EQ(credentials.access_key, "ASIA1234567890"); EXPECT_EQ(credentials.secret_key, "1234567890abcdef"); EXPECT_EQ(credentials.session_token, "token"); @@ -103,7 +111,8 @@ TEST(DefaultCredentialProviderTest, ConfigureEC2ProviderFromOptions) { // Force expiry and retrieve new credentials now += absl::Seconds(300); - url_to_response = DefaultEC2MetadataFlow("1234", "ASIA1234567890", "1234567890abcdef", "TOKEN", expiry); + url_to_response = DefaultEC2MetadataFlow("1234", "ASIA1234567890", + "1234567890abcdef", "TOKEN", expiry); // A new set of credentials is returned TENSORSTORE_ASSERT_OK_AND_ASSIGN(credentials, provider->GetCredentials()); @@ -124,10 +133,6 @@ TEST(DefaultCredentialProviderTest, ConfigureEC2ProviderFromOptions) { EXPECT_EQ(credentials.secret_key, ""); EXPECT_EQ(credentials.session_token, ""); ASSERT_EQ(credentials.expires_at, absl::InfiniteFuture()); - - } - - -} // namespace \ No newline at end of file +} // namespace \ No newline at end of file diff --git a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc index fa4c3c1d9..783edddaa 100644 --- a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc @@ -208,12 +208,12 @@ Result EC2MetadataCredentialProvider::GetCredentials() { } // Introduce a leeway of 60 seconds to avoid credential expiry conditions - auto expires_at = iam_credentials.expiration.value_or(default_timeout) - absl::Seconds(60); + auto expires_at = + iam_credentials.expiration.value_or(default_timeout) - absl::Seconds(60); return AwsCredentials{iam_credentials.access_key_id.value_or(""), iam_credentials.secret_access_key.value_or(""), - iam_credentials.token.value_or(""), - expires_at}; + iam_credentials.token.value_or(""), expires_at}; } } // namespace internal_kvstore_s3 diff --git a/tensorstore/kvstore/s3/credentials/ec2_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/ec2_credential_provider_test.cc index 4c230c9b1..e18da6ad8 100644 --- a/tensorstore/kvstore/s3/credentials/ec2_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/ec2_credential_provider_test.cc @@ -25,8 +25,8 @@ #include "absl/status/status.h" #include "absl/strings/cord.h" #include "absl/time/time.h" -#include "tensorstore/kvstore/s3/credentials/test_utils.h" #include "tensorstore/internal/http/http_response.h" +#include "tensorstore/kvstore/s3/credentials/test_utils.h" #include "tensorstore/util/result.h" #include "tensorstore/util/status_testutil.h" #include "tensorstore/util/str_cat.h" @@ -39,10 +39,11 @@ using ::tensorstore::internal_kvstore_s3::DefaultEC2MetadataFlow; using ::tensorstore::internal_kvstore_s3::EC2MetadataCredentialProvider; using ::tensorstore::internal_kvstore_s3::EC2MetadataMockTransport; - TEST(EC2MetadataCredentialProviderTest, CredentialRetrievalFlow) { auto expiry = absl::Now() + absl::Seconds(200); - auto url_to_response = DefaultEC2MetadataFlow("1234567890", "ASIA1234567890", "1234567890abcdef", "abcdef123456790", expiry); + auto url_to_response = + DefaultEC2MetadataFlow("1234567890", "ASIA1234567890", "1234567890abcdef", + "abcdef123456790", expiry); auto mock_transport = std::make_shared(url_to_response); diff --git a/tensorstore/kvstore/s3/credentials/environment_credential_provider.h b/tensorstore/kvstore/s3/credentials/environment_credential_provider.h index 701d08b94..bd5296866 100644 --- a/tensorstore/kvstore/s3/credentials/environment_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/environment_credential_provider.h @@ -28,7 +28,7 @@ class EnvironmentCredentialProvider : public AwsCredentialProvider { Result GetCredentials() override; }; -} // namespace internal_kvstore_s3 -} // namespace tensorstore +} // namespace internal_kvstore_s3 +} // namespace tensorstore -#endif // TENSORSTORE_KVSTORE_S3_CREDENTIALS_ENVIRONMENT_CREDENTIAL_PROVIDER_H \ No newline at end of file +#endif // TENSORSTORE_KVSTORE_S3_CREDENTIALS_ENVIRONMENT_CREDENTIAL_PROVIDER_H \ No newline at end of file diff --git a/tensorstore/kvstore/s3/credentials/file_credential_provider.cc b/tensorstore/kvstore/s3/credentials/file_credential_provider.cc index 7f5fd0cc9..1a5a92905 100644 --- a/tensorstore/kvstore/s3/credentials/file_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/file_credential_provider.cc @@ -78,9 +78,9 @@ Result GetAwsCredentialsFileName() { /// https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html#cli-configure-files-format Result FileCredentialProvider::GetCredentials() { TENSORSTORE_ASSIGN_OR_RETURN(auto filename, ([&]() -> Result { - if(!filename_.empty()) return filename_; - return GetAwsCredentialsFileName(); - }())); + if (!filename_.empty()) return filename_; + return GetAwsCredentialsFileName(); + }())); std::ifstream ifs(filename.c_str()); if (!ifs) { @@ -89,8 +89,8 @@ Result FileCredentialProvider::GetCredentials() { } auto profile = !profile_.empty() - ? std::string(profile_) - : GetEnv(kEnvAwsProfile).value_or(kDefaultProfile); + ? std::string(profile_) + : GetEnv(kEnvAwsProfile).value_or(kDefaultProfile); auto credentials = AwsCredentials{}; auto section_name = std::string{}; diff --git a/tensorstore/kvstore/s3/credentials/file_credential_provider.h b/tensorstore/kvstore/s3/credentials/file_credential_provider.h index 323cbbdfe..acb9bea93 100644 --- a/tensorstore/kvstore/s3/credentials/file_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/file_credential_provider.h @@ -40,11 +40,11 @@ class FileCredentialProvider : public AwsCredentialProvider { public: FileCredentialProvider(std::string filename, std::string profile) - : filename_(std::move(filename)), profile_(std::move(profile)) {} + : filename_(std::move(filename)), profile_(std::move(profile)) {} Result GetCredentials() override; }; -} // namespace internal_kvstore_s3 -} // namespace tensorstore +} // namespace internal_kvstore_s3 +} // namespace tensorstore -#endif // TENSORSTORE_KVSTORE_S3_CREDENTIALS_FILE_CREDENTIAL_PROVIDER_H \ No newline at end of file +#endif // TENSORSTORE_KVSTORE_S3_CREDENTIALS_FILE_CREDENTIAL_PROVIDER_H \ No newline at end of file diff --git a/tensorstore/kvstore/s3/credentials/file_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/file_credential_provider_test.cc index 5989c4dc0..5561675a3 100644 --- a/tensorstore/kvstore/s3/credentials/file_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/file_credential_provider_test.cc @@ -114,25 +114,25 @@ TEST_F(FileCredentialProviderTest, ASSERT_FALSE(provider.GetCredentials().ok()); } -TEST_F(FileCredentialProviderTest, - ProviderAwsCredentialsFromFileOverride) { +TEST_F(FileCredentialProviderTest, ProviderAwsCredentialsFromFileOverride) { TestData test_data; auto credentials_filename = test_data.WriteCredentialsFile(); - auto provider = std::make_unique(credentials_filename, ""); - TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, provider->GetCredentials()); + auto provider = + std::make_unique(credentials_filename, ""); + TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, + provider->GetCredentials()); ASSERT_EQ(credentials.access_key, "AKIAIOSFODNN7EXAMPLE"); ASSERT_EQ(credentials.secret_key, "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); ASSERT_EQ(credentials.session_token, "abcdef1234567890"); ASSERT_EQ(credentials.expires_at, absl::InfiniteFuture()); - provider = std::make_unique(credentials_filename, "alice"); + provider = + std::make_unique(credentials_filename, "alice"); TENSORSTORE_ASSERT_OK_AND_ASSIGN(credentials, provider->GetCredentials()); ASSERT_EQ(credentials.access_key, "AKIAIOSFODNN6EXAMPLE"); ASSERT_EQ(credentials.secret_key, "wJalrXUtnFEMI/K7MDENG/bPxRfiCZEXAMPLEKEY"); ASSERT_EQ(credentials.session_token, ""); ASSERT_EQ(credentials.expires_at, absl::InfiniteFuture()); - } - } // namespace diff --git a/tensorstore/kvstore/s3/credentials/test_utils.cc b/tensorstore/kvstore/s3/credentials/test_utils.cc index c27da7021..3ba484fc9 100644 --- a/tensorstore/kvstore/s3/credentials/test_utils.cc +++ b/tensorstore/kvstore/s3/credentials/test_utils.cc @@ -26,13 +26,10 @@ namespace tensorstore { namespace internal_kvstore_s3 { Future EC2MetadataMockTransport::IssueRequest( - const internal_http::HttpRequest& request, - absl::Cord payload, - absl::Duration request_timeout, - absl::Duration connect_timeout) { - + const internal_http::HttpRequest& request, absl::Cord payload, + absl::Duration request_timeout, absl::Duration connect_timeout) { ABSL_LOG(INFO) << request; - if(auto it = url_to_response_.find(StrCat(request.method, " ", request.url)); + if (auto it = url_to_response_.find(StrCat(request.method, " ", request.url)); it != url_to_response_.end()) { return it->second; } @@ -40,39 +37,40 @@ Future EC2MetadataMockTransport::IssueRequest( return internal_http::HttpResponse{404, absl::Cord(), {}}; } -absl::flat_hash_map DefaultEC2MetadataFlow( - const std::string & api_token, - const std::string & access_key, - const std::string & secret_key, - const std::string & session_token, - const absl::Time & expires_at) { - +absl::flat_hash_map +DefaultEC2MetadataFlow(const std::string& api_token, + const std::string& access_key, + const std::string& secret_key, + const std::string& session_token, + const absl::Time& expires_at) { return absl::flat_hash_map{ - {"POST http://169.254.169.254/latest/api/token", - internal_http::HttpResponse{200, absl::Cord{api_token}}}, - {"GET http://169.254.169.254/latest/meta-data/iam/", - internal_http::HttpResponse{200, absl::Cord{"info"}, - {{"x-aws-ec2-metadata-token", api_token}}}}, - {"GET http://169.254.169.254/latest/meta-data/iam/security-credentials/", - internal_http::HttpResponse{200, absl::Cord{"mock-iam-role"}, - {{"x-aws-ec2-metadata-token", api_token}}}}, - {"GET " - "http://169.254.169.254/latest/meta-data/iam/security-credentials/" - "mock-iam-role", - internal_http::HttpResponse{200, absl::Cord( - absl::StrFormat(R"({ + {"POST http://169.254.169.254/latest/api/token", + internal_http::HttpResponse{200, absl::Cord{api_token}}}, + {"GET http://169.254.169.254/latest/meta-data/iam/", + internal_http::HttpResponse{ + 200, absl::Cord{"info"}, {{"x-aws-ec2-metadata-token", api_token}}}}, + {"GET http://169.254.169.254/latest/meta-data/iam/security-credentials/", + internal_http::HttpResponse{200, + absl::Cord{"mock-iam-role"}, + {{"x-aws-ec2-metadata-token", api_token}}}}, + {"GET " + "http://169.254.169.254/latest/meta-data/iam/security-credentials/" + "mock-iam-role", + internal_http::HttpResponse{ + 200, + absl::Cord(absl::StrFormat( + R"({ "Code": "Success", "AccessKeyId": "%s", "SecretAccessKey": "%s", "Token": "%s", "Expiration": "%s" - })", access_key, secret_key, session_token, - absl::FormatTime("%Y-%m-%d%ET%H:%M:%E*S%Ez", - expires_at, absl::UTCTimeZone()))), - {{"x-aws-ec2-metadata-token", api_token}}}}}; + })", + access_key, secret_key, session_token, + absl::FormatTime("%Y-%m-%d%ET%H:%M:%E*S%Ez", expires_at, + absl::UTCTimeZone()))), + {{"x-aws-ec2-metadata-token", api_token}}}}}; } - - -} // namespace internal_kvstore_s3 -} // namespace tensorstore \ No newline at end of file +} // namespace internal_kvstore_s3 +} // namespace tensorstore \ No newline at end of file diff --git a/tensorstore/kvstore/s3/credentials/test_utils.h b/tensorstore/kvstore/s3/credentials/test_utils.h index f9db84633..d66816fca 100644 --- a/tensorstore/kvstore/s3/credentials/test_utils.h +++ b/tensorstore/kvstore/s3/credentials/test_utils.h @@ -14,9 +14,9 @@ #include +#include "absl/container/flat_hash_map.h" #include "absl/log/absl_log.h" #include "absl/strings/cord.h" -#include "absl/container/flat_hash_map.h" #include "absl/time/time.h" #include "tensorstore/internal/http/http_transport.h" #include "tensorstore/util/str_cat.h" @@ -30,25 +30,26 @@ namespace internal_kvstore_s3 { class EC2MetadataMockTransport : public internal_http::HttpTransport { public: EC2MetadataMockTransport( - const absl::flat_hash_map& url_to_response) + const absl::flat_hash_map& + url_to_response) : url_to_response_(url_to_response) {} - Future IssueRequest(const internal_http::HttpRequest& request, - absl::Cord payload, - absl::Duration request_timeout, - absl::Duration connect_timeout) override; + Future IssueRequest( + const internal_http::HttpRequest& request, absl::Cord payload, + absl::Duration request_timeout, absl::Duration connect_timeout) override; - const absl::flat_hash_map& url_to_response_; + const absl::flat_hash_map& + url_to_response_; }; /// Return a Default EC2 Metadata Credential Retrieval Flow, suitable /// for passing to EC2MetadataMockTransport -absl::flat_hash_map DefaultEC2MetadataFlow( - const std::string & api_token, - const std::string & access_key, - const std::string & secret_key, - const std::string & session_token, - const absl::Time & expires_at); - -} // namespace internal_kvstore_s3 -} // namespace tensorstore +absl::flat_hash_map +DefaultEC2MetadataFlow(const std::string& api_token, + const std::string& access_key, + const std::string& secret_key, + const std::string& session_token, + const absl::Time& expires_at); + +} // namespace internal_kvstore_s3 +} // namespace tensorstore From e5ceaf35d5e56cabfbb23a823e1845625d20c065 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Mon, 13 Nov 2023 19:10:32 +0200 Subject: [PATCH 39/63] IWYU --- .../s3/credentials/aws_credential_provider.cc | 3 ++- .../s3/credentials/aws_credential_provider.h | 1 + .../s3/credentials/default_credential_provider.cc | 2 +- .../s3/credentials/default_credential_provider.h | 2 ++ .../default_credential_provider_test.cc | 3 +-- .../s3/credentials/ec2_credential_provider.cc | 5 +++++ .../s3/credentials/ec2_credential_provider.h | 1 + .../credentials/ec2_credential_provider_test.cc | 3 +-- .../environment_credential_provider.cc | 5 +++++ .../environment_credential_provider_test.cc | 3 ++- .../s3/credentials/file_credential_provider.cc | 4 +++- .../s3/credentials/file_credential_provider.h | 1 + .../credentials/file_credential_provider_test.cc | 3 ++- tensorstore/kvstore/s3/credentials/test_utils.cc | 3 +++ tensorstore/kvstore/s3/credentials/test_utils.h | 5 +++-- tensorstore/kvstore/s3/localstack_test.cc | 15 ++++++++++----- tensorstore/kvstore/s3/s3_endpoint.cc | 1 + tensorstore/kvstore/s3/s3_endpoint_test.cc | 2 ++ tensorstore/kvstore/s3/s3_key_value_store.cc | 11 ++++++++--- tensorstore/kvstore/s3/s3_key_value_store_test.cc | 13 +++++++++---- tensorstore/kvstore/s3/s3_metadata.cc | 1 + tensorstore/kvstore/s3/s3_metadata_test.cc | 2 +- tensorstore/kvstore/s3/s3_request_builder.cc | 2 ++ tensorstore/kvstore/s3/s3_request_builder.h | 1 + tensorstore/kvstore/s3/s3_request_builder_test.cc | 1 - tensorstore/kvstore/s3/s3_resource.cc | 4 +--- tensorstore/kvstore/s3/s3_resource.h | 3 +-- tensorstore/kvstore/s3/validate_test.cc | 2 ++ 28 files changed, 72 insertions(+), 30 deletions(-) diff --git a/tensorstore/kvstore/s3/credentials/aws_credential_provider.cc b/tensorstore/kvstore/s3/credentials/aws_credential_provider.cc index c5a15b098..3647a00b6 100644 --- a/tensorstore/kvstore/s3/credentials/aws_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/aws_credential_provider.cc @@ -14,12 +14,13 @@ #include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" +#include #include #include #include +#include #include -#include "absl/strings/str_cat.h" #include "absl/synchronization/mutex.h" #include "tensorstore/internal/http/http_transport.h" #include "tensorstore/internal/no_destructor.h" diff --git a/tensorstore/kvstore/s3/credentials/aws_credential_provider.h b/tensorstore/kvstore/s3/credentials/aws_credential_provider.h index 1fab927f2..40fe9797b 100644 --- a/tensorstore/kvstore/s3/credentials/aws_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/aws_credential_provider.h @@ -20,6 +20,7 @@ #include #include +#include "absl/time/clock.h" #include "absl/time/time.h" #include "tensorstore/internal/http/http_transport.h" #include "tensorstore/util/result.h" diff --git a/tensorstore/kvstore/s3/credentials/default_credential_provider.cc b/tensorstore/kvstore/s3/credentials/default_credential_provider.cc index 2f2dfbc89..8bb63dec2 100644 --- a/tensorstore/kvstore/s3/credentials/default_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/default_credential_provider.cc @@ -15,7 +15,7 @@ #include "tensorstore/kvstore/s3/credentials/default_credential_provider.h" #include -#include +#include #include "absl/synchronization/mutex.h" #include "absl/time/time.h" diff --git a/tensorstore/kvstore/s3/credentials/default_credential_provider.h b/tensorstore/kvstore/s3/credentials/default_credential_provider.h index 3e323af2d..42be4c1dd 100644 --- a/tensorstore/kvstore/s3/credentials/default_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/default_credential_provider.h @@ -21,6 +21,8 @@ #include "absl/base/thread_annotations.h" #include "absl/functional/function_ref.h" #include "absl/synchronization/mutex.h" +#include "absl/time/clock.h" +#include "absl/time/time.h" #include "tensorstore/internal/http/curl_transport.h" #include "tensorstore/internal/http/http_transport.h" #include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" diff --git a/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc index dc675ce64..d50fc3da6 100644 --- a/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc @@ -14,7 +14,6 @@ #include "tensorstore/kvstore/s3/credentials/default_credential_provider.h" -#include #include #include @@ -27,8 +26,8 @@ #include "tensorstore/internal/path.h" #include "tensorstore/internal/test_util.h" #include "tensorstore/kvstore/s3/credentials/test_utils.h" +#include "tensorstore/util/future.h" #include "tensorstore/util/status_testutil.h" -#include "tensorstore/util/str_cat.h" namespace { diff --git a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc index 783edddaa..dd23c3459 100644 --- a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc @@ -14,6 +14,7 @@ #include "tensorstore/kvstore/s3/credentials/ec2_credential_provider.h" +#include #include #include #include @@ -25,7 +26,10 @@ #include "absl/strings/cord.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_split.h" +#include "absl/time/clock.h" #include "absl/time/time.h" +#include "nlohmann/json.hpp" +#include "tensorstore/json_serialization_options_base.h" #include "tensorstore/internal/env.h" #include "tensorstore/internal/http/http_request.h" #include "tensorstore/internal/http/http_response.h" @@ -34,6 +38,7 @@ #include "tensorstore/internal/json_binding/bindable.h" #include "tensorstore/internal/json_binding/json_binding.h" #include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" +#include "tensorstore/util/future.h" #include "tensorstore/util/result.h" #include "tensorstore/util/status.h" #include "tensorstore/util/str_cat.h" diff --git a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h index 43e935cd8..4ba3277d9 100644 --- a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h @@ -16,6 +16,7 @@ #define TENSORSTORE_KVSTORE_S3_CREDENTIALS_EC2_METADATA_CREDENTIAL_PROVIDER_H #include +#include #include "tensorstore/internal/http/http_transport.h" #include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" diff --git a/tensorstore/kvstore/s3/credentials/ec2_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/ec2_credential_provider_test.cc index e18da6ad8..34005bd15 100644 --- a/tensorstore/kvstore/s3/credentials/ec2_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/ec2_credential_provider_test.cc @@ -21,15 +21,14 @@ #include #include "absl/container/flat_hash_map.h" -#include "absl/log/absl_log.h" #include "absl/status/status.h" #include "absl/strings/cord.h" +#include "absl/time/clock.h" #include "absl/time/time.h" #include "tensorstore/internal/http/http_response.h" #include "tensorstore/kvstore/s3/credentials/test_utils.h" #include "tensorstore/util/result.h" #include "tensorstore/util/status_testutil.h" -#include "tensorstore/util/str_cat.h" namespace { diff --git a/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc b/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc index 806743777..33bd835d9 100644 --- a/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc @@ -14,7 +14,12 @@ #include "tensorstore/kvstore/s3/credentials/environment_credential_provider.h" +#include +#include + +#include "absl/log/absl_log.h" #include "absl/status/status.h" +#include "absl/strings/str_cat.h" #include "absl/time/time.h" #include "tensorstore/internal/env.h" diff --git a/tensorstore/kvstore/s3/credentials/environment_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/environment_credential_provider_test.cc index e81888f50..049ef2e00 100644 --- a/tensorstore/kvstore/s3/credentials/environment_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/environment_credential_provider_test.cc @@ -14,13 +14,14 @@ #include "tensorstore/kvstore/s3/credentials/environment_credential_provider.h" -#include #include +#include #include #include "tensorstore/internal/env.h" #include "tensorstore/internal/test_util.h" +#include "tensorstore/util/future.h" #include "tensorstore/util/result.h" #include "tensorstore/util/status_testutil.h" diff --git a/tensorstore/kvstore/s3/credentials/file_credential_provider.cc b/tensorstore/kvstore/s3/credentials/file_credential_provider.cc index 1a5a92905..7af519e46 100644 --- a/tensorstore/kvstore/s3/credentials/file_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/file_credential_provider.cc @@ -15,12 +15,14 @@ #include "tensorstore/kvstore/s3/credentials/file_credential_provider.h" #include +#include #include #include +#include "absl/log/absl_log.h" #include "absl/status/status.h" +#include "absl/strings/ascii.h" #include "absl/strings/str_cat.h" -#include "absl/strings/strip.h" #include "absl/time/time.h" #include "tensorstore/internal/env.h" #include "tensorstore/internal/path.h" diff --git a/tensorstore/kvstore/s3/credentials/file_credential_provider.h b/tensorstore/kvstore/s3/credentials/file_credential_provider.h index acb9bea93..51873eb54 100644 --- a/tensorstore/kvstore/s3/credentials/file_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/file_credential_provider.h @@ -16,6 +16,7 @@ #define TENSORSTORE_KVSTORE_S3_CREDENTIALS_FILE_CREDENTIAL_PROVIDER_H #include +#include #include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" #include "tensorstore/util/result.h" diff --git a/tensorstore/kvstore/s3/credentials/file_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/file_credential_provider_test.cc index 5561675a3..f31d00cec 100644 --- a/tensorstore/kvstore/s3/credentials/file_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/file_credential_provider_test.cc @@ -14,12 +14,13 @@ #include "tensorstore/kvstore/s3/credentials/file_credential_provider.h" -#include #include #include +#include #include +#include "absl/time/time.h" #include "tensorstore/internal/env.h" #include "tensorstore/internal/path.h" #include "tensorstore/internal/test_util.h" diff --git a/tensorstore/kvstore/s3/credentials/test_utils.cc b/tensorstore/kvstore/s3/credentials/test_utils.cc index 3ba484fc9..44dec0e42 100644 --- a/tensorstore/kvstore/s3/credentials/test_utils.cc +++ b/tensorstore/kvstore/s3/credentials/test_utils.cc @@ -15,8 +15,11 @@ #include "tensorstore/kvstore/s3/credentials/test_utils.h" #include +#include +#include "absl/log/absl_log.h" #include "absl/strings/cord.h" +#include "absl/strings/str_format.h" #include "absl/time/time.h" #include "tensorstore/internal/http/http_request.h" #include "tensorstore/internal/http/http_response.h" diff --git a/tensorstore/kvstore/s3/credentials/test_utils.h b/tensorstore/kvstore/s3/credentials/test_utils.h index d66816fca..de684b09c 100644 --- a/tensorstore/kvstore/s3/credentials/test_utils.h +++ b/tensorstore/kvstore/s3/credentials/test_utils.h @@ -15,11 +15,12 @@ #include #include "absl/container/flat_hash_map.h" -#include "absl/log/absl_log.h" #include "absl/strings/cord.h" #include "absl/time/time.h" +#include "tensorstore/internal/http/http_request.h" +#include "tensorstore/internal/http/http_response.h" #include "tensorstore/internal/http/http_transport.h" -#include "tensorstore/util/str_cat.h" +#include "tensorstore/util/future.h" namespace tensorstore { namespace internal_kvstore_s3 { diff --git a/tensorstore/kvstore/s3/localstack_test.cc b/tensorstore/kvstore/s3/localstack_test.cc index 0dd21c8d3..64be68755 100644 --- a/tensorstore/kvstore/s3/localstack_test.cc +++ b/tensorstore/kvstore/s3/localstack_test.cc @@ -12,14 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include +#include #include #include -#include -#include #include +#include +#include +#include -#include -#include #include "absl/flags/flag.h" #include "absl/log/absl_check.h" #include "absl/log/absl_log.h" @@ -29,7 +30,6 @@ #include "absl/strings/str_format.h" #include "absl/time/clock.h" #include "absl/time/time.h" -#include #include "tensorstore/context.h" #include "tensorstore/internal/env.h" #include "tensorstore/internal/http/curl_transport.h" @@ -45,6 +45,11 @@ #include "tensorstore/util/future.h" #include "tensorstore/util/result.h" #include "tensorstore/util/status_testutil.h" +#include "absl/container/flat_hash_map.h" +#include "nlohmann/detail/json_ref.hpp" +#include "tensorstore/internal/http/http_transport.h" +#include "tensorstore/internal/json_fwd.h" +#include "tensorstore/kvstore/spec.h" // When provided with --localstack_binary, localstack_test will start // localstack in host mode (via package localstack[runtime]). diff --git a/tensorstore/kvstore/s3/s3_endpoint.cc b/tensorstore/kvstore/s3/s3_endpoint.cc index f308e839e..0da091baa 100644 --- a/tensorstore/kvstore/s3/s3_endpoint.cc +++ b/tensorstore/kvstore/s3/s3_endpoint.cc @@ -15,6 +15,7 @@ #include "tensorstore/kvstore/s3/s3_endpoint.h" #include +#include #include #include #include diff --git a/tensorstore/kvstore/s3/s3_endpoint_test.cc b/tensorstore/kvstore/s3/s3_endpoint_test.cc index 46515c56b..7ed485f3d 100644 --- a/tensorstore/kvstore/s3/s3_endpoint_test.cc +++ b/tensorstore/kvstore/s3/s3_endpoint_test.cc @@ -16,6 +16,7 @@ #include #include +#include #include #include @@ -28,6 +29,7 @@ #include "tensorstore/internal/http/http_response.h" #include "tensorstore/internal/http/http_transport.h" #include "tensorstore/util/future.h" +#include "tensorstore/util/result.h" #include "tensorstore/util/status_testutil.h" #include "tensorstore/util/str_cat.h" diff --git a/tensorstore/kvstore/s3/s3_key_value_store.cc b/tensorstore/kvstore/s3/s3_key_value_store.cc index 73691012c..8071a26d0 100644 --- a/tensorstore/kvstore/s3/s3_key_value_store.cc +++ b/tensorstore/kvstore/s3/s3_key_value_store.cc @@ -14,7 +14,6 @@ #include #include - #include #include #include @@ -24,6 +23,9 @@ #include #include #include +#include +#include +#include #include "absl/log/absl_log.h" #include "absl/status/status.h" @@ -51,7 +53,6 @@ #include "tensorstore/internal/source_location.h" #include "tensorstore/internal/uri_utils.h" #include "tensorstore/kvstore/byte_range.h" -#include "tensorstore/kvstore/driver.h" #include "tensorstore/kvstore/gcs/validate.h" #include "tensorstore/kvstore/gcs_http/rate_limiter.h" #include "tensorstore/kvstore/generation.h" @@ -77,7 +78,6 @@ #include "tensorstore/util/result.h" #include "tensorstore/util/status.h" #include "tensorstore/util/str_cat.h" - // specializations #include "tensorstore/internal/cache_key/std_optional.h" // IWYU pragma: keep #include "tensorstore/internal/json_binding/std_array.h" // IWYU pragma: keep @@ -85,6 +85,11 @@ #include "tensorstore/serialization/fwd.h" // IWYU pragma: keep #include "tensorstore/serialization/std_optional.h" // IWYU pragma: keep #include "tensorstore/util/garbage_collection/std_optional.h" // IWYU pragma: keep +#include "tensorstore/internal/json_binding/bindable.h" +#include "tensorstore/internal/poly/poly_impl.h" +#include "tensorstore/internal/poly/storage.h" +#include "tensorstore/json_serialization_options_base.h" +#include "tensorstore/kvstore/gcs_http/admission_queue.h" #ifndef TENSORSTORE_INTERNAL_S3_LOG_REQUESTS #define TENSORSTORE_INTERNAL_S3_LOG_REQUESTS 0 diff --git a/tensorstore/kvstore/s3/s3_key_value_store_test.cc b/tensorstore/kvstore/s3/s3_key_value_store_test.cc index a3b97f4d4..e9c2bc7a7 100644 --- a/tensorstore/kvstore/s3/s3_key_value_store_test.cc +++ b/tensorstore/kvstore/s3/s3_key_value_store_test.cc @@ -12,18 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include +#include #include #include -#include +#include +#include +#include +#include -#include -#include #include "absl/container/flat_hash_map.h" #include "absl/log/absl_log.h" #include "absl/status/status.h" #include "absl/strings/cord.h" #include "absl/time/time.h" -#include #include "tensorstore/context.h" #include "tensorstore/internal/http/curl_transport.h" #include "tensorstore/internal/http/http_request.h" @@ -39,6 +41,9 @@ #include "tensorstore/util/future.h" #include "tensorstore/util/status_testutil.h" #include "tensorstore/util/str_cat.h" +#include "tensorstore/kvstore/read_result.h" +#include "tensorstore/kvstore/spec.h" +#include "tensorstore/util/result.h" namespace kvstore = ::tensorstore::kvstore; diff --git a/tensorstore/kvstore/s3/s3_metadata.cc b/tensorstore/kvstore/s3/s3_metadata.cc index 549200a37..e762da29e 100644 --- a/tensorstore/kvstore/s3/s3_metadata.cc +++ b/tensorstore/kvstore/s3/s3_metadata.cc @@ -20,6 +20,7 @@ #include #include #include +#include #include "absl/status/status.h" #include "absl/strings/str_cat.h" diff --git a/tensorstore/kvstore/s3/s3_metadata_test.cc b/tensorstore/kvstore/s3/s3_metadata_test.cc index 4fb4f782e..0d10c8e60 100644 --- a/tensorstore/kvstore/s3/s3_metadata_test.cc +++ b/tensorstore/kvstore/s3/s3_metadata_test.cc @@ -14,8 +14,8 @@ #include "tensorstore/kvstore/s3/s3_metadata.h" -#include +#include #include #include #include diff --git a/tensorstore/kvstore/s3/s3_request_builder.cc b/tensorstore/kvstore/s3/s3_request_builder.cc index afd2c031f..2a03bae73 100644 --- a/tensorstore/kvstore/s3/s3_request_builder.cc +++ b/tensorstore/kvstore/s3/s3_request_builder.cc @@ -23,6 +23,7 @@ #include #include +#include #include #include #include @@ -36,6 +37,7 @@ #include "absl/strings/str_format.h" #include "absl/strings/str_join.h" #include "absl/time/time.h" +#include #include // IWYU pragma: keep #include #include "tensorstore/internal/digest/sha256.h" diff --git a/tensorstore/kvstore/s3/s3_request_builder.h b/tensorstore/kvstore/s3/s3_request_builder.h index eb57a55f7..ed750f891 100644 --- a/tensorstore/kvstore/s3/s3_request_builder.h +++ b/tensorstore/kvstore/s3/s3_request_builder.h @@ -18,6 +18,7 @@ /// \file /// S3 Request Builder +#include #include #include #include diff --git a/tensorstore/kvstore/s3/s3_request_builder_test.cc b/tensorstore/kvstore/s3/s3_request_builder_test.cc index 9b8016ae7..4417905be 100644 --- a/tensorstore/kvstore/s3/s3_request_builder_test.cc +++ b/tensorstore/kvstore/s3/s3_request_builder_test.cc @@ -15,7 +15,6 @@ #include "tensorstore/kvstore/s3/s3_request_builder.h" #include -#include #include #include diff --git a/tensorstore/kvstore/s3/s3_resource.cc b/tensorstore/kvstore/s3/s3_resource.cc index 637cbbf2b..0f1c6cffb 100644 --- a/tensorstore/kvstore/s3/s3_resource.cc +++ b/tensorstore/kvstore/s3/s3_resource.cc @@ -14,14 +14,13 @@ #include "tensorstore/kvstore/s3/s3_resource.h" -#include #include #include #include +#include #include "absl/base/call_once.h" #include "absl/flags/flag.h" -#include "absl/flags/marshalling.h" #include "absl/log/absl_log.h" #include "absl/time/time.h" #include "tensorstore/context.h" @@ -31,7 +30,6 @@ #include "tensorstore/kvstore/gcs_http/rate_limiter.h" #include "tensorstore/kvstore/gcs_http/scaling_rate_limiter.h" #include "tensorstore/util/result.h" - /// specializations #include "tensorstore/internal/cache_key/absl_time.h" // IWYU pragma: keep #include "tensorstore/internal/cache_key/std_optional.h" // IWYU pragma: keep diff --git a/tensorstore/kvstore/s3/s3_resource.h b/tensorstore/kvstore/s3/s3_resource.h index cbfa208cf..e5e6d7dc3 100644 --- a/tensorstore/kvstore/s3/s3_resource.h +++ b/tensorstore/kvstore/s3/s3_resource.h @@ -16,7 +16,6 @@ #define TENSORSTORE_KVSTORE_S3_S3_RESOURCE_H_ #include - #include #include @@ -24,13 +23,13 @@ #include "absl/time/time.h" #include "tensorstore/context.h" #include "tensorstore/context_resource_provider.h" +#include "tensorstore/json_serialization_options_base.h" #include "tensorstore/internal/json_binding/json_binding.h" #include "tensorstore/internal/json_binding/std_optional.h" #include "tensorstore/internal/retries_context_resource.h" #include "tensorstore/kvstore/gcs_http/admission_queue.h" #include "tensorstore/kvstore/gcs_http/rate_limiter.h" #include "tensorstore/util/result.h" - /// specializations #include "tensorstore/internal/json_binding/absl_time.h" #include "tensorstore/internal/json_binding/bindable.h" diff --git a/tensorstore/kvstore/s3/validate_test.cc b/tensorstore/kvstore/s3/validate_test.cc index 49e18b87a..9e173938a 100644 --- a/tensorstore/kvstore/s3/validate_test.cc +++ b/tensorstore/kvstore/s3/validate_test.cc @@ -14,6 +14,8 @@ #include "tensorstore/kvstore/s3/validate.h" +#include + #include namespace { From eda984259701fbbac6093c30dfb1585ccd36fd25 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Tue, 14 Nov 2023 12:38:00 +0200 Subject: [PATCH 40/63] Set test_utils library test_only=11 --- tensorstore/kvstore/s3/credentials/BUILD | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorstore/kvstore/s3/credentials/BUILD b/tensorstore/kvstore/s3/credentials/BUILD index 99b810678..c6a6934b3 100644 --- a/tensorstore/kvstore/s3/credentials/BUILD +++ b/tensorstore/kvstore/s3/credentials/BUILD @@ -48,7 +48,8 @@ tensorstore_cc_library( "//tensorstore/util:str_cat", "//tensorstore/util:result", "@com_google_absl//absl/container:flat_hash_map", - ] + ], + testonly = 1, ) tensorstore_cc_test( From e947d6332d0c66596eb24f8347c12f6f52e2b6bc Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Tue, 14 Nov 2023 15:43:36 +0200 Subject: [PATCH 41/63] IWYU reverts --- .../s3/credentials/default_credential_provider_test.cc | 5 ++--- .../s3/credentials/environment_credential_provider.cc | 2 ++ .../kvstore/s3/credentials/file_credential_provider.cc | 1 + tensorstore/kvstore/s3/s3_key_value_store.cc | 4 ---- tensorstore/kvstore/s3/validate_test.cc | 2 -- 5 files changed, 5 insertions(+), 9 deletions(-) diff --git a/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc index d50fc3da6..6d5aed9b3 100644 --- a/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc @@ -17,11 +17,12 @@ #include #include +#include #include #include "absl/container/flat_hash_map.h" #include "absl/strings/cord.h" -#include "tensorstore/internal/env.h" +#include "absl/time/time.h" #include "tensorstore/internal/http/http_response.h" #include "tensorstore/internal/path.h" #include "tensorstore/internal/test_util.h" @@ -33,8 +34,6 @@ namespace { using ::tensorstore::Future; using ::tensorstore::internal::JoinPath; -using ::tensorstore::internal::SetEnv; -using ::tensorstore::internal::UnsetEnv; using ::tensorstore::internal_http::HttpResponse; using ::tensorstore::internal_kvstore_s3::DefaultAwsCredentialsProvider; using ::tensorstore::internal_kvstore_s3::DefaultEC2MetadataFlow; diff --git a/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc b/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc index 33bd835d9..74bbb48fd 100644 --- a/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc @@ -21,7 +21,9 @@ #include "absl/status/status.h" #include "absl/strings/str_cat.h" #include "absl/time/time.h" +#include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" #include "tensorstore/internal/env.h" +#include "tensorstore/util/result.h" using ::tensorstore::internal::GetEnv; diff --git a/tensorstore/kvstore/s3/credentials/file_credential_provider.cc b/tensorstore/kvstore/s3/credentials/file_credential_provider.cc index 7af519e46..da2f36760 100644 --- a/tensorstore/kvstore/s3/credentials/file_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/file_credential_provider.cc @@ -24,6 +24,7 @@ #include "absl/strings/ascii.h" #include "absl/strings/str_cat.h" #include "absl/time/time.h" +#include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" #include "tensorstore/internal/env.h" #include "tensorstore/internal/path.h" #include "tensorstore/util/result.h" diff --git a/tensorstore/kvstore/s3/s3_key_value_store.cc b/tensorstore/kvstore/s3/s3_key_value_store.cc index 5970d58c3..19661c324 100644 --- a/tensorstore/kvstore/s3/s3_key_value_store.cc +++ b/tensorstore/kvstore/s3/s3_key_value_store.cc @@ -88,10 +88,6 @@ #include "tensorstore/serialization/std_optional.h" // IWYU pragma: keep #include "tensorstore/util/garbage_collection/std_optional.h" // IWYU pragma: keep #include "tensorstore/internal/json_binding/bindable.h" -#include "tensorstore/internal/poly/poly_impl.h" -#include "tensorstore/internal/poly/storage.h" -#include "tensorstore/json_serialization_options_base.h" -#include "tensorstore/kvstore/gcs_http/admission_queue.h" using ::tensorstore::internal::DataCopyConcurrencyResource; using ::tensorstore::internal::IntrusivePtr; diff --git a/tensorstore/kvstore/s3/validate_test.cc b/tensorstore/kvstore/s3/validate_test.cc index 9e173938a..49e18b87a 100644 --- a/tensorstore/kvstore/s3/validate_test.cc +++ b/tensorstore/kvstore/s3/validate_test.cc @@ -14,8 +14,6 @@ #include "tensorstore/kvstore/s3/validate.h" -#include - #include namespace { From 326a7254857d3e624008ec4e474cda60f1f54ac2 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Tue, 14 Nov 2023 16:22:01 +0200 Subject: [PATCH 42/63] Refactor aws_credential_provider library into smaller libraries --- tensorstore/kvstore/s3/BUILD | 2 +- tensorstore/kvstore/s3/credentials/BUILD | 85 +++++++++++----- .../s3/credentials/aws_credential_provider.cc | 96 ------------------- .../s3/credentials/aws_credential_provider.h | 11 --- .../default_credential_provider.cc | 61 ++++++++++++ .../credentials/default_credential_provider.h | 13 +++ tensorstore/kvstore/s3/s3_key_value_store.cc | 2 +- 7 files changed, 137 insertions(+), 133 deletions(-) delete mode 100644 tensorstore/kvstore/s3/credentials/aws_credential_provider.cc diff --git a/tensorstore/kvstore/s3/BUILD b/tensorstore/kvstore/s3/BUILD index 3ed2b142f..d650cfdd3 100644 --- a/tensorstore/kvstore/s3/BUILD +++ b/tensorstore/kvstore/s3/BUILD @@ -46,7 +46,7 @@ tensorstore_cc_library( "//tensorstore/kvstore:key_range", "//tensorstore/kvstore/gcs:validate", "//tensorstore/kvstore/gcs_http:rate_limiter", - "//tensorstore/kvstore/s3/credentials:aws_credential_provider", + "//tensorstore/kvstore/s3/credentials:default_credential_provider", "//tensorstore/serialization", "//tensorstore/util:executor", "//tensorstore/util:future", diff --git a/tensorstore/kvstore/s3/credentials/BUILD b/tensorstore/kvstore/s3/credentials/BUILD index c6a6934b3..624f5f3a5 100644 --- a/tensorstore/kvstore/s3/credentials/BUILD +++ b/tensorstore/kvstore/s3/credentials/BUILD @@ -7,28 +7,65 @@ licenses(["notice"]) tensorstore_cc_library( name = "aws_credential_provider", - srcs = [ - "aws_credential_provider.cc", - "default_credential_provider.cc", - "environment_credential_provider.cc", - "file_credential_provider.cc", - "ec2_credential_provider.cc"], - hdrs = [ - "aws_credential_provider.h", - "default_credential_provider.h", - "environment_credential_provider.h", - "file_credential_provider.h", - "ec2_credential_provider.h"], + hdrs = ["aws_credential_provider.h"], +) + +tensorstore_cc_library( + name = "environment_credential_provider", + hdrs = ["environment_credential_provider.h"], + srcs = ["environment_credential_provider.cc"], + deps = [ + ":aws_credential_provider", + "//tensorstore/internal:env", + "//tensorstore/util:result", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/status", + ] +) + +tensorstore_cc_library( + name = "file_credential_provider", + hdrs = ["file_credential_provider.h"], + srcs = ["file_credential_provider.cc"], + deps = [ + ":aws_credential_provider", + "//tensorstore/internal:env", + "//tensorstore/internal:path", + "//tensorstore/util:result", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/status", + ] +) + +tensorstore_cc_library( + name = "ec2_credential_provider", + hdrs = ["ec2_credential_provider.h"], + srcs = ["ec2_credential_provider.cc"], + deps = [ + ":aws_credential_provider", + "//tensorstore/internal:env", + "//tensorstore/util:result", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/status", + "//tensorstore/internal/http", + "//tensorstore/internal/http:curl_transport", + "//tensorstore/internal/json", + "//tensorstore/internal/json_binding", + "//tensorstore/internal/json_binding:absl_time", + "//tensorstore/internal/json_binding:bindable", + ] +) + +tensorstore_cc_library( + name = "default_credential_provider", + srcs = ["default_credential_provider.cc"], + hdrs = ["default_credential_provider.h"], deps = [ - "//tensorstore/internal:env", + ":aws_credential_provider", + ":ec2_credential_provider", + ":environment_credential_provider", + ":file_credential_provider", "//tensorstore/internal:no_destructor", - "//tensorstore/internal:path", - "//tensorstore/internal/http", - "//tensorstore/internal/http:curl_transport", - "//tensorstore/internal/json", - "//tensorstore/internal/json_binding", - "//tensorstore/internal/json_binding:absl_time", - "//tensorstore/internal/json_binding:bindable", "//tensorstore/util:result", "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", @@ -57,7 +94,7 @@ tensorstore_cc_test( size = "small", srcs = ["default_credential_provider_test.cc"], deps = [ - ":aws_credential_provider", + ":default_credential_provider", ":test_utils", "//tensorstore/internal:test_util", "//tensorstore/util:result", @@ -72,7 +109,7 @@ tensorstore_cc_test( size = "small", srcs = ["environment_credential_provider_test.cc"], deps = [ - ":aws_credential_provider", + ":environment_credential_provider", "//tensorstore/internal:test_util", "//tensorstore/util:result", "//tensorstore/util:status_testutil", @@ -86,7 +123,7 @@ tensorstore_cc_test( size = "small", srcs = ["file_credential_provider_test.cc"], deps = [ - ":aws_credential_provider", + ":file_credential_provider", "//tensorstore/internal:path", "//tensorstore/internal:test_util", "//tensorstore/util:result", @@ -101,7 +138,7 @@ tensorstore_cc_test( size = "small", srcs = ["ec2_credential_provider_test.cc"], deps = [ - ":aws_credential_provider", + ":ec2_credential_provider", ":test_utils", "//tensorstore/internal/http", "//tensorstore/util:result", diff --git a/tensorstore/kvstore/s3/credentials/aws_credential_provider.cc b/tensorstore/kvstore/s3/credentials/aws_credential_provider.cc deleted file mode 100644 index 3647a00b6..000000000 --- a/tensorstore/kvstore/s3/credentials/aws_credential_provider.cc +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2023 The TensorStore Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" - -#include -#include -#include -#include -#include -#include - -#include "absl/synchronization/mutex.h" -#include "tensorstore/internal/http/http_transport.h" -#include "tensorstore/internal/no_destructor.h" -#include "tensorstore/kvstore/s3/credentials/default_credential_provider.h" -#include "tensorstore/util/result.h" - -using ::tensorstore::Result; - -namespace tensorstore { -namespace internal_kvstore_s3 { -namespace { - -/// Return a ChainedCredentialProvider that attempts to retrieve credentials -/// from -/// 1. AWS Environment Variables, e.g. AWS_ACCESS_KEY_ID -/// 2. Shared Credential File, e.g. $HOME/.aws/credentials -/// 3. EC2 Metadata Server -Result> GetDefaultAwsCredentialProvider( - std::string_view profile, - std::shared_ptr transport) { - return std::make_unique( - DefaultAwsCredentialsProvider::Options{ - {}, std::string{profile}, transport}); -} - -struct AwsCredentialProviderRegistry { - std::vector> providers; - absl::Mutex mutex; -}; - -AwsCredentialProviderRegistry& GetAwsProviderRegistry() { - static internal::NoDestructor registry; - return *registry; -} - -} // namespace - -void RegisterAwsCredentialProviderProvider(AwsCredentialProviderFn provider, - int priority) { - auto& registry = GetAwsProviderRegistry(); - absl::WriterMutexLock lock(®istry.mutex); - registry.providers.emplace_back(priority, std::move(provider)); - std::sort(registry.providers.begin(), registry.providers.end(), - [](const auto& a, const auto& b) { return a.first < b.first; }); -} - -/// Obtain a credential provider from a series of registered and default -/// providers -/// -/// Providers are returned in the following order: -/// 1. Any registered providers that supply valid credentials -/// 2. Environment variable provider if valid credential can be obtained from -/// AWS_* environment variables -/// 3. File provider containing credentials from the $HOME/.aws/credentials -/// file. -/// The `profile` variable overrides the default profile in this file. -/// 4. EC2 Metadata server. The `transport` variable overrides the default -/// HttpTransport. -Result> GetAwsCredentialProvider( - std::string_view profile, - std::shared_ptr transport) { - auto& registry = GetAwsProviderRegistry(); - absl::WriterMutexLock lock(®istry.mutex); - for (const auto& provider : registry.providers) { - auto credentials = provider.second(); - if (credentials.ok()) return credentials; - } - - return GetDefaultAwsCredentialProvider(profile, transport); -} - -} // namespace internal_kvstore_s3 -} // namespace tensorstore diff --git a/tensorstore/kvstore/s3/credentials/aws_credential_provider.h b/tensorstore/kvstore/s3/credentials/aws_credential_provider.h index 40fe9797b..d36828b63 100644 --- a/tensorstore/kvstore/s3/credentials/aws_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/aws_credential_provider.h @@ -22,7 +22,6 @@ #include "absl/time/clock.h" #include "absl/time/time.h" -#include "tensorstore/internal/http/http_transport.h" #include "tensorstore/util/result.h" namespace tensorstore { @@ -63,16 +62,6 @@ class AwsCredentialProvider { virtual Result GetCredentials() = 0; }; -using AwsCredentialProviderFn = - std::function>()>; - -void RegisterAwsCredentialProviderProvider(AwsCredentialProviderFn provider, - int priority); - -Result> GetAwsCredentialProvider( - std::string_view profile, - std::shared_ptr transport); - } // namespace internal_kvstore_s3 } // namespace tensorstore diff --git a/tensorstore/kvstore/s3/credentials/default_credential_provider.cc b/tensorstore/kvstore/s3/credentials/default_credential_provider.cc index 8bb63dec2..ef03880ba 100644 --- a/tensorstore/kvstore/s3/credentials/default_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/default_credential_provider.cc @@ -19,6 +19,7 @@ #include "absl/synchronization/mutex.h" #include "absl/time/time.h" +#include "tensorstore/internal/no_destructor.h" #include "tensorstore/internal/http/http_transport.h" #include "tensorstore/kvstore/s3/credentials/ec2_credential_provider.h" #include "tensorstore/kvstore/s3/credentials/environment_credential_provider.h" @@ -27,6 +28,66 @@ namespace tensorstore { namespace internal_kvstore_s3 { +namespace { + +/// Return a ChainedCredentialProvider that attempts to retrieve credentials +/// from +/// 1. AWS Environment Variables, e.g. AWS_ACCESS_KEY_ID +/// 2. Shared Credential File, e.g. $HOME/.aws/credentials +/// 3. EC2 Metadata Server +Result> GetDefaultAwsCredentialProvider( + std::string_view profile, + std::shared_ptr transport) { + return std::make_unique( + DefaultAwsCredentialsProvider::Options{ + {}, std::string{profile}, transport}); +} + +struct AwsCredentialProviderRegistry { + std::vector> providers; + absl::Mutex mutex; +}; + +AwsCredentialProviderRegistry& GetAwsProviderRegistry() { + static internal::NoDestructor registry; + return *registry; +} + +} // namespace + +void RegisterAwsCredentialProviderProvider(AwsCredentialProviderFn provider, + int priority) { + auto& registry = GetAwsProviderRegistry(); + absl::WriterMutexLock lock(®istry.mutex); + registry.providers.emplace_back(priority, std::move(provider)); + std::sort(registry.providers.begin(), registry.providers.end(), + [](const auto& a, const auto& b) { return a.first < b.first; }); +} + +/// Obtain a credential provider from a series of registered and default +/// providers +/// +/// Providers are returned in the following order: +/// 1. Any registered providers that supply valid credentials +/// 2. Environment variable provider if valid credential can be obtained from +/// AWS_* environment variables +/// 3. File provider containing credentials from the $HOME/.aws/credentials +/// file. +/// The `profile` variable overrides the default profile in this file. +/// 4. EC2 Metadata server. The `transport` variable overrides the default +/// HttpTransport. +Result> GetAwsCredentialProvider( + std::string_view profile, + std::shared_ptr transport) { + auto& registry = GetAwsProviderRegistry(); + absl::WriterMutexLock lock(®istry.mutex); + for (const auto& provider : registry.providers) { + auto credentials = provider.second(); + if (credentials.ok()) return credentials; + } + + return GetDefaultAwsCredentialProvider(profile, transport); +} DefaultAwsCredentialsProvider::DefaultAwsCredentialsProvider( Options options, absl::FunctionRef clock) diff --git a/tensorstore/kvstore/s3/credentials/default_credential_provider.h b/tensorstore/kvstore/s3/credentials/default_credential_provider.h index 42be4c1dd..0fb221485 100644 --- a/tensorstore/kvstore/s3/credentials/default_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/default_credential_provider.h @@ -15,8 +15,10 @@ #ifndef TENSORSTORE_KVSTORE_S3_CREDENTIALS_DEFAULT_CREDENTIAL_PROVIDER #define TENSORSTORE_KVSTORE_S3_CREDENTIALS_DEFAULT_CREDENTIAL_PROVIDER +#include #include #include +#include #include "absl/base/thread_annotations.h" #include "absl/functional/function_ref.h" @@ -67,6 +69,17 @@ class DefaultAwsCredentialsProvider : public AwsCredentialProvider { AwsCredentials credentials_ ABSL_GUARDED_BY(mutex_); }; +using AwsCredentialProviderFn = + std::function>()>; + +void RegisterAwsCredentialProviderProvider(AwsCredentialProviderFn provider, + int priority); + +Result> GetAwsCredentialProvider( + std::string_view profile, + std::shared_ptr transport); + + } // namespace internal_kvstore_s3 } // namespace tensorstore diff --git a/tensorstore/kvstore/s3/s3_key_value_store.cc b/tensorstore/kvstore/s3/s3_key_value_store.cc index 19661c324..df1dd6660 100644 --- a/tensorstore/kvstore/s3/s3_key_value_store.cc +++ b/tensorstore/kvstore/s3/s3_key_value_store.cc @@ -62,7 +62,7 @@ #include "tensorstore/kvstore/operations.h" #include "tensorstore/kvstore/read_result.h" #include "tensorstore/kvstore/registry.h" -#include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" +#include "tensorstore/kvstore/s3/credentials/default_credential_provider.h" #include "tensorstore/kvstore/s3/s3_endpoint.h" #include "tensorstore/kvstore/s3/s3_metadata.h" #include "tensorstore/kvstore/s3/s3_request_builder.h" From af52f4162f97726d717dd4f280af4e62753a37eb Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Tue, 14 Nov 2023 16:29:45 +0200 Subject: [PATCH 43/63] File credential provider style nits --- .../credentials/file_credential_provider.cc | 24 +++++-------------- 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/tensorstore/kvstore/s3/credentials/file_credential_provider.cc b/tensorstore/kvstore/s3/credentials/file_credential_provider.cc index da2f36760..94b71debb 100644 --- a/tensorstore/kvstore/s3/credentials/file_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/file_credential_provider.cc @@ -53,27 +53,15 @@ static constexpr char kEnvAwsProfile[] = "AWS_PROFILE"; static constexpr char kDefaultProfile[] = "default"; Result GetAwsCredentialsFileName() { - std::string result; - - if (auto credentials_file = GetEnv(kEnvAwsCredentialsFile); - credentials_file) { - result = *credentials_file; - } else { - if (auto home_dir = GetEnv("HOME"); home_dir) { - result = JoinPath(*home_dir, kDefaultAwsCredentialsFilePath); - } else { + auto credentials_file = GetEnv(kEnvAwsCredentialsFile); + if(!credentials_file) { + auto home_dir = GetEnv("HOME"); + if(!home_dir) { return absl::NotFoundError("Could not read $HOME"); } + return JoinPath(*home_dir, kDefaultAwsCredentialsFilePath); } - - if (auto fstream = std::ifstream(result.c_str()); !fstream.good()) { - return absl::NotFoundError( - absl::StrCat("Could not find the credentials file at " - "location [", - result, "]")); - } - - return result; + return *credentials_file; } } // namespace From 9636486d97c81b389eacf66c20cf449328557312 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Tue, 14 Nov 2023 16:32:04 +0200 Subject: [PATCH 44/63] Environment credential provider style nits --- .../s3/credentials/environment_credential_provider.cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc b/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc index 74bbb48fd..0e01158ec 100644 --- a/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc @@ -42,6 +42,15 @@ static constexpr char kEnvAwsSessionToken[] = "AWS_SESSION_TOKEN"; } // namespace Result EnvironmentCredentialProvider::GetCredentials() { + auto access_key = GetEnv(kEnvAwsAccessKeyId); + if(!access_key) { + return absl::NotFoundError(absl::StrCat(kEnvAwsAccessKeyId, " not set")); + } + + ABSL_LOG_FIRST_N(INFO, 1) + << "Using Environment Variable " << kEnvAwsAccessKeyId; + + if (auto access_key = GetEnv(kEnvAwsAccessKeyId); access_key) { ABSL_LOG_FIRST_N(INFO, 1) << "Using Environment Variable " << kEnvAwsAccessKeyId; From 4a07bb761d8825306dbf942fe1dff7b2f86064ff Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Tue, 14 Nov 2023 16:33:21 +0200 Subject: [PATCH 45/63] Remove use of future from environment_credential_provider_test.cc --- .../s3/credentials/environment_credential_provider_test.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/tensorstore/kvstore/s3/credentials/environment_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/environment_credential_provider_test.cc index 049ef2e00..434bfbb88 100644 --- a/tensorstore/kvstore/s3/credentials/environment_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/environment_credential_provider_test.cc @@ -21,13 +21,11 @@ #include "tensorstore/internal/env.h" #include "tensorstore/internal/test_util.h" -#include "tensorstore/util/future.h" #include "tensorstore/util/result.h" #include "tensorstore/util/status_testutil.h" namespace { -using ::tensorstore::Future; using ::tensorstore::internal::GetEnv; using ::tensorstore::internal::SetEnv; using ::tensorstore::internal::UnsetEnv; From 6eaa6191203c4576c0a1c360bbf8d983bd48a556 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Tue, 14 Nov 2023 16:36:13 +0200 Subject: [PATCH 46/63] AwsCredentials updates --- .../s3/credentials/aws_credential_provider.h | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tensorstore/kvstore/s3/credentials/aws_credential_provider.h b/tensorstore/kvstore/s3/credentials/aws_credential_provider.h index d36828b63..1d6279af8 100644 --- a/tensorstore/kvstore/s3/credentials/aws_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/aws_credential_provider.h @@ -36,18 +36,17 @@ namespace internal_kvstore_s3 { /// https://docs.aws.amazon.com/STS/latest/APIReference/welcome.html struct AwsCredentials { /// AWS_ACCESS_KEY_ID - std::string access_key = {}; + std::string access_key; /// AWS_SECRET_KEY_ID - std::string secret_key = {}; + std::string secret_key; /// AWS_SESSION_TOKEN - std::string session_token = {}; + std::string session_token; /// Expiration date absl::Time expires_at = absl::InfinitePast(); - /// Anonymous credentials expiring duration seconds from now - static AwsCredentials Anonymous( - const absl::Duration& duration = absl::InfiniteDuration()) { - return AwsCredentials{{}, {}, {}, absl::Now() + duration}; + /// Anonymous credentials that do not expire + static AwsCredentials Anonymous() { + return AwsCredentials{{}, {}, {}, absl::InfiniteFuture()}; } bool IsAnonymous() const { return access_key.empty(); } From a3101a5037db2d820f9b87311712782ff9113c9d Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Tue, 14 Nov 2023 16:50:10 +0200 Subject: [PATCH 47/63] Add FileCredentialProvider.Get{FileName,Profile} accessors --- .../credentials/file_credential_provider.cc | 27 +++++++++---------- .../s3/credentials/file_credential_provider.h | 8 ++++++ .../file_credential_provider_test.cc | 13 +++++++++ 3 files changed, 34 insertions(+), 14 deletions(-) diff --git a/tensorstore/kvstore/s3/credentials/file_credential_provider.cc b/tensorstore/kvstore/s3/credentials/file_credential_provider.cc index 94b71debb..cc826be7c 100644 --- a/tensorstore/kvstore/s3/credentials/file_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/file_credential_provider.cc @@ -68,21 +68,20 @@ Result GetAwsCredentialsFileName() { /// https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html#cli-configure-files-format Result FileCredentialProvider::GetCredentials() { - TENSORSTORE_ASSIGN_OR_RETURN(auto filename, ([&]() -> Result { - if (!filename_.empty()) return filename_; - return GetAwsCredentialsFileName(); - }())); + if(filename_.empty()) { + TENSORSTORE_ASSIGN_OR_RETURN(filename_, GetAwsCredentialsFileName()); + } + + if(profile_.empty()) { + profile_ = GetEnv(kEnvAwsProfile).value_or(kDefaultProfile); + } - std::ifstream ifs(filename.c_str()); + std::ifstream ifs(filename_.c_str()); if (!ifs) { return absl::NotFoundError( - absl::StrCat("Could not open credentials file [", filename, "]")); + absl::StrCat("Could not open credentials file [", filename_, "]")); } - auto profile = !profile_.empty() - ? std::string(profile_) - : GetEnv(kEnvAwsProfile).value_or(kDefaultProfile); - auto credentials = AwsCredentials{}; auto section_name = std::string{}; auto line = std::string{}; @@ -101,7 +100,7 @@ Result FileCredentialProvider::GetCredentials() { } // Look for key=value pairs if we're in the appropriate profile - if (section_name == profile) { + if (section_name == profile_) { profile_found = true; if (auto pos = sline.find('='); pos != std::string::npos) { auto key = absl::StripAsciiWhitespace(sline.substr(0, pos)); @@ -119,14 +118,14 @@ Result FileCredentialProvider::GetCredentials() { } if (!profile_found) { - return absl::NotFoundError(absl::StrCat("Profile [", profile, + return absl::NotFoundError(absl::StrCat("Profile [", profile_, "] not found " "in credentials file [", - filename, "]")); + filename_, "]")); } ABSL_LOG_FIRST_N(INFO, 1) - << "Using profile [" << profile << "] in file [" << filename << "]"; + << "Using profile [" << profile_ << "] in file [" << filename_ << "]"; credentials.expires_at = absl::InfiniteFuture(); return credentials; diff --git a/tensorstore/kvstore/s3/credentials/file_credential_provider.h b/tensorstore/kvstore/s3/credentials/file_credential_provider.h index 51873eb54..0f4687a4b 100644 --- a/tensorstore/kvstore/s3/credentials/file_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/file_credential_provider.h @@ -43,6 +43,14 @@ class FileCredentialProvider : public AwsCredentialProvider { FileCredentialProvider(std::string filename, std::string profile) : filename_(std::move(filename)), profile_(std::move(profile)) {} Result GetCredentials() override; + + const std::string & GetFileName() const { + return filename_; + } + + const std::string & GetProfile() const { + return profile_; + } }; } // namespace internal_kvstore_s3 diff --git a/tensorstore/kvstore/s3/credentials/file_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/file_credential_provider_test.cc index f31d00cec..073f9105a 100644 --- a/tensorstore/kvstore/s3/credentials/file_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/file_credential_provider_test.cc @@ -70,6 +70,8 @@ TEST_F(FileCredentialProviderTest, ProviderAwsCredentialsFromFileDefault) { SetEnv("AWS_SHARED_CREDENTIALS_FILE", credentials_filename.c_str()); auto provider = FileCredentialProvider("", ""); TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, provider.GetCredentials()); + ASSERT_EQ(provider.GetFileName(), credentials_filename); + ASSERT_EQ(provider.GetProfile(), "default"); ASSERT_EQ(credentials.access_key, "AKIAIOSFODNN7EXAMPLE"); ASSERT_EQ(credentials.secret_key, "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); ASSERT_EQ(credentials.session_token, "abcdef1234567890"); @@ -84,6 +86,8 @@ TEST_F(FileCredentialProviderTest, SetEnv("AWS_SHARED_CREDENTIALS_FILE", credentials_filename.c_str()); auto provider = FileCredentialProvider("", "alice"); TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, provider.GetCredentials()); + ASSERT_EQ(provider.GetFileName(), credentials_filename); + ASSERT_EQ(provider.GetProfile(), "alice"); ASSERT_EQ(credentials.access_key, "AKIAIOSFODNN6EXAMPLE"); ASSERT_EQ(credentials.secret_key, "wJalrXUtnFEMI/K7MDENG/bPxRfiCZEXAMPLEKEY"); ASSERT_EQ(credentials.session_token, ""); @@ -98,6 +102,8 @@ TEST_F(FileCredentialProviderTest, ProviderAwsCredentialsFromFileProfileEnv) { SetEnv("AWS_PROFILE", "alice"); auto provider = FileCredentialProvider("", ""); TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, provider.GetCredentials()); + ASSERT_EQ(provider.GetFileName(), credentials_filename); + ASSERT_EQ(provider.GetProfile(), "alice"); ASSERT_EQ(credentials.access_key, "AKIAIOSFODNN6EXAMPLE"); ASSERT_EQ(credentials.secret_key, "wJalrXUtnFEMI/K7MDENG/bPxRfiCZEXAMPLEKEY"); ASSERT_EQ(credentials.session_token, ""); @@ -113,6 +119,9 @@ TEST_F(FileCredentialProviderTest, SetEnv("AWS_PROFILE", "bob"); auto provider = FileCredentialProvider("", ""); ASSERT_FALSE(provider.GetCredentials().ok()); + ASSERT_EQ(provider.GetFileName(), credentials_filename); + ASSERT_EQ(provider.GetProfile(), "bob"); + } TEST_F(FileCredentialProviderTest, ProviderAwsCredentialsFromFileOverride) { @@ -122,6 +131,8 @@ TEST_F(FileCredentialProviderTest, ProviderAwsCredentialsFromFileOverride) { std::make_unique(credentials_filename, ""); TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, provider->GetCredentials()); + ASSERT_EQ(provider->GetFileName(), credentials_filename); + ASSERT_EQ(provider->GetProfile(), "default"); ASSERT_EQ(credentials.access_key, "AKIAIOSFODNN7EXAMPLE"); ASSERT_EQ(credentials.secret_key, "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); ASSERT_EQ(credentials.session_token, "abcdef1234567890"); @@ -130,6 +141,8 @@ TEST_F(FileCredentialProviderTest, ProviderAwsCredentialsFromFileOverride) { provider = std::make_unique(credentials_filename, "alice"); TENSORSTORE_ASSERT_OK_AND_ASSIGN(credentials, provider->GetCredentials()); + ASSERT_EQ(provider->GetFileName(), credentials_filename); + ASSERT_EQ(provider->GetProfile(), "alice"); ASSERT_EQ(credentials.access_key, "AKIAIOSFODNN6EXAMPLE"); ASSERT_EQ(credentials.secret_key, "wJalrXUtnFEMI/K7MDENG/bPxRfiCZEXAMPLEKEY"); ASSERT_EQ(credentials.session_token, ""); From 963894540d483bc68552e46b0258353bcea37cac Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Tue, 14 Nov 2023 18:31:50 +0200 Subject: [PATCH 48/63] EnvironmentCredentialProvider style nits --- .../environment_credential_provider.cc | 28 ++++++------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc b/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc index 0e01158ec..e805723dd 100644 --- a/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc @@ -46,29 +46,17 @@ Result EnvironmentCredentialProvider::GetCredentials() { if(!access_key) { return absl::NotFoundError(absl::StrCat(kEnvAwsAccessKeyId, " not set")); } - ABSL_LOG_FIRST_N(INFO, 1) << "Using Environment Variable " << kEnvAwsAccessKeyId; - - - if (auto access_key = GetEnv(kEnvAwsAccessKeyId); access_key) { - ABSL_LOG_FIRST_N(INFO, 1) - << "Using Environment Variable " << kEnvAwsAccessKeyId; - auto credentials = AwsCredentials{*access_key}; - - if (auto secret_key = GetEnv(kEnvAwsSecretAccessKey); secret_key) { - credentials.secret_key = *secret_key; - } - - if (auto session_token = GetEnv(kEnvAwsSessionToken); session_token) { - credentials.session_token = *session_token; - } - - credentials.expires_at = absl::InfiniteFuture(); - return credentials; + auto credentials = AwsCredentials{*access_key}; + if (auto secret_key = GetEnv(kEnvAwsSecretAccessKey); secret_key) { + credentials.secret_key = *secret_key; } - - return absl::NotFoundError(absl::StrCat(kEnvAwsAccessKeyId, " not set")); + if (auto session_token = GetEnv(kEnvAwsSessionToken); session_token) { + credentials.session_token = *session_token; + } + credentials.expires_at = absl::InfiniteFuture(); + return credentials; } } // namespace internal_kvstore_s3 From 44936fae3f1b29598bfac08bb3282fb4e6723d16 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Tue, 14 Nov 2023 18:45:11 +0200 Subject: [PATCH 49/63] Support metadata_endpoint in the AwsCredentialsResource --- .../s3/credentials/default_credential_provider.cc | 15 ++++++++++----- .../s3/credentials/default_credential_provider.h | 10 +++++++--- .../default_credential_provider_test.cc | 2 +- .../s3/credentials/ec2_credential_provider.h | 10 +++++++++- .../credentials/ec2_credential_provider_test.cc | 6 +++--- .../s3/credentials/file_credential_provider.h | 10 ++++++---- tensorstore/kvstore/s3/s3_key_value_store.cc | 8 +++++--- tensorstore/kvstore/s3/schema.yml | 10 ++++++++++ 8 files changed, 51 insertions(+), 20 deletions(-) diff --git a/tensorstore/kvstore/s3/credentials/default_credential_provider.cc b/tensorstore/kvstore/s3/credentials/default_credential_provider.cc index ef03880ba..643fed055 100644 --- a/tensorstore/kvstore/s3/credentials/default_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/default_credential_provider.cc @@ -36,11 +36,13 @@ namespace { /// 2. Shared Credential File, e.g. $HOME/.aws/credentials /// 3. EC2 Metadata Server Result> GetDefaultAwsCredentialProvider( + std::string_view filename, std::string_view profile, + std::string_view metadata_endpoint, std::shared_ptr transport) { return std::make_unique( DefaultAwsCredentialsProvider::Options{ - {}, std::string{profile}, transport}); + std::string{filename}, std::string{profile}, std::string{metadata_endpoint}, transport}); } struct AwsCredentialProviderRegistry { @@ -77,7 +79,9 @@ void RegisterAwsCredentialProviderProvider(AwsCredentialProviderFn provider, /// 4. EC2 Metadata server. The `transport` variable overrides the default /// HttpTransport. Result> GetAwsCredentialProvider( + std::string_view filename, std::string_view profile, + std::string_view metadata_endpoint, std::shared_ptr transport) { auto& registry = GetAwsProviderRegistry(); absl::WriterMutexLock lock(®istry.mutex); @@ -86,7 +90,7 @@ Result> GetAwsCredentialProvider( if (credentials.ok()) return credentials; } - return GetDefaultAwsCredentialProvider(profile, transport); + return GetDefaultAwsCredentialProvider(filename, profile, metadata_endpoint, transport); } DefaultAwsCredentialsProvider::DefaultAwsCredentialsProvider( @@ -124,8 +128,8 @@ Result DefaultAwsCredentialsProvider::GetCredentials() { } // 2. Shared Credential File, e.g. $HOME/.aws/credentials - provider_ = std::make_unique(options_.filename, - options_.profile); + provider_ = std::make_unique( + options_.filename, options_.profile); credentials_result = provider_->GetCredentials(); if (credentials_result.ok()) { credentials_ = credentials_result.value(); @@ -134,7 +138,8 @@ Result DefaultAwsCredentialsProvider::GetCredentials() { // 3. EC2 Metadata Server provider_ = - std::make_unique(options_.transport); + std::make_unique( + options_.endpoint, options_.transport); credentials_result = provider_->GetCredentials(); if (credentials_result.ok()) { credentials_ = credentials_result.value(); diff --git a/tensorstore/kvstore/s3/credentials/default_credential_provider.h b/tensorstore/kvstore/s3/credentials/default_credential_provider.h index 0fb221485..33b206987 100644 --- a/tensorstore/kvstore/s3/credentials/default_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/default_credential_provider.h @@ -49,15 +49,17 @@ class DefaultAwsCredentialsProvider : public AwsCredentialProvider { /// /// 1. Shared Credential Filename /// 2. Shared Credential File Profile + /// 3. EC2 Metadata Server Endpoint /// 3. Http Transport for querying the EC2 Metadata Server struct Options { - std::string filename = {}; - std::string profile = {}; + std::string filename; + std::string profile; + std::string endpoint; std::shared_ptr transport; }; DefaultAwsCredentialsProvider( - Options options = {{}, {}, internal_http::GetDefaultHttpTransport()}, + Options options = {{}, {}, {}, internal_http::GetDefaultHttpTransport()}, absl::FunctionRef clock = absl::Now); Result GetCredentials() override; @@ -76,7 +78,9 @@ void RegisterAwsCredentialProviderProvider(AwsCredentialProviderFn provider, int priority); Result> GetAwsCredentialProvider( + std::string_view filename, std::string_view profile, + std::string_view metadata_endpoint, std::shared_ptr transport); diff --git a/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc index 6d5aed9b3..e34e97cc4 100644 --- a/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc @@ -86,7 +86,7 @@ TEST(DefaultCredentialProviderTest, ConfigureEC2ProviderFromOptions) { std::make_shared(url_to_response); auto provider = std::make_unique( - Options{{}, {}, mock_transport}, stuck_clock); + Options{{}, {}, {}, mock_transport}, stuck_clock); TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, provider->GetCredentials()); EXPECT_EQ(credentials.access_key, "ASIA1234567890"); diff --git a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h index 4ba3277d9..8e9896ba7 100644 --- a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h @@ -16,6 +16,8 @@ #define TENSORSTORE_KVSTORE_S3_CREDENTIALS_EC2_METADATA_CREDENTIAL_PROVIDER_H #include +#include +#include #include #include "tensorstore/internal/http/http_transport.h" @@ -29,12 +31,18 @@ namespace internal_kvstore_s3 { class EC2MetadataCredentialProvider : public AwsCredentialProvider { public: EC2MetadataCredentialProvider( + std::string_view endpoint, std::shared_ptr transport) - : transport_(std::move(transport)) {} + : endpoint_(endpoint), transport_(std::move(transport)) {} Result GetCredentials() override; + inline const std::string & GetEndpoint() const { + return endpoint_; + } + private: + std::string endpoint_; std::shared_ptr transport_; }; diff --git a/tensorstore/kvstore/s3/credentials/ec2_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/ec2_credential_provider_test.cc index 34005bd15..3a1a7e9d0 100644 --- a/tensorstore/kvstore/s3/credentials/ec2_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/ec2_credential_provider_test.cc @@ -47,7 +47,7 @@ TEST(EC2MetadataCredentialProviderTest, CredentialRetrievalFlow) { auto mock_transport = std::make_shared(url_to_response); auto provider = - std::make_shared(mock_transport); + std::make_shared("", mock_transport); TENSORSTORE_CHECK_OK_AND_ASSIGN(auto credentials, provider->GetCredentials()); ASSERT_EQ(credentials.access_key, "ASIA1234567890"); ASSERT_EQ(credentials.secret_key, "1234567890abcdef"); @@ -68,7 +68,7 @@ TEST(EC2MetadataCredentialProviderTest, NoIamRolesInSecurityCredentials) { auto mock_transport = std::make_shared(url_to_response); auto provider = - std::make_shared(mock_transport); + std::make_shared("", mock_transport); ASSERT_FALSE(provider->GetCredentials()); EXPECT_THAT(provider->GetCredentials().status().ToString(), ::testing::HasSubstr("Empty EC2 Role list")); @@ -97,7 +97,7 @@ TEST(EC2MetadataCredentialProviderTest, UnsuccessfulJsonResponse) { auto mock_transport = std::make_shared(url_to_response); auto provider = - std::make_shared(mock_transport); + std::make_shared("", mock_transport); auto credentials = provider->GetCredentials(); EXPECT_THAT(credentials.status(), MatchesStatus(absl::StatusCode::kNotFound)); diff --git a/tensorstore/kvstore/s3/credentials/file_credential_provider.h b/tensorstore/kvstore/s3/credentials/file_credential_provider.h index 0f4687a4b..c18d81617 100644 --- a/tensorstore/kvstore/s3/credentials/file_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/file_credential_provider.h @@ -16,6 +16,7 @@ #define TENSORSTORE_KVSTORE_S3_CREDENTIALS_FILE_CREDENTIAL_PROVIDER_H #include +#include #include #include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" @@ -40,15 +41,16 @@ class FileCredentialProvider : public AwsCredentialProvider { std::string profile_; public: - FileCredentialProvider(std::string filename, std::string profile) - : filename_(std::move(filename)), profile_(std::move(profile)) {} + FileCredentialProvider(std::string_view filename, std::string_view profile) + : filename_(filename), profile_(profile) {} + Result GetCredentials() override; - const std::string & GetFileName() const { + inline const std::string & GetFileName() const { return filename_; } - const std::string & GetProfile() const { + inline const std::string & GetProfile() const { return profile_; } }; diff --git a/tensorstore/kvstore/s3/s3_key_value_store.cc b/tensorstore/kvstore/s3/s3_key_value_store.cc index df1dd6660..2ae53a63b 100644 --- a/tensorstore/kvstore/s3/s3_key_value_store.cc +++ b/tensorstore/kvstore/s3/s3_key_value_store.cc @@ -209,8 +209,9 @@ struct AwsCredentialsResource struct Spec { std::string profile; std::string filename; + std::string metadata_endpoint; constexpr static auto ApplyMembers = [](auto&& x, auto f) { - return f(x.profile, x.filename); + return f(x.profile, x.filename, x.metadata_endpoint); }; }; @@ -226,7 +227,8 @@ struct AwsCredentialsResource static constexpr auto JsonBinder() { return jb::Object( jb::Member("profile", jb::Projection<&Spec::profile>()), - jb::Member("filename", jb::Projection<&Spec::filename>()) + jb::Member("filename", jb::Projection<&Spec::filename>()), + jb::Member("metadata_endpoint", jb::Projection<&Spec::metadata_endpoint>()) ); } @@ -234,7 +236,7 @@ struct AwsCredentialsResource const Spec& spec, internal::ContextResourceCreationContext context) const { auto result = GetAwsCredentialProvider( - spec.profile, internal_http::GetDefaultHttpTransport()); + spec.profile, spec.filename, spec.metadata_endpoint, internal_http::GetDefaultHttpTransport()); if (!result.ok() && absl::IsNotFound(result.status())) { return Resource{spec, nullptr}; } diff --git a/tensorstore/kvstore/s3/schema.yml b/tensorstore/kvstore/s3/schema.yml index 9a7bb3dc5..bd6efe991 100644 --- a/tensorstore/kvstore/s3/schema.yml +++ b/tensorstore/kvstore/s3/schema.yml @@ -116,6 +116,16 @@ definitions: description: |- The profile name in the :file:`~/.aws/credentials` file, when used. Overrides the :envvar:`AWS_PROFILE` environment variables. + filename: + type: string + description: |- + The filename containing credentials. + Overrides the :envvar:`AWS_SHARED_CREDENTIALS_FILE` environment variable. + metadata_endpoint: + type: string + description: |- + The endpoint of the metadata server. + Overrides the :envvar:`AWS_EC2_METADATA_SERVICE_ENDPOINT` environment variable. experimental_s3_rate_limiter: $id: Context.experimental_s3_rate_limiter description: |- From ff2e4ed626f7d6884f92042ad8b6013acb60d4bc Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Tue, 14 Nov 2023 18:47:16 +0200 Subject: [PATCH 50/63] docstring nit --- .../kvstore/s3/credentials/default_credential_provider.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorstore/kvstore/s3/credentials/default_credential_provider.cc b/tensorstore/kvstore/s3/credentials/default_credential_provider.cc index 643fed055..ee655786a 100644 --- a/tensorstore/kvstore/s3/credentials/default_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/default_credential_provider.cc @@ -30,7 +30,7 @@ namespace tensorstore { namespace internal_kvstore_s3 { namespace { -/// Return a ChainedCredentialProvider that attempts to retrieve credentials +/// Return a DefaultCredentialProvider that attempts to retrieve credentials /// from /// 1. AWS Environment Variables, e.g. AWS_ACCESS_KEY_ID /// 2. Shared Credential File, e.g. $HOME/.aws/credentials From 4c2b8da883c9b31f1bd5e57da481ac211fbfd397 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Tue, 14 Nov 2023 18:56:44 +0200 Subject: [PATCH 51/63] Set default timeout period once credentials have been retrieved --- .../s3/credentials/ec2_credential_provider.cc | 20 +++++++++---------- .../s3/credentials/ec2_credential_provider.h | 3 --- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc index dd23c3459..75591f166 100644 --- a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc @@ -116,11 +116,11 @@ inline constexpr auto EC2CredentialsResponseBinder = jb::Object( jb::Projection(&EC2CredentialsResponse::expiration))); // Obtain a metadata token for communicating with the api server. -Result GetEC2ApiToken(internal_http::HttpTransport& transport) { +Result GetEC2ApiToken(std::string_view endpoint, internal_http::HttpTransport& transport) { // Obtain Metadata server API tokens with a TTL of 21600 seconds auto token_request = HttpRequestBuilder("POST", - tensorstore::StrCat(GetEC2MetadataServiceEndpoint(), + tensorstore::StrCat(endpoint, "/latest/api/token")) .AddHeader("x-aws-ec2-metadata-token-ttl-seconds: 21600") .BuildRequest(); @@ -138,10 +138,6 @@ Result GetEC2ApiToken(internal_http::HttpTransport& transport) { } // namespace -// Returns whether the EC2 Metadata Server is available. -bool IsEC2MetadataServiceAvailable(internal_http::HttpTransport& transport) { - return GetEC2ApiToken(transport).ok(); -} /// Obtains AWS Credentials from the EC2Metadata. /// @@ -156,16 +152,17 @@ bool IsEC2MetadataServiceAvailable(internal_http::HttpTransport& transport) { /// 3. Obtain the associated credentials from path /// "/latest/meta-data/iam/security-credentials/". Result EC2MetadataCredentialProvider::GetCredentials() { - auto default_timeout = absl::Now() + kDefaultTimeout; + if(endpoint_.empty()) { + endpoint_ = GetEC2MetadataServiceEndpoint(); + } - // Obtain an API token for communicating with the EC2 Metadata server - TENSORSTORE_ASSIGN_OR_RETURN(auto api_token, GetEC2ApiToken(*transport_)); + TENSORSTORE_ASSIGN_OR_RETURN(auto api_token, GetEC2ApiToken(endpoint_, *transport_)); auto token_header = tensorstore::StrCat(kMetadataTokenHeader, api_token); auto iam_role_request = HttpRequestBuilder("GET", - tensorstore::StrCat(GetEC2MetadataServiceEndpoint(), + tensorstore::StrCat(endpoint_, kIamCredentialsPath)) .AddHeader(token_header) .BuildRequest(); @@ -184,7 +181,7 @@ Result EC2MetadataCredentialProvider::GetCredentials() { } auto iam_credentials_request_url = tensorstore::StrCat( - GetEC2MetadataServiceEndpoint(), kIamCredentialsPath, iam_roles[0]); + endpoint_, kIamCredentialsPath, iam_roles[0]); auto iam_credentials_request = HttpRequestBuilder("GET", iam_credentials_request_url) @@ -213,6 +210,7 @@ Result EC2MetadataCredentialProvider::GetCredentials() { } // Introduce a leeway of 60 seconds to avoid credential expiry conditions + auto default_timeout = absl::Now() + kDefaultTimeout; auto expires_at = iam_credentials.expiration.value_or(default_timeout) - absl::Seconds(60); diff --git a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h index 8e9896ba7..c0bd8608c 100644 --- a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h @@ -46,9 +46,6 @@ class EC2MetadataCredentialProvider : public AwsCredentialProvider { std::shared_ptr transport_; }; -// Returns whether the EC2 Metadata Server is available. -bool IsEC2MetadataServiceAvailable(internal_http::HttpTransport& transport); - } // namespace internal_kvstore_s3 } // namespace tensorstore From a9a4ad7bc56d890cd87db6adf72b8c15159bb7ac Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Tue, 14 Nov 2023 19:55:13 +0200 Subject: [PATCH 52/63] Test metadata service endpoint configuration --- .../default_credential_provider_test.cc | 12 ++-- .../ec2_credential_provider_test.cc | 68 ++++++++++++++----- .../kvstore/s3/credentials/test_utils.cc | 13 ++-- .../kvstore/s3/credentials/test_utils.h | 3 +- 4 files changed, 67 insertions(+), 29 deletions(-) diff --git a/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc index e34e97cc4..9b574b1ee 100644 --- a/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc @@ -41,6 +41,8 @@ using ::tensorstore::internal_kvstore_s3::EC2MetadataMockTransport; using Options = ::tensorstore::internal_kvstore_s3::DefaultAwsCredentialsProvider::Options; +static constexpr char endpoint[] = "http://endpoint"; + class CredentialFileFactory : public tensorstore::internal::ScopedTemporaryDirectory { public: @@ -79,14 +81,14 @@ TEST(DefaultCredentialProviderTest, ConfigureEC2ProviderFromOptions) { auto now = absl::Now(); auto stuck_clock = [&]() -> absl::Time { return now; }; auto expiry = now + absl::Seconds(200); - auto url_to_response = DefaultEC2MetadataFlow( + auto url_to_response = DefaultEC2MetadataFlow(endpoint, "1234", "ASIA1234567890", "1234567890abcdef", "token", expiry); auto mock_transport = std::make_shared(url_to_response); auto provider = std::make_unique( - Options{{}, {}, {}, mock_transport}, stuck_clock); + Options{{}, {}, endpoint, mock_transport}, stuck_clock); TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, provider->GetCredentials()); EXPECT_EQ(credentials.access_key, "ASIA1234567890"); @@ -96,7 +98,7 @@ TEST(DefaultCredentialProviderTest, ConfigureEC2ProviderFromOptions) { /// Force failure on credential retrieval url_to_response = absl::flat_hash_map{ - {"POST http://169.254.169.254/latest/api/token", + {"POST http://endpoint/latest/api/token", HttpResponse{404, absl::Cord{""}}}, }; @@ -109,7 +111,7 @@ TEST(DefaultCredentialProviderTest, ConfigureEC2ProviderFromOptions) { // Force expiry and retrieve new credentials now += absl::Seconds(300); - url_to_response = DefaultEC2MetadataFlow("1234", "ASIA1234567890", + url_to_response = DefaultEC2MetadataFlow(endpoint, "1234", "ASIA1234567890", "1234567890abcdef", "TOKEN", expiry); // A new set of credentials is returned @@ -121,7 +123,7 @@ TEST(DefaultCredentialProviderTest, ConfigureEC2ProviderFromOptions) { /// Force failure on credential retrieval url_to_response = absl::flat_hash_map{ - {"POST http://169.254.169.254/latest/api/token", + {"POST http://endpoint/latest/api/token", HttpResponse{404, absl::Cord{""}}}, }; diff --git a/tensorstore/kvstore/s3/credentials/ec2_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/ec2_credential_provider_test.cc index 3a1a7e9d0..5e46732b8 100644 --- a/tensorstore/kvstore/s3/credentials/ec2_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/ec2_credential_provider_test.cc @@ -25,11 +25,15 @@ #include "absl/strings/cord.h" #include "absl/time/clock.h" #include "absl/time/time.h" +#include "tensorstore/internal/env.h" #include "tensorstore/internal/http/http_response.h" #include "tensorstore/kvstore/s3/credentials/test_utils.h" #include "tensorstore/util/result.h" #include "tensorstore/util/status_testutil.h" +using ::tensorstore::internal::UnsetEnv; +using ::tensorstore::internal::SetEnv; + namespace { using ::tensorstore::MatchesStatus; @@ -38,66 +42,98 @@ using ::tensorstore::internal_kvstore_s3::DefaultEC2MetadataFlow; using ::tensorstore::internal_kvstore_s3::EC2MetadataCredentialProvider; using ::tensorstore::internal_kvstore_s3::EC2MetadataMockTransport; -TEST(EC2MetadataCredentialProviderTest, CredentialRetrievalFlow) { +static constexpr char endpoint[] = "http://169.254.169.254"; +static constexpr char api_token[] = "1234567890"; +static constexpr char access_key[] = "ASIA1234567890"; +static constexpr char secret_key[] = "1234567890abcdef"; +static constexpr char session_token[] = "abcdef123456790"; + +class EC2MetadataCredentialProviderTest : public ::testing::Test { + protected: + void SetUp() override { UnsetEnv("AWS_EC2_METADATA_SERVICE_ENDPOINT"); } +}; + + +TEST_F(EC2MetadataCredentialProviderTest, CredentialRetrievalFlow) { + auto expiry = absl::Now() + absl::Seconds(200); + auto url_to_response = + DefaultEC2MetadataFlow(endpoint, api_token, access_key, secret_key, + session_token, expiry); + + auto mock_transport = + std::make_shared(url_to_response); + auto provider = + std::make_shared(endpoint, mock_transport); + TENSORSTORE_CHECK_OK_AND_ASSIGN(auto credentials, provider->GetCredentials()); + ASSERT_EQ(credentials.access_key, access_key); + ASSERT_EQ(credentials.secret_key, secret_key); + ASSERT_EQ(credentials.session_token, session_token); + // expiry less the 60s leeway + ASSERT_EQ(credentials.expires_at, expiry - absl::Seconds(60)); +} + +TEST_F(EC2MetadataCredentialProviderTest, EnvironmentVariableMetadataServer) { + SetEnv("AWS_EC2_METADATA_SERVICE_ENDPOINT", "http://endpoint"); auto expiry = absl::Now() + absl::Seconds(200); auto url_to_response = - DefaultEC2MetadataFlow("1234567890", "ASIA1234567890", "1234567890abcdef", - "abcdef123456790", expiry); + DefaultEC2MetadataFlow("http://endpoint", api_token, access_key, secret_key, + session_token, expiry); auto mock_transport = std::make_shared(url_to_response); auto provider = std::make_shared("", mock_transport); TENSORSTORE_CHECK_OK_AND_ASSIGN(auto credentials, provider->GetCredentials()); - ASSERT_EQ(credentials.access_key, "ASIA1234567890"); - ASSERT_EQ(credentials.secret_key, "1234567890abcdef"); - ASSERT_EQ(credentials.session_token, "abcdef123456790"); + ASSERT_EQ(credentials.access_key, access_key); + ASSERT_EQ(credentials.secret_key, secret_key); + ASSERT_EQ(credentials.session_token, session_token); // expiry less the 60s leeway ASSERT_EQ(credentials.expires_at, expiry - absl::Seconds(60)); } -TEST(EC2MetadataCredentialProviderTest, NoIamRolesInSecurityCredentials) { + +TEST_F(EC2MetadataCredentialProviderTest, NoIamRolesInSecurityCredentials) { auto url_to_response = absl::flat_hash_map{ {"POST http://169.254.169.254/latest/api/token", - HttpResponse{200, absl::Cord{"1234567890"}}}, + HttpResponse{200, absl::Cord{api_token}}}, {"GET http://169.254.169.254/latest/meta-data/iam/security-credentials/", HttpResponse{ - 200, absl::Cord{""}, {{"x-aws-ec2-metadata-token", "1234567890"}}}}, + 200, absl::Cord{""}, {{"x-aws-ec2-metadata-token", api_token}}}}, }; auto mock_transport = std::make_shared(url_to_response); auto provider = - std::make_shared("", mock_transport); + std::make_shared(endpoint, mock_transport); ASSERT_FALSE(provider->GetCredentials()); EXPECT_THAT(provider->GetCredentials().status().ToString(), ::testing::HasSubstr("Empty EC2 Role list")); } -TEST(EC2MetadataCredentialProviderTest, UnsuccessfulJsonResponse) { +TEST_F(EC2MetadataCredentialProviderTest, UnsuccessfulJsonResponse) { // Test that "Code" != "Success" parsing succeeds auto url_to_response = absl::flat_hash_map{ {"POST http://169.254.169.254/latest/api/token", - HttpResponse{200, absl::Cord{"1234567890"}}}, + HttpResponse{200, absl::Cord{api_token}}}, {"GET http://169.254.169.254/latest/meta-data/iam/", HttpResponse{200, absl::Cord{"info"}, - {{"x-aws-ec2-metadata-token", "1234567890"}}}}, + {{"x-aws-ec2-metadata-token", api_token}}}}, {"GET http://169.254.169.254/latest/meta-data/iam/security-credentials/", HttpResponse{200, absl::Cord{"mock-iam-role"}, - {{"x-aws-ec2-metadata-token", "1234567890"}}}}, + {{"x-aws-ec2-metadata-token", api_token}}}}, {"GET " "http://169.254.169.254/latest/meta-data/iam/security-credentials/" "mock-iam-role", HttpResponse{200, absl::Cord(R"({"Code": "EntirelyUnsuccessful"})"), - {{"x-aws-ec2-metadata-token", "1234567890"}}}}}; + {{"x-aws-ec2-metadata-token", api_token}}}}}; auto mock_transport = std::make_shared(url_to_response); auto provider = - std::make_shared("", mock_transport); + std::make_shared(endpoint, mock_transport); auto credentials = provider->GetCredentials(); EXPECT_THAT(credentials.status(), MatchesStatus(absl::StatusCode::kNotFound)); diff --git a/tensorstore/kvstore/s3/credentials/test_utils.cc b/tensorstore/kvstore/s3/credentials/test_utils.cc index 44dec0e42..93293a7e0 100644 --- a/tensorstore/kvstore/s3/credentials/test_utils.cc +++ b/tensorstore/kvstore/s3/credentials/test_utils.cc @@ -41,24 +41,23 @@ Future EC2MetadataMockTransport::IssueRequest( } absl::flat_hash_map -DefaultEC2MetadataFlow(const std::string& api_token, +DefaultEC2MetadataFlow(const std::string& endpoint, + const std::string& api_token, const std::string& access_key, const std::string& secret_key, const std::string& session_token, const absl::Time& expires_at) { return absl::flat_hash_map{ - {"POST http://169.254.169.254/latest/api/token", + {absl::StrFormat("POST %s/latest/api/token", endpoint), internal_http::HttpResponse{200, absl::Cord{api_token}}}, - {"GET http://169.254.169.254/latest/meta-data/iam/", + {absl::StrFormat("GET %s/latest/meta-data/iam/", endpoint), internal_http::HttpResponse{ 200, absl::Cord{"info"}, {{"x-aws-ec2-metadata-token", api_token}}}}, - {"GET http://169.254.169.254/latest/meta-data/iam/security-credentials/", + {absl::StrFormat("GET %s/latest/meta-data/iam/security-credentials/", endpoint), internal_http::HttpResponse{200, absl::Cord{"mock-iam-role"}, {{"x-aws-ec2-metadata-token", api_token}}}}, - {"GET " - "http://169.254.169.254/latest/meta-data/iam/security-credentials/" - "mock-iam-role", + {absl::StrFormat("GET %s/latest/meta-data/iam/security-credentials/mock-iam-role", endpoint), internal_http::HttpResponse{ 200, absl::Cord(absl::StrFormat( diff --git a/tensorstore/kvstore/s3/credentials/test_utils.h b/tensorstore/kvstore/s3/credentials/test_utils.h index de684b09c..73a74b33b 100644 --- a/tensorstore/kvstore/s3/credentials/test_utils.h +++ b/tensorstore/kvstore/s3/credentials/test_utils.h @@ -46,7 +46,8 @@ class EC2MetadataMockTransport : public internal_http::HttpTransport { /// Return a Default EC2 Metadata Credential Retrieval Flow, suitable /// for passing to EC2MetadataMockTransport absl::flat_hash_map -DefaultEC2MetadataFlow(const std::string& api_token, +DefaultEC2MetadataFlow(const std::string& endpoint, + const std::string& api_token, const std::string& access_key, const std::string& secret_key, const std::string& session_token, From 1d39c46c75d0bc4acca291a1f04069804d91d050 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Wed, 15 Nov 2023 09:19:30 +0200 Subject: [PATCH 53/63] More EC2 Metadata Endpoint Testing --- .../ec2_credential_provider_test.cc | 37 +++++++++++++++---- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/tensorstore/kvstore/s3/credentials/ec2_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/ec2_credential_provider_test.cc index 5e46732b8..58b69c93f 100644 --- a/tensorstore/kvstore/s3/credentials/ec2_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/ec2_credential_provider_test.cc @@ -42,7 +42,8 @@ using ::tensorstore::internal_kvstore_s3::DefaultEC2MetadataFlow; using ::tensorstore::internal_kvstore_s3::EC2MetadataCredentialProvider; using ::tensorstore::internal_kvstore_s3::EC2MetadataMockTransport; -static constexpr char endpoint[] = "http://169.254.169.254"; +static constexpr char default_endpoint[] = "http://169.254.169.254"; +static constexpr char custom_endpoint[] = "http://custom.endpoint"; static constexpr char api_token[] = "1234567890"; static constexpr char access_key[] = "ASIA1234567890"; static constexpr char secret_key[] = "1234567890abcdef"; @@ -57,14 +58,15 @@ class EC2MetadataCredentialProviderTest : public ::testing::Test { TEST_F(EC2MetadataCredentialProviderTest, CredentialRetrievalFlow) { auto expiry = absl::Now() + absl::Seconds(200); auto url_to_response = - DefaultEC2MetadataFlow(endpoint, api_token, access_key, secret_key, + DefaultEC2MetadataFlow(default_endpoint, api_token, access_key, secret_key, session_token, expiry); auto mock_transport = std::make_shared(url_to_response); auto provider = - std::make_shared(endpoint, mock_transport); + std::make_shared("", mock_transport); TENSORSTORE_CHECK_OK_AND_ASSIGN(auto credentials, provider->GetCredentials()); + ASSERT_EQ(provider->GetEndpoint(), default_endpoint); ASSERT_EQ(credentials.access_key, access_key); ASSERT_EQ(credentials.secret_key, secret_key); ASSERT_EQ(credentials.session_token, session_token); @@ -73,10 +75,10 @@ TEST_F(EC2MetadataCredentialProviderTest, CredentialRetrievalFlow) { } TEST_F(EC2MetadataCredentialProviderTest, EnvironmentVariableMetadataServer) { - SetEnv("AWS_EC2_METADATA_SERVICE_ENDPOINT", "http://endpoint"); + SetEnv("AWS_EC2_METADATA_SERVICE_ENDPOINT", custom_endpoint); auto expiry = absl::Now() + absl::Seconds(200); auto url_to_response = - DefaultEC2MetadataFlow("http://endpoint", api_token, access_key, secret_key, + DefaultEC2MetadataFlow(custom_endpoint, api_token, access_key, secret_key, session_token, expiry); auto mock_transport = @@ -84,6 +86,26 @@ TEST_F(EC2MetadataCredentialProviderTest, EnvironmentVariableMetadataServer) { auto provider = std::make_shared("", mock_transport); TENSORSTORE_CHECK_OK_AND_ASSIGN(auto credentials, provider->GetCredentials()); + ASSERT_EQ(provider->GetEndpoint(), custom_endpoint); + ASSERT_EQ(credentials.access_key, access_key); + ASSERT_EQ(credentials.secret_key, secret_key); + ASSERT_EQ(credentials.session_token, session_token); + // expiry less the 60s leeway + ASSERT_EQ(credentials.expires_at, expiry - absl::Seconds(60)); +} + +TEST_F(EC2MetadataCredentialProviderTest, InjectedMetadataServer) { + auto expiry = absl::Now() + absl::Seconds(200); + auto url_to_response = + DefaultEC2MetadataFlow(custom_endpoint, api_token, access_key, secret_key, + session_token, expiry); + + auto mock_transport = + std::make_shared(url_to_response); + auto provider = + std::make_shared(custom_endpoint, mock_transport); + TENSORSTORE_CHECK_OK_AND_ASSIGN(auto credentials, provider->GetCredentials()); + ASSERT_EQ(provider->GetEndpoint(), custom_endpoint); ASSERT_EQ(credentials.access_key, access_key); ASSERT_EQ(credentials.secret_key, secret_key); ASSERT_EQ(credentials.session_token, session_token); @@ -104,8 +126,9 @@ TEST_F(EC2MetadataCredentialProviderTest, NoIamRolesInSecurityCredentials) { auto mock_transport = std::make_shared(url_to_response); auto provider = - std::make_shared(endpoint, mock_transport); + std::make_shared("", mock_transport); ASSERT_FALSE(provider->GetCredentials()); + ASSERT_EQ(provider->GetEndpoint(), default_endpoint); EXPECT_THAT(provider->GetCredentials().status().ToString(), ::testing::HasSubstr("Empty EC2 Role list")); } @@ -133,7 +156,7 @@ TEST_F(EC2MetadataCredentialProviderTest, UnsuccessfulJsonResponse) { auto mock_transport = std::make_shared(url_to_response); auto provider = - std::make_shared(endpoint, mock_transport); + std::make_shared("", mock_transport); auto credentials = provider->GetCredentials(); EXPECT_THAT(credentials.status(), MatchesStatus(absl::StatusCode::kNotFound)); From c3bebf68074cb880d92cdaad6152afad727d8c98 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Wed, 15 Nov 2023 15:42:18 +0200 Subject: [PATCH 54/63] Apply clangd IWYU suggestions --- .../kvstore/s3/credentials/aws_credential_provider.h | 4 ---- .../s3/credentials/default_credential_provider_test.cc | 1 - .../kvstore/s3/credentials/ec2_credential_provider.cc | 3 --- .../s3/credentials/environment_credential_provider.h | 1 + tensorstore/kvstore/s3/s3_endpoint_test.cc | 1 - tensorstore/kvstore/s3/s3_key_value_store.cc | 6 ------ tensorstore/kvstore/s3/s3_key_value_store_test.cc | 5 ----- tensorstore/kvstore/s3/s3_metadata_test.cc | 2 -- tensorstore/kvstore/s3/s3_request_builder.h | 1 - tensorstore/kvstore/s3/s3_resource.cc | 1 - tensorstore/kvstore/s3/s3_resource.h | 4 ---- 11 files changed, 1 insertion(+), 28 deletions(-) diff --git a/tensorstore/kvstore/s3/credentials/aws_credential_provider.h b/tensorstore/kvstore/s3/credentials/aws_credential_provider.h index 1d6279af8..03d9d85c8 100644 --- a/tensorstore/kvstore/s3/credentials/aws_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/aws_credential_provider.h @@ -15,12 +15,8 @@ #ifndef TENSORSTORE_KVSTORE_S3_CREDENTIALS_AWS_CREDENTIAL_PROVIDER #define TENSORSTORE_KVSTORE_S3_CREDENTIALS_AWS_CREDENTIAL_PROVIDER -#include -#include #include -#include -#include "absl/time/clock.h" #include "absl/time/time.h" #include "tensorstore/util/result.h" diff --git a/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc index 9b574b1ee..d3037b444 100644 --- a/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc @@ -17,7 +17,6 @@ #include #include -#include #include #include "absl/container/flat_hash_map.h" diff --git a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc index 75591f166..66b8bfb19 100644 --- a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc @@ -14,8 +14,6 @@ #include "tensorstore/kvstore/s3/credentials/ec2_credential_provider.h" -#include -#include #include #include #include @@ -29,7 +27,6 @@ #include "absl/time/clock.h" #include "absl/time/time.h" #include "nlohmann/json.hpp" -#include "tensorstore/json_serialization_options_base.h" #include "tensorstore/internal/env.h" #include "tensorstore/internal/http/http_request.h" #include "tensorstore/internal/http/http_response.h" diff --git a/tensorstore/kvstore/s3/credentials/environment_credential_provider.h b/tensorstore/kvstore/s3/credentials/environment_credential_provider.h index bd5296866..6432cb850 100644 --- a/tensorstore/kvstore/s3/credentials/environment_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/environment_credential_provider.h @@ -16,6 +16,7 @@ #define TENSORSTORE_KVSTORE_S3_CREDENTIALS_ENVIRONMENT_CREDENTIAL_PROVIDER_H #include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" + #include "tensorstore/util/result.h" namespace tensorstore { diff --git a/tensorstore/kvstore/s3/s3_endpoint_test.cc b/tensorstore/kvstore/s3/s3_endpoint_test.cc index 7ed485f3d..f6c1ba327 100644 --- a/tensorstore/kvstore/s3/s3_endpoint_test.cc +++ b/tensorstore/kvstore/s3/s3_endpoint_test.cc @@ -29,7 +29,6 @@ #include "tensorstore/internal/http/http_response.h" #include "tensorstore/internal/http/http_transport.h" #include "tensorstore/util/future.h" -#include "tensorstore/util/result.h" #include "tensorstore/util/status_testutil.h" #include "tensorstore/util/str_cat.h" diff --git a/tensorstore/kvstore/s3/s3_key_value_store.cc b/tensorstore/kvstore/s3/s3_key_value_store.cc index 2ae53a63b..f4b531399 100644 --- a/tensorstore/kvstore/s3/s3_key_value_store.cc +++ b/tensorstore/kvstore/s3/s3_key_value_store.cc @@ -13,7 +13,6 @@ // limitations under the License. #include -#include #include #include #include @@ -23,9 +22,6 @@ #include #include #include -#include -#include -#include #include "absl/base/attributes.h" #include "absl/log/absl_log.h" @@ -41,7 +37,6 @@ #include "tensorstore/context_resource_provider.h" #include "tensorstore/internal/data_copy_concurrency_resource.h" #include "tensorstore/internal/digest/sha256.h" -#include "tensorstore/internal/http/curl_transport.h" #include "tensorstore/internal/http/http_request.h" #include "tensorstore/internal/http/http_response.h" #include "tensorstore/internal/http/http_transport.h" @@ -87,7 +82,6 @@ #include "tensorstore/serialization/fwd.h" // IWYU pragma: keep #include "tensorstore/serialization/std_optional.h" // IWYU pragma: keep #include "tensorstore/util/garbage_collection/std_optional.h" // IWYU pragma: keep -#include "tensorstore/internal/json_binding/bindable.h" using ::tensorstore::internal::DataCopyConcurrencyResource; using ::tensorstore::internal::IntrusivePtr; diff --git a/tensorstore/kvstore/s3/s3_key_value_store_test.cc b/tensorstore/kvstore/s3/s3_key_value_store_test.cc index e9c2bc7a7..5c88b8253 100644 --- a/tensorstore/kvstore/s3/s3_key_value_store_test.cc +++ b/tensorstore/kvstore/s3/s3_key_value_store_test.cc @@ -16,10 +16,7 @@ #include #include #include -#include -#include #include -#include #include "absl/container/flat_hash_map.h" #include "absl/log/absl_log.h" @@ -41,8 +38,6 @@ #include "tensorstore/util/future.h" #include "tensorstore/util/status_testutil.h" #include "tensorstore/util/str_cat.h" -#include "tensorstore/kvstore/read_result.h" -#include "tensorstore/kvstore/spec.h" #include "tensorstore/util/result.h" namespace kvstore = ::tensorstore::kvstore; diff --git a/tensorstore/kvstore/s3/s3_metadata_test.cc b/tensorstore/kvstore/s3/s3_metadata_test.cc index 0d10c8e60..9de7fec12 100644 --- a/tensorstore/kvstore/s3/s3_metadata_test.cc +++ b/tensorstore/kvstore/s3/s3_metadata_test.cc @@ -14,8 +14,6 @@ #include "tensorstore/kvstore/s3/s3_metadata.h" - -#include #include #include #include diff --git a/tensorstore/kvstore/s3/s3_request_builder.h b/tensorstore/kvstore/s3/s3_request_builder.h index ed750f891..eb57a55f7 100644 --- a/tensorstore/kvstore/s3/s3_request_builder.h +++ b/tensorstore/kvstore/s3/s3_request_builder.h @@ -18,7 +18,6 @@ /// \file /// S3 Request Builder -#include #include #include #include diff --git a/tensorstore/kvstore/s3/s3_resource.cc b/tensorstore/kvstore/s3/s3_resource.cc index 0f1c6cffb..04f1fe5f0 100644 --- a/tensorstore/kvstore/s3/s3_resource.cc +++ b/tensorstore/kvstore/s3/s3_resource.cc @@ -17,7 +17,6 @@ #include #include #include -#include #include "absl/base/call_once.h" #include "absl/flags/flag.h" diff --git a/tensorstore/kvstore/s3/s3_resource.h b/tensorstore/kvstore/s3/s3_resource.h index e5e6d7dc3..9fd19c752 100644 --- a/tensorstore/kvstore/s3/s3_resource.h +++ b/tensorstore/kvstore/s3/s3_resource.h @@ -23,16 +23,12 @@ #include "absl/time/time.h" #include "tensorstore/context.h" #include "tensorstore/context_resource_provider.h" -#include "tensorstore/json_serialization_options_base.h" #include "tensorstore/internal/json_binding/json_binding.h" #include "tensorstore/internal/json_binding/std_optional.h" #include "tensorstore/internal/retries_context_resource.h" #include "tensorstore/kvstore/gcs_http/admission_queue.h" #include "tensorstore/kvstore/gcs_http/rate_limiter.h" #include "tensorstore/util/result.h" -/// specializations -#include "tensorstore/internal/json_binding/absl_time.h" -#include "tensorstore/internal/json_binding/bindable.h" namespace tensorstore { namespace internal_kvstore_s3 { From 8b23ef025fe3eac164822148a64e6022938363c6 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Wed, 15 Nov 2023 16:01:21 +0200 Subject: [PATCH 55/63] clang-format --- .../default_credential_provider.cc | 23 +++++++++---------- .../credentials/default_credential_provider.h | 4 +--- .../default_credential_provider_test.cc | 4 ++-- .../s3/credentials/ec2_credential_provider.cc | 19 ++++++++------- .../s3/credentials/ec2_credential_provider.h | 4 +--- .../ec2_credential_provider_test.cc | 17 ++++++-------- .../environment_credential_provider.cc | 4 ++-- .../environment_credential_provider.h | 1 - .../credentials/file_credential_provider.cc | 10 ++++---- .../s3/credentials/file_credential_provider.h | 10 +++----- .../file_credential_provider_test.cc | 1 - .../kvstore/s3/credentials/test_utils.cc | 7 ++++-- 12 files changed, 46 insertions(+), 58 deletions(-) diff --git a/tensorstore/kvstore/s3/credentials/default_credential_provider.cc b/tensorstore/kvstore/s3/credentials/default_credential_provider.cc index ee655786a..eaa6aec85 100644 --- a/tensorstore/kvstore/s3/credentials/default_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/default_credential_provider.cc @@ -19,8 +19,8 @@ #include "absl/synchronization/mutex.h" #include "absl/time/time.h" -#include "tensorstore/internal/no_destructor.h" #include "tensorstore/internal/http/http_transport.h" +#include "tensorstore/internal/no_destructor.h" #include "tensorstore/kvstore/s3/credentials/ec2_credential_provider.h" #include "tensorstore/kvstore/s3/credentials/environment_credential_provider.h" #include "tensorstore/kvstore/s3/credentials/file_credential_provider.h" @@ -36,13 +36,13 @@ namespace { /// 2. Shared Credential File, e.g. $HOME/.aws/credentials /// 3. EC2 Metadata Server Result> GetDefaultAwsCredentialProvider( - std::string_view filename, - std::string_view profile, + std::string_view filename, std::string_view profile, std::string_view metadata_endpoint, std::shared_ptr transport) { return std::make_unique( DefaultAwsCredentialsProvider::Options{ - std::string{filename}, std::string{profile}, std::string{metadata_endpoint}, transport}); + std::string{filename}, std::string{profile}, + std::string{metadata_endpoint}, transport}); } struct AwsCredentialProviderRegistry { @@ -79,8 +79,7 @@ void RegisterAwsCredentialProviderProvider(AwsCredentialProviderFn provider, /// 4. EC2 Metadata server. The `transport` variable overrides the default /// HttpTransport. Result> GetAwsCredentialProvider( - std::string_view filename, - std::string_view profile, + std::string_view filename, std::string_view profile, std::string_view metadata_endpoint, std::shared_ptr transport) { auto& registry = GetAwsProviderRegistry(); @@ -90,7 +89,8 @@ Result> GetAwsCredentialProvider( if (credentials.ok()) return credentials; } - return GetDefaultAwsCredentialProvider(filename, profile, metadata_endpoint, transport); + return GetDefaultAwsCredentialProvider(filename, profile, metadata_endpoint, + transport); } DefaultAwsCredentialsProvider::DefaultAwsCredentialsProvider( @@ -128,8 +128,8 @@ Result DefaultAwsCredentialsProvider::GetCredentials() { } // 2. Shared Credential File, e.g. $HOME/.aws/credentials - provider_ = std::make_unique( - options_.filename, options_.profile); + provider_ = std::make_unique(options_.filename, + options_.profile); credentials_result = provider_->GetCredentials(); if (credentials_result.ok()) { credentials_ = credentials_result.value(); @@ -137,9 +137,8 @@ Result DefaultAwsCredentialsProvider::GetCredentials() { } // 3. EC2 Metadata Server - provider_ = - std::make_unique( - options_.endpoint, options_.transport); + provider_ = std::make_unique( + options_.endpoint, options_.transport); credentials_result = provider_->GetCredentials(); if (credentials_result.ok()) { credentials_ = credentials_result.value(); diff --git a/tensorstore/kvstore/s3/credentials/default_credential_provider.h b/tensorstore/kvstore/s3/credentials/default_credential_provider.h index 33b206987..21cddd16e 100644 --- a/tensorstore/kvstore/s3/credentials/default_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/default_credential_provider.h @@ -78,12 +78,10 @@ void RegisterAwsCredentialProviderProvider(AwsCredentialProviderFn provider, int priority); Result> GetAwsCredentialProvider( - std::string_view filename, - std::string_view profile, + std::string_view filename, std::string_view profile, std::string_view metadata_endpoint, std::shared_ptr transport); - } // namespace internal_kvstore_s3 } // namespace tensorstore diff --git a/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc index d3037b444..43650a736 100644 --- a/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc @@ -80,8 +80,8 @@ TEST(DefaultCredentialProviderTest, ConfigureEC2ProviderFromOptions) { auto now = absl::Now(); auto stuck_clock = [&]() -> absl::Time { return now; }; auto expiry = now + absl::Seconds(200); - auto url_to_response = DefaultEC2MetadataFlow(endpoint, - "1234", "ASIA1234567890", "1234567890abcdef", "token", expiry); + auto url_to_response = DefaultEC2MetadataFlow( + endpoint, "1234", "ASIA1234567890", "1234567890abcdef", "token", expiry); auto mock_transport = std::make_shared(url_to_response); diff --git a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc index 66b8bfb19..afa8187fb 100644 --- a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc @@ -113,12 +113,12 @@ inline constexpr auto EC2CredentialsResponseBinder = jb::Object( jb::Projection(&EC2CredentialsResponse::expiration))); // Obtain a metadata token for communicating with the api server. -Result GetEC2ApiToken(std::string_view endpoint, internal_http::HttpTransport& transport) { +Result GetEC2ApiToken(std::string_view endpoint, + internal_http::HttpTransport& transport) { // Obtain Metadata server API tokens with a TTL of 21600 seconds auto token_request = HttpRequestBuilder("POST", - tensorstore::StrCat(endpoint, - "/latest/api/token")) + tensorstore::StrCat(endpoint, "/latest/api/token")) .AddHeader("x-aws-ec2-metadata-token-ttl-seconds: 21600") .BuildRequest(); @@ -135,7 +135,6 @@ Result GetEC2ApiToken(std::string_view endpoint, internal_http::Http } // namespace - /// Obtains AWS Credentials from the EC2Metadata. /// /// https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html#instancedata-meta-data-retrieval-examples @@ -149,18 +148,18 @@ Result GetEC2ApiToken(std::string_view endpoint, internal_http::Http /// 3. Obtain the associated credentials from path /// "/latest/meta-data/iam/security-credentials/". Result EC2MetadataCredentialProvider::GetCredentials() { - if(endpoint_.empty()) { + if (endpoint_.empty()) { endpoint_ = GetEC2MetadataServiceEndpoint(); } - TENSORSTORE_ASSIGN_OR_RETURN(auto api_token, GetEC2ApiToken(endpoint_, *transport_)); + TENSORSTORE_ASSIGN_OR_RETURN(auto api_token, + GetEC2ApiToken(endpoint_, *transport_)); auto token_header = tensorstore::StrCat(kMetadataTokenHeader, api_token); auto iam_role_request = HttpRequestBuilder("GET", - tensorstore::StrCat(endpoint_, - kIamCredentialsPath)) + tensorstore::StrCat(endpoint_, kIamCredentialsPath)) .AddHeader(token_header) .BuildRequest(); @@ -177,8 +176,8 @@ Result EC2MetadataCredentialProvider::GetCredentials() { return absl::NotFoundError("Empty EC2 Role list"); } - auto iam_credentials_request_url = tensorstore::StrCat( - endpoint_, kIamCredentialsPath, iam_roles[0]); + auto iam_credentials_request_url = + tensorstore::StrCat(endpoint_, kIamCredentialsPath, iam_roles[0]); auto iam_credentials_request = HttpRequestBuilder("GET", iam_credentials_request_url) diff --git a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h index c0bd8608c..138ca5c06 100644 --- a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h @@ -37,9 +37,7 @@ class EC2MetadataCredentialProvider : public AwsCredentialProvider { Result GetCredentials() override; - inline const std::string & GetEndpoint() const { - return endpoint_; - } + inline const std::string& GetEndpoint() const { return endpoint_; } private: std::string endpoint_; diff --git a/tensorstore/kvstore/s3/credentials/ec2_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/ec2_credential_provider_test.cc index 58b69c93f..1f6ff42ce 100644 --- a/tensorstore/kvstore/s3/credentials/ec2_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/ec2_credential_provider_test.cc @@ -31,8 +31,8 @@ #include "tensorstore/util/result.h" #include "tensorstore/util/status_testutil.h" -using ::tensorstore::internal::UnsetEnv; using ::tensorstore::internal::SetEnv; +using ::tensorstore::internal::UnsetEnv; namespace { @@ -54,12 +54,11 @@ class EC2MetadataCredentialProviderTest : public ::testing::Test { void SetUp() override { UnsetEnv("AWS_EC2_METADATA_SERVICE_ENDPOINT"); } }; - TEST_F(EC2MetadataCredentialProviderTest, CredentialRetrievalFlow) { auto expiry = absl::Now() + absl::Seconds(200); auto url_to_response = - DefaultEC2MetadataFlow(default_endpoint, api_token, access_key, secret_key, - session_token, expiry); + DefaultEC2MetadataFlow(default_endpoint, api_token, access_key, + secret_key, session_token, expiry); auto mock_transport = std::make_shared(url_to_response); @@ -102,8 +101,8 @@ TEST_F(EC2MetadataCredentialProviderTest, InjectedMetadataServer) { auto mock_transport = std::make_shared(url_to_response); - auto provider = - std::make_shared(custom_endpoint, mock_transport); + auto provider = std::make_shared( + custom_endpoint, mock_transport); TENSORSTORE_CHECK_OK_AND_ASSIGN(auto credentials, provider->GetCredentials()); ASSERT_EQ(provider->GetEndpoint(), custom_endpoint); ASSERT_EQ(credentials.access_key, access_key); @@ -113,7 +112,6 @@ TEST_F(EC2MetadataCredentialProviderTest, InjectedMetadataServer) { ASSERT_EQ(credentials.expires_at, expiry - absl::Seconds(60)); } - TEST_F(EC2MetadataCredentialProviderTest, NoIamRolesInSecurityCredentials) { auto url_to_response = absl::flat_hash_map{ {"POST http://169.254.169.254/latest/api/token", @@ -139,9 +137,8 @@ TEST_F(EC2MetadataCredentialProviderTest, UnsuccessfulJsonResponse) { {"POST http://169.254.169.254/latest/api/token", HttpResponse{200, absl::Cord{api_token}}}, {"GET http://169.254.169.254/latest/meta-data/iam/", - HttpResponse{200, - absl::Cord{"info"}, - {{"x-aws-ec2-metadata-token", api_token}}}}, + HttpResponse{ + 200, absl::Cord{"info"}, {{"x-aws-ec2-metadata-token", api_token}}}}, {"GET http://169.254.169.254/latest/meta-data/iam/security-credentials/", HttpResponse{200, absl::Cord{"mock-iam-role"}, diff --git a/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc b/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc index e805723dd..4423847ff 100644 --- a/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/environment_credential_provider.cc @@ -21,8 +21,8 @@ #include "absl/status/status.h" #include "absl/strings/str_cat.h" #include "absl/time/time.h" -#include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" #include "tensorstore/internal/env.h" +#include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" #include "tensorstore/util/result.h" using ::tensorstore::internal::GetEnv; @@ -43,7 +43,7 @@ static constexpr char kEnvAwsSessionToken[] = "AWS_SESSION_TOKEN"; Result EnvironmentCredentialProvider::GetCredentials() { auto access_key = GetEnv(kEnvAwsAccessKeyId); - if(!access_key) { + if (!access_key) { return absl::NotFoundError(absl::StrCat(kEnvAwsAccessKeyId, " not set")); } ABSL_LOG_FIRST_N(INFO, 1) diff --git a/tensorstore/kvstore/s3/credentials/environment_credential_provider.h b/tensorstore/kvstore/s3/credentials/environment_credential_provider.h index 6432cb850..bd5296866 100644 --- a/tensorstore/kvstore/s3/credentials/environment_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/environment_credential_provider.h @@ -16,7 +16,6 @@ #define TENSORSTORE_KVSTORE_S3_CREDENTIALS_ENVIRONMENT_CREDENTIAL_PROVIDER_H #include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" - #include "tensorstore/util/result.h" namespace tensorstore { diff --git a/tensorstore/kvstore/s3/credentials/file_credential_provider.cc b/tensorstore/kvstore/s3/credentials/file_credential_provider.cc index cc826be7c..3a8b11387 100644 --- a/tensorstore/kvstore/s3/credentials/file_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/file_credential_provider.cc @@ -24,9 +24,9 @@ #include "absl/strings/ascii.h" #include "absl/strings/str_cat.h" #include "absl/time/time.h" -#include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" #include "tensorstore/internal/env.h" #include "tensorstore/internal/path.h" +#include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" #include "tensorstore/util/result.h" using ::tensorstore::internal::GetEnv; @@ -54,9 +54,9 @@ static constexpr char kDefaultProfile[] = "default"; Result GetAwsCredentialsFileName() { auto credentials_file = GetEnv(kEnvAwsCredentialsFile); - if(!credentials_file) { + if (!credentials_file) { auto home_dir = GetEnv("HOME"); - if(!home_dir) { + if (!home_dir) { return absl::NotFoundError("Could not read $HOME"); } return JoinPath(*home_dir, kDefaultAwsCredentialsFilePath); @@ -68,11 +68,11 @@ Result GetAwsCredentialsFileName() { /// https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html#cli-configure-files-format Result FileCredentialProvider::GetCredentials() { - if(filename_.empty()) { + if (filename_.empty()) { TENSORSTORE_ASSIGN_OR_RETURN(filename_, GetAwsCredentialsFileName()); } - if(profile_.empty()) { + if (profile_.empty()) { profile_ = GetEnv(kEnvAwsProfile).value_or(kDefaultProfile); } diff --git a/tensorstore/kvstore/s3/credentials/file_credential_provider.h b/tensorstore/kvstore/s3/credentials/file_credential_provider.h index c18d81617..90140387f 100644 --- a/tensorstore/kvstore/s3/credentials/file_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/file_credential_provider.h @@ -42,17 +42,13 @@ class FileCredentialProvider : public AwsCredentialProvider { public: FileCredentialProvider(std::string_view filename, std::string_view profile) - : filename_(filename), profile_(profile) {} + : filename_(filename), profile_(profile) {} Result GetCredentials() override; - inline const std::string & GetFileName() const { - return filename_; - } + inline const std::string& GetFileName() const { return filename_; } - inline const std::string & GetProfile() const { - return profile_; - } + inline const std::string& GetProfile() const { return profile_; } }; } // namespace internal_kvstore_s3 diff --git a/tensorstore/kvstore/s3/credentials/file_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/file_credential_provider_test.cc index 073f9105a..99d0c282d 100644 --- a/tensorstore/kvstore/s3/credentials/file_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/file_credential_provider_test.cc @@ -121,7 +121,6 @@ TEST_F(FileCredentialProviderTest, ASSERT_FALSE(provider.GetCredentials().ok()); ASSERT_EQ(provider.GetFileName(), credentials_filename); ASSERT_EQ(provider.GetProfile(), "bob"); - } TEST_F(FileCredentialProviderTest, ProviderAwsCredentialsFromFileOverride) { diff --git a/tensorstore/kvstore/s3/credentials/test_utils.cc b/tensorstore/kvstore/s3/credentials/test_utils.cc index 93293a7e0..935c61dcc 100644 --- a/tensorstore/kvstore/s3/credentials/test_utils.cc +++ b/tensorstore/kvstore/s3/credentials/test_utils.cc @@ -53,11 +53,14 @@ DefaultEC2MetadataFlow(const std::string& endpoint, {absl::StrFormat("GET %s/latest/meta-data/iam/", endpoint), internal_http::HttpResponse{ 200, absl::Cord{"info"}, {{"x-aws-ec2-metadata-token", api_token}}}}, - {absl::StrFormat("GET %s/latest/meta-data/iam/security-credentials/", endpoint), + {absl::StrFormat("GET %s/latest/meta-data/iam/security-credentials/", + endpoint), internal_http::HttpResponse{200, absl::Cord{"mock-iam-role"}, {{"x-aws-ec2-metadata-token", api_token}}}}, - {absl::StrFormat("GET %s/latest/meta-data/iam/security-credentials/mock-iam-role", endpoint), + {absl::StrFormat( + "GET %s/latest/meta-data/iam/security-credentials/mock-iam-role", + endpoint), internal_http::HttpResponse{ 200, absl::Cord(absl::StrFormat( From 968d3bd57e5fe8606d3542d65c17369a088d232a Mon Sep 17 00:00:00 2001 From: Laramie Leavitt Date: Wed, 15 Nov 2023 02:27:23 -0800 Subject: [PATCH 56/63] Fix subprocess_test on windows+mingw: include PATH in the environment PiperOrigin-RevId: 582597962 Change-Id: I51d0277367494f530fc21bb6dcda3d9013033cdd --- tensorstore/internal/os/subprocess_test.cc | 12 +++++++----- tensorstore/internal/os/subprocess_win.cc | 10 ++++++++-- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/tensorstore/internal/os/subprocess_test.cc b/tensorstore/internal/os/subprocess_test.cc index ac8dcdae7..d148726aa 100644 --- a/tensorstore/internal/os/subprocess_test.cc +++ b/tensorstore/internal/os/subprocess_test.cc @@ -50,7 +50,6 @@ TEST(SubprocessTest, Join) { SubprocessOptions opts; opts.executable = *program_name; opts.args = {kSubprocessArg}; - auto child = SpawnSubprocess(opts); TENSORSTORE_ASSERT_OK(child.status()); @@ -101,8 +100,8 @@ TEST(SubprocessTest, Redirects) { SubprocessOptions opts; opts.executable = *program_name; opts.args = {kSubprocessArg}; - opts.env = absl::flat_hash_map( - {{"SUBPROCESS_TEST_ENV", "1"}}); + opts.env.emplace(::tensorstore::internal::GetEnvironmentMap()); + opts.env->insert_or_assign("SUBPROCESS_TEST_ENV", "1"); opts.stdout_action = SubprocessOptions::Redirect{out_file}; opts.stderr_action = SubprocessOptions::Redirect{out_file}; @@ -137,8 +136,11 @@ TEST(SubprocessTest, Env) { SubprocessOptions opts; opts.executable = *program_name; opts.args = {"--env=SUBPROCESS_TEST_ENV"}; - opts.env = absl::flat_hash_map( - {{"SUBPROCESS_TEST_ENV", "1"}}); + opts.env = absl::flat_hash_map({ +#ifdef _WIN32 + {"PATH", ::tensorstore::internal::GetEnv("PATH").value_or("")}, +#endif + {"SUBPROCESS_TEST_ENV", "1"}}); auto child = SpawnSubprocess(opts); ASSERT_TRUE(child.ok()); diff --git a/tensorstore/internal/os/subprocess_win.cc b/tensorstore/internal/os/subprocess_win.cc index 7dafa4924..476e1d785 100644 --- a/tensorstore/internal/os/subprocess_win.cc +++ b/tensorstore/internal/os/subprocess_win.cc @@ -266,8 +266,15 @@ Result SpawnSubprocess(const SubprocessOptions& options) { return absl::InvalidArgumentError("SpawnSubprocess: path too large."); } + // Build the environment block. std::wstring env; if (options.env.has_value()) { + if (!options.env->count("PATH")) { + // Windows uses the PATH when searching for DLLs: + // https://learn.microsoft.com/en-us/windows/win32/dlls/dynamic-link-library-search-order + ABSL_LOG(INFO) << "SpawnSubprocess: environment missing PATH; Some DLLs " + "may fail to load."; + } env = BuildEnvironmentBlock(*options.env); } @@ -289,8 +296,7 @@ Result SpawnSubprocess(const SubprocessOptions& options) { std::vector handles_to_close; std::vector handles_to_inherit; - auto status = - SetupHandles(options, ex, handles_to_close, handles_to_inherit); + auto status = SetupHandles(options, ex, handles_to_close, handles_to_inherit); std::unique_ptr ex_storage; From 6ed2030b4d0673cc4468a8d445e13accf494cfc7 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Wed, 15 Nov 2023 19:33:01 +0200 Subject: [PATCH 57/63] Test Anonymous Credentials and Environment Credentials --- .../default_credential_provider_test.cc | 84 +++++++++++++++++-- 1 file changed, 76 insertions(+), 8 deletions(-) diff --git a/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc index 43650a736..160f3bf7a 100644 --- a/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc @@ -17,21 +17,22 @@ #include #include +#include +#include #include #include "absl/container/flat_hash_map.h" #include "absl/strings/cord.h" #include "absl/time/time.h" +#include "tensorstore/internal/env.h" #include "tensorstore/internal/http/http_response.h" #include "tensorstore/internal/path.h" #include "tensorstore/internal/test_util.h" #include "tensorstore/kvstore/s3/credentials/test_utils.h" -#include "tensorstore/util/future.h" #include "tensorstore/util/status_testutil.h" namespace { -using ::tensorstore::Future; using ::tensorstore::internal::JoinPath; using ::tensorstore::internal_http::HttpResponse; using ::tensorstore::internal_kvstore_s3::DefaultAwsCredentialsProvider; @@ -39,6 +40,8 @@ using ::tensorstore::internal_kvstore_s3::DefaultEC2MetadataFlow; using ::tensorstore::internal_kvstore_s3::EC2MetadataMockTransport; using Options = ::tensorstore::internal_kvstore_s3::DefaultAwsCredentialsProvider::Options; +using ::tensorstore::internal::SetEnv; +using ::tensorstore::internal::UnsetEnv; static constexpr char endpoint[] = "http://endpoint"; @@ -59,9 +62,64 @@ class CredentialFileFactory } }; +class DefaultCredentialProviderTest : public ::testing::Test { + protected: + void SetUp() override { + UnsetEnv("AWS_ACCESS_KEY_ID"); + UnsetEnv("AWS_SECRET_KEY_ID"); + UnsetEnv("AWS_SESSION_TOKEN"); + } +}; + +TEST_F(DefaultCredentialProviderTest, AnonymousCredentials) { + /// Force failure on credential retrieval + auto url_to_response = absl::flat_hash_map{ + {"POST http://endpoint/latest/api/token", + HttpResponse{404, absl::Cord{""}}}, + }; + + auto mock_transport = std::make_shared(url_to_response); + auto provider = std::make_unique( + Options{{}, {}, {}, mock_transport}); + + TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, provider->GetCredentials()); + EXPECT_TRUE(credentials.IsAnonymous()); + EXPECT_EQ(credentials.expires_at, absl::InfiniteFuture()); + + // Idempotent + TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials2, provider->GetCredentials()); + EXPECT_TRUE(credentials2.IsAnonymous()); + EXPECT_EQ(credentials2.expires_at, absl::InfiniteFuture()); + +} + + +TEST_F(DefaultCredentialProviderTest, EnvironmentCredentialIdempotency) { + SetEnv("AWS_ACCESS_KEY_ID", "access"); + SetEnv("AWS_SECRET_ACCESS_KEY", "secret"); + SetEnv("AWS_SESSION_TOKEN", "token"); + + auto provider = std::make_unique(); + TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, + provider->GetCredentials()); + EXPECT_EQ(credentials.access_key, "access"); + EXPECT_EQ(credentials.secret_key, "secret"); + EXPECT_EQ(credentials.session_token, "token"); + EXPECT_EQ(credentials.expires_at, absl::InfiniteFuture()); + + // Expect idempotency as environment credentials never expire + TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials2, + provider->GetCredentials()); + EXPECT_EQ(credentials.access_key, credentials2.access_key); + EXPECT_EQ(credentials.secret_key, credentials2.secret_key); + EXPECT_EQ(credentials.session_token, credentials2.session_token); + EXPECT_EQ(credentials.expires_at, credentials2.expires_at); +} + + /// Test configuration of FileCredentialProvider from /// DefaultAwsCredentialsProvider::Options -TEST(DefaultCredentialProviderTest, ConfigureFileProviderFromOptions) { +TEST_F(DefaultCredentialProviderTest, ConfigureFileProviderFromOptions) { auto factory = CredentialFileFactory{}; auto credentials_file = factory.WriteCredentialsFile(); @@ -72,11 +130,21 @@ TEST(DefaultCredentialProviderTest, ConfigureFileProviderFromOptions) { EXPECT_EQ(credentials.access_key, "AKIAIOSFODNN6EXAMPLE"); EXPECT_EQ(credentials.secret_key, "wJalrXUtnFEMI/K7MDENG/bPxRfiCZEXAMPLEKEY"); EXPECT_EQ(credentials.session_token, "abcdef1234567890"); + EXPECT_EQ(credentials.expires_at, absl::InfiniteFuture()); + + // Expect idempotency as file credentials never expire + TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials2, + provider->GetCredentials()); + EXPECT_EQ(credentials.access_key, credentials2.access_key); + EXPECT_EQ(credentials.secret_key, credentials2.secret_key); + EXPECT_EQ(credentials.session_token, credentials2.session_token); + EXPECT_EQ(credentials.expires_at, credentials2.expires_at); + } /// Test configuration of EC2MetaDataProvider from /// DefaultAwsCredentialsProvider::Options -TEST(DefaultCredentialProviderTest, ConfigureEC2ProviderFromOptions) { +TEST_F(DefaultCredentialProviderTest, ConfigureEC2ProviderFromOptions) { auto now = absl::Now(); auto stuck_clock = [&]() -> absl::Time { return now; }; auto expiry = now + absl::Seconds(200); @@ -93,7 +161,7 @@ TEST(DefaultCredentialProviderTest, ConfigureEC2ProviderFromOptions) { EXPECT_EQ(credentials.access_key, "ASIA1234567890"); EXPECT_EQ(credentials.secret_key, "1234567890abcdef"); EXPECT_EQ(credentials.session_token, "token"); - ASSERT_EQ(credentials.expires_at, expiry - absl::Seconds(60)); + EXPECT_EQ(credentials.expires_at, expiry - absl::Seconds(60)); /// Force failure on credential retrieval url_to_response = absl::flat_hash_map{ @@ -106,7 +174,7 @@ TEST(DefaultCredentialProviderTest, ConfigureEC2ProviderFromOptions) { EXPECT_EQ(credentials.access_key, "ASIA1234567890"); EXPECT_EQ(credentials.secret_key, "1234567890abcdef"); EXPECT_EQ(credentials.session_token, "token"); - ASSERT_EQ(credentials.expires_at, expiry - absl::Seconds(60)); + EXPECT_EQ(credentials.expires_at, expiry - absl::Seconds(60)); // Force expiry and retrieve new credentials now += absl::Seconds(300); @@ -118,7 +186,7 @@ TEST(DefaultCredentialProviderTest, ConfigureEC2ProviderFromOptions) { EXPECT_EQ(credentials.access_key, "ASIA1234567890"); EXPECT_EQ(credentials.secret_key, "1234567890abcdef"); EXPECT_EQ(credentials.session_token, "TOKEN"); - ASSERT_EQ(credentials.expires_at, expiry - absl::Seconds(60)); + EXPECT_EQ(credentials.expires_at, expiry - absl::Seconds(60)); /// Force failure on credential retrieval url_to_response = absl::flat_hash_map{ @@ -131,7 +199,7 @@ TEST(DefaultCredentialProviderTest, ConfigureEC2ProviderFromOptions) { EXPECT_EQ(credentials.access_key, ""); EXPECT_EQ(credentials.secret_key, ""); EXPECT_EQ(credentials.session_token, ""); - ASSERT_EQ(credentials.expires_at, absl::InfiniteFuture()); + EXPECT_EQ(credentials.expires_at, absl::InfiniteFuture()); } } // namespace \ No newline at end of file From 840bc0de6495c6ce28a3e317cc54a912f76e60e2 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Wed, 15 Nov 2023 20:05:49 +0200 Subject: [PATCH 58/63] Credentials IWYU scan --- .../kvstore/s3/credentials/default_credential_provider.cc | 5 +++++ .../kvstore/s3/credentials/ec2_credential_provider.cc | 1 - .../s3/credentials/environment_credential_provider_test.cc | 7 ++----- .../kvstore/s3/credentials/file_credential_provider.h | 1 - tensorstore/kvstore/s3/credentials/test_utils.cc | 1 + 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/tensorstore/kvstore/s3/credentials/default_credential_provider.cc b/tensorstore/kvstore/s3/credentials/default_credential_provider.cc index eaa6aec85..a4fce3a86 100644 --- a/tensorstore/kvstore/s3/credentials/default_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/default_credential_provider.cc @@ -14,9 +14,14 @@ #include "tensorstore/kvstore/s3/credentials/default_credential_provider.h" +#include #include +#include +#include #include +#include +#include "absl/functional/function_ref.h" #include "absl/synchronization/mutex.h" #include "absl/time/time.h" #include "tensorstore/internal/http/http_transport.h" diff --git a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc index afa8187fb..0e5b43f91 100644 --- a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc +++ b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.cc @@ -35,7 +35,6 @@ #include "tensorstore/internal/json_binding/bindable.h" #include "tensorstore/internal/json_binding/json_binding.h" #include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" -#include "tensorstore/util/future.h" #include "tensorstore/util/result.h" #include "tensorstore/util/status.h" #include "tensorstore/util/str_cat.h" diff --git a/tensorstore/kvstore/s3/credentials/environment_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/environment_credential_provider_test.cc index 434bfbb88..f0120592e 100644 --- a/tensorstore/kvstore/s3/credentials/environment_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/environment_credential_provider_test.cc @@ -14,19 +14,16 @@ #include "tensorstore/kvstore/s3/credentials/environment_credential_provider.h" -#include - -#include #include +#include + #include "tensorstore/internal/env.h" -#include "tensorstore/internal/test_util.h" #include "tensorstore/util/result.h" #include "tensorstore/util/status_testutil.h" namespace { -using ::tensorstore::internal::GetEnv; using ::tensorstore::internal::SetEnv; using ::tensorstore::internal::UnsetEnv; using ::tensorstore::internal_kvstore_s3::EnvironmentCredentialProvider; diff --git a/tensorstore/kvstore/s3/credentials/file_credential_provider.h b/tensorstore/kvstore/s3/credentials/file_credential_provider.h index 90140387f..cc8108cb6 100644 --- a/tensorstore/kvstore/s3/credentials/file_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/file_credential_provider.h @@ -17,7 +17,6 @@ #include #include -#include #include "tensorstore/kvstore/s3/credentials/aws_credential_provider.h" #include "tensorstore/util/result.h" diff --git a/tensorstore/kvstore/s3/credentials/test_utils.cc b/tensorstore/kvstore/s3/credentials/test_utils.cc index 935c61dcc..6c25b4ed0 100644 --- a/tensorstore/kvstore/s3/credentials/test_utils.cc +++ b/tensorstore/kvstore/s3/credentials/test_utils.cc @@ -17,6 +17,7 @@ #include #include +#include "absl/container/flat_hash_map.h" #include "absl/log/absl_log.h" #include "absl/strings/cord.h" #include "absl/strings/str_format.h" From 3812b8b3a6da52810541cd4b1f9d563ff58f8d0c Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Wed, 15 Nov 2023 20:06:09 +0200 Subject: [PATCH 59/63] some autos --- .../s3/credentials/default_credential_provider_test.cc | 8 +------- .../s3/credentials/file_credential_provider_test.cc | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc index 160f3bf7a..7222f21ca 100644 --- a/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc @@ -72,12 +72,7 @@ class DefaultCredentialProviderTest : public ::testing::Test { }; TEST_F(DefaultCredentialProviderTest, AnonymousCredentials) { - /// Force failure on credential retrieval - auto url_to_response = absl::flat_hash_map{ - {"POST http://endpoint/latest/api/token", - HttpResponse{404, absl::Cord{""}}}, - }; - + auto url_to_response = absl::flat_hash_map(); auto mock_transport = std::make_shared(url_to_response); auto provider = std::make_unique( Options{{}, {}, {}, mock_transport}); @@ -90,7 +85,6 @@ TEST_F(DefaultCredentialProviderTest, AnonymousCredentials) { TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials2, provider->GetCredentials()); EXPECT_TRUE(credentials2.IsAnonymous()); EXPECT_EQ(credentials2.expires_at, absl::InfiniteFuture()); - } diff --git a/tensorstore/kvstore/s3/credentials/file_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/file_credential_provider_test.cc index 99d0c282d..ee09df942 100644 --- a/tensorstore/kvstore/s3/credentials/file_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/file_credential_provider_test.cc @@ -81,7 +81,7 @@ TEST_F(FileCredentialProviderTest, ProviderAwsCredentialsFromFileDefault) { TEST_F(FileCredentialProviderTest, ProviderAwsCredentialsFromFileProfileOverride) { TestData test_data; - std::string credentials_filename = test_data.WriteCredentialsFile(); + auto credentials_filename = test_data.WriteCredentialsFile(); SetEnv("AWS_SHARED_CREDENTIALS_FILE", credentials_filename.c_str()); auto provider = FileCredentialProvider("", "alice"); From 24271d73a38394a2c647e58c08ab9714d4b6bfc4 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Wed, 15 Nov 2023 20:45:54 +0200 Subject: [PATCH 60/63] Cut whitespace down --- tensorstore/kvstore/s3/credentials/file_credential_provider.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/tensorstore/kvstore/s3/credentials/file_credential_provider.h b/tensorstore/kvstore/s3/credentials/file_credential_provider.h index cc8108cb6..61dd4d7fc 100644 --- a/tensorstore/kvstore/s3/credentials/file_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/file_credential_provider.h @@ -44,9 +44,7 @@ class FileCredentialProvider : public AwsCredentialProvider { : filename_(filename), profile_(profile) {} Result GetCredentials() override; - inline const std::string& GetFileName() const { return filename_; } - inline const std::string& GetProfile() const { return profile_; } }; From 77bf8cff0fe6240d2ebea19df93581f4d4549916 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Wed, 15 Nov 2023 20:50:57 +0200 Subject: [PATCH 61/63] clang-format --- .../default_credential_provider_test.cc | 28 +++++++++---------- .../s3/credentials/ec2_credential_provider.h | 1 - .../environment_credential_provider_test.cc | 4 +-- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc index 7222f21ca..fc2edfbad 100644 --- a/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/default_credential_provider_test.cc @@ -34,14 +34,14 @@ namespace { using ::tensorstore::internal::JoinPath; +using ::tensorstore::internal::SetEnv; +using ::tensorstore::internal::UnsetEnv; using ::tensorstore::internal_http::HttpResponse; using ::tensorstore::internal_kvstore_s3::DefaultAwsCredentialsProvider; using ::tensorstore::internal_kvstore_s3::DefaultEC2MetadataFlow; using ::tensorstore::internal_kvstore_s3::EC2MetadataMockTransport; using Options = ::tensorstore::internal_kvstore_s3::DefaultAwsCredentialsProvider::Options; -using ::tensorstore::internal::SetEnv; -using ::tensorstore::internal::UnsetEnv; static constexpr char endpoint[] = "http://endpoint"; @@ -63,31 +63,33 @@ class CredentialFileFactory }; class DefaultCredentialProviderTest : public ::testing::Test { - protected: - void SetUp() override { - UnsetEnv("AWS_ACCESS_KEY_ID"); - UnsetEnv("AWS_SECRET_KEY_ID"); - UnsetEnv("AWS_SESSION_TOKEN"); - } + protected: + void SetUp() override { + UnsetEnv("AWS_ACCESS_KEY_ID"); + UnsetEnv("AWS_SECRET_KEY_ID"); + UnsetEnv("AWS_SESSION_TOKEN"); + } }; TEST_F(DefaultCredentialProviderTest, AnonymousCredentials) { auto url_to_response = absl::flat_hash_map(); - auto mock_transport = std::make_shared(url_to_response); + auto mock_transport = + std::make_shared(url_to_response); auto provider = std::make_unique( Options{{}, {}, {}, mock_transport}); - TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, provider->GetCredentials()); + TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials, + provider->GetCredentials()); EXPECT_TRUE(credentials.IsAnonymous()); EXPECT_EQ(credentials.expires_at, absl::InfiniteFuture()); // Idempotent - TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials2, provider->GetCredentials()); + TENSORSTORE_ASSERT_OK_AND_ASSIGN(auto credentials2, + provider->GetCredentials()); EXPECT_TRUE(credentials2.IsAnonymous()); EXPECT_EQ(credentials2.expires_at, absl::InfiniteFuture()); } - TEST_F(DefaultCredentialProviderTest, EnvironmentCredentialIdempotency) { SetEnv("AWS_ACCESS_KEY_ID", "access"); SetEnv("AWS_SECRET_ACCESS_KEY", "secret"); @@ -110,7 +112,6 @@ TEST_F(DefaultCredentialProviderTest, EnvironmentCredentialIdempotency) { EXPECT_EQ(credentials.expires_at, credentials2.expires_at); } - /// Test configuration of FileCredentialProvider from /// DefaultAwsCredentialsProvider::Options TEST_F(DefaultCredentialProviderTest, ConfigureFileProviderFromOptions) { @@ -133,7 +134,6 @@ TEST_F(DefaultCredentialProviderTest, ConfigureFileProviderFromOptions) { EXPECT_EQ(credentials.secret_key, credentials2.secret_key); EXPECT_EQ(credentials.session_token, credentials2.session_token); EXPECT_EQ(credentials.expires_at, credentials2.expires_at); - } /// Test configuration of EC2MetaDataProvider from diff --git a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h index 138ca5c06..0ff213d51 100644 --- a/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h +++ b/tensorstore/kvstore/s3/credentials/ec2_credential_provider.h @@ -36,7 +36,6 @@ class EC2MetadataCredentialProvider : public AwsCredentialProvider { : endpoint_(endpoint), transport_(std::move(transport)) {} Result GetCredentials() override; - inline const std::string& GetEndpoint() const { return endpoint_; } private: diff --git a/tensorstore/kvstore/s3/credentials/environment_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/environment_credential_provider_test.cc index f0120592e..3d273e950 100644 --- a/tensorstore/kvstore/s3/credentials/environment_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/environment_credential_provider_test.cc @@ -14,10 +14,10 @@ #include "tensorstore/kvstore/s3/credentials/environment_credential_provider.h" -#include - #include +#include + #include "tensorstore/internal/env.h" #include "tensorstore/util/result.h" #include "tensorstore/util/status_testutil.h" From 104015324961fc632fd8df9cc7e28bc790823537 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Wed, 15 Nov 2023 20:58:46 +0200 Subject: [PATCH 62/63] Remove unnecessary includes in environment_credential_provider_test.cc --- .../s3/credentials/environment_credential_provider_test.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/tensorstore/kvstore/s3/credentials/environment_credential_provider_test.cc b/tensorstore/kvstore/s3/credentials/environment_credential_provider_test.cc index 3d273e950..0bf3db64f 100644 --- a/tensorstore/kvstore/s3/credentials/environment_credential_provider_test.cc +++ b/tensorstore/kvstore/s3/credentials/environment_credential_provider_test.cc @@ -16,10 +16,7 @@ #include -#include - #include "tensorstore/internal/env.h" -#include "tensorstore/util/result.h" #include "tensorstore/util/status_testutil.h" namespace { From 4295487ae0bceff91da2b3c9a03c8111248747b8 Mon Sep 17 00:00:00 2001 From: Simon Perkins Date: Wed, 15 Nov 2023 21:18:08 +0200 Subject: [PATCH 63/63] Fix schema.yml indent --- tensorstore/kvstore/s3/schema.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorstore/kvstore/s3/schema.yml b/tensorstore/kvstore/s3/schema.yml index bd6efe991..aaa02ab50 100644 --- a/tensorstore/kvstore/s3/schema.yml +++ b/tensorstore/kvstore/s3/schema.yml @@ -123,9 +123,9 @@ definitions: Overrides the :envvar:`AWS_SHARED_CREDENTIALS_FILE` environment variable. metadata_endpoint: type: string - description: |- - The endpoint of the metadata server. - Overrides the :envvar:`AWS_EC2_METADATA_SERVICE_ENDPOINT` environment variable. + description: |- + The endpoint of the metadata server. + Overrides the :envvar:`AWS_EC2_METADATA_SERVICE_ENDPOINT` environment variable. experimental_s3_rate_limiter: $id: Context.experimental_s3_rate_limiter description: |-