diff --git a/core/pom.xml b/core/pom.xml index e5d10ea..9021056 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -4,7 +4,7 @@ blobs com.expedia.www - 1.0.0-SNAPSHOT + 1.0.3-SNAPSHOT 4.0.0 diff --git a/haystack-blobs/blobs-agent-client/pom.xml b/haystack-blobs/blobs-agent-client/pom.xml index f984c0a..b97b664 100644 --- a/haystack-blobs/blobs-agent-client/pom.xml +++ b/haystack-blobs/blobs-agent-client/pom.xml @@ -5,7 +5,7 @@ haystack-blobs com.expedia.www - 1.0.0-SNAPSHOT + 1.0.3-SNAPSHOT 4.0.0 blobs-agent-client @@ -37,25 +37,19 @@ protobuf-java ${protobuf.version} + + io.grpc - grpc-core - ${grpc.version} - - - io.grpc - grpc-protobuf - ${grpc.version} - - - io.grpc - grpc-stub + grpc-all ${grpc.version} + provided io.grpc grpc-netty-shaded ${grpc.version} + provided @@ -99,7 +93,7 @@ com.expedia.www blobs-grpc-models - 1.0.0-SNAPSHOT + ${parent.version} compile diff --git a/haystack-blobs/blobs-agent-dispatchers/pom.xml b/haystack-blobs/blobs-agent-dispatchers/pom.xml index c8bc89f..cd60aac 100644 --- a/haystack-blobs/blobs-agent-dispatchers/pom.xml +++ b/haystack-blobs/blobs-agent-dispatchers/pom.xml @@ -5,7 +5,7 @@ haystack-blobs com.expedia.www - 1.0.0-SNAPSHOT + 1.0.3-SNAPSHOT 4.0.0 blobs-agent-dispatchers diff --git a/haystack-blobs/blobs-agent-dispatchers/src/main/java/com/expedia/www/haystack/agent/blobs/dispatcher/s3/S3Dispatcher.java b/haystack-blobs/blobs-agent-dispatchers/src/main/java/com/expedia/www/haystack/agent/blobs/dispatcher/s3/S3Dispatcher.java index ee39438..4fab679 100644 --- a/haystack-blobs/blobs-agent-dispatchers/src/main/java/com/expedia/www/haystack/agent/blobs/dispatcher/s3/S3Dispatcher.java +++ b/haystack-blobs/blobs-agent-dispatchers/src/main/java/com/expedia/www/haystack/agent/blobs/dispatcher/s3/S3Dispatcher.java @@ -21,6 +21,8 @@ import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; +import com.amazonaws.auth.profile.internal.securitytoken.RoleInfo; +import com.amazonaws.auth.profile.internal.securitytoken.STSProfileCredentialsServiceProvider; import com.amazonaws.client.builder.AwsClientBuilder; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; @@ -73,6 +75,8 @@ public class S3Dispatcher implements BlobDispatcher, AutoCloseable { private final static String AWS_SERVICE_ENDPOINT = "service.endpoint"; private final static String AWS_PATH_STYLE_ACCESS_ENABLED = "path.style.access.enabled"; private final static String AWS_DISABLE_CHUNKED_ENCODING = "disable.chunked.encoding"; + private final static String AWS_USE_STS_ARN = "use.sts.arn"; + private final static String AWS_STS_ARN_ROLE = "sts.arn.role"; private final static String SHOULD_WAIT_FOR_UPLOAD = "should.wait.for.upload"; @@ -271,7 +275,15 @@ private static AmazonS3 getS3Client(Config config, ClientConfiguration clientCon @VisibleForTesting static AWSCredentialsProvider buildCredentialProvider(final Config config) { - if (config.hasPath(AWS_ACCESS_KEY) && config.hasPath(AWS_SECRET_KEY)) { + if (config.hasPath(AWS_USE_STS_ARN) && config.getBoolean(AWS_USE_STS_ARN)) { + LOGGER.info("using STS profile credential service provider"); + Validate.isTrue(config.hasPath(AWS_STS_ARN_ROLE), "AWS STS Assume-Role should be present when enabled"); + + return new STSProfileCredentialsServiceProvider( + new RoleInfo().withRoleArn(config.getString(AWS_STS_ARN_ROLE)) + .withRoleSessionName("haystack-monitoring-blobs-agent")); + + } else if (config.hasPath(AWS_ACCESS_KEY) && config.hasPath(AWS_SECRET_KEY)) { LOGGER.info("using static aws credential provider with access and secret key for s3 dispatcher"); return new AWSStaticCredentialsProvider( new BasicAWSCredentials(config.getString(AWS_ACCESS_KEY), config.getString(AWS_SECRET_KEY))); diff --git a/haystack-blobs/blobs-agent-dispatchers/src/test/scala/com/expedia/www/haystack/agent/blobs/dispatcher/s3/S3DispatcherSpec.scala b/haystack-blobs/blobs-agent-dispatchers/src/test/scala/com/expedia/www/haystack/agent/blobs/dispatcher/s3/S3DispatcherSpec.scala index cfca16c..32a87f7 100644 --- a/haystack-blobs/blobs-agent-dispatchers/src/test/scala/com/expedia/www/haystack/agent/blobs/dispatcher/s3/S3DispatcherSpec.scala +++ b/haystack-blobs/blobs-agent-dispatchers/src/test/scala/com/expedia/www/haystack/agent/blobs/dispatcher/s3/S3DispatcherSpec.scala @@ -3,6 +3,7 @@ package com.expedia.www.haystack.agent.blobs.dispatcher.s3 import java.io.{ByteArrayInputStream, InputStream} import java.util.Optional +import com.amazonaws.auth.profile.internal.securitytoken.STSProfileCredentialsServiceProvider import com.amazonaws.auth.{AWSStaticCredentialsProvider, DefaultAWSCredentialsProviderChain} import com.amazonaws.services.s3.AmazonS3Client import com.amazonaws.services.s3.model.{ObjectMetadata, PutObjectRequest, S3Object, S3ObjectInputStream} @@ -259,6 +260,42 @@ class S3DispatcherSpec extends FunSpec with GivenWhenThen with BeforeAndAfter wi caught.getMessage should include("RateLimit is hit with outstanding(pending) requests=0") } + it("should throw error while building the credential provider using unavailable STS assume-role") { + When("given the complete configuration") + val config = ConfigFactory.parseString( + """ + |bucket.name = "haystack" + |max.outstanding.requests = 50 + |should.wait.for.upload = false + |use.sts.arn = true + """.stripMargin) + + And("credential provider is build") + val caught = intercept[Exception]{ + S3Dispatcher.buildCredentialProvider(config) + } + caught should not be null + caught.getMessage should include("AWS STS Assume-Role should be present when enabled") + } + + it("should build the credential provider using STS assume-role") { + When("given the complete configuration") + val config = ConfigFactory.parseString( + """ + |bucket.name = "haystack" + |max.outstanding.requests = 50 + |should.wait.for.upload = false + |use.sts.arn = true + |sts.arn.role = "role/tempArnRole" + """.stripMargin) + + And("credential provider is build") + val provider = S3Dispatcher.buildCredentialProvider(config) + + Then("it should be the instance of STSProfileCredentialsServiceProvider") + provider.isInstanceOf[STSProfileCredentialsServiceProvider] shouldBe true + } + it("should build the credential provider using access and secret key") { When("given the complete configuration") val config = ConfigFactory.parseString( diff --git a/haystack-blobs/blobs-agent-server/pom.xml b/haystack-blobs/blobs-agent-server/pom.xml index 075567e..8ba7795 100644 --- a/haystack-blobs/blobs-agent-server/pom.xml +++ b/haystack-blobs/blobs-agent-server/pom.xml @@ -5,7 +5,7 @@ haystack-blobs com.expedia.www - 1.0.0-SNAPSHOT + 1.0.3-SNAPSHOT 4.0.0 blobs-agent-server @@ -41,6 +41,20 @@ FastInfoset + + + io.grpc + grpc-services + ${grpc.version} + provided + + + io.grpc + grpc-all + ${grpc.version} + provided + + org.scala-lang diff --git a/haystack-blobs/blobs-grpc-models/pom.xml b/haystack-blobs/blobs-grpc-models/pom.xml index 8cb35da..24fa46c 100644 --- a/haystack-blobs/blobs-grpc-models/pom.xml +++ b/haystack-blobs/blobs-grpc-models/pom.xml @@ -5,7 +5,7 @@ haystack-blobs com.expedia.www - 1.0.0-SNAPSHOT + 1.0.3-SNAPSHOT 4.0.0 blobs-grpc-models @@ -19,6 +19,12 @@ 3.3.1 provided + + io.grpc + grpc-all + ${grpc.version} + compile + diff --git a/haystack-blobs/pom.xml b/haystack-blobs/pom.xml index caa90d9..f96bb79 100644 --- a/haystack-blobs/pom.xml +++ b/haystack-blobs/pom.xml @@ -5,7 +5,7 @@ blobs com.expedia.www - 1.0.0-SNAPSHOT + 1.0.3-SNAPSHOT 4.0.0 @@ -38,19 +38,6 @@ commons-lang3 - - - io.grpc - grpc-all - ${grpc.version} - - - - io.grpc - grpc-services - ${grpc.version} - - org.easymock easymock diff --git a/haystack-blobs/span-blob-context/pom.xml b/haystack-blobs/span-blob-context/pom.xml index 60f6fd7..9bd89a9 100644 --- a/haystack-blobs/span-blob-context/pom.xml +++ b/haystack-blobs/span-blob-context/pom.xml @@ -5,7 +5,7 @@ haystack-blobs com.expedia.www - 1.0.0-SNAPSHOT + 1.0.3-SNAPSHOT 4.0.0 span-blob-context diff --git a/pom.xml b/pom.xml index 9be738e..017cd88 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ 4.0.0 com.expedia.www blobs - 1.0.0-SNAPSHOT + 1.0.3-SNAPSHOT pom diff --git a/stores/file-store/pom.xml b/stores/file-store/pom.xml index da4ff08..7b13637 100644 --- a/stores/file-store/pom.xml +++ b/stores/file-store/pom.xml @@ -5,7 +5,7 @@ blob-stores com.expedia.www - 1.0.0-SNAPSHOT + 1.0.3-SNAPSHOT 4.0.0 diff --git a/stores/pom.xml b/stores/pom.xml index 22db196..38c9f67 100644 --- a/stores/pom.xml +++ b/stores/pom.xml @@ -5,7 +5,7 @@ blobs com.expedia.www - 1.0.0-SNAPSHOT + 1.0.3-SNAPSHOT 4.0.0 blob-stores diff --git a/stores/s3-store/pom.xml b/stores/s3-store/pom.xml index d545385..76c2c66 100644 --- a/stores/s3-store/pom.xml +++ b/stores/s3-store/pom.xml @@ -4,7 +4,7 @@ blob-stores com.expedia.www - 1.0.0-SNAPSHOT + 1.0.3-SNAPSHOT 4.0.0