Improvements (#15)

* Scope change for dependencies in haystack-blobs * Added STS Assume role for S3Dispatcher * Version bump to 1.0.3-SNAPSHOT * Scope change for grpc dependencies * Version bump to 1.0.3-SNAPSHOT(all modules)
ExpediaDotCom · Aug 17, 2019 · e38ac99 · e38ac99
1 parent f7d8b33
commit e38ac99
Show file tree

Hide file tree

Showing 13 changed files with 87 additions and 37 deletions.
diff --git a/core/pom.xml b/core/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <artifactId>blobs</artifactId>
         <groupId>com.expedia.www</groupId>
-        <version>1.0.0-SNAPSHOT</version>
+        <version>1.0.3-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>

diff --git a/haystack-blobs/blobs-agent-client/pom.xml b/haystack-blobs/blobs-agent-client/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>haystack-blobs</artifactId>
         <groupId>com.expedia.www</groupId>
-        <version>1.0.0-SNAPSHOT</version>
+        <version>1.0.3-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>blobs-agent-client</artifactId>
@@ -37,25 +37,19 @@
             <artifactId>protobuf-java</artifactId>
             <version>${protobuf.version}</version>
         </dependency>
+
+        <!-- grpc -->
         <dependency>
             <groupId>io.grpc</groupId>
-            <artifactId>grpc-core</artifactId>
-            <version>${grpc.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>io.grpc</groupId>
-            <artifactId>grpc-protobuf</artifactId>
-            <version>${grpc.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>io.grpc</groupId>
-            <artifactId>grpc-stub</artifactId>
+            <artifactId>grpc-all</artifactId>
             <version>${grpc.version}</version>
+            <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>io.grpc</groupId>
             <artifactId>grpc-netty-shaded</artifactId>
             <version>${grpc.version}</version>
+            <scope>provided</scope>
         </dependency>
 
         <!-- test -->
@@ -99,7 +93,7 @@
         <dependency>
             <groupId>com.expedia.www</groupId>
             <artifactId>blobs-grpc-models</artifactId>
-            <version>1.0.0-SNAPSHOT</version>
+            <version>${parent.version}</version>
             <scope>compile</scope>
         </dependency>
     </dependencies>

diff --git a/haystack-blobs/blobs-agent-dispatchers/pom.xml b/haystack-blobs/blobs-agent-dispatchers/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>haystack-blobs</artifactId>
         <groupId>com.expedia.www</groupId>
-        <version>1.0.0-SNAPSHOT</version>
+        <version>1.0.3-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>blobs-agent-dispatchers</artifactId>

diff --git a/...tchers/src/main/java/com/expedia/www/haystack/agent/blobs/dispatcher/s3/S3Dispatcher.java b/...tchers/src/main/java/com/expedia/www/haystack/agent/blobs/dispatcher/s3/S3Dispatcher.java
@@ -21,6 +21,8 @@
 import com.amazonaws.auth.AWSStaticCredentialsProvider;
 import com.amazonaws.auth.BasicAWSCredentials;
 import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
+import com.amazonaws.auth.profile.internal.securitytoken.RoleInfo;
+import com.amazonaws.auth.profile.internal.securitytoken.STSProfileCredentialsServiceProvider;
 import com.amazonaws.client.builder.AwsClientBuilder;
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3ClientBuilder;
@@ -73,6 +75,8 @@ public class S3Dispatcher implements BlobDispatcher, AutoCloseable {
     private final static String AWS_SERVICE_ENDPOINT = "service.endpoint";
     private final static String AWS_PATH_STYLE_ACCESS_ENABLED = "path.style.access.enabled";
     private final static String AWS_DISABLE_CHUNKED_ENCODING = "disable.chunked.encoding";
+    private final static String AWS_USE_STS_ARN = "use.sts.arn";
+    private final static String AWS_STS_ARN_ROLE = "sts.arn.role";
 
     private final static String SHOULD_WAIT_FOR_UPLOAD = "should.wait.for.upload";
 
@@ -271,7 +275,15 @@ private static AmazonS3 getS3Client(Config config, ClientConfiguration clientCon
 
     @VisibleForTesting
     static AWSCredentialsProvider buildCredentialProvider(final Config config) {
-        if (config.hasPath(AWS_ACCESS_KEY) && config.hasPath(AWS_SECRET_KEY)) {
+        if (config.hasPath(AWS_USE_STS_ARN) && config.getBoolean(AWS_USE_STS_ARN)) {
+            LOGGER.info("using STS profile credential service provider");
+            Validate.isTrue(config.hasPath(AWS_STS_ARN_ROLE), "AWS STS Assume-Role should be present when enabled");
+
+            return new STSProfileCredentialsServiceProvider(
+                    new RoleInfo().withRoleArn(config.getString(AWS_STS_ARN_ROLE))
+                            .withRoleSessionName("haystack-monitoring-blobs-agent"));
+
+        } else if (config.hasPath(AWS_ACCESS_KEY) && config.hasPath(AWS_SECRET_KEY)) {
             LOGGER.info("using static aws credential provider with access and secret key for s3 dispatcher");
             return new AWSStaticCredentialsProvider(
                     new BasicAWSCredentials(config.getString(AWS_ACCESS_KEY), config.getString(AWS_SECRET_KEY)));

diff --git a/.../src/test/scala/com/expedia/www/haystack/agent/blobs/dispatcher/s3/S3DispatcherSpec.scala b/.../src/test/scala/com/expedia/www/haystack/agent/blobs/dispatcher/s3/S3DispatcherSpec.scala
@@ -3,6 +3,7 @@ package com.expedia.www.haystack.agent.blobs.dispatcher.s3
 import java.io.{ByteArrayInputStream, InputStream}
 import java.util.Optional
 
+import com.amazonaws.auth.profile.internal.securitytoken.STSProfileCredentialsServiceProvider
 import com.amazonaws.auth.{AWSStaticCredentialsProvider, DefaultAWSCredentialsProviderChain}
 import com.amazonaws.services.s3.AmazonS3Client
 import com.amazonaws.services.s3.model.{ObjectMetadata, PutObjectRequest, S3Object, S3ObjectInputStream}
@@ -259,6 +260,42 @@ class S3DispatcherSpec extends FunSpec with GivenWhenThen with BeforeAndAfter wi
       caught.getMessage should include("RateLimit is hit with outstanding(pending) requests=0")
     }
 
+    it("should throw error while building the credential provider using unavailable STS assume-role") {
+      When("given the complete configuration")
+      val config = ConfigFactory.parseString(
+        """
+          |bucket.name = "haystack"
+          |max.outstanding.requests = 50
+          |should.wait.for.upload = false
+          |use.sts.arn = true
+        """.stripMargin)
+
+      And("credential provider is build")
+      val caught = intercept[Exception]{
+        S3Dispatcher.buildCredentialProvider(config)
+      }
+      caught should not be null
+      caught.getMessage should include("AWS STS Assume-Role should be present when enabled")
+    }
+
+    it("should build the credential provider using STS assume-role") {
+      When("given the complete configuration")
+      val config = ConfigFactory.parseString(
+        """
+          |bucket.name = "haystack"
+          |max.outstanding.requests = 50
+          |should.wait.for.upload = false
+          |use.sts.arn = true
+          |sts.arn.role = "role/tempArnRole"
+        """.stripMargin)
+
+      And("credential provider is build")
+      val provider = S3Dispatcher.buildCredentialProvider(config)
+
+      Then("it should be the instance of STSProfileCredentialsServiceProvider")
+      provider.isInstanceOf[STSProfileCredentialsServiceProvider] shouldBe true
+    }
+
     it("should build the credential provider using access and secret key") {
       When("given the complete configuration")
       val config = ConfigFactory.parseString(

diff --git a/haystack-blobs/blobs-agent-server/pom.xml b/haystack-blobs/blobs-agent-server/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>haystack-blobs</artifactId>
         <groupId>com.expedia.www</groupId>
-        <version>1.0.0-SNAPSHOT</version>
+        <version>1.0.3-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>blobs-agent-server</artifactId>
@@ -41,6 +41,20 @@
             <artifactId>FastInfoset</artifactId>
         </dependency>
 
+        <!--grpc-->
+        <dependency>
+            <groupId>io.grpc</groupId>
+            <artifactId>grpc-services</artifactId>
+            <version>${grpc.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>io.grpc</groupId>
+            <artifactId>grpc-all</artifactId>
+            <version>${grpc.version}</version>
+            <scope>provided</scope>
+        </dependency>
+
         <!--Test-->
         <dependency>
             <groupId>org.scala-lang</groupId>

diff --git a/haystack-blobs/blobs-grpc-models/pom.xml b/haystack-blobs/blobs-grpc-models/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>haystack-blobs</artifactId>
         <groupId>com.expedia.www</groupId>
-        <version>1.0.0-SNAPSHOT</version>
+        <version>1.0.3-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>blobs-grpc-models</artifactId>
@@ -19,6 +19,12 @@
             <version>3.3.1</version>
             <scope>provided</scope>
         </dependency>
+        <dependency>
+            <groupId>io.grpc</groupId>
+            <artifactId>grpc-all</artifactId>
+            <version>${grpc.version}</version>
+            <scope>compile</scope>
+        </dependency>
     </dependencies>
 
     <build>

diff --git a/haystack-blobs/pom.xml b/haystack-blobs/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>blobs</artifactId>
         <groupId>com.expedia.www</groupId>
-        <version>1.0.0-SNAPSHOT</version>
+        <version>1.0.3-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
@@ -38,19 +38,6 @@
             <artifactId>commons-lang3</artifactId>
         </dependency>
 
-        <!-- grpc -->
-        <dependency>
-            <groupId>io.grpc</groupId>
-            <artifactId>grpc-all</artifactId>
-            <version>${grpc.version}</version>
-        </dependency>
-
-        <dependency>
-            <groupId>io.grpc</groupId>
-            <artifactId>grpc-services</artifactId>
-            <version>${grpc.version}</version>
-        </dependency>
-
         <dependency>
             <groupId>org.easymock</groupId>
             <artifactId>easymock</artifactId>

diff --git a/haystack-blobs/span-blob-context/pom.xml b/haystack-blobs/span-blob-context/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>haystack-blobs</artifactId>
         <groupId>com.expedia.www</groupId>
-        <version>1.0.0-SNAPSHOT</version>
+        <version>1.0.3-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>span-blob-context</artifactId>

diff --git a/pom.xml b/pom.xml
@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.expedia.www</groupId>
     <artifactId>blobs</artifactId>
-    <version>1.0.0-SNAPSHOT</version>
+    <version>1.0.3-SNAPSHOT</version>
     <packaging>pom</packaging>
 
     <modules>

diff --git a/stores/file-store/pom.xml b/stores/file-store/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>blob-stores</artifactId>
         <groupId>com.expedia.www</groupId>
-        <version>1.0.0-SNAPSHOT</version>
+        <version>1.0.3-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>

diff --git a/stores/pom.xml b/stores/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>blobs</artifactId>
         <groupId>com.expedia.www</groupId>
-        <version>1.0.0-SNAPSHOT</version>
+        <version>1.0.3-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>blob-stores</artifactId>

diff --git a/stores/s3-store/pom.xml b/stores/s3-store/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <artifactId>blob-stores</artifactId>
         <groupId>com.expedia.www</groupId>
-        <version>1.0.0-SNAPSHOT</version>
+        <version>1.0.3-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>