diff --git a/athena-aws-cmdb/pom.xml b/athena-aws-cmdb/pom.xml index 30b2a094ef..90c1ad7b59 100644 --- a/athena-aws-cmdb/pom.xml +++ b/athena-aws-cmdb/pom.xml @@ -3,7 +3,7 @@ aws-athena-query-federation com.amazonaws - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 4.0.0 athena-aws-cmdb @@ -12,7 +12,7 @@ com.amazonaws aws-athena-federation-sdk - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 withdep diff --git a/athena-cloudera-hive/pom.xml b/athena-cloudera-hive/pom.xml index 6c410eb9c7..7ab0f7525e 100644 --- a/athena-cloudera-hive/pom.xml +++ b/athena-cloudera-hive/pom.xml @@ -3,7 +3,7 @@ aws-athena-query-federation com.amazonaws - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 4.0.0 athena-cloudera-hive @@ -15,7 +15,7 @@ com.amazonaws athena-federation-integ-test - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 test diff --git a/athena-cloudera-impala/pom.xml b/athena-cloudera-impala/pom.xml index 727b895327..e25d94d2ba 100644 --- a/athena-cloudera-impala/pom.xml +++ b/athena-cloudera-impala/pom.xml @@ -3,7 +3,7 @@ aws-athena-query-federation com.amazonaws - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 4.0.0 athena-cloudera-impala @@ -15,7 +15,7 @@ com.amazonaws athena-federation-integ-test - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 test @@ -35,6 +35,7 @@ test-jar test + org.mockito mockito-core diff --git a/athena-cloudwatch-metrics/pom.xml b/athena-cloudwatch-metrics/pom.xml index f191d9670d..6c8bff216e 100644 --- a/athena-cloudwatch-metrics/pom.xml +++ b/athena-cloudwatch-metrics/pom.xml @@ -3,7 +3,7 @@ aws-athena-query-federation com.amazonaws - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 4.0.0 athena-cloudwatch-metrics @@ -12,7 +12,7 @@ com.amazonaws aws-athena-federation-sdk - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 withdep diff --git a/athena-cloudwatch/pom.xml b/athena-cloudwatch/pom.xml index 7256280bff..79757d6167 100644 --- a/athena-cloudwatch/pom.xml +++ b/athena-cloudwatch/pom.xml @@ -3,7 +3,7 @@ aws-athena-query-federation com.amazonaws - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 4.0.0 athena-cloudwatch @@ -12,13 +12,13 @@ com.amazonaws aws-athena-federation-sdk - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 withdep com.amazonaws athena-federation-integ-test - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 test diff --git a/athena-datalakegen2/pom.xml b/athena-datalakegen2/pom.xml index e11f22f93d..c72c6c4813 100644 --- a/athena-datalakegen2/pom.xml +++ b/athena-datalakegen2/pom.xml @@ -3,7 +3,7 @@ aws-athena-query-federation com.amazonaws - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 4.0.0 athena-datalakegen2 @@ -12,7 +12,7 @@ com.amazonaws athena-federation-integ-test - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 test diff --git a/athena-db2-as400/pom.xml b/athena-db2-as400/pom.xml index 76ccaeeb67..ab204d5b37 100644 --- a/athena-db2-as400/pom.xml +++ b/athena-db2-as400/pom.xml @@ -3,7 +3,7 @@ aws-athena-query-federation com.amazonaws - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 4.0.0 athena-db2-as400 @@ -12,7 +12,7 @@ com.amazonaws athena-federation-integ-test - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 test diff --git a/athena-db2/pom.xml b/athena-db2/pom.xml index ced3939ac2..3859aae0ed 100644 --- a/athena-db2/pom.xml +++ b/athena-db2/pom.xml @@ -3,7 +3,7 @@ aws-athena-query-federation com.amazonaws - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 4.0.0 athena-db2 @@ -12,7 +12,7 @@ com.amazonaws athena-federation-integ-test - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 test @@ -59,6 +59,7 @@ ${mockito.version} test + diff --git a/athena-docdb/pom.xml b/athena-docdb/pom.xml index a59bef0626..5dd645c20a 100644 --- a/athena-docdb/pom.xml +++ b/athena-docdb/pom.xml @@ -3,7 +3,7 @@ aws-athena-query-federation com.amazonaws - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 4.0.0 athena-docdb @@ -12,7 +12,7 @@ com.amazonaws aws-athena-federation-sdk - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 withdep @@ -25,7 +25,7 @@ com.amazonaws athena-federation-integ-test - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 test diff --git a/athena-dynamodb/pom.xml b/athena-dynamodb/pom.xml index 139f23cc4d..fd44faefe9 100644 --- a/athena-dynamodb/pom.xml +++ b/athena-dynamodb/pom.xml @@ -3,7 +3,7 @@ aws-athena-query-federation com.amazonaws - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 4.0.0 athena-dynamodb @@ -12,13 +12,13 @@ com.amazonaws aws-athena-federation-sdk - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 withdep com.amazonaws athena-federation-integ-test - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 test @@ -89,7 +89,7 @@ com.amazonaws aws-athena-federation-sdk - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 test-jar test diff --git a/athena-elasticsearch/pom.xml b/athena-elasticsearch/pom.xml index abccf91365..fdd3cb3222 100644 --- a/athena-elasticsearch/pom.xml +++ b/athena-elasticsearch/pom.xml @@ -3,7 +3,7 @@ aws-athena-query-federation com.amazonaws - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 4.0.0 athena-elasticsearch @@ -12,7 +12,7 @@ com.amazonaws aws-athena-federation-sdk - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 withdep @@ -25,7 +25,7 @@ com.amazonaws athena-federation-integ-test - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 test @@ -191,7 +191,7 @@ com.amazonaws aws-athena-federation-sdk - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 test-jar test diff --git a/athena-example/pom.xml b/athena-example/pom.xml index 54fc091281..e3a84da25e 100644 --- a/athena-example/pom.xml +++ b/athena-example/pom.xml @@ -3,7 +3,7 @@ aws-athena-query-federation com.amazonaws - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 4.0.0 athena-example @@ -12,7 +12,7 @@ com.amazonaws aws-athena-federation-sdk - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 withdep diff --git a/athena-federation-integ-test/README.md b/athena-federation-integ-test/README.md index 2c761f9229..5440cb091d 100644 --- a/athena-federation-integ-test/README.md +++ b/athena-federation-integ-test/README.md @@ -36,7 +36,7 @@ in most **pom.xml** files (e.g. com.amazonaws athena-federation-integ-test - Current version of the SDK (e.g. 2022.47.1-arfaraj-qpt-1) + Current version of the SDK (e.g. 2022.47.1) test ``` diff --git a/athena-federation-integ-test/pom.xml b/athena-federation-integ-test/pom.xml index f449a450cd..581ba4a332 100644 --- a/athena-federation-integ-test/pom.xml +++ b/athena-federation-integ-test/pom.xml @@ -3,11 +3,11 @@ aws-athena-query-federation com.amazonaws - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 4.0.0 athena-federation-integ-test - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 jar Amazon Athena Query Federation Integ Test diff --git a/athena-federation-sdk-tools/pom.xml b/athena-federation-sdk-tools/pom.xml index c3140c1379..30eb909a1c 100644 --- a/athena-federation-sdk-tools/pom.xml +++ b/athena-federation-sdk-tools/pom.xml @@ -3,18 +3,18 @@ aws-athena-query-federation com.amazonaws - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 4.0.0 athena-federation-sdk-tools jar Amazon Athena Query Federation SDK Tools - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 com.amazonaws aws-athena-federation-sdk - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 withdep diff --git a/athena-federation-sdk/.attach_pid29964 b/athena-federation-sdk/.attach_pid29964 deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/athena-federation-sdk/athena-federation-sdk.yaml b/athena-federation-sdk/athena-federation-sdk.yaml index a1d0a0e35e..8499d0bd67 100644 --- a/athena-federation-sdk/athena-federation-sdk.yaml +++ b/athena-federation-sdk/athena-federation-sdk.yaml @@ -10,7 +10,7 @@ Metadata: Labels: - athena-federation HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' - SemanticVersion: 2022.47.1-arfaraj-qpt-1 + SemanticVersion: 2022.47.1 SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' Parameters: AthenaCatalogName: @@ -47,7 +47,7 @@ Resources: spill_prefix: !Ref SpillPrefix FunctionName: !Ref AthenaCatalogName Handler: "com.amazonaws.athena.connector.lambda.examples.ExampleCompositeHandler" - CodeUri: "./target/aws-athena-federation-sdk-2022.47.1-arfaraj-qpt-1-withdep.jar" + CodeUri: "./target/aws-athena-federation-sdk-2022.47.1-withdep.jar" Description: "This connector enables Amazon Athena to communicate with a randomly generated data source." Runtime: java11 Timeout: !Ref LambdaTimeout diff --git a/athena-federation-sdk/pom.xml b/athena-federation-sdk/pom.xml index fa3ec83716..a70be75972 100644 --- a/athena-federation-sdk/pom.xml +++ b/athena-federation-sdk/pom.xml @@ -4,11 +4,11 @@ com.amazonaws aws-athena-query-federation - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 com.amazonaws aws-athena-federation-sdk - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 jar Amazon Athena Query Federation SDK The Athena Query Federation SDK defines a set of interfaces and wire protocols that you can implement to enable Athena to delegate portions of it's query execution plan to code that you deploy/write. @@ -314,23 +314,6 @@ - - org.apache.maven.plugins - maven-source-plugin - 3.2.1 - - - attach-sources - - jar-no-fork - - - - - true - false - - org.apache.maven.plugins maven-jar-plugin @@ -349,6 +332,7 @@ ${mvn.checkstyle.version} checkstyle.xml + UTF-8 true false false diff --git a/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/domain/Split.java b/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/domain/Split.java index b188c7e883..2011f2d6e0 100644 --- a/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/domain/Split.java +++ b/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/domain/Split.java @@ -239,17 +239,6 @@ public Builder withEncryptionKey(EncryptionKey key) return this; } - /** - * Apply a map of properties to the split - * @param properties - * @return - */ - public Builder applyProperties(Map properties) - { - properties.entrySet().forEach(entry -> add(entry.getKey(), entry.getValue())); - return this; - } - /** * Builds the Split * diff --git a/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/domain/predicate/Constraints.java b/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/domain/predicate/Constraints.java index 2553991b5c..42c7434743 100644 --- a/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/domain/predicate/Constraints.java +++ b/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/domain/predicate/Constraints.java @@ -22,18 +22,13 @@ import com.amazonaws.athena.connector.lambda.domain.predicate.expression.FederationExpression; import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonGetter; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSetter; import com.google.common.base.Objects; import java.util.Collections; import java.util.List; import java.util.Map; -import static java.util.Objects.requireNonNull; - /** * Container which holds and maps column names to the corresponding constraint (e.g. ValueSet). * @@ -45,7 +40,6 @@ * supplying connectors with a more complete view of the query and its predicate. We expect a future release to provide * full predicates to connectors and lets the connector decide which parts of the predicate it is capable of applying */ -@JsonIgnoreProperties(value = {"queryPassthroughArguments"}, allowGetters = true, allowSetters = true) public class Constraints implements AutoCloseable { @@ -55,7 +49,6 @@ public class Constraints private List expression; private final List orderByClause; private long limit; - private Map queryPassthroughArguments; @Deprecated public Constraints(Map summary) @@ -73,7 +66,6 @@ public Constraints(@JsonProperty("summary") Map summary, this.expression = expression; this.orderByClause = orderByClause; this.limit = limit; - this.queryPassthroughArguments = Collections.emptyMap(); } /** @@ -110,23 +102,6 @@ public boolean hasNonEmptyOrderByClause() { return this.orderByClause != null && this.orderByClause.size() > 0; } - - @JsonGetter("queryPassthroughArguments") - public Map getQueryPassthroughArguments() - { - return this.queryPassthroughArguments; - } - - @JsonSetter("queryPassthroughArguments") - public void setQueryPassthroughArguments(Map queryPassthroughArguments) - { - this.queryPassthroughArguments = requireNonNull(queryPassthroughArguments, "queryPassthroughArguments is null"); - } - - public boolean isQueryPassThrough() - { - return !this.queryPassthroughArguments.isEmpty(); - } @Override public boolean equals(Object o) @@ -143,8 +118,7 @@ public boolean equals(Object o) return Objects.equal(this.summary, that.summary) && Objects.equal(this.expression, that.expression) && Objects.equal(this.orderByClause, that.orderByClause) && - Objects.equal(this.limit, that.limit) && - Objects.equal(this.getQueryPassthroughArguments(), that.getQueryPassthroughArguments()); + Objects.equal(this.limit, that.limit); } @Override @@ -155,7 +129,6 @@ public String toString() "expression=" + expression + "orderByClause=" + orderByClause + "limit=" + limit + - "queryPassthroughArguments=" + queryPassthroughArguments + '}'; } diff --git a/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/handlers/MetadataHandler.java b/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/handlers/MetadataHandler.java index 0cecb91fb1..7fa137e77e 100644 --- a/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/handlers/MetadataHandler.java +++ b/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/handlers/MetadataHandler.java @@ -30,7 +30,6 @@ import com.amazonaws.athena.connector.lambda.data.SchemaBuilder; import com.amazonaws.athena.connector.lambda.data.SimpleBlockWriter; import com.amazonaws.athena.connector.lambda.data.SupportedTypes; -import com.amazonaws.athena.connector.lambda.domain.Split; import com.amazonaws.athena.connector.lambda.domain.predicate.ConstraintEvaluator; import com.amazonaws.athena.connector.lambda.domain.spill.S3SpillLocation; import com.amazonaws.athena.connector.lambda.domain.spill.SpillLocation; @@ -78,7 +77,6 @@ import java.io.InputStream; import java.io.OutputStream; import java.util.Collections; -import java.util.Map; import java.util.UUID; import static com.amazonaws.athena.connector.lambda.handlers.AthenaExceptionFilter.ATHENA_EXCEPTION_FILTER; @@ -272,7 +270,7 @@ protected final void doHandleRequest(BlockAllocator allocator, } return; case GET_TABLE: - try (GetTableResponse response = resolveDoGetTableImplementation(allocator, (GetTableRequest) req)) { + try (GetTableResponse response = doGetTable(allocator, (GetTableRequest) req)) { logger.info("doHandleRequest: response[{}]", response); assertNotNull(response); assertTypes(response); @@ -328,32 +326,6 @@ public abstract ListSchemasResponse doListSchemaNames(final BlockAllocator alloc public abstract ListTablesResponse doListTables(final BlockAllocator allocator, final ListTablesRequest request) throws Exception; - private GetTableResponse resolveDoGetTableImplementation(final BlockAllocator allocator, final GetTableRequest request) - throws Exception - { - logger.info("resolveDoGetTableImplementation: resolving implementation - isQueryPassthrough[{}]", request.isQueryPassthrough()); - if (request.isQueryPassthrough()) { - return doGetQueryPassthroughSchema(allocator, request); - } - return doGetTable(allocator, request); - } - - /** - * Used to get definition (field names, types, descriptions, etc...) of a Query PassThrough. - * - * @param allocator Tool for creating and managing Apache Arrow Blocks. - * @param request Provides details on who made the request and which Athena catalog, database, and table they are querying. - * @return A GetTableResponse which primarily contains: - * 1. An Apache Arrow Schema object describing the table's columns, types, and descriptions. - * 2. A Set of partition column names (or empty if the table isn't partitioned). - */ - public GetTableResponse doGetQueryPassthroughSchema(final BlockAllocator allocator, final GetTableRequest request) - throws Exception - { - //todo; maybe we need a better name for this method, - throw new UnsupportedOperationException("Not implemented"); - } - /** * Used to get definition (field names, types, descriptions, etc...) of a Table. * @@ -550,21 +522,4 @@ private void assertTypes(GetTableResponse response) SupportedTypes.assertSupported(next); } } - - /** - * Helper function that provides a single partition for Query Pass-Through - * - */ - protected GetSplitsResponse setupQueryPassthroughSplit(GetSplitsRequest request) - { - //Every split must have a unique location if we wish to spill to avoid failures - SpillLocation spillLocation = makeSpillLocation(request); - - //Since this is QPT query we return a fixed split. - Map qptArguments = request.getConstraints().getQueryPassthroughArguments(); - return new GetSplitsResponse(request.getCatalogName(), - Split.newBuilder(spillLocation, makeEncryptionKey()) - .applyProperties(qptArguments) - .build()); - } } diff --git a/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/handlers/SerDeVersion.java b/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/handlers/SerDeVersion.java index 31ba79b3af..e31111dae2 100644 --- a/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/handlers/SerDeVersion.java +++ b/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/handlers/SerDeVersion.java @@ -30,5 +30,5 @@ public class SerDeVersion { private SerDeVersion() {} - public static final int SERDE_VERSION = 5; + public static final int SERDE_VERSION = 4; } diff --git a/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/metadata/GetTableRequest.java b/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/metadata/GetTableRequest.java index 49874bc483..691ae281c3 100644 --- a/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/metadata/GetTableRequest.java +++ b/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/metadata/GetTableRequest.java @@ -23,26 +23,18 @@ import com.amazonaws.athena.connector.lambda.domain.TableName; import com.amazonaws.athena.connector.lambda.security.FederatedIdentity; import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonGetter; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSetter; import com.google.common.base.Objects; -import java.util.Collections; -import java.util.Map; - import static java.util.Objects.requireNonNull; /** * Represents the input of a GetTable operation. */ -@JsonIgnoreProperties(value = {"queryPassthroughArguments"}, allowGetters = true, allowSetters = true) public class GetTableRequest extends MetadataRequest { private final TableName tableName; - private Map queryPassthroughArguments; /** * Constructs a new GetTableRequest object. @@ -61,7 +53,6 @@ public GetTableRequest(@JsonProperty("identity") FederatedIdentity identity, super(identity, MetadataRequestType.GET_TABLE, queryId, catalogName); requireNonNull(tableName, "tableName is null"); this.tableName = tableName; - this.queryPassthroughArguments = Collections.emptyMap(); } /** @@ -81,29 +72,12 @@ public void close() //No Op } - @JsonGetter("queryPassthroughArguments") - public Map getQueryPassthroughArguments() - { - return this.queryPassthroughArguments; - } - - @JsonSetter("queryPassthroughArguments") - public void setQueryPassthroughArguments(Map queryPassthroughArguments) - { - this.queryPassthroughArguments = requireNonNull(queryPassthroughArguments, "queryPassthroughArguments is null"); - } - - public boolean isQueryPassthrough() - { - return !this.queryPassthroughArguments.isEmpty(); - } @Override public String toString() { return "GetTableRequest{" + "queryId=" + getQueryId() + ", tableName=" + tableName + - "queryPassthroughArguments=" + queryPassthroughArguments + '}'; } @@ -121,13 +95,12 @@ public boolean equals(Object o) return Objects.equal(this.tableName, that.tableName) && Objects.equal(this.getRequestType(), that.getRequestType()) && - Objects.equal(this.getCatalogName(), that.getCatalogName()) && - Objects.equal(this.getQueryPassthroughArguments(), that.getQueryPassthroughArguments()); + Objects.equal(this.getCatalogName(), that.getCatalogName()); } @Override public int hashCode() { - return Objects.hashCode(tableName, getRequestType(), getCatalogName(), getQueryPassthroughArguments()); + return Objects.hashCode(tableName, getRequestType(), getCatalogName()); } } diff --git a/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/metadata/MetadataRequestType.java b/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/metadata/MetadataRequestType.java index fdc4a7b2df..4e75062f73 100644 --- a/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/metadata/MetadataRequestType.java +++ b/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/metadata/MetadataRequestType.java @@ -39,10 +39,6 @@ public enum MetadataRequestType * For fetching a table's metadata including schema and partition columns. */ GET_TABLE, - /** - * For fetching a Query Passthrough metadata including schema. - */ - GET_QUERY_PASSTHROUGH_SCHEMA, /** * For fetching a table's partitioning layout. diff --git a/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/metadata/optimizations/qpt/QueryPassthrough.java b/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/metadata/optimizations/qpt/QueryPassthrough.java deleted file mode 100644 index 50d9e0e48d..0000000000 --- a/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/metadata/optimizations/qpt/QueryPassthrough.java +++ /dev/null @@ -1,66 +0,0 @@ -/*- - * #%L - * Amazon Athena Query Federation SDK - * %% - * Copyright (C) 2019 - 2023 Amazon Web Services - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ -package com.amazonaws.athena.connector.lambda.metadata.optimizations.qpt; - -import com.amazonaws.athena.connector.lambda.metadata.optimizations.OptimizationSubType; - -import java.util.AbstractMap; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; -/** - * This class describes Query Passthrough Signature; - * Schema Name; is where the function will reside in the catalog attaching this namespace - * Name; is the table function name of the QPT; - * Arguments; list of all arguments that this QPT is expecting to have - * - */ -public enum QueryPassthrough { - QUERY_PASSTHROUGH_NAME("query_passthrough_name"), - QUERY_PASSTHROUGH_SCHEMA_NAME("query_passthrough_schema_name"), - QUERY_PASSTHROUGH_ARGUMENTS("query_passthrough_arguments"); - private final String signature; - - QueryPassthrough(String signature) - { - this.signature = signature; - } - - public String getSignature() - { - return signature; - } - - public final Map.Entry> withSchemaName(String namespace) - { - return new AbstractMap.SimpleImmutableEntry<>(this.getSignature(), Arrays.asList(new OptimizationSubType(namespace, Collections.emptyList()))); - } - - public final Map.Entry> withName(String name) - { - return new AbstractMap.SimpleImmutableEntry<>(this.getSignature(), Arrays.asList(new OptimizationSubType(name, Collections.emptyList()))); - } - - public final Map.Entry> withArguments(List arguments) - { - return new AbstractMap.SimpleImmutableEntry<>(this.getSignature(), Arrays.asList(new OptimizationSubType(this.getSignature(), arguments))); - } -} diff --git a/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/metadata/optimizations/qpt/QueryPassthroughSignature.java b/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/metadata/optimizations/qpt/QueryPassthroughSignature.java deleted file mode 100644 index d47c3428f2..0000000000 --- a/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/metadata/optimizations/qpt/QueryPassthroughSignature.java +++ /dev/null @@ -1,65 +0,0 @@ -/*- - * #%L - * Amazon Athena Query Federation SDK - * %% - * Copyright (C) 2019 - 2023 Amazon Web Services - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ -package com.amazonaws.athena.connector.lambda.metadata.optimizations.qpt; - -import org.apache.commons.lang3.StringUtils; - -import java.util.List; -import java.util.Map; - -public interface QueryPassthroughSignature -{ - /** - * - * @return QPT Function's name - */ - public String getName(); - - /** - * - * @return QPT Function's domain - */ - public String getDomain(); - - /** - * - * @return QPT Function's Arguments - */ - public List arguments(); - - /** - * a static method to verify that the arguments returned by the engine are the same that the connector defined - * @param engineQptArguments - * @param clientQptArguments - * @throws IllegalArgumentException - */ - static void verifyQueryPassthroughArguments(Map engineQptArguments, List clientQptArguments) - throws IllegalArgumentException - { - for (String argument : clientQptArguments) { - if (!engineQptArguments.containsKey(argument)) { - throw new IllegalArgumentException("Missing Query Passthrough Argument: " + argument); - } - if (StringUtils.isEmpty(engineQptArguments.get(argument))) { - throw new IllegalArgumentException("Missing Query Passthrough Value for Argument: " + argument); - } - } - } -} diff --git a/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/serde/VersionedObjectMapperFactory.java b/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/serde/VersionedObjectMapperFactory.java index 57b0b67b0a..5f0a69a70c 100644 --- a/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/serde/VersionedObjectMapperFactory.java +++ b/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/serde/VersionedObjectMapperFactory.java @@ -24,7 +24,6 @@ import com.amazonaws.athena.connector.lambda.serde.v2.ObjectMapperFactoryV2; import com.amazonaws.athena.connector.lambda.serde.v3.ObjectMapperFactoryV3; import com.amazonaws.athena.connector.lambda.serde.v4.ObjectMapperFactoryV4; -import com.amazonaws.athena.connector.lambda.serde.v5.ObjectMapperFactoryV5; import com.fasterxml.jackson.databind.ObjectMapper; /** @@ -63,8 +62,6 @@ public static ObjectMapper create(BlockAllocator allocator, int version) return ObjectMapperFactoryV3.create(allocator); case 4: return ObjectMapperFactoryV4.create(allocator); - case 5: - return ObjectMapperFactoryV5.create(allocator); default: throw new IllegalArgumentException("No serde version " + version); } diff --git a/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/serde/v2/GetTableRequestSerDe.java b/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/serde/v2/GetTableRequestSerDe.java index 67709c2cef..02e4b232aa 100644 --- a/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/serde/v2/GetTableRequestSerDe.java +++ b/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/serde/v2/GetTableRequestSerDe.java @@ -27,20 +27,16 @@ import com.amazonaws.athena.connector.lambda.serde.FederatedIdentitySerDe; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonToken; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.SerializerProvider; import java.io.IOException; -import java.util.HashMap; -import java.util.Map; import static java.util.Objects.requireNonNull; public final class GetTableRequestSerDe { private static final String TABLE_NAME_FIELD = "tableName"; - private static final String QUERY_PASSTHROUGH_ARGUMENTS = "queryPassthroughArguments"; private GetTableRequestSerDe() {} @@ -64,9 +60,6 @@ protected void doRequestSerialize(FederationRequest federationRequest, JsonGener jgen.writeFieldName(TABLE_NAME_FIELD); tableNameSerializer.serialize(getTableRequest.getTableName(), jgen, provider); - if (!getTableRequest.getQueryPassthroughArguments().isEmpty()) { - writeStringMap(jgen, QUERY_PASSTHROUGH_ARGUMENTS, getTableRequest.getQueryPassthroughArguments()); - } } } @@ -89,19 +82,7 @@ protected MetadataRequest doRequestDeserialize(JsonParser jparser, Deserializati assertFieldName(jparser, TABLE_NAME_FIELD); TableName tableName = tableNameDeserializer.deserialize(jparser, ctxt); - // This will insure backward compatibility given that QPT arguments are optional - Map queryPassthroughArguments = new HashMap<>(); - if (jparser.nextToken() == JsonToken.FIELD_NAME - && jparser.getCurrentName().equals(QUERY_PASSTHROUGH_ARGUMENTS)) { - validateObjectStart(jparser.nextToken()); - while (jparser.nextToken() != JsonToken.END_OBJECT) { - queryPassthroughArguments.put(jparser.getCurrentName(), jparser.getValueAsString()); - } - } - - GetTableRequest getTableRequest = new GetTableRequest(identity, queryId, catalogName, tableName); - getTableRequest.setQueryPassthroughArguments(queryPassthroughArguments); - return getTableRequest; + return new GetTableRequest(identity, queryId, catalogName, tableName); } } } diff --git a/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/serde/v4/ConstraintsSerDeV4.java b/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/serde/v4/ConstraintsSerDeV4.java index 98b84b895e..615d6b0264 100644 --- a/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/serde/v4/ConstraintsSerDeV4.java +++ b/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/serde/v4/ConstraintsSerDeV4.java @@ -36,7 +36,6 @@ import com.google.common.collect.ImmutableMap; import java.io.IOException; -import java.util.HashMap; import java.util.Map; import static java.util.Objects.requireNonNull; @@ -47,7 +46,6 @@ public final class ConstraintsSerDeV4 private static final String EXPRESSION_FIELD = "expression"; private static final String ORDER_BY_CLAUSE = "orderByClause"; private static final String LIMIT_FIELD = "limit"; - private static final String QUERY_PASSTHROUGH_ARGUMENTS = "queryPassthroughArguments"; private ConstraintsSerDeV4() {} @@ -90,9 +88,6 @@ public void doSerialize(Constraints constraints, JsonGenerator jgen, SerializerP jgen.writeEndArray(); jgen.writeNumberField(LIMIT_FIELD, constraints.getLimit()); - if (constraints.isQueryPassThrough()) { - writeStringMap(jgen, QUERY_PASSTHROUGH_ARGUMENTS, constraints.getQueryPassthroughArguments()); - } } } @@ -141,22 +136,11 @@ public Constraints doDeserialize(JsonParser jparser, DeserializationContext ctxt orderByClauseBuilder.add(orderByFieldDeserializer.doDeserialize(jparser, ctxt)); validateObjectEnd(jparser); } + long limit = getNextLongField(jparser, LIMIT_FIELD); - // This will insure backward compatibility given that QPT arguments are optional - Map queryPassthroughArguments = new HashMap<>(); - if (jparser.nextToken() == JsonToken.FIELD_NAME - && jparser.getCurrentName().equals(QUERY_PASSTHROUGH_ARGUMENTS)) { - validateObjectStart(jparser.nextToken()); - while (jparser.nextToken() != JsonToken.END_OBJECT) { - queryPassthroughArguments.put(jparser.getCurrentName(), jparser.getValueAsString()); - } - } - - Constraints constraints = new Constraints(summaryMap.build(), federationExpression.build(), orderByClauseBuilder.build(), limit); - constraints.setQueryPassthroughArguments(queryPassthroughArguments); - return constraints; + return new Constraints(summaryMap.build(), federationExpression.build(), orderByClauseBuilder.build(), limit); } } } diff --git a/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/serde/v5/ConstraintsSerDeV5.java b/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/serde/v5/ConstraintsSerDeV5.java deleted file mode 100644 index 6de72b7208..0000000000 --- a/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/serde/v5/ConstraintsSerDeV5.java +++ /dev/null @@ -1,158 +0,0 @@ -/*- - * #%L - * Amazon Athena Query Federation SDK - * %% - * Copyright (C) 2019 - 2020 Amazon Web Services - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ -package com.amazonaws.athena.connector.lambda.serde.v5; - -import com.amazonaws.athena.connector.lambda.domain.predicate.Constraints; -import com.amazonaws.athena.connector.lambda.domain.predicate.OrderByField; -import com.amazonaws.athena.connector.lambda.domain.predicate.ValueSet; -import com.amazonaws.athena.connector.lambda.domain.predicate.expression.FederationExpression; -import com.amazonaws.athena.connector.lambda.serde.BaseDeserializer; -import com.amazonaws.athena.connector.lambda.serde.BaseSerializer; -import com.amazonaws.athena.connector.lambda.serde.VersionedSerDe; -import com.amazonaws.athena.connector.lambda.serde.v2.ValueSetSerDe; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonToken; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.SerializerProvider; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -import static java.util.Objects.requireNonNull; - -public final class ConstraintsSerDeV5 -{ - private static final String SUMMARY_FIELD = "summary"; - private static final String EXPRESSION_FIELD = "expression"; - private static final String ORDER_BY_CLAUSE = "orderByClause"; - private static final String LIMIT_FIELD = "limit"; - private static final String QUERY_PASSTHROUGH_ARGUMENTS = "queryPassthroughArguments"; - - private ConstraintsSerDeV5() {} - - public static final class Serializer extends BaseSerializer implements VersionedSerDe.Serializer - { - private final ValueSetSerDe.Serializer valueSetSerializer; - private final VersionedSerDe.Serializer federationExpressionSerializer; - private final VersionedSerDe.Serializer orderByFieldSerializer; - public Serializer(ValueSetSerDe.Serializer valueSetSerializer, - VersionedSerDe.Serializer federationExpressionSerializer, - VersionedSerDe.Serializer orderByFieldSerializer) - { - super(Constraints.class); - this.valueSetSerializer = requireNonNull(valueSetSerializer, "valueSetSerDe is null"); - this.federationExpressionSerializer = requireNonNull(federationExpressionSerializer, "federationExpressionSerDe is null"); - this.orderByFieldSerializer = requireNonNull(orderByFieldSerializer, "orderByFieldSerDe is null"); - } - - @Override - public void doSerialize(Constraints constraints, JsonGenerator jgen, SerializerProvider provider) - throws IOException - { - jgen.writeObjectFieldStart(SUMMARY_FIELD); - for (Map.Entry entry : constraints.getSummary().entrySet()) { - jgen.writeFieldName(entry.getKey()); - valueSetSerializer.serialize(entry.getValue(), jgen, provider); - } - jgen.writeEndObject(); - - jgen.writeArrayFieldStart(EXPRESSION_FIELD); - for (FederationExpression federationExpression : constraints.getExpression()) { - federationExpressionSerializer.serialize(federationExpression, jgen, provider); - } - jgen.writeEndArray(); - - jgen.writeArrayFieldStart(ORDER_BY_CLAUSE); - for (OrderByField orderByField : constraints.getOrderByClause()) { - orderByFieldSerializer.serialize(orderByField, jgen, provider); - } - jgen.writeEndArray(); - - jgen.writeNumberField(LIMIT_FIELD, constraints.getLimit()); - - writeStringMap(jgen, QUERY_PASSTHROUGH_ARGUMENTS, constraints.getQueryPassthroughArguments()); - } - } - - public static final class Deserializer extends BaseDeserializer implements VersionedSerDe.Deserializer - { - private final ValueSetSerDe.Deserializer valueSetDeserializer; - private final VersionedSerDe.Deserializer federationExpressionDeserializer; - private final VersionedSerDe.Deserializer orderByFieldDeserializer; - - public Deserializer(ValueSetSerDe.Deserializer valueSetDeserializer, - VersionedSerDe.Deserializer federationExpressionDeserializer, - VersionedSerDe.Deserializer orderByFieldDeserializer) - { - super(Constraints.class); - this.valueSetDeserializer = requireNonNull(valueSetDeserializer, "valueSetSerDe is null"); - this.federationExpressionDeserializer = requireNonNull(federationExpressionDeserializer, "federationExpressionSerDe is null"); - this.orderByFieldDeserializer = requireNonNull(orderByFieldDeserializer, "orderByFieldSerDe is null"); - } - - @Override - public Constraints doDeserialize(JsonParser jparser, DeserializationContext ctxt) - throws IOException - { - assertFieldName(jparser, SUMMARY_FIELD); - validateObjectStart(jparser.nextToken()); - ImmutableMap.Builder summaryMap = ImmutableMap.builder(); - while (jparser.nextToken() != JsonToken.END_OBJECT) { - String column = jparser.getCurrentName(); - summaryMap.put(column, valueSetDeserializer.deserialize(jparser, ctxt)); - } - - assertFieldName(jparser, EXPRESSION_FIELD); - validateArrayStart(jparser); - ImmutableList.Builder federationExpression = ImmutableList.builder(); - while (jparser.nextToken() != JsonToken.END_ARRAY) { - validateObjectStart(jparser.getCurrentToken()); - federationExpression.add(federationExpressionDeserializer.doDeserialize(jparser, ctxt)); - validateObjectEnd(jparser); - } - - assertFieldName(jparser, ORDER_BY_CLAUSE); - validateArrayStart(jparser); - ImmutableList.Builder orderByClauseBuilder = ImmutableList.builder(); - while (jparser.nextToken() != JsonToken.END_ARRAY) { - validateObjectStart(jparser.getCurrentToken()); - orderByClauseBuilder.add(orderByFieldDeserializer.doDeserialize(jparser, ctxt)); - validateObjectEnd(jparser); - } - - long limit = getNextLongField(jparser, LIMIT_FIELD); - - Map queryPassthroughArguments = new HashMap<>(); - assertFieldName(jparser, QUERY_PASSTHROUGH_ARGUMENTS); - validateObjectStart(jparser.nextToken()); - while (jparser.nextToken() != JsonToken.END_OBJECT) { - queryPassthroughArguments.put(jparser.getCurrentName(), jparser.getValueAsString()); - } - - Constraints constraints = new Constraints(summaryMap.build(), federationExpression.build(), orderByClauseBuilder.build(), limit); - constraints.setQueryPassthroughArguments(queryPassthroughArguments); - return constraints; - } - } -} diff --git a/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/serde/v5/FederationRequestSerDeV5.java b/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/serde/v5/FederationRequestSerDeV5.java deleted file mode 100644 index 4bdba5698f..0000000000 --- a/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/serde/v5/FederationRequestSerDeV5.java +++ /dev/null @@ -1,95 +0,0 @@ -/*- - * #%L - * Amazon Athena Query Federation SDK - * %% - * Copyright (C) 2019 - 2022 Amazon Web Services - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ -package com.amazonaws.athena.connector.lambda.serde.v5; - -import com.amazonaws.athena.connector.lambda.request.FederationRequest; -import com.amazonaws.athena.connector.lambda.serde.DelegatingDeserializer; -import com.amazonaws.athena.connector.lambda.serde.DelegatingSerializer; -import com.amazonaws.athena.connector.lambda.serde.PingRequestSerDe; -import com.amazonaws.athena.connector.lambda.serde.TypedDeserializer; -import com.amazonaws.athena.connector.lambda.serde.TypedSerializer; -import com.amazonaws.athena.connector.lambda.serde.VersionedSerDe; -import com.amazonaws.athena.connector.lambda.serde.v2.GetSplitsRequestSerDe; -import com.amazonaws.athena.connector.lambda.serde.v2.GetTableLayoutRequestSerDe; -import com.amazonaws.athena.connector.lambda.serde.v2.ListSchemasRequestSerDe; -import com.amazonaws.athena.connector.lambda.serde.v2.ListTablesRequestSerDe; -import com.amazonaws.athena.connector.lambda.serde.v2.ReadRecordsRequestSerDe; -import com.amazonaws.athena.connector.lambda.serde.v2.UserDefinedFunctionRequestSerDe; -import com.amazonaws.athena.connector.lambda.serde.v4.GetDataSourceCapabilitiesRequestSerDeV4; -import com.google.common.collect.ImmutableSet; - -public class FederationRequestSerDeV5 -{ - private FederationRequestSerDeV5() {} - - public static final class Serializer extends DelegatingSerializer implements VersionedSerDe.Serializer - { - public Serializer( - PingRequestSerDe.Serializer pingSerializer, - ListSchemasRequestSerDe.Serializer listSchemasSerializer, - ListTablesRequestSerDe.Serializer listTablesSerializer, - GetTableRequestSerDeV5.Serializer getTableSerializer, - GetTableLayoutRequestSerDe.Serializer getTableLayoutSerializer, - GetSplitsRequestSerDe.Serializer getSplitsSerializer, - ReadRecordsRequestSerDe.Serializer readRecordsSerializer, - UserDefinedFunctionRequestSerDe.Serializer userDefinedFunctionSerializer, - GetDataSourceCapabilitiesRequestSerDeV4.Serializer getDataSourceCapabilitiesSerializer) - { - super(FederationRequest.class, ImmutableSet.>builder() - .add(pingSerializer) - .add(listSchemasSerializer) - .add(listTablesSerializer) - .add(getTableSerializer) - .add(getTableLayoutSerializer) - .add(getSplitsSerializer) - .add(readRecordsSerializer) - .add(userDefinedFunctionSerializer) - .add(getDataSourceCapabilitiesSerializer) - .build()); - } - } - - public static final class Deserializer extends DelegatingDeserializer implements VersionedSerDe.Deserializer - { - public Deserializer( - PingRequestSerDe.Deserializer pingDeserializer, - ListSchemasRequestSerDe.Deserializer listSchemasDeserializer, - ListTablesRequestSerDe.Deserializer listTablesDeserializer, - GetTableRequestSerDeV5.Deserializer getTableDeserializer, - GetTableLayoutRequestSerDe.Deserializer getTableLayoutDeserializer, - GetSplitsRequestSerDe.Deserializer getSplitsDeserializer, - ReadRecordsRequestSerDe.Deserializer readRecordsDeserializer, - UserDefinedFunctionRequestSerDe.Deserializer userDefinedFunctionDeserializer, - GetDataSourceCapabilitiesRequestSerDeV4.Deserializer getDataSourceCapabilitiesDeserializer) - { - super(FederationRequest.class, ImmutableSet.>builder() - .add(pingDeserializer) - .add(listSchemasDeserializer) - .add(listTablesDeserializer) - .add(getTableDeserializer) - .add(getTableLayoutDeserializer) - .add(getSplitsDeserializer) - .add(readRecordsDeserializer) - .add(userDefinedFunctionDeserializer) - .add(getDataSourceCapabilitiesDeserializer) - .build()); - } - } -} diff --git a/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/serde/v5/GetTableRequestSerDeV5.java b/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/serde/v5/GetTableRequestSerDeV5.java deleted file mode 100644 index beacddcab1..0000000000 --- a/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/serde/v5/GetTableRequestSerDeV5.java +++ /dev/null @@ -1,105 +0,0 @@ -/*- - * #%L - * Amazon Athena Query Federation SDK - * %% - * Copyright (C) 2019 - 2020 Amazon Web Services - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ -package com.amazonaws.athena.connector.lambda.serde.v5; - -import com.amazonaws.athena.connector.lambda.domain.TableName; -import com.amazonaws.athena.connector.lambda.metadata.GetTableRequest; -import com.amazonaws.athena.connector.lambda.metadata.MetadataRequest; -import com.amazonaws.athena.connector.lambda.request.FederationRequest; -import com.amazonaws.athena.connector.lambda.security.FederatedIdentity; -import com.amazonaws.athena.connector.lambda.serde.FederatedIdentitySerDe; -import com.amazonaws.athena.connector.lambda.serde.v2.MetadataRequestDeserializer; -import com.amazonaws.athena.connector.lambda.serde.v2.MetadataRequestSerializer; -import com.amazonaws.athena.connector.lambda.serde.v2.TableNameSerDe; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonToken; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.SerializerProvider; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -import static java.util.Objects.requireNonNull; - -public final class GetTableRequestSerDeV5 -{ - private static final String TABLE_NAME_FIELD = "tableName"; - private static final String QUERY_PASSTHROUGH_ARGUMENTS = "queryPassthroughArguments"; - - private GetTableRequestSerDeV5() {} - - public static final class Serializer extends MetadataRequestSerializer - { - private final FederatedIdentitySerDe.Serializer identitySerializer; - private final TableNameSerDe.Serializer tableNameSerializer; - - public Serializer(FederatedIdentitySerDe.Serializer identitySerializer, TableNameSerDe.Serializer tableNameSerializer) - { - super(GetTableRequest.class, identitySerializer); - this.identitySerializer = requireNonNull(identitySerializer, "identitySerializer is null"); - this.tableNameSerializer = requireNonNull(tableNameSerializer, "tableNameSerializer is null"); - } - - @Override - protected void doRequestSerialize(FederationRequest federationRequest, JsonGenerator jgen, SerializerProvider provider) - throws IOException - { - GetTableRequest getTableRequest = (GetTableRequest) federationRequest; - - jgen.writeFieldName(TABLE_NAME_FIELD); - tableNameSerializer.serialize(getTableRequest.getTableName(), jgen, provider); - writeStringMap(jgen, QUERY_PASSTHROUGH_ARGUMENTS, getTableRequest.getQueryPassthroughArguments()); - } - } - - public static final class Deserializer extends MetadataRequestDeserializer - { - private final FederatedIdentitySerDe.Deserializer identityDeserializer; - private final TableNameSerDe.Deserializer tableNameDeserializer; - - public Deserializer(FederatedIdentitySerDe.Deserializer identityDeserializer, TableNameSerDe.Deserializer tableNameDeserializer) - { - super(GetTableRequest.class, identityDeserializer); - this.identityDeserializer = requireNonNull(identityDeserializer, "identityDeserializer is null"); - this.tableNameDeserializer = requireNonNull(tableNameDeserializer, "tableNameDeserializer is null"); - } - - @Override - protected MetadataRequest doRequestDeserialize(JsonParser jparser, DeserializationContext ctxt, FederatedIdentity identity, String queryId, String catalogName) - throws IOException - { - assertFieldName(jparser, TABLE_NAME_FIELD); - TableName tableName = tableNameDeserializer.deserialize(jparser, ctxt); - - Map queryPassthroughArguments = new HashMap<>(); - assertFieldName(jparser, QUERY_PASSTHROUGH_ARGUMENTS); - validateObjectStart(jparser.nextToken()); - while (jparser.nextToken() != JsonToken.END_OBJECT) { - queryPassthroughArguments.put(jparser.getCurrentName(), jparser.getValueAsString()); - } - - GetTableRequest getTableRequest = new GetTableRequest(identity, queryId, catalogName, tableName); - getTableRequest.setQueryPassthroughArguments(queryPassthroughArguments); - return getTableRequest; - } - } -} diff --git a/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/serde/v5/ObjectMapperFactoryV5.java b/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/serde/v5/ObjectMapperFactoryV5.java deleted file mode 100644 index d47dc71c23..0000000000 --- a/athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/serde/v5/ObjectMapperFactoryV5.java +++ /dev/null @@ -1,378 +0,0 @@ -/*- - * #%L - * Amazon Athena Query Federation SDK - * %% - * Copyright (C) 2019 - 2020 Amazon Web Services - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ -package com.amazonaws.athena.connector.lambda.serde.v5; - -import com.amazonaws.athena.connector.lambda.data.Block; -import com.amazonaws.athena.connector.lambda.data.BlockAllocator; -import com.amazonaws.athena.connector.lambda.domain.predicate.Constraints; -import com.amazonaws.athena.connector.lambda.domain.predicate.OrderByField; -import com.amazonaws.athena.connector.lambda.domain.predicate.expression.FederationExpression; -import com.amazonaws.athena.connector.lambda.domain.predicate.functions.FunctionName; -import com.amazonaws.athena.connector.lambda.metadata.optimizations.OptimizationSubType; -import com.amazonaws.athena.connector.lambda.request.FederationRequest; -import com.amazonaws.athena.connector.lambda.request.FederationResponse; -import com.amazonaws.athena.connector.lambda.serde.FederatedIdentitySerDe; -import com.amazonaws.athena.connector.lambda.serde.PingRequestSerDe; -import com.amazonaws.athena.connector.lambda.serde.PingResponseSerDe; -import com.amazonaws.athena.connector.lambda.serde.VersionedSerDe; -import com.amazonaws.athena.connector.lambda.serde.v2.AllOrNoneValueSetSerDe; -import com.amazonaws.athena.connector.lambda.serde.v2.ArrowTypeSerDe; -import com.amazonaws.athena.connector.lambda.serde.v2.EncryptionKeySerDe; -import com.amazonaws.athena.connector.lambda.serde.v2.EquatableValueSetSerDe; -import com.amazonaws.athena.connector.lambda.serde.v2.GetSplitsRequestSerDe; -import com.amazonaws.athena.connector.lambda.serde.v2.GetSplitsResponseSerDe; -import com.amazonaws.athena.connector.lambda.serde.v2.GetTableLayoutRequestSerDe; -import com.amazonaws.athena.connector.lambda.serde.v2.GetTableLayoutResponseSerDe; -import com.amazonaws.athena.connector.lambda.serde.v2.GetTableResponseSerDe; -import com.amazonaws.athena.connector.lambda.serde.v2.LambdaFunctionExceptionSerDe; -import com.amazonaws.athena.connector.lambda.serde.v2.ListSchemasRequestSerDe; -import com.amazonaws.athena.connector.lambda.serde.v2.ListSchemasResponseSerDe; -import com.amazonaws.athena.connector.lambda.serde.v2.ListTablesRequestSerDe; -import com.amazonaws.athena.connector.lambda.serde.v2.ListTablesResponseSerDe; -import com.amazonaws.athena.connector.lambda.serde.v2.MarkerSerDe; -import com.amazonaws.athena.connector.lambda.serde.v2.RangeSerDe; -import com.amazonaws.athena.connector.lambda.serde.v2.ReadRecordsRequestSerDe; -import com.amazonaws.athena.connector.lambda.serde.v2.ReadRecordsResponseSerDe; -import com.amazonaws.athena.connector.lambda.serde.v2.RemoteReadRecordsResponseSerDe; -import com.amazonaws.athena.connector.lambda.serde.v2.S3SpillLocationSerDe; -import com.amazonaws.athena.connector.lambda.serde.v2.SortedRangeSetSerDe; -import com.amazonaws.athena.connector.lambda.serde.v2.SpillLocationSerDe; -import com.amazonaws.athena.connector.lambda.serde.v2.SplitSerDe; -import com.amazonaws.athena.connector.lambda.serde.v2.TableNameSerDe; -import com.amazonaws.athena.connector.lambda.serde.v2.UserDefinedFunctionRequestSerDe; -import com.amazonaws.athena.connector.lambda.serde.v2.UserDefinedFunctionResponseSerDe; -import com.amazonaws.athena.connector.lambda.serde.v2.ValueSetSerDe; -import com.amazonaws.athena.connector.lambda.serde.v4.BlockSerDeV4; -import com.amazonaws.athena.connector.lambda.serde.v4.ConstantExpressionSerDeV4; -import com.amazonaws.athena.connector.lambda.serde.v4.FederationExpressionSerDeV4; -import com.amazonaws.athena.connector.lambda.serde.v4.FederationResponseSerDeV4; -import com.amazonaws.athena.connector.lambda.serde.v4.FunctionCallExpressionSerDeV4; -import com.amazonaws.athena.connector.lambda.serde.v4.FunctionNameSerDeV4; -import com.amazonaws.athena.connector.lambda.serde.v4.GetDataSourceCapabilitiesRequestSerDeV4; -import com.amazonaws.athena.connector.lambda.serde.v4.GetDataSourceCapabilitiesResponseSerDeV4; -import com.amazonaws.athena.connector.lambda.serde.v4.OptimizationSubTypeSerDeV4; -import com.amazonaws.athena.connector.lambda.serde.v4.OrderByFieldSerDeV4; -import com.amazonaws.athena.connector.lambda.serde.v4.SchemaSerDeV4; -import com.amazonaws.athena.connector.lambda.serde.v4.VariableExpressionSerDeV4; -import com.amazonaws.services.lambda.invoke.LambdaFunctionException; -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.databind.BeanDescription; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.JavaType; -import com.fasterxml.jackson.databind.JsonDeserializer; -import com.fasterxml.jackson.databind.JsonMappingException; -import com.fasterxml.jackson.databind.JsonSerializer; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializerProvider; -import com.fasterxml.jackson.databind.cfg.DeserializerFactoryConfig; -import com.fasterxml.jackson.databind.cfg.SerializerFactoryConfig; -import com.fasterxml.jackson.databind.deser.BeanDeserializerFactory; -import com.fasterxml.jackson.databind.deser.DefaultDeserializationContext; -import com.fasterxml.jackson.databind.deser.DeserializerFactory; -import com.fasterxml.jackson.databind.deser.Deserializers; -import com.fasterxml.jackson.databind.module.SimpleDeserializers; -import com.fasterxml.jackson.databind.module.SimpleSerializers; -import com.fasterxml.jackson.databind.ser.BeanSerializerFactory; -import com.fasterxml.jackson.databind.ser.SerializerFactory; -import com.fasterxml.jackson.databind.ser.Serializers; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import org.apache.arrow.vector.types.pojo.Schema; - -public class ObjectMapperFactoryV5 -{ - private static final JsonFactory JSON_FACTORY = new JsonFactory(); - private static final String LAMDA_EXCEPTION_CLASS_NAME = LambdaFunctionException.class.getName(); - - private static final SerializerFactory SERIALIZER_FACTORY; - - static { - // Serializers can be static since they don't need a BlockAllocator - ImmutableList> sers = ImmutableList.of(createRequestSerializer(), createResponseSerializer()); - SimpleSerializers serializers = new SimpleSerializers(sers); - SerializerFactoryConfig config = new SerializerFactoryConfig().withAdditionalSerializers(serializers); - SERIALIZER_FACTORY = new StrictSerializerFactory(config); - } - - private ObjectMapperFactoryV5() {} - - /** - * Custom SerializerFactory that *only* uses the custom serializers that we inject into the {@link ObjectMapper}. - */ - private static class StrictSerializerFactory extends BeanSerializerFactory - { - private StrictSerializerFactory(SerializerFactoryConfig config) - { - super(config); - } - - @Override - public StrictSerializerFactory withConfig(SerializerFactoryConfig config) - { - if (_factoryConfig == config) { - return this; - } - return new StrictSerializerFactory(config); - } - - @Override - @SuppressWarnings("unchecked") - public JsonSerializer createSerializer(SerializerProvider prov, JavaType origType) - throws JsonMappingException - { - for (Serializers serializers : customSerializers()) { - JsonSerializer ser = serializers.findSerializer(prov.getConfig(), origType, null); - if (ser != null) { - return (JsonSerializer) ser; - } - } - throw new IllegalArgumentException("No explicitly configured serializer for " + origType); - } - } - - /** - * Custom DeserializerFactory that *only* uses the custom deserializers that we inject into the {@link ObjectMapper}. - */ - private static class StrictDeserializerFactory extends BeanDeserializerFactory - { - private StrictDeserializerFactory(DeserializerFactoryConfig config) - { - super(config); - } - - @Override - public DeserializerFactory withConfig(DeserializerFactoryConfig config) - { - if (_factoryConfig == config) { - return this; - } - return new StrictDeserializerFactory(config); - } - - @Override - @SuppressWarnings("unchecked") - public JsonDeserializer createBeanDeserializer(DeserializationContext ctxt, JavaType type, BeanDescription beanDesc) - throws JsonMappingException - { - for (Deserializers d : _factoryConfig.deserializers()) { - JsonDeserializer deser = d.findBeanDeserializer(type, ctxt.getConfig(), beanDesc); - if (deser != null) { - return (JsonDeserializer) deser; - } - } - throw new IllegalArgumentException("No explicitly configured deserializer for " + type); - } - } - - /** - * Locked down ObjectMapper that only uses the serializers/deserializers provided and does not fall back to annotation or reflection - * based serialization. - */ - private static class StrictObjectMapper extends ObjectMapper - { - private StrictObjectMapper(BlockAllocator allocator) - { - super(JSON_FACTORY); - _serializerFactory = SERIALIZER_FACTORY; - - ImmutableMap, JsonDeserializer> desers = ImmutableMap.of( - FederationRequest.class, createRequestDeserializer(allocator), - FederationResponse.class, createResponseDeserializer(allocator), - LambdaFunctionException.class, new LambdaFunctionExceptionSerDe.Deserializer()); - SimpleDeserializers deserializers = new SimpleDeserializers(desers); - DeserializerFactoryConfig dConfig = new DeserializerFactoryConfig().withAdditionalDeserializers(deserializers); - _deserializationContext = new DefaultDeserializationContext.Impl(new StrictDeserializerFactory(dConfig)); - // required by LambdaInvokerFactory - disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); - } - } - - public static ObjectMapper create(BlockAllocator allocator) - { - return new StrictObjectMapper(allocator); - } - - private static FederationRequestSerDeV5.Serializer createRequestSerializer() - { - FederatedIdentitySerDe.Serializer identity = new FederatedIdentitySerDe.Serializer(); - TableNameSerDe.Serializer tableName = new TableNameSerDe.Serializer(); - VersionedSerDe.Serializer schema = new SchemaSerDeV4.Serializer(); - VersionedSerDe.Serializer block = new BlockSerDeV4.Serializer(schema); - ArrowTypeSerDe.Serializer arrowType = new ArrowTypeSerDe.Serializer(); - MarkerSerDe.Serializer marker = new MarkerSerDe.Serializer(block); - RangeSerDe.Serializer range = new RangeSerDe.Serializer(marker); - EquatableValueSetSerDe.Serializer equatableValueSet = new EquatableValueSetSerDe.Serializer(block); - SortedRangeSetSerDe.Serializer sortedRangeSet = new SortedRangeSetSerDe.Serializer(arrowType, range); - AllOrNoneValueSetSerDe.Serializer allOrNoneValueSet = new AllOrNoneValueSetSerDe.Serializer(arrowType); - ValueSetSerDe.Serializer valueSet = new ValueSetSerDe.Serializer(equatableValueSet, sortedRangeSet, allOrNoneValueSet); - VersionedSerDe.Serializer functionName = new FunctionNameSerDeV4.Serializer(); - ConstantExpressionSerDeV4.Serializer constantExpression = new ConstantExpressionSerDeV4.Serializer(block, arrowType); - FunctionCallExpressionSerDeV4.Serializer functionCallExpression = new FunctionCallExpressionSerDeV4.Serializer(functionName, arrowType); - VariableExpressionSerDeV4.Serializer variableExpression = new VariableExpressionSerDeV4.Serializer(arrowType); - VersionedSerDe.Serializer federationExpression = new FederationExpressionSerDeV4.Serializer(constantExpression, functionCallExpression, variableExpression); - functionCallExpression.setFederationExpressionSerializer(federationExpression); - VersionedSerDe.Serializer orderByField = new OrderByFieldSerDeV4.Serializer(); - VersionedSerDe.Serializer constraints = new ConstraintsSerDeV5.Serializer(valueSet, federationExpression, orderByField); - S3SpillLocationSerDe.Serializer s3SpillLocation = new S3SpillLocationSerDe.Serializer(); - SpillLocationSerDe.Serializer spillLocation = new SpillLocationSerDe.Serializer(s3SpillLocation); - EncryptionKeySerDe.Serializer encryptionKey = new EncryptionKeySerDe.Serializer(); - SplitSerDe.Serializer split = new SplitSerDe.Serializer(spillLocation, encryptionKey); - PingRequestSerDe.Serializer ping = new PingRequestSerDe.Serializer(identity); - ListSchemasRequestSerDe.Serializer listSchemas = new ListSchemasRequestSerDe.Serializer(identity); - ListTablesRequestSerDe.Serializer listTables = new ListTablesRequestSerDe.Serializer(identity); - GetTableRequestSerDeV5.Serializer getTable = new GetTableRequestSerDeV5.Serializer(identity, tableName); - GetTableLayoutRequestSerDe.Serializer getTableLayout = new GetTableLayoutRequestSerDe.Serializer(identity, tableName, constraints, schema); - GetSplitsRequestSerDe.Serializer getSplits = new GetSplitsRequestSerDe.Serializer(identity, tableName, block, constraints); - ReadRecordsRequestSerDe.Serializer readRecords = new ReadRecordsRequestSerDe.Serializer(identity, tableName, constraints, schema, split); - UserDefinedFunctionRequestSerDe.Serializer userDefinedFunction = new UserDefinedFunctionRequestSerDe.Serializer(identity, block, schema); - GetDataSourceCapabilitiesRequestSerDeV4.Serializer getDataSourceCapabilities = new GetDataSourceCapabilitiesRequestSerDeV4.Serializer(identity); - return new FederationRequestSerDeV5.Serializer( - ping, - listSchemas, - listTables, - getTable, - getTableLayout, - getSplits, - readRecords, - userDefinedFunction, - getDataSourceCapabilities); - } - - private static FederationRequestSerDeV5.Deserializer createRequestDeserializer(BlockAllocator allocator) - { - FederatedIdentitySerDe.Deserializer identity = new FederatedIdentitySerDe.Deserializer(); - TableNameSerDe.Deserializer tableName = new TableNameSerDe.Deserializer(); - VersionedSerDe.Deserializer schema = new SchemaSerDeV4.Deserializer(); - VersionedSerDe.Deserializer block = new BlockSerDeV4.Deserializer(allocator, schema); - ArrowTypeSerDe.Deserializer arrowType = new ArrowTypeSerDe.Deserializer(); - MarkerSerDe.Deserializer marker = new MarkerSerDe.Deserializer(block); - RangeSerDe.Deserializer range = new RangeSerDe.Deserializer(marker); - EquatableValueSetSerDe.Deserializer equatableValueSet = new EquatableValueSetSerDe.Deserializer(block); - SortedRangeSetSerDe.Deserializer sortedRangeSet = new SortedRangeSetSerDe.Deserializer(arrowType, range); - AllOrNoneValueSetSerDe.Deserializer allOrNoneValueSet = new AllOrNoneValueSetSerDe.Deserializer(arrowType); - ValueSetSerDe.Deserializer valueSet = new ValueSetSerDe.Deserializer(equatableValueSet, sortedRangeSet, allOrNoneValueSet); - - VersionedSerDe.Deserializer functionName = new FunctionNameSerDeV4.Deserializer(); - ConstantExpressionSerDeV4.Deserializer constantExpression = new ConstantExpressionSerDeV4.Deserializer(block, arrowType); - FunctionCallExpressionSerDeV4.Deserializer functionCallExpression = new FunctionCallExpressionSerDeV4.Deserializer(functionName, arrowType); - VariableExpressionSerDeV4.Deserializer variableExpression = new VariableExpressionSerDeV4.Deserializer(arrowType); - VersionedSerDe.Deserializer federationExpression = new FederationExpressionSerDeV4.Deserializer(constantExpression, functionCallExpression, variableExpression); - functionCallExpression.setFederationExpressionSerializer(federationExpression); - VersionedSerDe.Deserializer orderByField = new OrderByFieldSerDeV4.Deserializer(); - VersionedSerDe.Deserializer constraints = new ConstraintsSerDeV5.Deserializer(valueSet, federationExpression, orderByField); - - S3SpillLocationSerDe.Deserializer s3SpillLocation = new S3SpillLocationSerDe.Deserializer(); - SpillLocationSerDe.Deserializer spillLocation = new SpillLocationSerDe.Deserializer(s3SpillLocation); - EncryptionKeySerDe.Deserializer encryptionKey = new EncryptionKeySerDe.Deserializer(); - SplitSerDe.Deserializer split = new SplitSerDe.Deserializer(spillLocation, encryptionKey); - - PingRequestSerDe.Deserializer ping = new PingRequestSerDe.Deserializer(identity); - ListSchemasRequestSerDe.Deserializer listSchemas = new ListSchemasRequestSerDe.Deserializer(identity); - ListTablesRequestSerDe.Deserializer listTables = new ListTablesRequestSerDe.Deserializer(identity); - GetTableRequestSerDeV5.Deserializer getTable = new GetTableRequestSerDeV5.Deserializer(identity, tableName); - GetTableLayoutRequestSerDe.Deserializer getTableLayout = new GetTableLayoutRequestSerDe.Deserializer(identity, tableName, constraints, schema); - GetSplitsRequestSerDe.Deserializer getSplits = new GetSplitsRequestSerDe.Deserializer(identity, tableName, block, constraints); - ReadRecordsRequestSerDe.Deserializer readRecords = new ReadRecordsRequestSerDe.Deserializer(identity, tableName, constraints, schema, split); - UserDefinedFunctionRequestSerDe.Deserializer userDefinedFunction = new UserDefinedFunctionRequestSerDe.Deserializer(identity, block, schema); - GetDataSourceCapabilitiesRequestSerDeV4.Deserializer getDataSourceCapabilities = new GetDataSourceCapabilitiesRequestSerDeV4.Deserializer(identity); - - return new FederationRequestSerDeV5.Deserializer( - ping, - listSchemas, - listTables, - getTable, - getTableLayout, - getSplits, - readRecords, - userDefinedFunction, - getDataSourceCapabilities); - } - - private static FederationResponseSerDeV4.Serializer createResponseSerializer() - { - TableNameSerDe.Serializer tableName = new TableNameSerDe.Serializer(); - VersionedSerDe.Serializer schema = new SchemaSerDeV4.Serializer(); - VersionedSerDe.Serializer block = new BlockSerDeV4.Serializer(schema); - S3SpillLocationSerDe.Serializer s3SpillLocation = new S3SpillLocationSerDe.Serializer(); - SpillLocationSerDe.Serializer spillLocation = new SpillLocationSerDe.Serializer(s3SpillLocation); - EncryptionKeySerDe.Serializer encryptionKey = new EncryptionKeySerDe.Serializer(); - SplitSerDe.Serializer split = new SplitSerDe.Serializer(spillLocation, encryptionKey); - - PingResponseSerDe.Serializer ping = new PingResponseSerDe.Serializer(); - ListSchemasResponseSerDe.Serializer listSchemas = new ListSchemasResponseSerDe.Serializer(); - ListTablesResponseSerDe.Serializer listTables = new ListTablesResponseSerDe.Serializer(tableName); - GetTableResponseSerDe.Serializer getTable = new GetTableResponseSerDe.Serializer(tableName, schema); - GetTableLayoutResponseSerDe.Serializer getTableLayout = new GetTableLayoutResponseSerDe.Serializer(tableName, block); - GetSplitsResponseSerDe.Serializer getSplits = new GetSplitsResponseSerDe.Serializer(split); - ReadRecordsResponseSerDe.Serializer readRecords = new ReadRecordsResponseSerDe.Serializer(block); - RemoteReadRecordsResponseSerDe.Serializer remoteReadRecords = new RemoteReadRecordsResponseSerDe.Serializer(schema, spillLocation, encryptionKey); - UserDefinedFunctionResponseSerDe.Serializer userDefinedFunction = new UserDefinedFunctionResponseSerDe.Serializer(block); - VersionedSerDe.Serializer optimizationSubtype = new OptimizationSubTypeSerDeV4.Serializer(); - GetDataSourceCapabilitiesResponseSerDeV4.Serializer getDataSourceCapabilities = new GetDataSourceCapabilitiesResponseSerDeV4.Serializer(optimizationSubtype); - - return new FederationResponseSerDeV4.Serializer( - ping, - listSchemas, - listTables, - getTable, - getTableLayout, - getSplits, - readRecords, - remoteReadRecords, - userDefinedFunction, - getDataSourceCapabilities); - } - - private static FederationResponseSerDeV4.Deserializer createResponseDeserializer(BlockAllocator allocator) - { - TableNameSerDe.Deserializer tableName = new TableNameSerDe.Deserializer(); - VersionedSerDe.Deserializer schema = new SchemaSerDeV4.Deserializer(); - VersionedSerDe.Deserializer block = new BlockSerDeV4.Deserializer(allocator, schema); - S3SpillLocationSerDe.Deserializer s3SpillLocation = new S3SpillLocationSerDe.Deserializer(); - SpillLocationSerDe.Deserializer spillLocation = new SpillLocationSerDe.Deserializer(s3SpillLocation); - EncryptionKeySerDe.Deserializer encryptionKey = new EncryptionKeySerDe.Deserializer(); - SplitSerDe.Deserializer split = new SplitSerDe.Deserializer(spillLocation, encryptionKey); - - PingResponseSerDe.Deserializer ping = new PingResponseSerDe.Deserializer(); - ListSchemasResponseSerDe.Deserializer listSchemas = new ListSchemasResponseSerDe.Deserializer(); - ListTablesResponseSerDe.Deserializer listTables = new ListTablesResponseSerDe.Deserializer(tableName); - GetTableResponseSerDe.Deserializer getTable = new GetTableResponseSerDe.Deserializer(tableName, schema); - GetTableLayoutResponseSerDe.Deserializer getTableLayout = new GetTableLayoutResponseSerDe.Deserializer(tableName, block); - GetSplitsResponseSerDe.Deserializer getSplits = new GetSplitsResponseSerDe.Deserializer(split); - ReadRecordsResponseSerDe.Deserializer readRecords = new ReadRecordsResponseSerDe.Deserializer(block); - RemoteReadRecordsResponseSerDe.Deserializer remoteReadRecords = new RemoteReadRecordsResponseSerDe.Deserializer(schema, spillLocation, encryptionKey); - UserDefinedFunctionResponseSerDe.Deserializer userDefinedFunction = new UserDefinedFunctionResponseSerDe.Deserializer(block); - VersionedSerDe.Deserializer optimizationSubtype = new OptimizationSubTypeSerDeV4.Deserializer(); - GetDataSourceCapabilitiesResponseSerDeV4.Deserializer getDataSourceCapabilities = new GetDataSourceCapabilitiesResponseSerDeV4.Deserializer(optimizationSubtype); - - return new FederationResponseSerDeV4.Deserializer( - ping, - listSchemas, - listTables, - getTable, - getTableLayout, - getSplits, - readRecords, - remoteReadRecords, - userDefinedFunction, - getDataSourceCapabilities); - } -} diff --git a/athena-federation-sdk/src/test/java/com/amazonaws/athena/connector/lambda/handlers/CompositeHandlerTest.java b/athena-federation-sdk/src/test/java/com/amazonaws/athena/connector/lambda/handlers/CompositeHandlerTest.java index 0edcd92c06..89475c0361 100644 --- a/athena-federation-sdk/src/test/java/com/amazonaws/athena/connector/lambda/handlers/CompositeHandlerTest.java +++ b/athena-federation-sdk/src/test/java/com/amazonaws/athena/connector/lambda/handlers/CompositeHandlerTest.java @@ -29,7 +29,17 @@ import com.amazonaws.athena.connector.lambda.domain.predicate.Constraints; import com.amazonaws.athena.connector.lambda.domain.spill.S3SpillLocation; import com.amazonaws.athena.connector.lambda.domain.spill.SpillLocationVerifier; -import com.amazonaws.athena.connector.lambda.metadata.*; +import com.amazonaws.athena.connector.lambda.metadata.GetSplitsRequest; +import com.amazonaws.athena.connector.lambda.metadata.GetSplitsResponse; +import com.amazonaws.athena.connector.lambda.metadata.GetTableLayoutRequest; +import com.amazonaws.athena.connector.lambda.metadata.GetTableLayoutResponse; +import com.amazonaws.athena.connector.lambda.metadata.GetTableRequest; +import com.amazonaws.athena.connector.lambda.metadata.GetTableResponse; +import com.amazonaws.athena.connector.lambda.metadata.ListSchemasRequest; +import com.amazonaws.athena.connector.lambda.metadata.ListSchemasResponse; +import com.amazonaws.athena.connector.lambda.metadata.ListTablesRequest; +import com.amazonaws.athena.connector.lambda.metadata.ListTablesResponse; +import com.amazonaws.athena.connector.lambda.metadata.MetadataRequestType; import com.amazonaws.athena.connector.lambda.records.ReadRecordsRequest; import com.amazonaws.athena.connector.lambda.records.ReadRecordsResponse; import com.amazonaws.athena.connector.lambda.request.PingRequest; @@ -55,6 +65,7 @@ import java.util.UUID; import static com.amazonaws.athena.connector.lambda.domain.predicate.Constraints.DEFAULT_NO_LIMIT; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.mock; @@ -178,7 +189,6 @@ public void doGetTable() throws Exception { GetTableRequest req = mock(GetTableRequest.class); -// when(req.isQueryPassthrough()).thenReturn(false); when(req.getRequestType()).thenReturn(MetadataRequestType.GET_TABLE); compositeHandler.handleRequest(allocator, req, new ByteArrayOutputStream(), objectMapper); verify(mockMetadataHandler, times(1)).doGetTable(nullable(BlockAllocatorImpl.class), nullable(GetTableRequest.class)); diff --git a/athena-federation-sdk/src/test/java/com/amazonaws/athena/connector/lambda/serde/TypedSerDeTest.java b/athena-federation-sdk/src/test/java/com/amazonaws/athena/connector/lambda/serde/TypedSerDeTest.java index f83528d741..55f1e42ddc 100644 --- a/athena-federation-sdk/src/test/java/com/amazonaws/athena/connector/lambda/serde/TypedSerDeTest.java +++ b/athena-federation-sdk/src/test/java/com/amazonaws/athena/connector/lambda/serde/TypedSerDeTest.java @@ -30,8 +30,8 @@ import java.io.IOException; import java.util.Collections; +import java.util.List; -import static com.amazonaws.athena.connector.lambda.utils.TestUtils.SERDE_VERSION_FIVE; import static com.amazonaws.athena.connector.lambda.utils.TestUtils.SERDE_VERSION_FOUR; import static com.amazonaws.athena.connector.lambda.utils.TestUtils.SERDE_VERSION_TWO; @@ -41,7 +41,6 @@ public abstract class TypedSerDeTest protected BlockAllocator allocator; protected ObjectMapper mapper; protected ObjectMapper mapperV4; - protected ObjectMapper mapperV5; protected FederatedIdentity federatedIdentity = new FederatedIdentity("testArn", "0123456789", Collections.emptyMap(), Collections.emptyList()); protected String expectedSerDeText; protected T expected; @@ -52,10 +51,8 @@ public void before() allocator = new BlockAllocatorImpl("test-allocator-id"); mapper = VersionedObjectMapperFactory.create(allocator, SERDE_VERSION_TWO); mapperV4 = VersionedObjectMapperFactory.create(allocator, SERDE_VERSION_FOUR); - mapperV5 = VersionedObjectMapperFactory.create(allocator, SERDE_VERSION_FIVE); mapper.enable(SerializationFeature.INDENT_OUTPUT); mapperV4.enable(SerializationFeature.INDENT_OUTPUT); - mapperV5.enable(SerializationFeature.INDENT_OUTPUT); } @After diff --git a/athena-federation-sdk/src/test/java/com/amazonaws/athena/connector/lambda/serde/v2/GetTableRequestSerDeTest.java b/athena-federation-sdk/src/test/java/com/amazonaws/athena/connector/lambda/serde/v2/GetTableRequestSerDeTest.java index c44fb30e58..29d8a38589 100644 --- a/athena-federation-sdk/src/test/java/com/amazonaws/athena/connector/lambda/serde/v2/GetTableRequestSerDeTest.java +++ b/athena-federation-sdk/src/test/java/com/amazonaws/athena/connector/lambda/serde/v2/GetTableRequestSerDeTest.java @@ -33,7 +33,6 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; -import java.util.HashMap; import static org.junit.Assert.assertEquals; @@ -46,9 +45,6 @@ public void beforeTest() throws IOException { expected = new GetTableRequest(federatedIdentity, "test-query-id", "test-catalog", new TableName("test-schema", "test-table")); - HashMap queryPassthroughArguments = new HashMap<>(1); - queryPassthroughArguments.put("query", "SELECT * FROM DUMMY_TABLE;"); - ((GetTableRequest)expected).setQueryPassthroughArguments(queryPassthroughArguments); String expectedSerDeFile = utils.getResourceOrFail("serde/v2", "GetTableRequest.json"); expectedSerDeText = utils.readAllAsString(expectedSerDeFile).trim(); diff --git a/athena-federation-sdk/src/test/java/com/amazonaws/athena/connector/lambda/serde/v4/ReadRecordsRequestSerDeV4Test.java b/athena-federation-sdk/src/test/java/com/amazonaws/athena/connector/lambda/serde/v4/ReadRecordsRequestSerDeV4Test.java index 8bddd0418d..c6560f5b48 100644 --- a/athena-federation-sdk/src/test/java/com/amazonaws/athena/connector/lambda/serde/v4/ReadRecordsRequestSerDeV4Test.java +++ b/athena-federation-sdk/src/test/java/com/amazonaws/athena/connector/lambda/serde/v4/ReadRecordsRequestSerDeV4Test.java @@ -92,9 +92,6 @@ public void beforeTest() constraintsMap.put("col4", EquatableValueSet.newBuilder(allocator, Types.MinorType.FLOAT8.getType(), false, true).add(1.1D).build()); constraintsMap.put("col5", new AllOrNoneValueSet(Types.MinorType.FLOAT8.getType(), false, true)); - HashMap queryPassthroughArguments = new HashMap<>(1); - queryPassthroughArguments.put("query", "SELECT * FROM DUMMY_TABLE;"); - Block partitions = allocator.createBlock(schema); FederationExpression federationExpression = new FunctionCallExpression( @@ -115,7 +112,6 @@ public void beforeTest() ); Constraints constraints = new Constraints(constraintsMap, ImmutableList.of(federationExpression), orderByClause, DEFAULT_NO_LIMIT); - constraints.setQueryPassthroughArguments(queryPassthroughArguments); int num_partitions = 10; for (int i = 0; i < num_partitions; i++) { diff --git a/athena-federation-sdk/src/test/java/com/amazonaws/athena/connector/lambda/serde/v5/GetTableRequestSerDeV5Test.java b/athena-federation-sdk/src/test/java/com/amazonaws/athena/connector/lambda/serde/v5/GetTableRequestSerDeV5Test.java deleted file mode 100644 index 84de1e1807..0000000000 --- a/athena-federation-sdk/src/test/java/com/amazonaws/athena/connector/lambda/serde/v5/GetTableRequestSerDeV5Test.java +++ /dev/null @@ -1,89 +0,0 @@ -/*- - * #%L - * Amazon Athena Query Federation SDK - * %% - * Copyright (C) 2019 - 2020 Amazon Web Services - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ -package com.amazonaws.athena.connector.lambda.serde.v5; - -import com.amazonaws.athena.connector.lambda.domain.TableName; -import com.amazonaws.athena.connector.lambda.metadata.GetTableRequest; -import com.amazonaws.athena.connector.lambda.request.FederationRequest; -import com.amazonaws.athena.connector.lambda.serde.TypedSerDeTest; -import com.fasterxml.jackson.core.JsonEncoding; -import org.junit.Before; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.HashMap; - -import static org.junit.Assert.assertEquals; - -public class GetTableRequestSerDeV5Test extends TypedSerDeTest -{ - private static final Logger logger = LoggerFactory.getLogger(GetTableRequestSerDeV5Test.class); - - @Before - public void beforeTest() - throws IOException - { - expected = new GetTableRequest(federatedIdentity, "test-query-id", "test-catalog", new TableName("test-schema", "test-table")); - HashMap queryPassthroughArguments = new HashMap<>(1); - queryPassthroughArguments.put("query", "SELECT * FROM DUMMY_TABLE;"); - ((GetTableRequest)expected).setQueryPassthroughArguments(queryPassthroughArguments); - - String expectedSerDeFile = utils.getResourceOrFail("serde/v5", "GetTableRequest.json"); - expectedSerDeText = utils.readAllAsString(expectedSerDeFile).trim(); - } - - @Test - public void serialize() - throws IOException - { - logger.info("serialize: enter"); - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - - mapperV5.writeValue(outputStream, expected); - - String actual = new String(outputStream.toByteArray(), JsonEncoding.UTF8.getJavaName()); - logger.info("serialize: serialized text[{}]", actual); - - assertEquals(expectedSerDeText, actual); - - logger.info("serialize: exit"); - } - - @Test - public void deserialize() - throws IOException - { - logger.info("deserialize: enter"); - InputStream input = new ByteArrayInputStream(expectedSerDeText.getBytes()); - - GetTableRequest actual = (GetTableRequest) mapperV5.readValue(input, FederationRequest.class); - - logger.info("deserialize: deserialized[{}]", actual); - - assertEquals(expected, actual); - - logger.info("deserialize: exit"); - } -} diff --git a/athena-federation-sdk/src/test/java/com/amazonaws/athena/connector/lambda/serde/v5/ObjectMapperFactoryV5Test.java b/athena-federation-sdk/src/test/java/com/amazonaws/athena/connector/lambda/serde/v5/ObjectMapperFactoryV5Test.java deleted file mode 100644 index b3df328828..0000000000 --- a/athena-federation-sdk/src/test/java/com/amazonaws/athena/connector/lambda/serde/v5/ObjectMapperFactoryV5Test.java +++ /dev/null @@ -1,56 +0,0 @@ -/*- - * #%L - * Amazon Athena Query Federation SDK - * %% - * Copyright (C) 2019 - 2020 Amazon Web Services - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ -package com.amazonaws.athena.connector.lambda.serde.v5; - -import com.amazonaws.athena.connector.lambda.data.BlockAllocator; -import com.amazonaws.athena.connector.lambda.data.BlockAllocatorImpl; -import com.amazonaws.athena.connector.lambda.serde.VersionedObjectMapperFactory; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonMappingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.arrow.vector.types.pojo.ArrowType; -import org.junit.Test; - -import java.io.IOException; - -import static com.amazonaws.athena.connector.lambda.utils.TestUtils.SERDE_VERSION_FIVE; - -public class ObjectMapperFactoryV5Test -{ - @Test(expected = JsonMappingException.class) - public void testStrictSerializer() - throws JsonProcessingException - { - try (BlockAllocator allocator = new BlockAllocatorImpl()) { - ObjectMapper mapper = VersionedObjectMapperFactory.create(allocator, SERDE_VERSION_FIVE); - mapper.writeValueAsString(new ArrowType.Null()); - } - } - - @Test(expected = JsonMappingException.class) - public void testStrictDeserializer() - throws IOException - { - try (BlockAllocator allocator = new BlockAllocatorImpl()) { - ObjectMapper mapper = VersionedObjectMapperFactory.create(allocator, SERDE_VERSION_FIVE); - mapper.readValue("{\"@type\" : \"FloatingPoint\", \"precision\" : \"DOUBLE\"}", ArrowType.FloatingPoint.class); - } - } -} diff --git a/athena-federation-sdk/src/test/java/com/amazonaws/athena/connector/lambda/serde/v5/ReadRecordsRequestSerDeV5Test.java b/athena-federation-sdk/src/test/java/com/amazonaws/athena/connector/lambda/serde/v5/ReadRecordsRequestSerDeV5Test.java deleted file mode 100644 index 3d1d273738..0000000000 --- a/athena-federation-sdk/src/test/java/com/amazonaws/athena/connector/lambda/serde/v5/ReadRecordsRequestSerDeV5Test.java +++ /dev/null @@ -1,189 +0,0 @@ -/*- - * #%L - * Amazon Athena Query Federation SDK - * %% - * Copyright (C) 2019 - 2020 Amazon Web Services - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ -package com.amazonaws.athena.connector.lambda.serde.v5; - -import com.amazonaws.athena.connector.lambda.data.Block; -import com.amazonaws.athena.connector.lambda.data.BlockUtils; -import com.amazonaws.athena.connector.lambda.data.SchemaBuilder; -import com.amazonaws.athena.connector.lambda.domain.Split; -import com.amazonaws.athena.connector.lambda.domain.TableName; -import com.amazonaws.athena.connector.lambda.domain.predicate.AllOrNoneValueSet; -import com.amazonaws.athena.connector.lambda.domain.predicate.Constraints; -import com.amazonaws.athena.connector.lambda.domain.predicate.EquatableValueSet; -import com.amazonaws.athena.connector.lambda.domain.predicate.OrderByField; -import com.amazonaws.athena.connector.lambda.domain.predicate.Range; -import com.amazonaws.athena.connector.lambda.domain.predicate.SortedRangeSet; -import com.amazonaws.athena.connector.lambda.domain.predicate.ValueSet; -import com.amazonaws.athena.connector.lambda.domain.predicate.expression.ConstantExpression; -import com.amazonaws.athena.connector.lambda.domain.predicate.expression.FederationExpression; -import com.amazonaws.athena.connector.lambda.domain.predicate.expression.FunctionCallExpression; -import com.amazonaws.athena.connector.lambda.domain.predicate.expression.VariableExpression; -import com.amazonaws.athena.connector.lambda.domain.predicate.functions.StandardFunctions; -import com.amazonaws.athena.connector.lambda.domain.spill.S3SpillLocation; -import com.amazonaws.athena.connector.lambda.domain.spill.SpillLocation; -import com.amazonaws.athena.connector.lambda.records.ReadRecordsRequest; -import com.amazonaws.athena.connector.lambda.request.FederationRequest; -import com.amazonaws.athena.connector.lambda.security.EncryptionKey; -import com.amazonaws.athena.connector.lambda.serde.TypedSerDeTest; -import com.fasterxml.jackson.core.JsonEncoding; -import com.google.common.collect.ImmutableList; -import org.apache.arrow.vector.types.Types; -import org.apache.arrow.vector.types.pojo.ArrowType; -import org.apache.arrow.vector.types.pojo.Schema; -import org.junit.Before; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static com.amazonaws.athena.connector.lambda.domain.predicate.Constraints.DEFAULT_NO_LIMIT; -import static org.junit.Assert.assertEquals; - -public class ReadRecordsRequestSerDeV5Test extends TypedSerDeTest -{ - private static final Logger logger = LoggerFactory.getLogger(ReadRecordsRequestSerDeV5Test.class); - - @Before - public void beforeTest() - throws IOException - { - String yearCol = "year"; - String monthCol = "month"; - String dayCol = "day"; - - Schema schema = SchemaBuilder.newBuilder() - .addField(yearCol, new ArrowType.Int(32, true)) - .addField(monthCol, new ArrowType.Int(32, true)) - .addField(dayCol, new ArrowType.Int(32, true)) - .addField("col2", new ArrowType.Utf8()) - .addField("col3", Types.MinorType.FLOAT8.getType()) - .addField("col4", Types.MinorType.FLOAT8.getType()) - .addField("col5", Types.MinorType.FLOAT8.getType()) - .build(); - - Map constraintsMap = new HashMap<>(); - constraintsMap.put("col3", SortedRangeSet.copyOf(Types.MinorType.FLOAT8.getType(), - ImmutableList.of(Range.greaterThan(allocator, Types.MinorType.FLOAT8.getType(), -10000D)), false)); - constraintsMap.put("col4", EquatableValueSet.newBuilder(allocator, Types.MinorType.FLOAT8.getType(), false, true).add(1.1D).build()); - constraintsMap.put("col5", new AllOrNoneValueSet(Types.MinorType.FLOAT8.getType(), false, true)); - - HashMap queryPassthroughArguments = new HashMap<>(1); - queryPassthroughArguments.put("query", "SELECT * FROM DUMMY_TABLE;"); - - Block partitions = allocator.createBlock(schema); - - FederationExpression federationExpression = new FunctionCallExpression( - ArrowType.Bool.INSTANCE, - StandardFunctions.GREATER_THAN_OPERATOR_FUNCTION_NAME.getFunctionName(), - ImmutableList.of(new FunctionCallExpression( - Types.MinorType.FLOAT8.getType(), - StandardFunctions.ADD_FUNCTION_NAME.getFunctionName(), - ImmutableList.of(new VariableExpression("col3", Types.MinorType.FLOAT8.getType()), - new ConstantExpression( - BlockUtils.newBlock(allocator, "col1", new ArrowType.Int(32, true), ImmutableList.of(10)), - new ArrowType.Int(32, true)))), - new VariableExpression("col2", Types.MinorType.FLOAT8.getType()))); - - List orderByClause = ImmutableList.of( - new OrderByField("col3", OrderByField.Direction.ASC_NULLS_FIRST), - new OrderByField("col2", OrderByField.Direction.DESC_NULLS_FIRST) - ); - - Constraints constraints = new Constraints(constraintsMap, ImmutableList.of(federationExpression), orderByClause, DEFAULT_NO_LIMIT); - constraints.setQueryPassthroughArguments(queryPassthroughArguments); - - int num_partitions = 10; - for (int i = 0; i < num_partitions; i++) { - BlockUtils.setValue(partitions.getFieldVector(yearCol), i, 2016 + i); - BlockUtils.setValue(partitions.getFieldVector(monthCol), i, (i % 12) + 1); - BlockUtils.setValue(partitions.getFieldVector(dayCol), i, (i % 28) + 1); - } - partitions.setRowCount(num_partitions); - - SpillLocation spillLocation = S3SpillLocation.newBuilder() - .withBucket("athena-virtuoso-test") - .withPrefix("lambda-spill") - .withQueryId("test-query-id") - .withSplitId("test-split-id") - .withIsDirectory(true) - .build(); - EncryptionKey encryptionKey = new EncryptionKey("test-key".getBytes(), "test-nonce".getBytes()); - Split split = Split.newBuilder(spillLocation, encryptionKey) - .add("year", "2017") - .add("month", "11") - .add("day", "1") - .build(); - - expected = new ReadRecordsRequest(federatedIdentity, - "test-query-id", - "test-catalog", - new TableName("test-schema", "test-table"), - schema, - split, - constraints, - 100_000_000_000L, - 100_000_000_000L); - - - String expectedSerDeFile = utils.getResourceOrFail("serde/v5", "ReadRecordsRequest.json"); - expectedSerDeText = utils.readAllAsString(expectedSerDeFile).trim(); - } - - @Test - public void serialize() - throws Exception - { - logger.info("serialize: enter"); - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - - mapperV5.writeValue(outputStream, expected); - - String actual = new String(outputStream.toByteArray(), JsonEncoding.UTF8.getJavaName()); - logger.info("serialize: serialized text[{}]", actual); - - assertEquals(expectedSerDeText, actual); - expected.close(); - - logger.info("serialize: exit"); - } - - @Test - public void deserialize() - throws IOException - { - logger.info("deserialize: enter"); - InputStream input = new ByteArrayInputStream(expectedSerDeText.getBytes()); - - ReadRecordsRequest actual = (ReadRecordsRequest) mapperV5.readValue(input, FederationRequest.class); - - logger.info("deserialize: deserialized[{}]", actual); - - assertEquals(expected, actual); - - logger.info("deserialize: exit"); - } -} diff --git a/athena-federation-sdk/src/test/java/com/amazonaws/athena/connector/lambda/utils/TestUtils.java b/athena-federation-sdk/src/test/java/com/amazonaws/athena/connector/lambda/utils/TestUtils.java index 70fa45fb95..35777e3cf6 100644 --- a/athena-federation-sdk/src/test/java/com/amazonaws/athena/connector/lambda/utils/TestUtils.java +++ b/athena-federation-sdk/src/test/java/com/amazonaws/athena/connector/lambda/utils/TestUtils.java @@ -39,7 +39,6 @@ public class TestUtils public final static int SERDE_VERSION_TWO = 2; public final static int SERDE_VERSION_THREE = 3; public final static int SERDE_VERSION_FOUR = 4; - public final static int SERDE_VERSION_FIVE = 5; /** * Helper to retrieve resources from the class path and enforce they are found diff --git a/athena-federation-sdk/src/test/resources/serde/v2/GetTableRequest.json b/athena-federation-sdk/src/test/resources/serde/v2/GetTableRequest.json index 3a34af1054..8888b66e91 100644 --- a/athena-federation-sdk/src/test/resources/serde/v2/GetTableRequest.json +++ b/athena-federation-sdk/src/test/resources/serde/v2/GetTableRequest.json @@ -13,8 +13,5 @@ "tableName" : { "schemaName" : "test-schema", "tableName" : "test-table" - }, - "queryPassthroughArguments" : { - "query" : "SELECT * FROM DUMMY_TABLE;" } } \ No newline at end of file diff --git a/athena-federation-sdk/src/test/resources/serde/v4/ReadRecordsRequest.json b/athena-federation-sdk/src/test/resources/serde/v4/ReadRecordsRequest.json index e0455cc7ee..c010929a6c 100644 --- a/athena-federation-sdk/src/test/resources/serde/v4/ReadRecordsRequest.json +++ b/athena-federation-sdk/src/test/resources/serde/v4/ReadRecordsRequest.json @@ -135,10 +135,7 @@ "columnName" : "col2", "direction" : "DESC_NULLS_FIRST" } ], - "limit" : -1, - "queryPassthroughArguments" : { - "query" : "SELECT * FROM DUMMY_TABLE;" - } + "limit" : -1 }, "maxBlockSize" : "100000000000", "maxInlineBlockSize" : "100000000000" diff --git a/athena-federation-sdk/src/test/resources/serde/v5/GetTableRequest.json b/athena-federation-sdk/src/test/resources/serde/v5/GetTableRequest.json deleted file mode 100644 index 3a34af1054..0000000000 --- a/athena-federation-sdk/src/test/resources/serde/v5/GetTableRequest.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "@type" : "GetTableRequest", - "identity" : { - "id" : "UNKNOWN", - "principal" : "UNKNOWN", - "account" : "0123456789", - "arn" : "testArn", - "tags" : { }, - "groups" : [ ] - }, - "queryId" : "test-query-id", - "catalogName" : "test-catalog", - "tableName" : { - "schemaName" : "test-schema", - "tableName" : "test-table" - }, - "queryPassthroughArguments" : { - "query" : "SELECT * FROM DUMMY_TABLE;" - } -} \ No newline at end of file diff --git a/athena-federation-sdk/src/test/resources/serde/v5/ReadRecordsRequest.json b/athena-federation-sdk/src/test/resources/serde/v5/ReadRecordsRequest.json deleted file mode 100644 index e0455cc7ee..0000000000 --- a/athena-federation-sdk/src/test/resources/serde/v5/ReadRecordsRequest.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "@type" : "ReadRecordsRequest", - "identity" : { - "id" : "UNKNOWN", - "principal" : "UNKNOWN", - "account" : "0123456789", - "arn" : "testArn", - "tags" : { }, - "groups" : [ ] - }, - "queryId" : "test-catalog", - "catalogName" : "test-query-id", - "tableName" : { - "schemaName" : "test-schema", - "tableName" : "test-table" - }, - "schema" : "//////ABAAAQAAAAAAAKAA4ABgANAAgACgAAAAAABAAQAAAAAAEKAAwAAAAIAAQACgAAAAgAAAAIAAAAAAAAAAcAAABwAQAAJAEAAOwAAAC0AAAAdAAAADwAAAAEAAAAvv7//xQAAAAUAAAAFAAAAAAAAwEUAAAAAAAAAAAAAACW////AAACAAQAAABjb2w1AAAAAPL+//8UAAAAFAAAABQAAAAAAAMBFAAAAAAAAAAAAAAAyv///wAAAgAEAAAAY29sNAAAAAAm////FAAAABQAAAAcAAAAAAADARwAAAAAAAAAAAAAAAAABgAIAAYABgAAAAAAAgAEAAAAY29sMwAAAABi////FAAAABQAAAAYAAAAAAAFARQAAAAAAAAAAAAAAAQABAAEAAAABAAAAGNvbDIAAAAAlv///xQAAAAUAAAAFAAAAAAAAgEYAAAAAAAAAAAAAACE////AAAAASAAAAADAAAAZGF5AMr///8UAAAAFAAAABQAAAAAAAIBGAAAAAAAAAAAAAAAuP///wAAAAEgAAAABQAAAG1vbnRoABIAGAAUABMAEgAMAAAACAAEABIAAAAUAAAAFAAAABwAAAAAAAIBIAAAAAAAAAAAAAAACAAMAAgABwAIAAAAAAAAASAAAAAEAAAAeWVhcgAAAAAAAAAA", - "split" : { - "spillLocation" : { - "@type" : "S3SpillLocation", - "bucket" : "athena-virtuoso-test", - "key" : "lambda-spill/test-query-id/test-split-id", - "directory" : true - }, - "encryptionKey" : { - "key" : "dGVzdC1rZXk=", - "nonce" : "dGVzdC1ub25jZQ==" - }, - "properties" : { - "month" : "11", - "year" : "2017", - "day" : "1" - } - }, - "constraints" : { - "summary" : { - "col4" : { - "@type" : "EquatableValueSet", - "valueBlock" : { - "aId" : "test-allocator-id", - "schema" : "/////5AAAAAQAAAAAAAKAA4ABgANAAgACgAAAAAABAAQAAAAAAEKAAwAAAAIAAQACgAAAAgAAAAIAAAAAAAAAAEAAAAYAAAAAAASABgAFAATABIADAAAAAgABAASAAAAFAAAABQAAAAcAAAAAAADARwAAAAAAAAAAAAAAAAABgAIAAYABgAAAAAAAgAEAAAAY29sMQAAAAA=", - "records" : "/////4gAAAAUAAAAAAAAAAwAFgAOABUAEAAEAAwAAAAQAAAAAAAAAAAABAAQAAAAAAMKABgADAAIAAQACgAAABQAAAA4AAAAAQAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAQAAAAAAAAAIAAAAAAAAAAgAAAAAAAAAAAAAAAEAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAACamZmZmZnxPw==" - }, - "whiteList" : false, - "nullAllowed" : true - }, - "col5" : { - "@type" : "AllOrNoneValueSet", - "type" : { - "@type" : "FloatingPoint", - "precision" : "DOUBLE" - }, - "all" : false, - "nullAllowed" : true - }, - "col3" : { - "@type" : "SortedRangeSet", - "type" : { - "@type" : "FloatingPoint", - "precision" : "DOUBLE" - }, - "ranges" : [ { - "low" : { - "valueBlock" : { - "aId" : "test-allocator-id", - "schema" : "/////5AAAAAQAAAAAAAKAA4ABgANAAgACgAAAAAABAAQAAAAAAEKAAwAAAAIAAQACgAAAAgAAAAIAAAAAAAAAAEAAAAYAAAAAAASABgAFAATABIADAAAAAgABAASAAAAFAAAABQAAAAcAAAAAAADARwAAAAAAAAAAAAAAAAABgAIAAYABgAAAAAAAgAEAAAAY29sMQAAAAA=", - "records" : "/////4gAAAAUAAAAAAAAAAwAFgAOABUAEAAEAAwAAAAQAAAAAAAAAAAABAAQAAAAAAMKABgADAAIAAQACgAAABQAAAA4AAAAAQAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAQAAAAAAAAAIAAAAAAAAAAgAAAAAAAAAAAAAAAEAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAIjDwA==" - }, - "bound" : "ABOVE", - "nullValue" : false - }, - "high" : { - "valueBlock" : { - "aId" : "test-allocator-id", - "schema" : "/////5AAAAAQAAAAAAAKAA4ABgANAAgACgAAAAAABAAQAAAAAAEKAAwAAAAIAAQACgAAAAgAAAAIAAAAAAAAAAEAAAAYAAAAAAASABgAFAATABIADAAAAAgABAASAAAAFAAAABQAAAAcAAAAAAADARwAAAAAAAAAAAAAAAAABgAIAAYABgAAAAAAAgAEAAAAY29sMQAAAAA=", - "records" : "" - }, - "bound" : "BELOW", - "nullValue" : true - } - } ], - "nullAllowed" : false - } - }, - "expression" : [ { - "@type" : "FunctionCallExpression", - "type" : { - "@type" : "Bool" - }, - "functionName" : { - "functionName" : "$greater_than" - }, - "arguments" : [ { - "@type" : "FunctionCallExpression", - "type" : { - "@type" : "FloatingPoint", - "precision" : "DOUBLE" - }, - "functionName" : { - "functionName" : "$add" - }, - "arguments" : [ { - "@type" : "VariableExpression", - "columnName" : "col3", - "type" : { - "@type" : "FloatingPoint", - "precision" : "DOUBLE" - } - }, { - "@type" : "ConstantExpression", - "valueBlock" : { - "aId" : "test-allocator-id", - "schema" : "/////5gAAAAQAAAAAAAKAA4ABgANAAgACgAAAAAABAAQAAAAAAEKAAwAAAAIAAQACgAAAAgAAAAIAAAAAAAAAAEAAAAYAAAAAAASABgAFAATABIADAAAAAgABAASAAAAFAAAABQAAAAcAAAAAAACASAAAAAAAAAAAAAAAAgADAAIAAcACAAAAAAAAAEgAAAABAAAAGNvbDEAAAAAAAAAAA==", - "records" : "/////4gAAAAUAAAAAAAAAAwAFgAOABUAEAAEAAwAAAAQAAAAAAAAAAAABAAQAAAAAAMKABgADAAIAAQACgAAABQAAAA4AAAAAQAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAQAAAAAAAAAIAAAAAAAAAAQAAAAAAAAAAAAAAAEAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAKAAAAAAAAAA==" - }, - "type" : { - "@type" : "Int", - "bitWidth" : 32, - "isSigned" : true - } - } ] - }, { - "@type" : "VariableExpression", - "columnName" : "col2", - "type" : { - "@type" : "FloatingPoint", - "precision" : "DOUBLE" - } - } ] - } ], - "orderByClause" : [ { - "columnName" : "col3", - "direction" : "ASC_NULLS_FIRST" - }, { - "columnName" : "col2", - "direction" : "DESC_NULLS_FIRST" - } ], - "limit" : -1, - "queryPassthroughArguments" : { - "query" : "SELECT * FROM DUMMY_TABLE;" - } - }, - "maxBlockSize" : "100000000000", - "maxInlineBlockSize" : "100000000000" -} diff --git a/athena-gcs/pom.xml b/athena-gcs/pom.xml index 45788c74d0..0ad722f621 100644 --- a/athena-gcs/pom.xml +++ b/athena-gcs/pom.xml @@ -3,7 +3,7 @@ aws-athena-query-federation com.amazonaws - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 4.0.0 athena-gcs @@ -12,7 +12,7 @@ com.amazonaws aws-athena-federation-sdk - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 @@ -81,7 +81,7 @@ com.amazonaws athena-federation-integ-test - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 com.google.guava @@ -90,6 +90,7 @@ test + org.testng testng @@ -114,6 +115,7 @@ ${mockito.version} test + software.amazon.awscdk logs diff --git a/athena-google-bigquery/pom.xml b/athena-google-bigquery/pom.xml index 5892f22d58..05e9ed650b 100644 --- a/athena-google-bigquery/pom.xml +++ b/athena-google-bigquery/pom.xml @@ -3,7 +3,7 @@ aws-athena-query-federation com.amazonaws - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 4.0.0 athena-google-bigquery @@ -12,7 +12,7 @@ com.amazonaws athena-federation-integ-test - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 test diff --git a/athena-hbase/pom.xml b/athena-hbase/pom.xml index d438ae61b5..8387be7dfa 100644 --- a/athena-hbase/pom.xml +++ b/athena-hbase/pom.xml @@ -3,7 +3,7 @@ aws-athena-query-federation com.amazonaws - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 4.0.0 athena-hbase @@ -21,7 +21,7 @@ com.amazonaws aws-athena-federation-sdk - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 withdep @@ -34,7 +34,7 @@ com.amazonaws athena-federation-integ-test - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 withdep test diff --git a/athena-hortonworks-hive/pom.xml b/athena-hortonworks-hive/pom.xml index 44cdd3e0ce..b5d99f9ef0 100644 --- a/athena-hortonworks-hive/pom.xml +++ b/athena-hortonworks-hive/pom.xml @@ -3,7 +3,7 @@ aws-athena-query-federation com.amazonaws - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 4.0.0 athena-hortonworks-hive @@ -15,7 +15,7 @@ com.amazonaws athena-federation-integ-test - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 test @@ -47,6 +47,7 @@ ${mockito.version} test + com.amazonaws diff --git a/athena-jdbc/pom.xml b/athena-jdbc/pom.xml index 5ad6b93350..48e0650560 100644 --- a/athena-jdbc/pom.xml +++ b/athena-jdbc/pom.xml @@ -3,11 +3,11 @@ aws-athena-query-federation com.amazonaws - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 4.0.0 athena-jdbc - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 com.amazonaws @@ -93,7 +93,7 @@ com.amazonaws aws-athena-federation-sdk - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 withdep @@ -106,7 +106,7 @@ com.amazonaws athena-federation-integ-test - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 test @@ -268,23 +268,6 @@ - - org.apache.maven.plugins - maven-source-plugin - 3.2.1 - - - attach-sources - - jar-no-fork - - - - - true - false - - diff --git a/athena-jdbc/src/main/java/com/amazonaws/athena/connectors/jdbc/manager/JdbcMetadataHandler.java b/athena-jdbc/src/main/java/com/amazonaws/athena/connectors/jdbc/manager/JdbcMetadataHandler.java index ebaaea2e17..77939d826e 100644 --- a/athena-jdbc/src/main/java/com/amazonaws/athena/connectors/jdbc/manager/JdbcMetadataHandler.java +++ b/athena-jdbc/src/main/java/com/amazonaws/athena/connectors/jdbc/manager/JdbcMetadataHandler.java @@ -36,12 +36,10 @@ import com.amazonaws.athena.connector.lambda.metadata.ListSchemasResponse; import com.amazonaws.athena.connector.lambda.metadata.ListTablesRequest; import com.amazonaws.athena.connector.lambda.metadata.ListTablesResponse; -import com.amazonaws.athena.connector.lambda.metadata.optimizations.qpt.QueryPassthroughSignature; import com.amazonaws.athena.connectors.jdbc.connection.DatabaseConnectionConfig; import com.amazonaws.athena.connectors.jdbc.connection.JdbcConnectionFactory; import com.amazonaws.athena.connectors.jdbc.connection.JdbcCredentialProvider; import com.amazonaws.athena.connectors.jdbc.connection.RdsSecretsCredentialProvider; -import com.amazonaws.athena.connectors.jdbc.qpt.JdbcQueryPassthrough; import com.amazonaws.athena.connectors.jdbc.splits.Splitter; import com.amazonaws.athena.connectors.jdbc.splits.SplitterFactory; import com.amazonaws.services.athena.AmazonAthena; @@ -60,13 +58,10 @@ import java.sql.Connection; import java.sql.DatabaseMetaData; -import java.sql.PreparedStatement; import java.sql.ResultSet; -import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.Set; @@ -245,65 +240,6 @@ public GetTableResponse doGetTable(final BlockAllocator blockAllocator, final Ge } } - @Override - public GetTableResponse doGetQueryPassthroughSchema(final BlockAllocator blockAllocator, final GetTableRequest getTableRequest) - throws Exception - { - if (!getTableRequest.isQueryPassthrough()) { - throw new IllegalArgumentException("No Query passed through [{}]" + getTableRequest); - } - - JdbcQueryPassthrough qpt = JdbcQueryPassthrough.getInstance(); - QueryPassthroughSignature.verifyQueryPassthroughArguments( - getTableRequest.getQueryPassthroughArguments(), qpt.arguments()); - //For JDBC; generally speaking we only have one argument; namely "QUERY" -- Other connectors might need more - String customerPassedQuery = getTableRequest.getQueryPassthroughArguments().get(qpt.getQueryArgument()); - - try (Connection connection = getJdbcConnectionFactory().getConnection(getCredentialProvider())) { - PreparedStatement preparedStatement = connection.prepareStatement(customerPassedQuery); - ResultSetMetaData metadata = preparedStatement.getMetaData(); - if (metadata == null) { - throw new UnsupportedOperationException("Query not supported: ResultSetMetaData not available for query: " + customerPassedQuery); - } - SchemaBuilder schemaBuilder = SchemaBuilder.newBuilder(); - - for (int columnIndex = 1; columnIndex <= metadata.getColumnCount(); columnIndex++) { - String columnName = metadata.getColumnName(columnIndex); - - int precision = metadata.getPrecision(columnIndex); - int scale = metadata.getScale(columnIndex); - int decimalDigits = precision - scale; - - ArrowType columnType = JdbcArrowTypeConverter.toArrowType( - metadata.getColumnType(columnIndex), - metadata.getColumnDisplaySize(columnIndex), - decimalDigits, - configOptions); - - if (columnType != null && SupportedTypes.isSupported(columnType)) { - if (columnType instanceof ArrowType.List) { - schemaBuilder.addListField(columnName, getArrayArrowTypeFromTypeName( - metadata.getTableName(columnIndex), - metadata.getColumnDisplaySize(columnIndex), - decimalDigits)); - } - else { - schemaBuilder.addField(FieldBuilder.newBuilder(columnName, columnType).build()); - } - } - else { - // Default to VARCHAR ArrowType - LOGGER.warn("getSchema: Unable to map type for column[" + columnName + - "] to a supported type, attempted " + columnType + " - defaulting type to VARCHAR."); - schemaBuilder.addField(FieldBuilder.newBuilder(columnName, new ArrowType.Utf8()).build()); - } - } - - Schema schema = schemaBuilder.build(); - return new GetTableResponse(getTableRequest.getCatalogName(), getTableRequest.getTableName(), schema, Collections.emptySet()); - } - } - /** * While being a no-op by default, this function will be overriden by subclasses that support this search. * diff --git a/athena-jdbc/src/main/java/com/amazonaws/athena/connectors/jdbc/qpt/JdbcQueryPassthrough.java b/athena-jdbc/src/main/java/com/amazonaws/athena/connectors/jdbc/qpt/JdbcQueryPassthrough.java deleted file mode 100644 index 8239df1f6c..0000000000 --- a/athena-jdbc/src/main/java/com/amazonaws/athena/connectors/jdbc/qpt/JdbcQueryPassthrough.java +++ /dev/null @@ -1,82 +0,0 @@ -/*- - * #%L - * athena-jdbc - * %% - * Copyright (C) 2019 Amazon Web Services - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ -package com.amazonaws.athena.connectors.jdbc.qpt; - -import com.amazonaws.athena.connector.lambda.metadata.optimizations.qpt.QueryPassthroughSignature; - -import java.util.ArrayList; -import java.util.List; - -/** - * A Singleton class that implements QPT signature interface to define - * the JDBC Query Passthrough Function's signature that will be used - * to inform the engine how to define QPT Function for a JDBC connector - */ -public class JdbcQueryPassthrough implements QueryPassthroughSignature -{ - private String name; - private String domain; - private List arguments; - private static JdbcQueryPassthrough instance; - - static - { - JdbcQueryPassthrough.getInstance(); - } - - private JdbcQueryPassthrough() - { - this.name = "query"; - this.domain = "system"; - this.arguments = new ArrayList<>(1); - arguments.add("QUERY"); - } - - public static JdbcQueryPassthrough getInstance() - { - if (instance == null) { - instance = new JdbcQueryPassthrough(); - } - return instance; - } - - @Override - public String getName() - { - return name; - } - - @Override - public String getDomain() - { - return domain; - } - - @Override - public List arguments() - { - return arguments; - } - - public String getQueryArgument() - { - return this.arguments.get(0); - } -} diff --git a/athena-kafka/pom.xml b/athena-kafka/pom.xml index 51027ed27f..66190dea49 100644 --- a/athena-kafka/pom.xml +++ b/athena-kafka/pom.xml @@ -3,7 +3,7 @@ aws-athena-query-federation com.amazonaws - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 4.0.0 athena-kafka @@ -68,7 +68,7 @@ com.amazonaws athena-federation-integ-test - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 withdep test @@ -104,10 +104,11 @@ com.amazonaws aws-athena-federation-sdk - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 withdep + org.testng testng diff --git a/athena-msk/pom.xml b/athena-msk/pom.xml index 09d3ab8449..b55889e062 100644 --- a/athena-msk/pom.xml +++ b/athena-msk/pom.xml @@ -3,7 +3,7 @@ aws-athena-query-federation com.amazonaws - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 4.0.0 athena-msk @@ -68,7 +68,7 @@ com.amazonaws athena-federation-integ-test - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 withdep test @@ -110,7 +110,7 @@ com.amazonaws aws-athena-federation-sdk - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 withdep diff --git a/athena-mysql/pom.xml b/athena-mysql/pom.xml index 9a77171264..c3ae651f7a 100644 --- a/athena-mysql/pom.xml +++ b/athena-mysql/pom.xml @@ -3,27 +3,27 @@ aws-athena-query-federation com.amazonaws - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 4.0.0 athena-mysql - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 com.amazonaws athena-federation-integ-test - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 test com.amazonaws athena-jdbc - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 com.amazonaws athena-jdbc - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 test-jar test @@ -98,23 +98,6 @@ - - org.apache.maven.plugins - maven-source-plugin - 3.2.1 - - - attach-sources - - jar-no-fork - - - - - true - false - - diff --git a/athena-mysql/src/main/java/com/amazonaws/athena/connectors/mysql/MySqlMetadataHandler.java b/athena-mysql/src/main/java/com/amazonaws/athena/connectors/mysql/MySqlMetadataHandler.java index 6e6c963ee2..9fb4209e7d 100644 --- a/athena-mysql/src/main/java/com/amazonaws/athena/connectors/mysql/MySqlMetadataHandler.java +++ b/athena-mysql/src/main/java/com/amazonaws/athena/connectors/mysql/MySqlMetadataHandler.java @@ -39,8 +39,6 @@ import com.amazonaws.athena.connector.lambda.metadata.optimizations.OptimizationSubType; import com.amazonaws.athena.connector.lambda.metadata.optimizations.pushdown.ComplexExpressionPushdownSubType; import com.amazonaws.athena.connector.lambda.metadata.optimizations.pushdown.FilterPushdownSubType; -import com.amazonaws.athena.connector.lambda.metadata.optimizations.qpt.QueryPassthrough; -import com.amazonaws.athena.connector.lambda.metadata.optimizations.qpt.QueryPassthroughSignature; import com.amazonaws.athena.connectors.jdbc.connection.DatabaseConnectionConfig; import com.amazonaws.athena.connectors.jdbc.connection.DatabaseConnectionInfo; import com.amazonaws.athena.connectors.jdbc.connection.GenericJdbcConnectionFactory; @@ -48,7 +46,6 @@ import com.amazonaws.athena.connectors.jdbc.manager.JDBCUtil; import com.amazonaws.athena.connectors.jdbc.manager.JdbcMetadataHandler; import com.amazonaws.athena.connectors.jdbc.manager.PreparedStatementBuilder; -import com.amazonaws.athena.connectors.jdbc.qpt.JdbcQueryPassthrough; import com.amazonaws.services.athena.AmazonAthena; import com.amazonaws.services.secretsmanager.AWSSecretsManager; import com.google.common.annotations.VisibleForTesting; @@ -130,11 +127,9 @@ protected MySqlMetadataHandler( public GetDataSourceCapabilitiesResponse doGetDataSourceCapabilities(BlockAllocator allocator, GetDataSourceCapabilitiesRequest request) { ImmutableMap.Builder> capabilities = ImmutableMap.builder(); - capabilities.put(DataSourceOptimizations.SUPPORTS_FILTER_PUSHDOWN.withSupportedSubTypes( FilterPushdownSubType.SORTED_RANGE_SET, FilterPushdownSubType.NULLABLE_COMPARISON )); - capabilities.put(DataSourceOptimizations.SUPPORTS_COMPLEX_EXPRESSION_PUSHDOWN.withSupportedSubTypes( ComplexExpressionPushdownSubType.SUPPORTED_FUNCTION_EXPRESSION_TYPES .withSubTypeProperties(Arrays.stream(StandardFunctions.values()) @@ -142,11 +137,6 @@ public GetDataSourceCapabilitiesResponse doGetDataSourceCapabilities(BlockAlloca .toArray(String[]::new)) )); - QueryPassthroughSignature qptSignature = JdbcQueryPassthrough.getInstance(); - capabilities.put(QueryPassthrough.QUERY_PASSTHROUGH_SCHEMA_NAME.withSchemaName(qptSignature.getName())); - capabilities.put(QueryPassthrough.QUERY_PASSTHROUGH_NAME.withName(qptSignature.getDomain())); - capabilities.put(QueryPassthrough.QUERY_PASSTHROUGH_ARGUMENTS.withArguments(qptSignature.arguments())); - return new GetDataSourceCapabilitiesResponse(request.getCatalogName(), capabilities.build()); } @@ -202,10 +192,6 @@ public void getPartitions(final BlockWriter blockWriter, final GetTableLayoutReq public GetSplitsResponse doGetSplits( final BlockAllocator blockAllocator, final GetSplitsRequest getSplitsRequest) { - if (getSplitsRequest.getConstraints().isQueryPassThrough()) { - return setupQueryPassthroughSplit(getSplitsRequest); - } - LOGGER.info("{}: Catalog {}, table {}", getSplitsRequest.getQueryId(), getSplitsRequest.getTableName().getSchemaName(), getSplitsRequest.getTableName().getTableName()); int partitionContd = decodeContinuationToken(getSplitsRequest); Set splits = new HashSet<>(); diff --git a/athena-mysql/src/main/java/com/amazonaws/athena/connectors/mysql/MySqlRecordHandler.java b/athena-mysql/src/main/java/com/amazonaws/athena/connectors/mysql/MySqlRecordHandler.java index 8c359b7650..d662cacbcf 100644 --- a/athena-mysql/src/main/java/com/amazonaws/athena/connectors/mysql/MySqlRecordHandler.java +++ b/athena-mysql/src/main/java/com/amazonaws/athena/connectors/mysql/MySqlRecordHandler.java @@ -22,7 +22,6 @@ import com.amazonaws.athena.connector.lambda.domain.Split; import com.amazonaws.athena.connector.lambda.domain.TableName; import com.amazonaws.athena.connector.lambda.domain.predicate.Constraints; -import com.amazonaws.athena.connector.lambda.metadata.optimizations.qpt.QueryPassthroughSignature; import com.amazonaws.athena.connectors.jdbc.connection.DatabaseConnectionConfig; import com.amazonaws.athena.connectors.jdbc.connection.DatabaseConnectionInfo; import com.amazonaws.athena.connectors.jdbc.connection.GenericJdbcConnectionFactory; @@ -30,7 +29,6 @@ import com.amazonaws.athena.connectors.jdbc.manager.JDBCUtil; import com.amazonaws.athena.connectors.jdbc.manager.JdbcRecordHandler; import com.amazonaws.athena.connectors.jdbc.manager.JdbcSplitQueryBuilder; -import com.amazonaws.athena.connectors.jdbc.qpt.JdbcQueryPassthrough; import com.amazonaws.services.athena.AmazonAthena; import com.amazonaws.services.athena.AmazonAthenaClientBuilder; import com.amazonaws.services.s3.AmazonS3; @@ -97,18 +95,7 @@ public MySqlRecordHandler(DatabaseConnectionConfig databaseConnectionConfig, Jdb public PreparedStatement buildSplitSql(Connection jdbcConnection, String catalogName, TableName tableName, Schema schema, Constraints constraints, Split split) throws SQLException { - PreparedStatement preparedStatement; - - if (constraints.isQueryPassThrough()) { - JdbcQueryPassthrough qpt = JdbcQueryPassthrough.getInstance(); - QueryPassthroughSignature.verifyQueryPassthroughArguments( - constraints.getQueryPassthroughArguments(), qpt.arguments()); - String clientPassQuery = constraints.getQueryPassthroughArguments().get(qpt.getQueryArgument()); - preparedStatement = jdbcConnection.prepareStatement(clientPassQuery); - } - else { - preparedStatement = jdbcSplitQueryBuilder.buildSql(jdbcConnection, null, tableName.getSchemaName(), tableName.getTableName(), schema, constraints, split); - } + PreparedStatement preparedStatement = jdbcSplitQueryBuilder.buildSql(jdbcConnection, null, tableName.getSchemaName(), tableName.getTableName(), schema, constraints, split); // Disable fetching all rows. preparedStatement.setFetchSize(Integer.MIN_VALUE); diff --git a/athena-neptune/pom.xml b/athena-neptune/pom.xml index 9d3972dcee..c866fa343e 100644 --- a/athena-neptune/pom.xml +++ b/athena-neptune/pom.xml @@ -3,7 +3,7 @@ aws-athena-query-federation com.amazonaws - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 4.0.0 athena-neptune @@ -16,7 +16,7 @@ com.amazonaws aws-athena-federation-sdk - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 withdep diff --git a/athena-oracle/pom.xml b/athena-oracle/pom.xml index 4231243e04..b405b7379b 100644 --- a/athena-oracle/pom.xml +++ b/athena-oracle/pom.xml @@ -3,7 +3,7 @@ aws-athena-query-federation com.amazonaws - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 4.0.0 athena-oracle @@ -12,7 +12,7 @@ com.amazonaws athena-federation-integ-test - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 test @@ -58,6 +58,9 @@ ${mockito.version} test + + + diff --git a/athena-postgresql/pom.xml b/athena-postgresql/pom.xml index 3492562740..c2f0f7fadd 100644 --- a/athena-postgresql/pom.xml +++ b/athena-postgresql/pom.xml @@ -3,7 +3,7 @@ aws-athena-query-federation com.amazonaws - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 4.0.0 athena-postgresql @@ -13,7 +13,7 @@ com.amazonaws athena-federation-integ-test - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 test diff --git a/athena-redis/pom.xml b/athena-redis/pom.xml index a829f4ef98..2683a1f7c0 100644 --- a/athena-redis/pom.xml +++ b/athena-redis/pom.xml @@ -3,7 +3,7 @@ aws-athena-query-federation com.amazonaws - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 4.0.0 athena-redis @@ -93,7 +93,7 @@ com.amazonaws aws-athena-federation-sdk - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 withdep @@ -141,7 +141,7 @@ com.amazonaws athena-federation-integ-test - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 test diff --git a/athena-redshift/pom.xml b/athena-redshift/pom.xml index 67b53f1903..001fd892a7 100644 --- a/athena-redshift/pom.xml +++ b/athena-redshift/pom.xml @@ -3,7 +3,7 @@ aws-athena-query-federation com.amazonaws - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 4.0.0 athena-redshift @@ -17,7 +17,7 @@ com.amazonaws athena-federation-integ-test - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 test diff --git a/athena-saphana/pom.xml b/athena-saphana/pom.xml index 8e6b0a43c5..b638c3b03a 100644 --- a/athena-saphana/pom.xml +++ b/athena-saphana/pom.xml @@ -3,7 +3,7 @@ aws-athena-query-federation com.amazonaws - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 4.0.0 athena-saphana @@ -12,7 +12,7 @@ com.amazonaws athena-federation-integ-test - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 test diff --git a/athena-snowflake/pom.xml b/athena-snowflake/pom.xml index 604a067a34..094fde6555 100644 --- a/athena-snowflake/pom.xml +++ b/athena-snowflake/pom.xml @@ -3,7 +3,7 @@ aws-athena-query-federation com.amazonaws - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 4.0.0 athena-snowflake @@ -12,7 +12,7 @@ com.amazonaws athena-federation-integ-test - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 test diff --git a/athena-sqlserver/pom.xml b/athena-sqlserver/pom.xml index 9704e2495e..6bfab70343 100644 --- a/athena-sqlserver/pom.xml +++ b/athena-sqlserver/pom.xml @@ -3,7 +3,7 @@ aws-athena-query-federation com.amazonaws - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 4.0.0 athena-sqlserver @@ -12,7 +12,7 @@ com.amazonaws athena-federation-integ-test - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 test diff --git a/athena-synapse/pom.xml b/athena-synapse/pom.xml index 45e5172da4..caede53392 100644 --- a/athena-synapse/pom.xml +++ b/athena-synapse/pom.xml @@ -3,7 +3,7 @@ aws-athena-query-federation com.amazonaws - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 4.0.0 athena-synapse @@ -12,7 +12,7 @@ com.amazonaws athena-federation-integ-test - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 test diff --git a/athena-teradata/pom.xml b/athena-teradata/pom.xml index f129b60ea9..f643b8709d 100644 --- a/athena-teradata/pom.xml +++ b/athena-teradata/pom.xml @@ -3,7 +3,7 @@ aws-athena-query-federation com.amazonaws - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 4.0.0 athena-teradata @@ -12,7 +12,7 @@ com.amazonaws athena-federation-integ-test - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 test @@ -75,7 +75,7 @@ - + diff --git a/athena-timestream/pom.xml b/athena-timestream/pom.xml index 507876fdfe..a58b2c13c0 100644 --- a/athena-timestream/pom.xml +++ b/athena-timestream/pom.xml @@ -3,7 +3,7 @@ aws-athena-query-federation com.amazonaws - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 4.0.0 athena-timestream @@ -12,7 +12,7 @@ com.amazonaws aws-athena-federation-sdk - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 withdep @@ -25,7 +25,7 @@ com.amazonaws athena-federation-integ-test - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 withdep test diff --git a/athena-tpcds/pom.xml b/athena-tpcds/pom.xml index 557c7ec433..10490d0609 100644 --- a/athena-tpcds/pom.xml +++ b/athena-tpcds/pom.xml @@ -3,7 +3,7 @@ aws-athena-query-federation com.amazonaws - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 4.0.0 athena-tpcds @@ -12,7 +12,7 @@ com.amazonaws aws-athena-federation-sdk - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 withdep diff --git a/athena-udfs/pom.xml b/athena-udfs/pom.xml index cd433f68cb..a5234ec6aa 100644 --- a/athena-udfs/pom.xml +++ b/athena-udfs/pom.xml @@ -3,7 +3,7 @@ aws-athena-query-federation com.amazonaws - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 4.0.0 athena-udfs @@ -12,7 +12,7 @@ com.amazonaws aws-athena-federation-sdk - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 withdep diff --git a/athena-vertica/pom.xml b/athena-vertica/pom.xml index d47f4c7e48..3ab6c47d01 100644 --- a/athena-vertica/pom.xml +++ b/athena-vertica/pom.xml @@ -3,7 +3,7 @@ aws-athena-query-federation com.amazonaws - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 4.0.0 athena-vertica @@ -12,7 +12,7 @@ com.amazonaws aws-athena-federation-sdk - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 withdep diff --git a/pom.xml b/pom.xml index 8801142eda..76c61f604b 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.amazonaws aws-athena-query-federation pom - 2022.47.1-arfaraj-qpt-1 + 2022.47.1 AWS Athena Query Federation The Amazon Athena Query Federation SDK allows you to customize Amazon Athena with your own code. https://github.com/awslabs/aws-athena-query-federation @@ -87,40 +87,40 @@ athena-federation-sdk athena-federation-sdk-tools athena-federation-integ-test - - - athena-dynamodb - - - - - - - + athena-cloudwatch + athena-cloudwatch-metrics + athena-dynamodb + athena-aws-cmdb + athena-docdb + athena-redis + athena-hbase + athena-example + athena-elasticsearch + athena-tpcds athena-jdbc athena-mysql - - - - - - - - - - - - - - - - - - - - - - + athena-postgresql + athena-redshift + athena-udfs + athena-vertica + athena-timestream + athena-neptune + athena-google-bigquery + athena-oracle + athena-sqlserver + athena-synapse + athena-datalakegen2 + athena-snowflake + athena-teradata + athena-saphana + athena-cloudera-hive + athena-cloudera-impala + athena-hortonworks-hive + athena-db2 + athena-db2-as400 + athena-msk + athena-kafka + athena-gcs @@ -392,16 +392,11 @@ attach-sources - package jar-no-fork - - true - false - org.apache.maven.plugins