From b34f93060c0f4f3558bbaaba87d960ef9aa7c22e Mon Sep 17 00:00:00 2001 From: Vamshi Vijay Nakkirtha <46505183+vamshin@users.noreply.github.com> Date: Wed, 20 Jan 2021 18:55:48 -0800 Subject: [PATCH] Run KNN integ tests with security plugin enabled (#304) * security tests --- .github/workflows/CI.yml | 41 +++++ build.gradle | 4 + .../plugin-metadata/plugin-security.policy | 1 + .../knn/KNNRestTestCase.java | 3 +- .../knn/ODFERestTestCase.java | 159 ++++++++++++++++++ 5 files changed, 206 insertions(+), 2 deletions(-) create mode 100644 src/test/java/com/amazon/opendistroforelasticsearch/knn/ODFERestTestCase.java diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index a571054c..5ce8e90e 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -26,3 +26,44 @@ jobs: - name: Run build run: | ./gradlew build + + - name: Pull and Run Docker for security tests + run: | + plugin=`ls build/distributions/*.zip` + version=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-3` + plugin_version=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-4` + echo $version + cd .. + if docker pull opendistroforelasticsearch/opendistroforelasticsearch:$version + then + echo "FROM opendistroforelasticsearch/opendistroforelasticsearch:$version" >> Dockerfile + echo "RUN if [ -d /usr/share/elasticsearch/plugins/opendistro-knn ]; then /usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro-knn; fi" >> Dockerfile + echo "RUN yum -y update \ && yum -y groupinstall "Development Tools" \ && yum install -y unzip glibc.x86_64 cmake \ && yum clean all" >> Dockerfile + echo "RUN git clone --recursive --branch ${GITHUB_REF##*/} https://github.com/opendistro-for-elasticsearch/k-NN.git /usr/share/elasticsearch/k-NN \ " >> Dockerfile + echo "&& cd /usr/share/elasticsearch/k-NN/jni \ && sed -i 's/-march=native/-march=x86-64/g' external/nmslib/similarity_search/CMakeLists.txt \ && cmake . \ && make \ " >> Dockerfile + echo "&& mkdir /tmp/jni/ && cp release/*.so /tmp/jni/ && ls -ltr /tmp/jni/ \ && cp /tmp/jni/libKNNIndex*.so /usr/lib \ && rm -rf /usr/share/elasticsearch/k-NN" >> Dockerfile + echo "RUN cd /usr/share/elasticsearch/" >> Dockerfile + echo "ADD k-NN/build/distributions/opendistro-knn-$plugin_version.zip /tmp/" >> Dockerfile + echo "RUN /usr/share/elasticsearch/bin/elasticsearch-plugin install --batch file:/tmp/opendistro-knn-$plugin_version.zip" >> Dockerfile + docker build -t odfe-knn:test . + echo "imagePresent=true" >> $GITHUB_ENV + else + echo "imagePresent=false" >> $GITHUB_ENV + fi + - name: Run Docker Image + if: env.imagePresent == 'true' + run: | + cd .. + docker run -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" odfe-knn:test + sleep 90 + - name: Run kNN Test + if: env.imagePresent == 'true' + run: | + security=`curl -XGET https://localhost:9200/_cat/plugins?v -u admin:admin --insecure |grep opendistro_security|wc -l` + if [ $security -gt 0 ] + then + echo "Security plugin is available. Running tests in security mode" + ./gradlew :integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=admin + else + echo "Security plugin is NOT available. Skipping tests as they are already ran part of ./gradlew build" + fi diff --git a/build.gradle b/build.gradle index c59cf969..f82c38c7 100644 --- a/build.gradle +++ b/build.gradle @@ -165,6 +165,10 @@ integTest { // allows integration test classes to access test resource from project root path systemProperty('project.root', project.rootDir.absolutePath) + systemProperty "https", System.getProperty("https") + systemProperty "user", System.getProperty("user") + systemProperty "password", System.getProperty("password") + doFirst { // Tell the test JVM if the cluster JVM is running under a debugger so that tests can // use longer timeouts for requests. diff --git a/src/main/plugin-metadata/plugin-security.policy b/src/main/plugin-metadata/plugin-security.policy index af59cec1..8e773174 100644 --- a/src/main/plugin-metadata/plugin-security.policy +++ b/src/main/plugin-metadata/plugin-security.policy @@ -1,3 +1,4 @@ grant { permission java.lang.RuntimePermission "loadLibrary.KNNIndexV2_0_11"; + permission java.net.SocketPermission "*", "connect,resolve"; }; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/knn/KNNRestTestCase.java b/src/test/java/com/amazon/opendistroforelasticsearch/knn/KNNRestTestCase.java index 043939d6..afec1a3a 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/knn/KNNRestTestCase.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/knn/KNNRestTestCase.java @@ -33,7 +33,6 @@ import org.elasticsearch.index.query.functionscore.ScriptScoreQueryBuilder; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.script.Script; -import org.elasticsearch.test.rest.ESRestTestCase; import org.junit.AfterClass; import org.junit.Before; @@ -61,7 +60,7 @@ /** * Base class for integration tests for KNN plugin. Contains several methods for testing KNN ES functionality. */ -public class KNNRestTestCase extends ESRestTestCase { +public class KNNRestTestCase extends ODFERestTestCase { public static final String INDEX_NAME = "test_index"; public static final String FIELD_NAME = "test_field"; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/knn/ODFERestTestCase.java b/src/test/java/com/amazon/opendistroforelasticsearch/knn/ODFERestTestCase.java new file mode 100644 index 00000000..8b9575a2 --- /dev/null +++ b/src/test/java/com/amazon/opendistroforelasticsearch/knn/ODFERestTestCase.java @@ -0,0 +1,159 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.knn; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +import org.apache.http.Header; +import org.apache.http.HttpHost; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.message.BasicHeader; +import org.apache.http.ssl.SSLContextBuilder; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.common.xcontent.DeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.junit.After; + +/** + * ODFE integration test base class to support both security disabled and enabled ODFE cluster. + */ +public abstract class ODFERestTestCase extends ESRestTestCase { + + protected boolean isHttps() { + boolean isHttps = Optional.ofNullable(System.getProperty("https")).map("true"::equalsIgnoreCase).orElse(false); + if (isHttps) { + // currently only external cluster is supported for security enabled testing + if (!Optional.ofNullable(System.getProperty("tests.rest.cluster")).isPresent()) { + throw new RuntimeException("cluster url should be provided for security enabled testing"); + } + } + + return isHttps; + } + + @Override + protected String getProtocol() { + return isHttps() ? "https" : "http"; + } + + @Override + protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOException { + RestClientBuilder builder = RestClient.builder(hosts); + if (isHttps()) { + configureHttpsClient(builder, settings); + } else { + configureClient(builder, settings); + } + + builder.setStrictDeprecationMode(true); + return builder.build(); + } + + protected static void configureHttpsClient(RestClientBuilder builder, Settings settings) throws IOException { + Map headers = ThreadContext.buildDefaultHeaders(settings); + Header[] defaultHeaders = new Header[headers.size()]; + int i = 0; + for (Map.Entry entry : headers.entrySet()) { + defaultHeaders[i++] = new BasicHeader(entry.getKey(), entry.getValue()); + } + builder.setDefaultHeaders(defaultHeaders); + builder.setHttpClientConfigCallback(httpClientBuilder -> { + String userName = Optional + .ofNullable(System.getProperty("user")) + .orElseThrow(() -> new RuntimeException("user name is missing")); + String password = Optional + .ofNullable(System.getProperty("password")) + .orElseThrow(() -> new RuntimeException("password is missing")); + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(userName, password)); + try { + return httpClientBuilder + .setDefaultCredentialsProvider(credentialsProvider) + // disable the certificate since our testing cluster just uses the default security configuration + .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE) + .setSSLContext(SSLContextBuilder.create().loadTrustMaterial(null, (chains, authType) -> true).build()); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + + final String socketTimeoutString = settings.get(CLIENT_SOCKET_TIMEOUT); + final TimeValue socketTimeout = TimeValue + .parseTimeValue(socketTimeoutString == null ? "60s" : socketTimeoutString, CLIENT_SOCKET_TIMEOUT); + builder.setRequestConfigCallback(conf -> conf.setSocketTimeout(Math.toIntExact(socketTimeout.getMillis()))); + if (settings.hasValue(CLIENT_PATH_PREFIX)) { + builder.setPathPrefix(settings.get(CLIENT_PATH_PREFIX)); + } + } + + /** + * wipeAllIndices won't work since it cannot delete security index. Use wipeAllODFEIndices instead. + */ + @Override + protected boolean preserveIndicesUponCompletion() { + return true; + } + + @SuppressWarnings("unchecked") + @After + protected void wipeAllODFEIndices() throws IOException { + Response response = client().performRequest(new Request("GET", "/_cat/indices?format=json&expand_wildcards=all")); + XContentType xContentType = XContentType.fromMediaTypeOrFormat(response.getEntity().getContentType().getValue()); + try ( + XContentParser parser = xContentType + .xContent() + .createParser( + NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + response.getEntity().getContent() + ) + ) { + XContentParser.Token token = parser.nextToken(); + List> parserList = null; + if (token == XContentParser.Token.START_ARRAY) { + parserList = parser.listOrderedMap().stream().map(obj -> (Map) obj).collect(Collectors.toList()); + } else { + parserList = Collections.singletonList(parser.mapOrdered()); + } + + for (Map index : parserList) { + String indexName = (String) index.get("index"); + if (indexName != null && !".opendistro_security".equals(indexName)) { + client().performRequest(new Request("DELETE", "/" + indexName)); + } + } + } + } +} +