add es 7.x support

README.md

@@ -3,7 +3,7 @@
 [![Build Status](https://travis-ci.org/allegro/embedded-elasticsearch.svg?branch=master)](https://travis-ci.org/allegro/embedded-elasticsearch)
 [![Maven Central](https://maven-badges.herokuapp.com/maven-central/pl.allegro.tech/embedded-elasticsearch/badge.svg)](http://central.maven.org/maven2/pl/allegro/tech/embedded-elasticsearch)
 
-Small utility for creating integration tests that use Elasticsearch. Instead of using `Node` it downloads Elasticsearch in specified version and starts it in a separate process. It also allows you to install required plugins which is not possible when using `NodeBuilder`. Utility was tested with 1.x, 2.x, 5.x and 6.x versions of Elasticsearch.
+Small utility for creating integration tests that use Elasticsearch. Instead of using `Node` it downloads Elasticsearch in specified version and starts it in a separate process. It also allows you to install required plugins which is not possible when using `NodeBuilder`. Utility was tested with 1.x, 2.x, 5.x, 6.x and 7.x versions of Elasticsearch.
 
 ## Introduction
 
@@ -63,6 +63,7 @@ Available `IndexSettings.Builder` options
 | Method | Description |
 | ------------- | ------------- |
 | `withType(String type, String mapping)` | specify type and it's mappings |
+| `withMapping(String mapping)` | starting from Elasticseatch 7, there is no more types, so when using an ES version 7.0 and above this method should be used insted of withType method |
 | `withSettings(String settings)` | specify index settings |
 
 

core/src/main/java/pl/allegro/tech/embeddedelasticsearch/ElasticRestClient.java

@@ -164,10 +164,13 @@ private String indexMetadataJson(String indexName, String indexType, String id,
             joiner.add("\"_id\": \"" + id + "\"");
         }
 
-        if(routing != null) {
-            joiner.add("\"_routing\": \"" + routing + "\"");
+        if (routing != null) {
+            if (newESVersion()) {
+                joiner.add("\"routing\": \"" + routing + "\"");
+            } else {
+                joiner.add("\"_routing\": \"" + routing + "\"");
+            }
         }
-
         return "{ \"index\": {" + joiner.toString() + "} }";
     }
 
@@ -180,6 +183,20 @@ void refresh() {
         }
     }
 
+    private boolean newESVersion() {
+        HttpGet request = new HttpGet(url("/"));
+        return httpClient.execute(request, response -> {
+            JsonNode jsonNode;
+            try {
+                jsonNode = OBJECT_MAPPER.readTree(readBodySafely(response));
+            } catch (IOException e) {
+                return false;
+            }
+            String esV = jsonNode.get("version").get("number").asText();
+            return Integer.parseInt(esV.substring(0,1)) >= 7; //if version is 7 and above
+        });
+    }
+
     private void performBulkRequest(String requestUrl, String bulkRequestBody) {
         HttpPost request = new HttpPost(requestUrl);
         request.setHeader(new BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json"));

core/src/main/java/pl/allegro/tech/embeddedelasticsearch/IndexSettings.java

@@ -17,31 +17,65 @@ public class IndexSettings {
 
     private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
 
+    private final Optional<JsonNode> mappings;
     private final List<TypeWithMapping> types;
     private final Optional<JsonNode> settings;
     private final Optional<JsonNode> aliases;
-
+    private boolean includeTypeName=false;
     public static Builder builder() {
         return new Builder();
     }
 
-    public IndexSettings(List<TypeWithMapping> types, Optional<String> settings) {
+    private IndexSettings(List<TypeWithMapping> types, Optional<String> settings, Optional<String> aliases) {
+        this.mappings = rawToJson(Optional.of("{}"));
         this.types = types;
         this.settings = rawToJson(settings);
         this.aliases = Optional.empty();
     }
 
-    private IndexSettings(List<TypeWithMapping> types, Optional<String> settings, Optional<String> aliases) {
-        this.types = types;
+    public IndexSettings(Optional<String>  mapping, Optional<String> settings) {
+        includeTypeName = true;
+        this.mappings = rawToJson(mapping);
+        this.settings = rawToJson(settings);
+        this.aliases = Optional.empty();
+        this.types = new ArrayList<>();
+    }
+
+    private IndexSettings(Optional<String> mapping, Optional<String> settings, Optional<String> aliases) {
+        this.mappings = rawToJson(mapping);
+        includeTypeName = true;
         this.settings = rawToJson(settings);
         this.aliases = rawToJson(aliases);
+        this.types = new ArrayList<>();
     }
 
     public static class Builder {
 
-        private final List<TypeWithMapping> types = new ArrayList<>();
+        private Optional<String> mapping  = Optional.empty();
         private Optional<String> settings = Optional.empty();
         private Optional<String> aliases = Optional.empty();
+        private final List<TypeWithMapping> types = new ArrayList<>();
+
+        /**
+         * Type with mappings to create with index
+         *
+         * @param mapping mappings for created type
+         */
+        public Builder withMapping(Object mapping) throws IOException {
+            String mappingString;
+            if (mapping == null) {
+                return this;
+            }
+            else if (mapping instanceof InputStream) {
+                InputStream mappingStream = (InputStream) mapping;
+                mappingString = IOUtils.toString(mappingStream, UTF_8);
+            }
+            else {
+                mappingString = (String) mapping;
+            }
+            this.mapping = Optional.of(mappingString);
+            return this;
+        }
 
         /**
          * Specify type inside created index
@@ -53,6 +87,7 @@ public Builder withType(String type, InputStream mapping) throws IOException {
             return withType(type, IOUtils.toString(mapping, UTF_8));
         }
 
+
         /**
          * Type with mappings to create with index
          *
@@ -106,16 +141,23 @@ public Builder withAliases(String aliases) {
          * @return IndexSettings with specified parameters
          */
         public IndexSettings build() {
-            return new IndexSettings(types, settings, aliases);
+            return new IndexSettings(mapping, settings, aliases);
         }
     }
 
     public ObjectNode toJson() {
         ObjectNode objectNode = new ObjectMapper().createObjectNode();
         objectNode.set("settings", settings.orElse(OBJECT_MAPPER.createObjectNode()));
         objectNode.set("aliases", aliases.orElse(OBJECT_MAPPER.createObjectNode()));
-        ObjectNode mappingsObject = prepareMappingsObject();
-        objectNode.set("mappings", mappingsObject);
+
+        if (includeTypeName){
+            objectNode.set("mappings",mappings.orElse(OBJECT_MAPPER.createObjectNode()));
+        }
+        else {
+            ObjectNode mappingsObject = prepareMappingsObject();
+            objectNode.set("mappings", mappingsObject);
+        }
+
         return objectNode;
     }
 

core/src/main/java/pl/allegro/tech/embeddedelasticsearch/InstallFromVersion.java

@@ -39,7 +39,8 @@ private enum ElsDownloadUrl {
         ELS_1x("1.", "https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-{VERSION}.zip"),
         ELS_2x("2.", "https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/zip/elasticsearch/{VERSION}/elasticsearch-{VERSION}.zip"),
         ELS_5x("5.", "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{VERSION}.zip"),
-        ELS_6x("6.", ELS_5x.downloadUrl);
+        ELS_6x("6.", ELS_5x.downloadUrl),
+        ELS_7x("7.", "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{VERSION}-windows-x86_64.zip");
 
         String versionPrefix;
         String downloadUrl;

core/src/main/java/pl/allegro/tech/embeddedelasticsearch/TypeWithMapping.java

@@ -30,4 +30,4 @@ public String getType() {
     public JsonNode getMapping() {
         return mapping;
     }
-}
+}

es73-test/build.gradle

@@ -0,0 +1,10 @@
+dependencies {
+    testCompile project(':test-base')
+
+    testCompile group: 'org.elasticsearch', name: 'elasticsearch', version: '7.3.2'
+    testCompile group: 'org.elasticsearch.client', name: 'elasticsearch-rest-high-level-client', version: '7.3.2'
+    testCompile group: 'org.elasticsearch.client', name: 'elasticsearch-rest-client', version: '7.3.2'
+    testCompile group: 'org.locationtech.spatial4j', name: 'spatial4j', version: '0.6'
+    testCompile group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.6.2'
+    testCompile group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.6.2'
+}

es73-test/src/test/groovy/pl/allegro/tech/embeddedelasticsearch/EmbeddedElasticSpec.groovy

@@ -0,0 +1,125 @@
+package pl.allegro.tech.embeddedelasticsearch
+
+
+import org.apache.http.HttpHost
+import org.elasticsearch.action.get.GetRequest
+import org.elasticsearch.action.search.SearchRequest
+import org.elasticsearch.client.RequestOptions
+import org.elasticsearch.client.RestClient
+import org.elasticsearch.client.RestHighLevelClient
+import org.elasticsearch.index.query.QueryBuilders
+import org.elasticsearch.search.builder.SearchSourceBuilder
+
+import static java.util.concurrent.TimeUnit.MINUTES
+import static pl.allegro.tech.embeddedelasticsearch.PopularProperties.HTTP_PORT
+import static pl.allegro.tech.embeddedelasticsearch.SampleIndices.*
+
+class EmbeddedElasticSpec extends EmbeddedElasticCoreApiBaseSpec {
+
+    static final ELASTIC_VERSION = "7.3.2"
+    static final HTTP_PORT_VALUE = 9999
+    static final DOC_TYPE = "_doc"
+
+    static EmbeddedElastic embeddedElastic = EmbeddedElastic.builder()
+            .withElasticVersion(ELASTIC_VERSION)
+            .withSetting(HTTP_PORT, HTTP_PORT_VALUE)
+            .withEsJavaOpts("-Xms128m -Xmx512m")
+            .withTemplate(CARS_TEMPLATE_NAME, CARS_TEMPLATE_7x)
+            .withIndex(CARS_INDEX_NAME,CARS_INDEX_7x)
+            .withIndex(BOOKS_INDEX_NAME, BOOKS_INDEX)
+            .withStartTimeout(2, MINUTES)
+            .build()
+            .start()
+
+    static RestHighLevelClient client = createClient()
+
+    def setup() {
+        embeddedElastic.recreateIndices()
+    }
+
+    def cleanupSpec() {
+        client.close()
+        embeddedElastic.stop()
+    }
+
+    static RestHighLevelClient createClient() {
+        return new RestHighLevelClient(RestClient.builder(new HttpHost("localhost", HTTP_PORT_VALUE)))
+    }
+
+    @Override
+    void index(IndexRequest indexRequest) {
+        IndexRequest newIndexRequest = new IndexRequest.IndexRequestBuilder(indexRequest.getIndexName(),"_doc",indexRequest.getJson()).build()
+        index(Arrays.asList(newIndexRequest))
+    }
+
+    @Override
+    void index(List<IndexRequest> indexRequests) {
+        ArrayList<IndexRequest> newIndexRequests = new ArrayList<>()
+        for (IndexRequest newIndexRequest : indexRequests) {
+            newIndexRequests.add( new IndexRequest.IndexRequestBuilder(newIndexRequest.getIndexName(),DOC_TYPE,newIndexRequest.getJson()).withId(newIndexRequest.getId()).withRouting(newIndexRequest.getRouting()).build())
+        }
+        embeddedElastic.index(newIndexRequests)
+    }
+
+    @Override
+    void index(SampleIndices.PaperBook book) {
+        index(new IndexRequest.IndexRequestBuilder(BOOKS_INDEX_NAME, DOC_TYPE, toJson(book)).build())
+    }
+
+    @Override
+    void index(SampleIndices.Car car) {
+        index(new IndexRequest.IndexRequestBuilder(CARS_INDEX_NAME, DOC_TYPE, toJson(car)).build())
+    }
+
+    @Override
+    void index(String indexName, String indexType, Map idJsonMap) {
+        embeddedElastic.index(indexName, DOC_TYPE,idJsonMap)
+    }
+
+    @Override
+    List<String> fetchAllDocuments() {
+        fetchAllDocuments(CARS_INDEX_NAME) + fetchAllDocuments(BOOKS_INDEX_NAME)
+    }
+
+    @Override
+    List<String> fetchAllDocuments(String indexName) {
+        final searchRequest = new SearchRequest(indexName)
+                .source(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()));
+
+        client.search(searchRequest, RequestOptions.DEFAULT)
+                .hits.hits.toList()
+                .collect { it.sourceAsString }
+    }
+
+    @Override
+    List<String> fetchAllDocuments(String indexName, String typeName) {
+        fetchAllDocuments(indexName)
+    }
+
+    @Override
+    List<String> fetchAllDocuments(String indexName, String typeName, String routing) {
+        final searchRequest = new SearchRequest(indexName)
+                .routing(routing)
+                .source(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()))
+
+        client.search(searchRequest, RequestOptions.DEFAULT)
+                .hits.hits.toList()
+                .collect { it.sourceAsString }
+    }
+
+    @Override
+    List<String> searchByTerm(String indexName, String typeName, String fieldName, String value) {
+        final searchRequest = new SearchRequest()
+                .source(new SearchSourceBuilder().query(QueryBuilders.termQuery(fieldName, value)))
+
+        client.search(searchRequest, RequestOptions.DEFAULT)
+                .hits.hits.toList()
+                .collect { it.sourceAsString }
+    }
+
+    @Override
+    String getById(String indexName, String typeName, String id) {
+        final getRequest = new GetRequest(indexName, DOC_TYPE, id)
+        client.get(getRequest,RequestOptions.DEFAULT).sourceAsString
+    }
+}

es73-test/src/test/groovy/pl/allegro/tech/embeddedelasticsearch/PluginsInstallationSpec.groovy

@@ -0,0 +1,31 @@
+package pl.allegro.tech.embeddedelasticsearch
+
+import static java.util.concurrent.TimeUnit.MINUTES
+
+class PluginsInstallationSpec extends PluginsInstallationBaseSpec {
+
+    static final HTTP_PORT_VALUE = 9200
+
+    EmbeddedElastic.Builder baseEmbeddedElastic() {
+        return EmbeddedElastic.builder()
+                .withElasticVersion("6.3.0")
+                .withEsJavaOpts("-Xms128m -Xmx512m")
+                .withSetting(PopularProperties.HTTP_PORT, HTTP_PORT_VALUE)
+                .withStartTimeout(2, MINUTES)
+    }
+
+    @Override
+    String pluginByUrlUrl() {
+        return "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-stempel/analysis-stempel-6.3.0.zip"
+    }
+
+    @Override
+    String pluginByUrlName() {
+        return "analysis-stempel"
+    }
+
+    @Override
+    String pluginByName() {
+        return "discovery-file"
+    }
+}

es73-test/src/test/resources/audio-book-mapping.json

@@ -0,0 +1,21 @@
+{
+  "audio_book": {
+    "properties": {
+      "author": {
+        "type": "text",
+        "index": "false"
+      },
+      "title": {
+        "type": "text",
+        "index": "false"
+      },
+      "readBy": {
+        "type": "text",
+        "index": "false"
+      },
+      "description": {
+        "type": "text"
+      }
+    }
+  }
+}

es73-test/src/test/resources/car-mapping.json

@@ -0,0 +1,15 @@
+{
+  "properties": {
+    "manufacturer": {
+      "type": "text",
+      "index": "false"
+    },
+    "model": {
+      "type": "text",
+      "index": "true"
+    },
+    "description": {
+      "type": "text"
+    }
+  }
+}