storage: add Solr write timeout in the storage configuration file, #T…

…ASK-6981
opencb · Sep 24, 2024 · 8f7cec5 · 8f7cec5
1 parent 87cc4a6
commit 8f7cec5
Show file tree

Hide file tree

Showing 4 changed files with 38 additions and 35 deletions.
diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/SearchConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/SearchConfiguration.java
@@ -36,20 +36,29 @@ public class SearchConfiguration {
     private String manager;
     private boolean active;
     private int timeout;
+    private int writeTimeout;
     private int insertBatchSize;
 
     private static final String DEFAULT_MODE = "cloud";
     private static final boolean DEFAULT_ACTIVE = true;
     private static final int DEFAULT_TIMEOUT = 30000;
+    private static final int DEFAULT_WRITE_TIMEOUT = 120000;
     private static final int DEFAULT_INSERT_BATCH_SIZE = 10000;
 
 
     public SearchConfiguration() {
-        this(Collections.emptyList(), "", DEFAULT_MODE, "", "", "", DEFAULT_ACTIVE, DEFAULT_TIMEOUT, DEFAULT_INSERT_BATCH_SIZE);
+        this(Collections.emptyList(), "", DEFAULT_MODE, "", "", "", DEFAULT_ACTIVE, DEFAULT_TIMEOUT, DEFAULT_WRITE_TIMEOUT,
+                DEFAULT_INSERT_BATCH_SIZE);
     }
 
+    @Deprecated
     public SearchConfiguration(List<String> hosts, String configSet, String mode, String user, String password, String manager,
                                boolean active, int timeout, int insertBatchSize) {
+        this(hosts, configSet, mode, user, password, manager, active, timeout, DEFAULT_WRITE_TIMEOUT, insertBatchSize);
+    }
+
+    public SearchConfiguration(List<String> hosts, String configSet, String mode, String user, String password, String manager,
+                               boolean active, int timeout, int writeTimeout, int insertBatchSize) {
         this.hosts = hosts;
         this.configSet = configSet;
         this.mode = mode;
@@ -58,32 +67,25 @@ public SearchConfiguration(List<String> hosts, String configSet, String mode, St
         this.manager = manager;
         this.active = active;
         this.timeout = timeout;
+        this.writeTimeout = writeTimeout;
         this.insertBatchSize = insertBatchSize;
     }
 
     @Override
     public String toString() {
-        return "SearchConfiguration{" +
-                "hosts=" + hosts +
-                ", configSet='" + configSet + '\'' +
-                ", mode='" + mode + '\'' +
-                ", user='" + user + '\'' +
-                ", password='" + password + '\'' +
-                ", manager='" + manager + '\'' +
-                ", active=" + active +
-                ", timeout=" + timeout +
-                ", insertBatchSize=" + insertBatchSize +
-                '}';
-    }
-
-    @Deprecated
-    public String getHost() {
-        return String.join(",", getHosts());
-    }
-
-    @Deprecated
-    public SearchConfiguration setHost(String host) {
-        return setHosts(StringUtils.isEmpty(host) ? Collections.emptyList() : Arrays.asList(host.split(",")));
+        final StringBuilder sb = new StringBuilder("SearchConfiguration{");
+        sb.append("hosts=").append(hosts);
+        sb.append(", configSet='").append(configSet).append('\'');
+        sb.append(", mode='").append(mode).append('\'');
+        sb.append(", user='").append(user).append('\'');
+        sb.append(", password='").append(password).append('\'');
+        sb.append(", manager='").append(manager).append('\'');
+        sb.append(", active=").append(active);
+        sb.append(", timeout=").append(timeout);
+        sb.append(", writeTimeout=").append(writeTimeout);
+        sb.append(", insertBatchSize=").append(insertBatchSize);
+        sb.append('}');
+        return sb.toString();
     }
 
     public List<String> getHosts() {
@@ -99,8 +101,9 @@ public String getConfigSet() {
         return configSet;
     }
 
-    public void setConfigSet(String configSet) {
+    public SearchConfiguration setConfigSet(String configSet) {
         this.configSet = configSet;
+        return this;
     }
 
     public String getMode() {
@@ -157,14 +160,12 @@ public SearchConfiguration setTimeout(int timeout) {
         return this;
     }
 
-    @Deprecated
-    public int getRows() {
-        return insertBatchSize;
+    public int getWriteTimeout() {
+        return writeTimeout;
     }
 
-    @Deprecated
-    public SearchConfiguration setRows(int rows) {
-        this.insertBatchSize = rows;
+    public SearchConfiguration setWriteTimeout(int writeTimeout) {
+        this.writeTimeout = writeTimeout;
         return this;
     }
 

diff --git a/...main/java/org/opencb/opencga/storage/app/cli/client/executors/VariantCommandExecutor.java b/...main/java/org/opencb/opencga/storage/app/cli/client/executors/VariantCommandExecutor.java
@@ -36,28 +36,28 @@
 import org.opencb.commons.utils.FileUtils;
 import org.opencb.opencga.core.common.TimeUtils;
 import org.opencb.opencga.core.common.UriUtils;
+import org.opencb.opencga.core.config.storage.StorageEngineConfiguration;
+import org.opencb.opencga.core.models.common.mixins.GenericRecordAvroJsonMixin;
 import org.opencb.opencga.core.models.operations.variant.VariantAggregateFamilyParams;
 import org.opencb.opencga.core.models.operations.variant.VariantAggregateParams;
 import org.opencb.opencga.storage.app.cli.CommandExecutor;
 import org.opencb.opencga.storage.app.cli.GeneralCliOptions;
 import org.opencb.opencga.storage.app.cli.client.options.StorageVariantCommandOptions;
 import org.opencb.opencga.storage.core.StorageEngineFactory;
-import org.opencb.opencga.core.config.storage.StorageEngineConfiguration;
 import org.opencb.opencga.storage.core.exceptions.StorageEngineException;
 import org.opencb.opencga.storage.core.metadata.models.ProjectMetadata;
 import org.opencb.opencga.storage.core.variant.VariantStorageEngine;
 import org.opencb.opencga.storage.core.variant.VariantStorageOptions;
 import org.opencb.opencga.storage.core.variant.VariantStoragePipeline;
 import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam;
-import org.opencb.opencga.storage.core.variant.query.VariantQueryUtils;
 import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIterator;
 import org.opencb.opencga.storage.core.variant.annotation.DefaultVariantAnnotationManager;
 import org.opencb.opencga.storage.core.variant.annotation.VariantAnnotationManager;
 import org.opencb.opencga.storage.core.variant.annotation.VariantAnnotatorException;
 import org.opencb.opencga.storage.core.variant.io.VariantWriterFactory;
-import org.opencb.opencga.core.models.common.mixins.GenericRecordAvroJsonMixin;
-import org.opencb.opencga.storage.core.variant.search.solr.VariantSearchManager;
+import org.opencb.opencga.storage.core.variant.query.VariantQueryUtils;
 import org.opencb.opencga.storage.core.variant.search.solr.SolrVariantDBIterator;
+import org.opencb.opencga.storage.core.variant.search.solr.VariantSearchManager;
 import org.opencb.opencga.storage.core.variant.stats.DefaultVariantStatisticsManager;
 
 import java.io.*;
@@ -68,8 +68,8 @@
 import java.util.*;
 import java.util.function.Function;
 
-import static org.opencb.opencga.storage.app.cli.client.options.StorageVariantCommandOptions.AggregateFamilyCommandOptions.AGGREGATE_FAMILY_COMMAND;
 import static org.opencb.opencga.storage.app.cli.client.options.StorageVariantCommandOptions.AggregateCommandOptions.AGGREGATE_COMMAND;
+import static org.opencb.opencga.storage.app.cli.client.options.StorageVariantCommandOptions.AggregateFamilyCommandOptions.AGGREGATE_FAMILY_COMMAND;
 import static org.opencb.opencga.storage.app.cli.client.options.StorageVariantCommandOptions.GenericAnnotationDeleteCommandOptions.ANNOTATION_DELETE_COMMAND;
 import static org.opencb.opencga.storage.app.cli.client.options.StorageVariantCommandOptions.GenericAnnotationMetadataCommandOptions.ANNOTATION_METADATA_COMMAND;
 import static org.opencb.opencga.storage.app.cli.client.options.StorageVariantCommandOptions.GenericAnnotationQueryCommandOptions.ANNOTATION_QUERY_COMMAND;
@@ -732,7 +732,7 @@ private void search() throws Exception {
         String solrUrl = (searchOptions.solrUrl == null ? "http://localhost:8983/solr/" : searchOptions.solrUrl);
         String dbName = (searchOptions.dbName == null ? "variants" : searchOptions.dbName);
 
-        variantStorageEngine.getConfiguration().getSearch().setHost(solrUrl);
+        variantStorageEngine.getConfiguration().getSearch().setHosts(Collections.singletonList(solrUrl));
 
 //        VariantSearchManager variantSearchManager = new VariantSearchManager(solrUrl, dbName);
 //        VariantSearchManager variantSearchManager = new VariantSearchManager(variantStorageEngine.getStudyConfigurationManager(),

diff --git a/opencga-storage/opencga-storage-core/src/main/resources/storage-configuration.yml b/opencga-storage/opencga-storage-core/src/main/resources/storage-configuration.yml
@@ -31,6 +31,7 @@ search:
   configSet: "${OPENCGA.STORAGE.SEARCH.CONFIG_SET}"
   mode: "cloud"
   timeout: ${OPENCGA.STORAGE.SEARCH.TIMEOUT}
+  writeTimeout: ${OPENCGA.STORAGE.SEARCH.WRITE.TIMEOUT}
   insertBatchSize: 5000
 
 ## Clinical database for indexing the pathogenic variants reported.

diff --git a/pom.xml b/pom.xml
@@ -1397,6 +1397,7 @@
                 <OPENCGA.STORAGE.SEARCH.CONFIG_SET>opencga-variant-configset-${project.parent.version}
                 </OPENCGA.STORAGE.SEARCH.CONFIG_SET>
                 <OPENCGA.STORAGE.SEARCH.TIMEOUT>30000</OPENCGA.STORAGE.SEARCH.TIMEOUT>
+                <OPENCGA.STORAGE.SEARCH.WRITE.TIMEOUT>120000</OPENCGA.STORAGE.SEARCH.WRITE.TIMEOUT>
                 <OPENCGA.STORAGE.CLINICAL.HOST>http://localhost:8983/solr/</OPENCGA.STORAGE.CLINICAL.HOST>
                 <OPENCGA.STORAGE.CLINICAL.MANAGER>""</OPENCGA.STORAGE.CLINICAL.MANAGER>
                 <OPENCGA.STORAGE.CLINICAL.TIMEOUT>30000</OPENCGA.STORAGE.CLINICAL.TIMEOUT>