analysis: Do not use the scratchDir as intermediate folder for export…

… and walk. #TASK-6722
opencb · Nov 29, 2024 · 14c07d9 · 14c07d9
1 parent 98ce6f8
commit 14c07d9
Show file tree

Hide file tree

Showing 4 changed files with 16 additions and 49 deletions.
diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/VariantExportTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/VariantExportTool.java
@@ -20,18 +20,14 @@
 import org.opencb.commons.datastore.core.Query;
 import org.opencb.commons.datastore.core.QueryOptions;
 import org.opencb.opencga.analysis.tools.OpenCgaTool;
-import org.opencb.opencga.catalog.io.IOManager;
-import org.opencb.opencga.core.common.UriUtils;
 import org.opencb.opencga.core.models.common.Enums;
 import org.opencb.opencga.core.models.variant.VariantExportParams;
 import org.opencb.opencga.core.tools.annotations.Tool;
 import org.opencb.opencga.core.tools.annotations.ToolParams;
 import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam;
 import org.opencb.opencga.storage.core.variant.io.VariantWriterFactory;
 
-import java.net.URI;
 import java.nio.file.Path;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
@@ -70,12 +66,8 @@ protected List<String> getSteps() {
 
     @Override
     protected void run() throws Exception {
-        List<URI> uris = new ArrayList<>(2);
         step(ID, () -> {
-            // Use scratch directory to store intermediate files. Move files to final directory at the end
-            // The scratch directory is expected to be faster than the final directory
-            // This also avoids moving files to final directory if the tool fails
-            Path outDir = getScratchDir();
+            Path outDir = getOutDir();
             String outputFile = StringUtils.isEmpty(toolParams.getOutputFileName())
                     ? outDir.toString()
                     : outDir.resolve(toolParams.getOutputFileName()).toString();
@@ -84,18 +76,9 @@ protected void run() throws Exception {
             for (VariantQueryParam param : VariantQueryParam.values()) {
                 queryOptions.remove(param.key());
             }
-            uris.addAll(variantStorageManager.exportData(outputFile,
+            variantStorageManager.exportData(outputFile,
                     outputFormat,
-                    toolParams.getVariantsFile(), query, queryOptions, token));
-        });
-        step("move-files", () -> {
-            // Move files to final directory
-            IOManager ioManager = catalogManager.getIoManagerFactory().get(uris.get(0));
-            for (URI uri : uris) {
-                String fileName = UriUtils.fileName(uri);
-                logger.info("Moving file -- " + fileName);
-                ioManager.move(uri, getOutDir().resolve(fileName).toUri());
-            }
+                    toolParams.getVariantsFile(), query, queryOptions, token);
         });
     }
 }
diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/VariantWalkerTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/VariantWalkerTool.java
@@ -20,17 +20,13 @@
 import org.opencb.commons.datastore.core.Query;
 import org.opencb.commons.datastore.core.QueryOptions;
 import org.opencb.opencga.analysis.tools.OpenCgaTool;
-import org.opencb.opencga.catalog.io.IOManager;
-import org.opencb.opencga.core.common.UriUtils;
 import org.opencb.opencga.core.models.common.Enums;
 import org.opencb.opencga.core.models.variant.VariantWalkerParams;
 import org.opencb.opencga.core.tools.annotations.Tool;
 import org.opencb.opencga.core.tools.annotations.ToolParams;
 import org.opencb.opencga.storage.core.variant.io.VariantWriterFactory;
 
-import java.net.URI;
 import java.nio.file.Path;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
@@ -75,29 +71,14 @@ protected List<String> getSteps() {
 
     @Override
     protected void run() throws Exception {
-        List<URI> uris = new ArrayList<>(2);
         step(ID, () -> {
-            // Use scratch directory to store intermediate files. Move files to final directory at the end
-            // The scratch directory is expected to be faster than the final directory
-            // This also avoids moving files to final directory if the tool fails
-            Path outDir = getScratchDir();
+            Path outDir = getOutDir();
             String outputFile = outDir.resolve(toolParams.getOutputFileName()).toString();
             Query query = toolParams.toQuery();
             QueryOptions queryOptions = new QueryOptions().append(QueryOptions.INCLUDE, toolParams.getInclude())
                     .append(QueryOptions.EXCLUDE, toolParams.getExclude());
-            uris.addAll(variantStorageManager.walkData(outputFile,
-                    format, query, queryOptions, toolParams.getDockerImage(), toolParams.getCommandLine(), token));
-        });
-        step("move-files", () -> {
-            // Move files to final directory
-            if (!uris.isEmpty()) {
-                IOManager ioManager = catalogManager.getIoManagerFactory().get(uris.get(0));
-                for (URI uri : uris) {
-                    String fileName = UriUtils.fileName(uri);
-                    logger.info("Moving file -- " + fileName);
-                    ioManager.move(uri, getOutDir().resolve(fileName).toUri());
-                }
-            }
+            variantStorageManager.walkData(outputFile,
+                    format, query, queryOptions, toolParams.getDockerImage(), toolParams.getCommandLine(), token);
         });
     }
 }
diff --git a/...in/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java b/...in/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java
@@ -341,7 +341,7 @@ private void query(VariantCommandOptions.AbstractVariantQueryCommandOptions cliO
             queryOptions.putIfNotEmpty("annotations", cliOptions.genericVariantQueryOptions.annotations);
 
             VariantExportParams toolParams = new VariantExportParams(
-                    query, outdir,
+                    query,
                     cliOptions.outputFileName,
                     cliOptions.outputFileFormat,
                     cliOptions.variantsFile);

diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantExportParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantExportParams.java
@@ -16,12 +16,13 @@
 
 package org.opencb.opencga.core.models.variant;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import org.opencb.commons.datastore.core.Query;
 
 public class VariantExportParams extends VariantQueryParams {
     public static final String DESCRIPTION = "Variant export params";
-    private String outdir;
+
     private String outputFileName;
     private String outputFileFormat;
     private String variantsFile;
@@ -35,21 +36,23 @@ public class VariantExportParams extends VariantQueryParams {
     public VariantExportParams() {
     }
 
-    public VariantExportParams(Query query, String outdir, String outputFileName, String outputFileFormat,
+    public VariantExportParams(Query query, String outputFileName, String outputFileFormat,
                                String variantsFile) {
         super(query);
-        this.outdir = outdir;
         this.outputFileName = outputFileName;
         this.outputFileFormat = outputFileFormat;
         this.variantsFile = variantsFile;
     }
 
+    @Deprecated
+    @JsonIgnore
     public String getOutdir() {
-        return outdir;
+        return null;
     }
 
-    public VariantExportParams setOutdir(String outdir) {
-        this.outdir = outdir;
+    @Deprecated
+    @JsonIgnore
+    public VariantExportParams setOutdir(String unused) {
         return this;
     }