From 14c07d90c63da3a324a06673fbdecec27ee6fa20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Fri, 29 Nov 2024 12:27:55 +0000 Subject: [PATCH] analysis: Do not use the scratchDir as intermediate folder for export and walk. #TASK-6722 --- .../analysis/variant/VariantExportTool.java | 23 +++-------------- .../analysis/variant/VariantWalkerTool.java | 25 +++---------------- .../VariantInternalCommandExecutor.java | 2 +- .../models/variant/VariantExportParams.java | 15 ++++++----- 4 files changed, 16 insertions(+), 49 deletions(-) diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/VariantExportTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/VariantExportTool.java index b0a2005ac18..b1c49aeafa6 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/VariantExportTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/VariantExportTool.java @@ -20,8 +20,6 @@ import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.analysis.tools.OpenCgaTool; -import org.opencb.opencga.catalog.io.IOManager; -import org.opencb.opencga.core.common.UriUtils; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.variant.VariantExportParams; import org.opencb.opencga.core.tools.annotations.Tool; @@ -29,9 +27,7 @@ import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; import org.opencb.opencga.storage.core.variant.io.VariantWriterFactory; -import java.net.URI; import java.nio.file.Path; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -70,12 +66,8 @@ protected List getSteps() { @Override protected void run() throws Exception { - List uris = new ArrayList<>(2); step(ID, () -> { - // Use scratch directory to store intermediate files. Move files to final directory at the end - // The scratch directory is expected to be faster than the final directory - // This also avoids moving files to final directory if the tool fails - Path outDir = getScratchDir(); + Path outDir = getOutDir(); String outputFile = StringUtils.isEmpty(toolParams.getOutputFileName()) ? outDir.toString() : outDir.resolve(toolParams.getOutputFileName()).toString(); @@ -84,18 +76,9 @@ protected void run() throws Exception { for (VariantQueryParam param : VariantQueryParam.values()) { queryOptions.remove(param.key()); } - uris.addAll(variantStorageManager.exportData(outputFile, + variantStorageManager.exportData(outputFile, outputFormat, - toolParams.getVariantsFile(), query, queryOptions, token)); - }); - step("move-files", () -> { - // Move files to final directory - IOManager ioManager = catalogManager.getIoManagerFactory().get(uris.get(0)); - for (URI uri : uris) { - String fileName = UriUtils.fileName(uri); - logger.info("Moving file -- " + fileName); - ioManager.move(uri, getOutDir().resolve(fileName).toUri()); - } + toolParams.getVariantsFile(), query, queryOptions, token); }); } } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/VariantWalkerTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/VariantWalkerTool.java index 5ab80d4f57e..109a9d4b5a7 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/VariantWalkerTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/VariantWalkerTool.java @@ -20,17 +20,13 @@ import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.analysis.tools.OpenCgaTool; -import org.opencb.opencga.catalog.io.IOManager; -import org.opencb.opencga.core.common.UriUtils; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.variant.VariantWalkerParams; import org.opencb.opencga.core.tools.annotations.Tool; import org.opencb.opencga.core.tools.annotations.ToolParams; import org.opencb.opencga.storage.core.variant.io.VariantWriterFactory; -import java.net.URI; import java.nio.file.Path; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -75,29 +71,14 @@ protected List getSteps() { @Override protected void run() throws Exception { - List uris = new ArrayList<>(2); step(ID, () -> { - // Use scratch directory to store intermediate files. Move files to final directory at the end - // The scratch directory is expected to be faster than the final directory - // This also avoids moving files to final directory if the tool fails - Path outDir = getScratchDir(); + Path outDir = getOutDir(); String outputFile = outDir.resolve(toolParams.getOutputFileName()).toString(); Query query = toolParams.toQuery(); QueryOptions queryOptions = new QueryOptions().append(QueryOptions.INCLUDE, toolParams.getInclude()) .append(QueryOptions.EXCLUDE, toolParams.getExclude()); - uris.addAll(variantStorageManager.walkData(outputFile, - format, query, queryOptions, toolParams.getDockerImage(), toolParams.getCommandLine(), token)); - }); - step("move-files", () -> { - // Move files to final directory - if (!uris.isEmpty()) { - IOManager ioManager = catalogManager.getIoManagerFactory().get(uris.get(0)); - for (URI uri : uris) { - String fileName = UriUtils.fileName(uri); - logger.info("Moving file -- " + fileName); - ioManager.move(uri, getOutDir().resolve(fileName).toUri()); - } - } + variantStorageManager.walkData(outputFile, + format, query, queryOptions, toolParams.getDockerImage(), toolParams.getCommandLine(), token); }); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java index 7d20deccb09..f59e9178e9a 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java @@ -341,7 +341,7 @@ private void query(VariantCommandOptions.AbstractVariantQueryCommandOptions cliO queryOptions.putIfNotEmpty("annotations", cliOptions.genericVariantQueryOptions.annotations); VariantExportParams toolParams = new VariantExportParams( - query, outdir, + query, cliOptions.outputFileName, cliOptions.outputFileFormat, cliOptions.variantsFile); diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantExportParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantExportParams.java index 7e5c1870e8d..3f00adc7be3 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantExportParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantExportParams.java @@ -16,12 +16,13 @@ package org.opencb.opencga.core.models.variant; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; import org.opencb.commons.datastore.core.Query; public class VariantExportParams extends VariantQueryParams { public static final String DESCRIPTION = "Variant export params"; - private String outdir; + private String outputFileName; private String outputFileFormat; private String variantsFile; @@ -35,21 +36,23 @@ public class VariantExportParams extends VariantQueryParams { public VariantExportParams() { } - public VariantExportParams(Query query, String outdir, String outputFileName, String outputFileFormat, + public VariantExportParams(Query query, String outputFileName, String outputFileFormat, String variantsFile) { super(query); - this.outdir = outdir; this.outputFileName = outputFileName; this.outputFileFormat = outputFileFormat; this.variantsFile = variantsFile; } + @Deprecated + @JsonIgnore public String getOutdir() { - return outdir; + return null; } - public VariantExportParams setOutdir(String outdir) { - this.outdir = outdir; + @Deprecated + @JsonIgnore + public VariantExportParams setOutdir(String unused) { return this; }