From 321b5f1317602ccad1be17d1611fdc696b1ffae2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Mon, 19 Feb 2024 08:11:24 +0100 Subject: [PATCH 01/15] analysis: save the coverage BIGWIG file in the file internal alignment index of BAM, #TASK-5679, #TASK-5662 --- .../alignment/AlignmentCoverageAnalysis.java | 17 +++++++++++++++-- .../opencga/catalog/db/api/FileDBAdaptor.java | 2 ++ .../catalog/db/mongodb/FileMongoDBAdaptor.java | 2 +- .../opencga/catalog/managers/FileManager.java | 5 +++++ .../core/models/file/FileInternalAlignment.java | 16 ++++++++++++++-- 5 files changed, 37 insertions(+), 5 deletions(-) diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentCoverageAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentCoverageAnalysis.java index d008ff93e04..27e809f25c9 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentCoverageAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentCoverageAnalysis.java @@ -23,7 +23,10 @@ import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.alignment.CoverageIndexParams; import org.opencb.opencga.core.models.common.Enums; +import org.opencb.opencga.core.models.common.InternalStatus; import org.opencb.opencga.core.models.file.File; +import org.opencb.opencga.core.models.file.FileInternalCoverageIndex; +import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.tools.annotations.Tool; import org.opencb.opencga.core.tools.annotations.ToolParams; @@ -181,12 +184,22 @@ protected void run() throws Exception { bwPath = targetPath; logger.info("Coverage file was copied into the BAM folder: {}", bwPath); } else { - logger.info("Couldn't copy the coverage file into the BAM folder. The coverage file is in the job folder instead: {}", + logger.info("It could not copy the coverage file into the BAM folder. The coverage file is in the job folder instead: {}", bwPath); } // Link generated BIGWIG file and update samples info - AlignmentAnalysisUtils.linkAndUpdate(bamCatalogFile, bwPath, study, catalogManager, token); + File bwCatalogFile = AlignmentAnalysisUtils.linkAndUpdate(bamCatalogFile, bwPath, study, catalogManager, token); + + // Update BAM file internal in order to set the coverage index (BIGWIG) + FileInternalCoverageIndex fileCoverageIndex = new FileInternalCoverageIndex(new InternalStatus(InternalStatus.READY), + bwCatalogFile.getId(), "deeptools bamCoverage", coverageParams.getWindowSize()); + OpenCGAResult openCGAResult = catalogManager.getFileManager().updateFileInternalCoverageIndex(bamCatalogFile, + fileCoverageIndex, token); + if (openCGAResult.getNumUpdated() != 1) { + throw new ToolException("It could not update OpenCGA file catalog (alignment index " + bamCatalogFile.getId() + ") with" + + " the coverage index (BIGWIG) file '" + bwCatalogFile.getId() + "'"); + } }); } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java index 1ed15060d91..59c511e8d87 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java @@ -97,6 +97,8 @@ enum QueryParams implements QueryParam { INTERNAL_VARIANT_SECONDARY_INDEX_STATUS_ID("internal.variant.secondaryIndex.status.id", TEXT_ARRAY, ""), INTERNAL_ALIGNMENT_INDEX("internal.alignment.index", TEXT_ARRAY, ""), INTERNAL_ALIGNMENT_INDEX_STATUS_ID("internal.alignment.index.status.id", TEXT_ARRAY, ""), + INTERNAL_COVERAGE_INDEX("internal.alignment.coverage", TEXT_ARRAY, ""), + INTERNAL_COVERAGE_INDEX_STATUS_ID("internal.alignment.coverage.status.id", TEXT_ARRAY, ""), INTERNAL_MISSING_SAMPLES("internal.missingSamples", OBJECT, ""), INTERNAL_MISSING_SAMPLES_EXISTING("internal.missingSamples.existing", TEXT_ARRAY, ""), INTERNAL_MISSING_SAMPLES_NON_EXISTING("internal.missingSamples.nonExisting", TEXT_ARRAY, ""), diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java index 3e6e48743c6..20e09695777 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java @@ -712,7 +712,7 @@ private UpdateDocument getValidatedUpdateParams(ClientSession clientSession, lon String[] acceptedObjectParams = {QueryParams.INTERNAL_VARIANT_INDEX.key(), QueryParams.INTERNAL_VARIANT_ANNOTATION_INDEX.key(), QueryParams.INTERNAL_VARIANT_SECONDARY_INDEX.key(), QueryParams.INTERNAL_VARIANT_SECONDARY_ANNOTATION_INDEX.key(), - QueryParams.INTERNAL_ALIGNMENT_INDEX.key(), QueryParams.SOFTWARE.key(), + QueryParams.INTERNAL_ALIGNMENT_INDEX.key(), QueryParams.INTERNAL_COVERAGE_INDEX.key(), QueryParams.SOFTWARE.key(), QueryParams.EXPERIMENT.key(), QueryParams.STATUS.key(), QueryParams.INTERNAL_MISSING_SAMPLES.key(), QueryParams.QUALITY_CONTROL.key(), QueryParams.INTERNAL_STATUS.key()}; filterObjectParams(parameters, document.getSet(), acceptedObjectParams); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java index d7cbd3cc475..f747070ffeb 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java @@ -460,6 +460,11 @@ public OpenCGAResult updateFileInternalAlignmentIndex(File file, FileInternal return updateFileInternalField(file, index, FileDBAdaptor.QueryParams.INTERNAL_ALIGNMENT_INDEX.key(), token); } + public OpenCGAResult updateFileInternalCoverageIndex(File file, FileInternalCoverageIndex index, String token) + throws CatalogException { + return updateFileInternalField(file, index, FileDBAdaptor.QueryParams.INTERNAL_COVERAGE_INDEX.key(), token); + } + private OpenCGAResult updateFileInternalField(File file, Object value, String fieldKey, String token) throws CatalogException { return updateFileInternalField(file, value, Collections.singletonList(fieldKey), token); } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/file/FileInternalAlignment.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/file/FileInternalAlignment.java index 074c14e7eed..310fb78654b 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/file/FileInternalAlignment.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/file/FileInternalAlignment.java @@ -5,22 +5,25 @@ public class FileInternalAlignment { private FileInternalAlignmentIndex index; + private FileInternalCoverageIndex coverage; public FileInternalAlignment() { } - public FileInternalAlignment(FileInternalAlignmentIndex index) { + public FileInternalAlignment(FileInternalAlignmentIndex index, FileInternalCoverageIndex coverage) { this.index = index; + this.coverage = coverage; } public static FileInternalAlignment init() { - return new FileInternalAlignment(FileInternalAlignmentIndex.init()); + return new FileInternalAlignment(FileInternalAlignmentIndex.init(), FileInternalCoverageIndex.init()); } @Override public String toString() { final StringBuilder sb = new StringBuilder("FileInternalAlignment{"); sb.append("index=").append(index); + sb.append(", coverage=").append(coverage); sb.append('}'); return sb.toString(); } @@ -34,6 +37,15 @@ public FileInternalAlignment setIndex(FileInternalAlignmentIndex index) { return this; } + public FileInternalCoverageIndex getCoverage() { + return coverage; + } + + public FileInternalAlignment setCoverage(FileInternalCoverageIndex coverage) { + this.coverage = coverage; + return this; + } + @Override public boolean equals(Object o) { if (this == o) return true; From aff4ce57c78365a6d46f8ef6bcb16c625377b842 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Mon, 19 Feb 2024 08:16:36 +0100 Subject: [PATCH 02/15] models: add FileInternalCoverageIndex, #TASK-5679, #TASK-5662 --- .../file/FileInternalCoverageIndex.java | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 opencga-core/src/main/java/org/opencb/opencga/core/models/file/FileInternalCoverageIndex.java diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/file/FileInternalCoverageIndex.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/file/FileInternalCoverageIndex.java new file mode 100644 index 00000000000..49a867f2fde --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/file/FileInternalCoverageIndex.java @@ -0,0 +1,90 @@ +package org.opencb.opencga.core.models.file; + +import org.opencb.opencga.core.models.common.InternalStatus; + +import java.util.Objects; + +public class FileInternalCoverageIndex { + + private InternalStatus status; + private String fileId; + private String indexer; + private int windowSize; + + public FileInternalCoverageIndex() { + } + + public FileInternalCoverageIndex(InternalStatus status, String fileId, String indexer, int windowSize) { + this.status = status; + this.fileId = fileId; + this.indexer = indexer; + this.windowSize = windowSize; + } + + public static FileInternalCoverageIndex init() { + return new FileInternalCoverageIndex(null, "", "", 0); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("FileInternalCoverageIndex{"); + sb.append("status=").append(status); + sb.append(", fileId='").append(fileId).append('\''); + sb.append(", indexer='").append(indexer).append('\''); + sb.append(", windowSize=").append(windowSize); + sb.append('}'); + return sb.toString(); + } + + public InternalStatus getStatus() { + return status; + } + + public FileInternalCoverageIndex setStatus(InternalStatus status) { + this.status = status; + return this; + } + + public String getFileId() { + return fileId; + } + + public FileInternalCoverageIndex setFileId(String fileId) { + this.fileId = fileId; + return this; + } + + public String getIndexer() { + return indexer; + } + + public FileInternalCoverageIndex setIndexer(String indexer) { + this.indexer = indexer; + return this; + } + + public int getWindowSize() { + return windowSize; + } + + public FileInternalCoverageIndex setWindowSize(int windowSize) { + this.windowSize = windowSize; + return this; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + FileInternalCoverageIndex that = (FileInternalCoverageIndex) o; + return Objects.equals(status, that.status) && + Objects.equals(fileId, that.fileId) && + Objects.equals(indexer, that.indexer) && + Objects.equals(windowSize, that.windowSize); + } + + @Override + public int hashCode() { + return Objects.hash(status, fileId, indexer, windowSize); + } +} From f9da105214a2f2f6504f29dd5ee8ea321cc0b855 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Mon, 19 Feb 2024 10:30:23 +0100 Subject: [PATCH 03/15] analysis: implement overwrite for alignment index and coverage; and generate clients, #TASK-5679, #TASK-5662 --- .../alignment/AlignmentAnalysisUtils.java | 11 +- .../alignment/AlignmentCoverageAnalysis.java | 51 ++++---- .../alignment/AlignmentIndexOperation.java | 31 ++++- .../alignment/AlignmentStorageManager.java | 3 +- .../executors/AlignmentCommandExecutor.java | 123 +----------------- .../options/AlignmentCommandOptions.java | 28 ++-- .../app/cli/main/OpenCgaCompleter.java | 2 +- .../app/cli/main/OpencgaCliOptionsParser.java | 2 +- .../AnalysisAlignmentCommandExecutor.java | 6 +- .../AnalysisAlignmentCommandOptions.java | 24 ++-- opencga-client/src/main/R/R/Admin-methods.R | 2 +- .../src/main/R/R/Alignment-methods.R | 6 +- opencga-client/src/main/R/R/AllGenerics.R | 22 ++-- .../src/main/R/R/Clinical-methods.R | 4 +- opencga-client/src/main/R/R/Cohort-methods.R | 4 +- opencga-client/src/main/R/R/Family-methods.R | 4 +- opencga-client/src/main/R/R/File-methods.R | 4 +- opencga-client/src/main/R/R/GA4GH-methods.R | 2 +- .../src/main/R/R/Individual-methods.R | 4 +- opencga-client/src/main/R/R/Job-methods.R | 4 +- opencga-client/src/main/R/R/Meta-methods.R | 2 +- .../src/main/R/R/Operation-methods.R | 2 +- opencga-client/src/main/R/R/Panel-methods.R | 4 +- opencga-client/src/main/R/R/Project-methods.R | 4 +- opencga-client/src/main/R/R/Sample-methods.R | 4 +- opencga-client/src/main/R/R/Study-methods.R | 4 +- opencga-client/src/main/R/R/User-methods.R | 4 +- opencga-client/src/main/R/R/Variant-methods.R | 2 +- .../client/rest/clients/AdminClient.java | 2 +- .../client/rest/clients/AlignmentClient.java | 6 +- .../rest/clients/ClinicalAnalysisClient.java | 2 +- .../client/rest/clients/CohortClient.java | 2 +- .../rest/clients/DiseasePanelClient.java | 2 +- .../client/rest/clients/FamilyClient.java | 2 +- .../client/rest/clients/FileClient.java | 2 +- .../client/rest/clients/GA4GHClient.java | 2 +- .../client/rest/clients/IndividualClient.java | 2 +- .../client/rest/clients/JobClient.java | 2 +- .../client/rest/clients/MetaClient.java | 2 +- .../client/rest/clients/ProjectClient.java | 2 +- .../client/rest/clients/SampleClient.java | 2 +- .../client/rest/clients/StudyClient.java | 2 +- .../client/rest/clients/UserClient.java | 2 +- .../client/rest/clients/VariantClient.java | 2 +- .../rest/clients/VariantOperationClient.java | 2 +- opencga-client/src/main/javascript/Admin.js | 2 +- .../src/main/javascript/Alignment.js | 6 +- .../src/main/javascript/ClinicalAnalysis.js | 2 +- opencga-client/src/main/javascript/Cohort.js | 2 +- .../src/main/javascript/DiseasePanel.js | 2 +- opencga-client/src/main/javascript/Family.js | 2 +- opencga-client/src/main/javascript/File.js | 2 +- opencga-client/src/main/javascript/GA4GH.js | 2 +- .../src/main/javascript/Individual.js | 2 +- opencga-client/src/main/javascript/Job.js | 2 +- opencga-client/src/main/javascript/Meta.js | 2 +- opencga-client/src/main/javascript/Project.js | 2 +- opencga-client/src/main/javascript/Sample.js | 2 +- opencga-client/src/main/javascript/Study.js | 2 +- opencga-client/src/main/javascript/User.js | 2 +- opencga-client/src/main/javascript/Variant.js | 2 +- .../src/main/javascript/VariantOperation.js | 2 +- .../pyopencga/rest_clients/admin_client.py | 2 +- .../rest_clients/alignment_client.py | 8 +- .../rest_clients/clinical_analysis_client.py | 2 +- .../pyopencga/rest_clients/cohort_client.py | 2 +- .../rest_clients/disease_panel_client.py | 2 +- .../pyopencga/rest_clients/family_client.py | 2 +- .../pyopencga/rest_clients/file_client.py | 2 +- .../pyopencga/rest_clients/ga4gh_client.py | 2 +- .../rest_clients/individual_client.py | 2 +- .../pyopencga/rest_clients/job_client.py | 2 +- .../pyopencga/rest_clients/meta_client.py | 2 +- .../pyopencga/rest_clients/project_client.py | 2 +- .../pyopencga/rest_clients/sample_client.py | 2 +- .../pyopencga/rest_clients/study_client.py | 2 +- .../pyopencga/rest_clients/user_client.py | 2 +- .../pyopencga/rest_clients/variant_client.py | 2 +- .../rest_clients/variant_operation_client.py | 2 +- .../opencga/core/api/FieldConstants.java | 9 +- .../opencga/core/api/ParamConstants.java | 2 - .../alignment/AlignmentIndexParams.java | 21 +-- .../models/alignment/CoverageIndexParams.java | 46 ++++--- .../rest/analysis/AlignmentWebService.java | 4 +- 84 files changed, 246 insertions(+), 315 deletions(-) diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentAnalysisUtils.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentAnalysisUtils.java index cbd21b3df58..c1933c0e65d 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentAnalysisUtils.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentAnalysisUtils.java @@ -28,14 +28,13 @@ public class AlignmentAnalysisUtils { - public static File linkAndUpdate(File bamCatalogFile, Path outPath, String study, CatalogManager catalogManager, String token) + public static File linkAndUpdate(File bamCatalogFile, Path outPath, String jobId, String study, CatalogManager catalogManager, String token) throws CatalogException, ToolException { // Link BW file and update sample info - FileLinkParams fileLinkParams = new FileLinkParams().setUri(outPath.toString()); - if (Paths.get(bamCatalogFile.getPath()).getParent() != null) { - fileLinkParams.setPath(Paths.get(bamCatalogFile.getPath()).getParent().resolve(outPath.getFileName()).toString()); - } - OpenCGAResult fileResult = catalogManager.getFileManager().link(study, fileLinkParams, false, token); + FileLinkParams fileLinkParams = new FileLinkParams() + .setUri(outPath.toString()) + .setPath(Paths.get(jobId).resolve(outPath.getFileName()).toString()); + OpenCGAResult fileResult = catalogManager.getFileManager().link(study, fileLinkParams, true, token); if (fileResult.getNumResults() != 1) { throw new ToolException("It could not link OpenCGA file catalog file for '" + outPath + "'"); } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentCoverageAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentCoverageAnalysis.java index 27e809f25c9..eedabbda00f 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentCoverageAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentCoverageAnalysis.java @@ -26,7 +26,6 @@ import org.opencb.opencga.core.models.common.InternalStatus; import org.opencb.opencga.core.models.file.File; import org.opencb.opencga.core.models.file.FileInternalCoverageIndex; -import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.tools.annotations.Tool; import org.opencb.opencga.core.tools.annotations.ToolParams; @@ -43,7 +42,7 @@ public class AlignmentCoverageAnalysis extends OpenCgaToolScopeStudy { public static final String ID = "coverage-index-run"; - public static final String DESCRIPTION = "Compute the coverage from a given alignment file, e.g., create a " + public static final String DESCRIPTION = "Compute the coverage from a given BAM alignment file, e.g., create a " + AlignmentConstants.BIGWIG_EXTENSION + " file from a " + AlignmentConstants.BAM_EXTENSION + " file"; @ToolParams @@ -63,37 +62,36 @@ protected void check() throws Exception { // Checking BAM file ID try { - bamCatalogFile = catalogManager.getFileManager().get(getStudy(), coverageParams.getBamFileId(), QueryOptions.empty(), + bamCatalogFile = catalogManager.getFileManager().get(getStudy(), coverageParams.getFileId(), QueryOptions.empty(), getToken()).first(); if (bamCatalogFile == null) { - throw new ToolException("Could not find BAM file from ID '" + coverageParams.getBamFileId() + "'"); + throw new ToolException("Could not find BAM file from ID '" + coverageParams.getFileId() + "'"); } } catch (Exception e) { - throw new ToolException("Could not get BAM file from ID " + coverageParams.getBamFileId()); + throw new ToolException("Could not get BAM file from ID " + coverageParams.getFileId()); } // Check if the input file is .bam if (!bamCatalogFile.getName().endsWith(AlignmentConstants.BAM_EXTENSION)) { - throw new ToolException("Invalid input alignment file '" + coverageParams.getBamFileId() + "' (" + bamCatalogFile.getName() + throw new ToolException("Invalid input alignment file '" + coverageParams.getFileId() + "' (" + bamCatalogFile.getName() + "): it must be in BAM format"); } // Getting BAI file - String baiFileId = coverageParams.getBaiFileId(); + String baiFileId; + try { + baiFileId = bamCatalogFile.getInternal().getAlignment().getIndex().getFileId(); + } catch (Exception e) { + throw new ToolException("Could not get internal alignment index file Id from BAM file ID '" + bamCatalogFile.getId() + "'"); + } if (StringUtils.isEmpty(baiFileId)) { - // BAI file ID was not provided, looking for it - logger.info("BAI file ID was not provided, getting it from the internal alignment index of the BAM file ID {}", - bamCatalogFile.getId()); - try { - baiFileId = bamCatalogFile.getInternal().getAlignment().getIndex().getFileId(); - } catch (Exception e) { - throw new ToolException("Could not get internal alignment index file Id from BAM file ID '" + bamCatalogFile.getId()); - } + throw new ToolException("Could not find the alignment index file for the BAM file ID '" + bamCatalogFile.getId() + "'. Please," + + " create the alignment index file before computing the coverage"); } try { baiCatalogFile = catalogManager.getFileManager().get(getStudy(), baiFileId, QueryOptions.empty(), getToken()).first(); if (baiCatalogFile == null) { - throw new ToolException("Could not find BAI file from ID '" + coverageParams.getBaiFileId() + "'"); + throw new ToolException("Could not find BAI file from ID '" + baiFileId + "'"); } } catch (Exception e) { throw new ToolException("Could not get BAI file from file ID " + baiFileId); @@ -114,6 +112,18 @@ protected void check() throws Exception { coverageParams.setWindowSize(Integer.parseInt(COVERAGE_WINDOW_SIZE_DEFAULT)); logger.info("Window size is set to {}", coverageParams.getWindowSize()); } + + // Check overwrite + String bwFileId; + try { + bwFileId = bamCatalogFile.getInternal().getAlignment().getCoverage().getFileId(); + } catch (Exception e) { + bwFileId = null; + } + if (StringUtils.isNotEmpty(bwFileId) && !coverageParams.isOverwrite()) { + throw new ToolException("Coverage file ID '" + bwFileId + "' already exists for file ID '" + bamCatalogFile.getId() + + "'. To overwrite the BIGWIG file use the flag --overwrite"); + } } @Override @@ -189,17 +199,12 @@ protected void run() throws Exception { } // Link generated BIGWIG file and update samples info - File bwCatalogFile = AlignmentAnalysisUtils.linkAndUpdate(bamCatalogFile, bwPath, study, catalogManager, token); + File bwCatalogFile = AlignmentAnalysisUtils.linkAndUpdate(bamCatalogFile, bwPath, getJobId(), study, catalogManager, token); // Update BAM file internal in order to set the coverage index (BIGWIG) FileInternalCoverageIndex fileCoverageIndex = new FileInternalCoverageIndex(new InternalStatus(InternalStatus.READY), bwCatalogFile.getId(), "deeptools bamCoverage", coverageParams.getWindowSize()); - OpenCGAResult openCGAResult = catalogManager.getFileManager().updateFileInternalCoverageIndex(bamCatalogFile, - fileCoverageIndex, token); - if (openCGAResult.getNumUpdated() != 1) { - throw new ToolException("It could not update OpenCGA file catalog (alignment index " + bamCatalogFile.getId() + ") with" - + " the coverage index (BIGWIG) file '" + bwCatalogFile.getId() + "'"); - } + catalogManager.getFileManager().updateFileInternalCoverageIndex(bamCatalogFile, fileCoverageIndex, token); }); } } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentIndexOperation.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentIndexOperation.java index 38d89cf2b5c..44d9bd900a1 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentIndexOperation.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentIndexOperation.java @@ -16,6 +16,7 @@ package org.opencb.opencga.analysis.alignment; +import org.apache.commons.lang3.StringUtils; import org.opencb.biodata.tools.alignment.BamManager; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.analysis.tools.OpenCgaTool; @@ -23,7 +24,8 @@ import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.common.InternalStatus; -import org.opencb.opencga.core.models.file.*; +import org.opencb.opencga.core.models.file.File; +import org.opencb.opencga.core.models.file.FileInternalAlignmentIndex; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.tools.annotations.Tool; @@ -35,10 +37,12 @@ public class AlignmentIndexOperation extends OpenCgaTool { public static final String ID = "alignment-index-run"; - public static final String DESCRIPTION = "Index a given alignment file, e.g., create a .bai file from a .bam file"; + public static final String DESCRIPTION = "Index a given alignment file BAM/CRAM, e.g., create a " + AlignmentConstants.BAI_EXTENSION + + " file from a " + AlignmentConstants.BAM_EXTENSION + " file"; private String study; private String inputFile; + private boolean overwrite = false; private File inputCatalogFile; private Path inputPath; @@ -67,6 +71,18 @@ protected void check() throws Exception { throw new ToolException("Invalid input alignment file '" + inputFile + "': it must be in BAM or CRAM format"); } + // Check overwrite + String baiFileId; + try { + baiFileId = inputCatalogFile.getInternal().getAlignment().getIndex().getFileId(); + } catch (Exception e) { + baiFileId = null; + } + if (StringUtils.isNotEmpty(baiFileId) && !overwrite) { + throw new ToolException("Alignment index file ID '" + baiFileId + "' already exists for file ID '" + inputCatalogFile.getId() + + "'. To overwrite the alignment index file use the flag --overwrite"); + } + outputPath = getOutDir().resolve(filename + (filename.endsWith(AlignmentConstants.BAM_EXTENSION) ? AlignmentConstants.BAI_EXTENSION : AlignmentConstants.CRAI_EXTENSION)); } @@ -103,7 +119,7 @@ protected void run() throws Exception { } // Link generated BAI file and update samples info, related file - File baiCatalogFile = AlignmentAnalysisUtils.linkAndUpdate(inputCatalogFile, outputPath, study, catalogManager, token); + File baiCatalogFile = AlignmentAnalysisUtils.linkAndUpdate(inputCatalogFile, outputPath, getJobId(), study, catalogManager, token); // Update BAM file internal in order to set the alignment index (BAI) FileInternalAlignmentIndex fileAlignmentIndex = new FileInternalAlignmentIndex(new InternalStatus(InternalStatus.READY), @@ -129,4 +145,13 @@ public AlignmentIndexOperation setInputFile(String inputFile) { this.inputFile = inputFile; return this; } + + public boolean isOverwrite() { + return overwrite; + } + + public AlignmentIndexOperation setOverwrite(boolean overwrite) { + this.overwrite = overwrite; + return this; + } } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentStorageManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentStorageManager.java index 2aa314a9fd5..fd8a0cebad5 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentStorageManager.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentStorageManager.java @@ -91,7 +91,7 @@ public AlignmentStorageManager(CatalogManager catalogManager, StorageEngineFacto // INDEX //------------------------------------------------------------------------- - public void index(String study, String inputFile, String outdir, String token) throws ToolException { + public void index(String study, String inputFile, boolean overwrite, String outdir, String token) throws ToolException { ObjectMap params = new ObjectMap(); AlignmentIndexOperation indexOperation = new AlignmentIndexOperation(); @@ -99,6 +99,7 @@ public void index(String study, String inputFile, String outdir, String token) t indexOperation.setStudy(study); indexOperation.setInputFile(inputFile); + indexOperation.setOverwrite(overwrite); indexOperation.start(); } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/AlignmentCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/AlignmentCommandExecutor.java index 07ed1b36c1f..6afb5b6805a 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/AlignmentCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/AlignmentCommandExecutor.java @@ -50,7 +50,6 @@ public class AlignmentCommandExecutor extends InternalCommandExecutor { private AlignmentCommandOptions alignmentCommandOptions; private String jobId; -// private AlignmentStorageEngine alignmentStorageManager; public AlignmentCommandExecutor(AlignmentCommandOptions options) { super(options.analysisCommonOptions); @@ -76,18 +75,6 @@ public void execute() throws Exception { case "gene-coveratge-stats-run": geneCoverageStatsRun(); break; -// case "stats-run": -// statsRun(); -// break; -// case "flagstats-run": -// flagStatsRun(); -// break; -// case "fastqcmetrics-run": -// fastQcMetricsRun(); -// break; -// case "hsmetrics-run": -// hsMetricsRun(); -// break; case "coverage-index-run": coverageRun(); break; @@ -119,38 +106,12 @@ public void execute() throws Exception { private void indexRun() throws Exception { AlignmentCommandOptions.IndexAlignmentCommandOptions cliOptions = alignmentCommandOptions.indexAlignmentCommandOptions; - AlignmentStorageManager alignmentManager = new AlignmentStorageManager(catalogManager, storageEngineFactory, alignmentCommandOptions.internalJobOptions.jobId); + AlignmentStorageManager alignmentManager = new AlignmentStorageManager(catalogManager, storageEngineFactory, + alignmentCommandOptions.internalJobOptions.jobId); - alignmentManager.index(cliOptions.study, cliOptions.file, cliOptions.outdir, cliOptions.commonOptions.token); + alignmentManager.index(cliOptions.study, cliOptions.file, cliOptions.overwrite, cliOptions.outdir, cliOptions.commonOptions.token); } - -// private void query() throws InterruptedException, CatalogException, IOException { -// ObjectMap objectMap = new ObjectMap(); -// objectMap.putIfNotNull("sid", alignmentCommandOptions.queryAlignmentCommandOptions.commonOptions.token); -// objectMap.putIfNotNull("study", alignmentCommandOptions.queryAlignmentCommandOptions.study); -// objectMap.putIfNotNull(AlignmentDBAdaptor.QueryParams.REGION.key(), alignmentCommandOptions.queryAlignmentCommandOptions.region); -// objectMap.putIfNotNull(AlignmentDBAdaptor.QueryParams.MIN_MAPQ.key(), -// alignmentCommandOptions.queryAlignmentCommandOptions.minMappingQuality); -// objectMap.putIfNotNull(AlignmentDBAdaptor.QueryParams.CONTAINED.key(), -// alignmentCommandOptions.queryAlignmentCommandOptions.contained); -// objectMap.putIfNotNull(AlignmentDBAdaptor.QueryParams.MD_FIELD.key(), -// alignmentCommandOptions.queryAlignmentCommandOptions.mdField); -// objectMap.putIfNotNull(AlignmentDBAdaptor.QueryParams.BIN_QUALITIES.key(), -// alignmentCommandOptions.queryAlignmentCommandOptions.binQualities); -// objectMap.putIfNotNull(QueryOptions.LIMIT, alignmentCommandOptions.queryAlignmentCommandOptions.limit); -// objectMap.putIfNotNull(QueryOptions.SKIP, alignmentCommandOptions.queryAlignmentCommandOptions.skip); -// objectMap.putIfNotNull(QueryOptions.COUNT, alignmentCommandOptions.queryAlignmentCommandOptions.count); -// -// OpenCGAClient openCGAClient = new OpenCGAClient(clientConfiguration); -// RestResponse alignments = openCGAClient.getAlignmentClient() -// .query(alignmentCommandOptions.queryAlignmentCommandOptions.fileId, objectMap); -// -// for (ReadAlignment readAlignment : alignments.allResults()) { -// System.out.println(readAlignment); -// } -// } - private void qcRun() throws ToolException { AlignmentCommandOptions.QcAlignmentCommandOptions cliOptions = alignmentCommandOptions.qcAlignmentCommandOptions; @@ -179,64 +140,13 @@ private void geneCoverageStatsRun() throws ToolException { toolRunner.execute(AlignmentGeneCoverageStatsAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); } -// private void statsRun() throws ToolException { -// AlignmentCommandOptions.StatsAlignmentCommandOptions cliOptions = alignmentCommandOptions.statsAlignmentCommandOptions; -// -// ObjectMap params = new AlignmentStatsParams( -// cliOptions.file, -// cliOptions.outdir -// ).toObjectMap(cliOptions.commonOptions.params) -// .append(ParamConstants.STUDY_PARAM, cliOptions.study); -// -// toolRunner.execute(AlignmentStatsAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); -// } -// -// private void flagStatsRun() throws ToolException { -// AlignmentCommandOptions.FlagStatsAlignmentCommandOptions cliOptions = alignmentCommandOptions.flagStatsAlignmentCommandOptions; -// -// ObjectMap params = new AlignmentFlagStatsParams( -// cliOptions.file, -// cliOptions.outdir -// ).toObjectMap(cliOptions.commonOptions.params) -// .append(ParamConstants.STUDY_PARAM, cliOptions.study); -// -// toolRunner.execute(AlignmentFlagStatsAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); -// } -// -// private void fastQcMetricsRun() throws ToolException { -// AlignmentCommandOptions.FastQcMetricsAlignmentCommandOptions cliOptions = alignmentCommandOptions -// .fastQcMetricsAlignmentCommandOptions; -// -// ObjectMap params = new AlignmentFastQcMetricsParams( -// cliOptions.file, -// cliOptions.outdir -// ).toObjectMap(cliOptions.commonOptions.params) -// .append(ParamConstants.STUDY_PARAM, cliOptions.study); -// -// toolRunner.execute(AlignmentFastQcMetricsAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); -// } -// -// private void hsMetricsRun() throws ToolException { -// AlignmentCommandOptions.HsMetricsAlignmentCommandOptions cliOptions = alignmentCommandOptions.hsMetricsAlignmentCommandOptions; -// -// ObjectMap params = new AlignmentHsMetricsParams( -// cliOptions.bamFile, -// cliOptions.bedFile, -// cliOptions.dictFile, -// cliOptions.outdir -// ).toObjectMap(cliOptions.commonOptions.params) -// .append(ParamConstants.STUDY_PARAM, cliOptions.study); -// -// toolRunner.execute(AlignmentHsMetricsAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); -// } - private void coverageRun() throws ToolException { AlignmentCommandOptions.CoverageAlignmentCommandOptions cliOptions = alignmentCommandOptions.coverageAlignmentCommandOptions; ObjectMap params = new CoverageIndexParams( cliOptions.bamFileId, - cliOptions.baiFileId, - cliOptions.windowSize + cliOptions.windowSize, + cliOptions.overwrite ).toObjectMap(cliOptions.commonOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study); toolRunner.execute(AlignmentCoverageAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); @@ -323,27 +233,4 @@ private void picard() throws Exception { toolRunner.execute(PicardWrapperAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); } - - //------------------------------------------------------------------------- - // M I S C E L A N E O U S M E T H O D S - //------------------------------------------------------------------------- - - private void addParam(Map map, String key, Object value) { - if (value == null) { - return; - } - - if (value instanceof String) { - if (!((String) value).isEmpty()) { - map.put(key, (String) value); - } - } else if (value instanceof Integer) { - map.put(key, Integer.toString((int) value)); - } else if (value instanceof Boolean) { - map.put(key, Boolean.toString((boolean) value)); - } else { - throw new UnsupportedOperationException(); - } - } - } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/AlignmentCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/AlignmentCommandOptions.java index 628988b9302..b0d4f6bb39b 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/AlignmentCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/AlignmentCommandOptions.java @@ -17,6 +17,8 @@ package org.opencb.opencga.app.cli.internal.options; import com.beust.jcommander.*; +import org.opencb.opencga.analysis.alignment.AlignmentCoverageAnalysis; +import org.opencb.opencga.analysis.alignment.AlignmentIndexOperation; import org.opencb.opencga.analysis.wrappers.bwa.BwaWrapperAnalysis; import org.opencb.opencga.analysis.wrappers.deeptools.DeeptoolsWrapperAnalysis; import org.opencb.opencga.analysis.wrappers.fastqc.FastqcWrapperAnalysis; @@ -24,6 +26,7 @@ import org.opencb.opencga.analysis.wrappers.samtools.SamtoolsWrapperAnalysis; import org.opencb.opencga.app.cli.GeneralCliOptions; import org.opencb.opencga.app.cli.internal.InternalCliOptionsParser; +import org.opencb.opencga.core.api.FieldConstants; import java.util.HashMap; import java.util.Map; @@ -40,10 +43,6 @@ public class AlignmentCommandOptions { public QueryAlignmentCommandOptions queryAlignmentCommandOptions; public QcAlignmentCommandOptions qcAlignmentCommandOptions; public GeneCoverageStatsAlignmentCommandOptions geneCoverageStatsAlignmentCommandOptions; -// public StatsAlignmentCommandOptions statsAlignmentCommandOptions; -// public FlagStatsAlignmentCommandOptions flagStatsAlignmentCommandOptions; -// public FastQcMetricsAlignmentCommandOptions fastQcMetricsAlignmentCommandOptions; -// public HsMetricsAlignmentCommandOptions hsMetricsAlignmentCommandOptions; public CoverageAlignmentCommandOptions coverageAlignmentCommandOptions; public CoverageQueryAlignmentCommandOptions coverageQueryAlignmentCommandOptions; public CoverageRatioAlignmentCommandOptions coverageRatioAlignmentCommandOptions; @@ -76,10 +75,6 @@ public AlignmentCommandOptions(GeneralCliOptions.CommonCommandOptions analysisCo this.queryAlignmentCommandOptions = new QueryAlignmentCommandOptions(); this.qcAlignmentCommandOptions = new QcAlignmentCommandOptions(); this.geneCoverageStatsAlignmentCommandOptions = new GeneCoverageStatsAlignmentCommandOptions(); -// this.statsAlignmentCommandOptions = new StatsAlignmentCommandOptions(); -// this.flagStatsAlignmentCommandOptions = new FlagStatsAlignmentCommandOptions(); -// this.fastQcMetricsAlignmentCommandOptions = new FastQcMetricsAlignmentCommandOptions(); -// this.hsMetricsAlignmentCommandOptions = new HsMetricsAlignmentCommandOptions(); this.coverageAlignmentCommandOptions = new CoverageAlignmentCommandOptions(); this.coverageQueryAlignmentCommandOptions = new CoverageQueryAlignmentCommandOptions(); this.coverageRatioAlignmentCommandOptions = new CoverageRatioAlignmentCommandOptions(); @@ -92,7 +87,7 @@ public AlignmentCommandOptions(GeneralCliOptions.CommonCommandOptions analysisCo this.picardCommandOptions = new PicardCommandOptions(); } - @Parameters(commandNames = {"index"}, commandDescription = ALIGNMENT_INDEX_DESCRIPTION) + @Parameters(commandNames = {"index"}, commandDescription = AlignmentIndexOperation.DESCRIPTION) public class IndexAlignmentCommandOptions extends GeneralCliOptions.StudyOption { @ParametersDelegate @@ -104,9 +99,12 @@ public class IndexAlignmentCommandOptions extends GeneralCliOptions.StudyOption @ParametersDelegate public Object internalJobOptions = internalJobOptionsObject; - @Parameter(names = {"--file"}, description = FILE_ID_DESCRIPTION, required = true, arity = 1) + @Parameter(names = {"--file-id"}, description = FieldConstants.ALIGNMENT_BAM_FILE_ID_DESCRIPTION, required = true, arity = 1) public String file; + @Parameter(names = {"--overwrite"}, description = FieldConstants.ALIGNMENT_OVERWRITE_DESCRIPTION, arity = 0) + public boolean overwrite; + @Parameter(names = {"-o", "--outdir"}, description = OUTPUT_DIRECTORY_DESCRIPTION) public String outdir; } @@ -320,7 +318,7 @@ public class GeneCoverageStatsAlignmentCommandOptions extends GeneralCliOptions. public String outdir; } - @Parameters(commandNames = {"coverage-index-run"}, commandDescription = ALIGNMENT_COVERAGE_DESCRIPTION) + @Parameters(commandNames = {"coverage-index-run"}, commandDescription = AlignmentCoverageAnalysis.DESCRIPTION) public class CoverageAlignmentCommandOptions extends GeneralCliOptions.StudyOption { @ParametersDelegate @@ -332,15 +330,15 @@ public class CoverageAlignmentCommandOptions extends GeneralCliOptions.StudyOpti @ParametersDelegate public Object internalJobOptions = internalJobOptionsObject; - @Parameter(names = {"--bam-file-id"}, description = "BAM file ID", required = true, arity = 1) + @Parameter(names = {"--file-id"}, description = FieldConstants.ALIGNMENT_BAM_FILE_ID_DESCRIPTION, required = true, arity = 1) public String bamFileId; - @Parameter(names = {"--bai-file-id"}, description = "BAI file ID; if not provided, it will search the most recent one", arity = 1) - public String baiFileId; - @Parameter(names = {"--window-size"}, description = COVERAGE_WINDOW_SIZE_DESCRIPTION, arity = 1) public int windowSize = 1; + @Parameter(names = {"--overwrite"}, description = FieldConstants.ALIGNMENT_OVERWRITE_DESCRIPTION, arity = 0) + public boolean overwrite; + @Parameter(names = {"-o", "--outdir"}, description = OUTPUT_DIRECTORY_DESCRIPTION) public String outdir; } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java index 48db1bde2fd..91c83dd3458 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java @@ -1,5 +1,5 @@ /* -* Copyright 2015-2024-02-14 OpenCB +* Copyright 2015-2024-02-19 OpenCB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java index af1660a9e33..34091c6c743 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java @@ -1,5 +1,5 @@ /* -* Copyright 2015-2024-02-14 OpenCB +* Copyright 2015-2024-02-19 OpenCB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisAlignmentCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisAlignmentCommandExecutor.java index 217b7421f32..bf253173991 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisAlignmentCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisAlignmentCommandExecutor.java @@ -178,9 +178,9 @@ private RestResponse runCoverageIndex() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), CoverageIndexParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "bamFileId",commandOptions.bamFileId, true); - putNestedIfNotEmpty(beanParams, "baiFileId",commandOptions.baiFileId, true); + putNestedIfNotEmpty(beanParams, "fileId",commandOptions.fileId, true); putNestedIfNotNull(beanParams, "windowSize",commandOptions.windowSize, true); + putNestedIfNotNull(beanParams, "overwrite",commandOptions.overwrite, true); coverageIndexParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -387,7 +387,7 @@ private RestResponse runIndex() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), AlignmentIndexParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "file",commandOptions.file, true); + putNestedIfNotEmpty(beanParams, "fileId",commandOptions.fileId, true); putNestedIfNotNull(beanParams, "overwrite",commandOptions.overwrite, true); alignmentIndexParams = JacksonUtils.getDefaultObjectMapper().copy() diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisAlignmentCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisAlignmentCommandOptions.java index cb5e4f77e45..19dbef7dfc4 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisAlignmentCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisAlignmentCommandOptions.java @@ -115,7 +115,7 @@ public class RunBwaCommandOptions { } - @Parameters(commandNames = {"coverage-index-run"}, commandDescription ="Compute coverage for a list of alignment files") + @Parameters(commandNames = {"coverage-index-run"}, commandDescription ="Compute the coverage from a given BAM alignment file, e.g., create a .bw file from a .bam file") public class RunCoverageIndexCommandOptions { @ParametersDelegate @@ -142,14 +142,14 @@ public class RunCoverageIndexCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; - @Parameter(names = {"--bam-file-id"}, description = "The body web service bamFileId parameter", required = false, arity = 1) - public String bamFileId; + @Parameter(names = {"--file-id"}, description = "Alignment file ID (in format BAM or CRAM)", required = false, arity = 1) + public String fileId; - @Parameter(names = {"--bai-file-id"}, description = "The body web service baiFileId parameter", required = false, arity = 1) - public String baiFileId; + @Parameter(names = {"--window-size"}, description = "Size of the bins, in bases, for the output of the BIGWIG file", required = false, arity = 1) + public Integer windowSize = 50; - @Parameter(names = {"--window-size"}, description = "The body web service windowSize parameter", required = false, arity = 1) - public Integer windowSize; + @Parameter(names = {"--overwrite"}, description = "Overwrite file", required = false, help = true, arity = 0) + public boolean overwrite = false; } @@ -360,7 +360,7 @@ public class RunFastqcCommandOptions { } - @Parameters(commandNames = {"index-run"}, commandDescription ="Index alignment file") + @Parameters(commandNames = {"index-run"}, commandDescription ="Index a given alignment file BAM/CRAM, e.g., create a .bai file from a .bam file") public class RunIndexCommandOptions { @ParametersDelegate @@ -387,10 +387,10 @@ public class RunIndexCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; - @Parameter(names = {"--file"}, description = "The body web service file parameter", required = false, arity = 1) - public String file; + @Parameter(names = {"--file-id"}, description = "Alignment file ID (in format BAM or CRAM)", required = false, arity = 1) + public String fileId; - @Parameter(names = {"--overwrite"}, description = "The body web service overwrite parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--overwrite"}, description = "Overwrite file", required = false, help = true, arity = 0) public boolean overwrite = false; } @@ -580,7 +580,5 @@ public class RunSamtoolsCommandOptions { @DynamicParameter(names = {"--samtools-params"}, description = "The body web service samtoolsParams parameter. Use: --samtools-params key=value", required = false) public java.util.Map samtoolsParams = new HashMap<>(); //Dynamic parameters must be initialized; - } - } \ No newline at end of file diff --git a/opencga-client/src/main/R/R/Admin-methods.R b/opencga-client/src/main/R/R/Admin-methods.R index c3041969f51..e572676578b 100644 --- a/opencga-client/src/main/R/R/Admin-methods.R +++ b/opencga-client/src/main/R/R/Admin-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-02-14 +# Autogenerated on: 2024-02-19 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/R/R/Alignment-methods.R b/opencga-client/src/main/R/R/Alignment-methods.R index 53cd5d9802c..adf29a2a38c 100644 --- a/opencga-client/src/main/R/R/Alignment-methods.R +++ b/opencga-client/src/main/R/R/Alignment-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-02-14 +# Autogenerated on: 2024-02-19 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -55,7 +55,7 @@ setMethod("alignmentClient", "OpencgaR", function(OpencgaR, endpointName, params subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), #' @section Endpoint /{apiVersion}/analysis/alignment/coverage/index/run: - #' Compute coverage for a list of alignment files. + #' Compute the coverage from a given BAM alignment file, e.g., create a .bw file from a .bam file. #' @param study study. #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. @@ -143,7 +143,7 @@ setMethod("alignmentClient", "OpencgaR", function(OpencgaR, endpointName, params subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), #' @section Endpoint /{apiVersion}/analysis/alignment/index/run: - #' Index alignment file. + #' Index a given alignment file BAM/CRAM, e.g., create a .bai file from a .bam file. #' @param study study. #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. diff --git a/opencga-client/src/main/R/R/AllGenerics.R b/opencga-client/src/main/R/R/AllGenerics.R index f9fdb51fe6f..ff9ada80c02 100644 --- a/opencga-client/src/main/R/R/AllGenerics.R +++ b/opencga-client/src/main/R/R/AllGenerics.R @@ -1,51 +1,51 @@ # ############################################################################## ## UserClient -setGeneric("userClient", function(OpencgaR, user, users, filterId, endpointName, params=NULL, ...) +setGeneric("userClient", function(OpencgaR, user, filterId, users, endpointName, params=NULL, ...) standardGeneric("userClient")) # ############################################################################## ## ProjectClient -setGeneric("projectClient", function(OpencgaR, project, projects, endpointName, params=NULL, ...) +setGeneric("projectClient", function(OpencgaR, projects, project, endpointName, params=NULL, ...) standardGeneric("projectClient")) # ############################################################################## ## StudyClient -setGeneric("studyClient", function(OpencgaR, group, variableSet, studies, templateId, study, members, endpointName, params=NULL, ...) +setGeneric("studyClient", function(OpencgaR, templateId, variableSet, studies, members, study, group, endpointName, params=NULL, ...) standardGeneric("studyClient")) # ############################################################################## ## FileClient -setGeneric("fileClient", function(OpencgaR, folder, annotationSet, file, files, members, endpointName, params=NULL, ...) +setGeneric("fileClient", function(OpencgaR, annotationSet, files, folder, file, members, endpointName, params=NULL, ...) standardGeneric("fileClient")) # ############################################################################## ## JobClient -setGeneric("jobClient", function(OpencgaR, job, members, jobs, endpointName, params=NULL, ...) +setGeneric("jobClient", function(OpencgaR, members, jobs, job, endpointName, params=NULL, ...) standardGeneric("jobClient")) # ############################################################################## ## SampleClient -setGeneric("sampleClient", function(OpencgaR, samples, annotationSet, members, sample, endpointName, params=NULL, ...) +setGeneric("sampleClient", function(OpencgaR, members, sample, annotationSet, samples, endpointName, params=NULL, ...) standardGeneric("sampleClient")) # ############################################################################## ## IndividualClient -setGeneric("individualClient", function(OpencgaR, individuals, members, annotationSet, individual, endpointName, params=NULL, ...) +setGeneric("individualClient", function(OpencgaR, members, individual, annotationSet, individuals, endpointName, params=NULL, ...) standardGeneric("individualClient")) # ############################################################################## ## FamilyClient -setGeneric("familyClient", function(OpencgaR, families, family, members, annotationSet, endpointName, params=NULL, ...) +setGeneric("familyClient", function(OpencgaR, members, annotationSet, family, families, endpointName, params=NULL, ...) standardGeneric("familyClient")) # ############################################################################## ## CohortClient -setGeneric("cohortClient", function(OpencgaR, cohort, members, annotationSet, cohorts, endpointName, params=NULL, ...) +setGeneric("cohortClient", function(OpencgaR, members, annotationSet, cohort, cohorts, endpointName, params=NULL, ...) standardGeneric("cohortClient")) # ############################################################################## ## PanelClient -setGeneric("panelClient", function(OpencgaR, panels, members, endpointName, params=NULL, ...) +setGeneric("panelClient", function(OpencgaR, members, panels, endpointName, params=NULL, ...) standardGeneric("panelClient")) # ############################################################################## @@ -60,7 +60,7 @@ setGeneric("variantClient", function(OpencgaR, endpointName, params=NULL, ...) # ############################################################################## ## ClinicalClient -setGeneric("clinicalClient", function(OpencgaR, interpretation, clinicalAnalysis, interpretations, annotationSet, clinicalAnalyses, members, endpointName, params=NULL, ...) +setGeneric("clinicalClient", function(OpencgaR, clinicalAnalyses, interpretation, annotationSet, interpretations, members, clinicalAnalysis, endpointName, params=NULL, ...) standardGeneric("clinicalClient")) # ############################################################################## diff --git a/opencga-client/src/main/R/R/Clinical-methods.R b/opencga-client/src/main/R/R/Clinical-methods.R index ecd04d6d74d..cefe09aaac0 100644 --- a/opencga-client/src/main/R/R/Clinical-methods.R +++ b/opencga-client/src/main/R/R/Clinical-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-02-14 +# Autogenerated on: 2024-02-19 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -61,7 +61,7 @@ #' [*]: Required parameter #' @export -setMethod("clinicalClient", "OpencgaR", function(OpencgaR, interpretation, clinicalAnalysis, interpretations, annotationSet, clinicalAnalyses, members, endpointName, params=NULL, ...) { +setMethod("clinicalClient", "OpencgaR", function(OpencgaR, clinicalAnalyses, interpretation, annotationSet, interpretations, members, clinicalAnalysis, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/analysis/clinical/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/Cohort-methods.R b/opencga-client/src/main/R/R/Cohort-methods.R index b4cbb071860..b2083be84d3 100644 --- a/opencga-client/src/main/R/R/Cohort-methods.R +++ b/opencga-client/src/main/R/R/Cohort-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-02-14 +# Autogenerated on: 2024-02-19 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -39,7 +39,7 @@ #' [*]: Required parameter #' @export -setMethod("cohortClient", "OpencgaR", function(OpencgaR, cohort, members, annotationSet, cohorts, endpointName, params=NULL, ...) { +setMethod("cohortClient", "OpencgaR", function(OpencgaR, members, annotationSet, cohort, cohorts, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/cohorts/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/Family-methods.R b/opencga-client/src/main/R/R/Family-methods.R index 9f965314e0f..78128f35c38 100644 --- a/opencga-client/src/main/R/R/Family-methods.R +++ b/opencga-client/src/main/R/R/Family-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-02-14 +# Autogenerated on: 2024-02-19 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -38,7 +38,7 @@ #' [*]: Required parameter #' @export -setMethod("familyClient", "OpencgaR", function(OpencgaR, families, family, members, annotationSet, endpointName, params=NULL, ...) { +setMethod("familyClient", "OpencgaR", function(OpencgaR, members, annotationSet, family, families, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/families/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/File-methods.R b/opencga-client/src/main/R/R/File-methods.R index fd0ad7f691c..adb97b96505 100644 --- a/opencga-client/src/main/R/R/File-methods.R +++ b/opencga-client/src/main/R/R/File-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-02-14 +# Autogenerated on: 2024-02-19 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -54,7 +54,7 @@ #' [*]: Required parameter #' @export -setMethod("fileClient", "OpencgaR", function(OpencgaR, folder, annotationSet, file, files, members, endpointName, params=NULL, ...) { +setMethod("fileClient", "OpencgaR", function(OpencgaR, annotationSet, files, folder, file, members, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/files/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/GA4GH-methods.R b/opencga-client/src/main/R/R/GA4GH-methods.R index d9d2c5689e4..77ad34ad5a3 100644 --- a/opencga-client/src/main/R/R/GA4GH-methods.R +++ b/opencga-client/src/main/R/R/GA4GH-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-02-14 +# Autogenerated on: 2024-02-19 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/R/R/Individual-methods.R b/opencga-client/src/main/R/R/Individual-methods.R index 72d5e0839dc..cb0e74109de 100644 --- a/opencga-client/src/main/R/R/Individual-methods.R +++ b/opencga-client/src/main/R/R/Individual-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-02-14 +# Autogenerated on: 2024-02-19 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -39,7 +39,7 @@ #' [*]: Required parameter #' @export -setMethod("individualClient", "OpencgaR", function(OpencgaR, individuals, members, annotationSet, individual, endpointName, params=NULL, ...) { +setMethod("individualClient", "OpencgaR", function(OpencgaR, members, individual, annotationSet, individuals, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/individuals/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/Job-methods.R b/opencga-client/src/main/R/R/Job-methods.R index edfb52fbaff..1c664f80e8d 100644 --- a/opencga-client/src/main/R/R/Job-methods.R +++ b/opencga-client/src/main/R/R/Job-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-02-14 +# Autogenerated on: 2024-02-19 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -40,7 +40,7 @@ #' [*]: Required parameter #' @export -setMethod("jobClient", "OpencgaR", function(OpencgaR, job, members, jobs, endpointName, params=NULL, ...) { +setMethod("jobClient", "OpencgaR", function(OpencgaR, members, jobs, job, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/jobs/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/Meta-methods.R b/opencga-client/src/main/R/R/Meta-methods.R index 730500d7b4a..4d913eb7009 100644 --- a/opencga-client/src/main/R/R/Meta-methods.R +++ b/opencga-client/src/main/R/R/Meta-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-02-14 +# Autogenerated on: 2024-02-19 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/R/R/Operation-methods.R b/opencga-client/src/main/R/R/Operation-methods.R index 11e89656de2..8c4cf77b9c0 100644 --- a/opencga-client/src/main/R/R/Operation-methods.R +++ b/opencga-client/src/main/R/R/Operation-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-02-14 +# Autogenerated on: 2024-02-19 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/R/R/Panel-methods.R b/opencga-client/src/main/R/R/Panel-methods.R index 3d1c5bfb8d3..ff584f8fb1e 100644 --- a/opencga-client/src/main/R/R/Panel-methods.R +++ b/opencga-client/src/main/R/R/Panel-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-02-14 +# Autogenerated on: 2024-02-19 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -36,7 +36,7 @@ #' [*]: Required parameter #' @export -setMethod("panelClient", "OpencgaR", function(OpencgaR, panels, members, endpointName, params=NULL, ...) { +setMethod("panelClient", "OpencgaR", function(OpencgaR, members, panels, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/panels/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/Project-methods.R b/opencga-client/src/main/R/R/Project-methods.R index ebbcb80dd92..564515f7f66 100644 --- a/opencga-client/src/main/R/R/Project-methods.R +++ b/opencga-client/src/main/R/R/Project-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-02-14 +# Autogenerated on: 2024-02-19 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -34,7 +34,7 @@ #' [*]: Required parameter #' @export -setMethod("projectClient", "OpencgaR", function(OpencgaR, project, projects, endpointName, params=NULL, ...) { +setMethod("projectClient", "OpencgaR", function(OpencgaR, projects, project, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/projects/create: diff --git a/opencga-client/src/main/R/R/Sample-methods.R b/opencga-client/src/main/R/R/Sample-methods.R index c5f0bbd3357..03b6788fbb6 100644 --- a/opencga-client/src/main/R/R/Sample-methods.R +++ b/opencga-client/src/main/R/R/Sample-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-02-14 +# Autogenerated on: 2024-02-19 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -39,7 +39,7 @@ #' [*]: Required parameter #' @export -setMethod("sampleClient", "OpencgaR", function(OpencgaR, samples, annotationSet, members, sample, endpointName, params=NULL, ...) { +setMethod("sampleClient", "OpencgaR", function(OpencgaR, members, sample, annotationSet, samples, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/samples/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/Study-methods.R b/opencga-client/src/main/R/R/Study-methods.R index c86c0da79b1..87fb4b00209 100644 --- a/opencga-client/src/main/R/R/Study-methods.R +++ b/opencga-client/src/main/R/R/Study-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-02-14 +# Autogenerated on: 2024-02-19 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -46,7 +46,7 @@ #' [*]: Required parameter #' @export -setMethod("studyClient", "OpencgaR", function(OpencgaR, group, variableSet, studies, templateId, study, members, endpointName, params=NULL, ...) { +setMethod("studyClient", "OpencgaR", function(OpencgaR, templateId, variableSet, studies, members, study, group, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/studies/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/User-methods.R b/opencga-client/src/main/R/R/User-methods.R index fb403c674e6..a88d0a01cdc 100644 --- a/opencga-client/src/main/R/R/User-methods.R +++ b/opencga-client/src/main/R/R/User-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-02-14 +# Autogenerated on: 2024-02-19 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -38,7 +38,7 @@ #' [*]: Required parameter #' @export -setMethod("userClient", "OpencgaR", function(OpencgaR, user, users, filterId, endpointName, params=NULL, ...) { +setMethod("userClient", "OpencgaR", function(OpencgaR, user, filterId, users, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/users/login: diff --git a/opencga-client/src/main/R/R/Variant-methods.R b/opencga-client/src/main/R/R/Variant-methods.R index d114e5d4882..bf160b91166 100644 --- a/opencga-client/src/main/R/R/Variant-methods.R +++ b/opencga-client/src/main/R/R/Variant-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-02-14 +# Autogenerated on: 2024-02-19 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AdminClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AdminClient.java index d4fcd4b1d60..744d3f7e9e9 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AdminClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AdminClient.java @@ -36,7 +36,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-02-14 +* Autogenerated on: 2024-02-19 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AlignmentClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AlignmentClient.java index c6c8c84fd8b..e54fa6c1382 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AlignmentClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AlignmentClient.java @@ -40,7 +40,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-02-14 +* Autogenerated on: 2024-02-19 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -77,7 +77,7 @@ public RestResponse runBwa(BwaWrapperParams data, ObjectMap params) throws } /** - * Compute coverage for a list of alignment files. + * Compute the coverage from a given BAM alignment file, e.g., create a .bw file from a .bam file. * @param data Coverage computation parameters. * @param params Map containing any of the following optional parameters. * study: study. @@ -214,7 +214,7 @@ public RestResponse runFastqc(FastqcWrapperParams data, ObjectMap params) t } /** - * Index alignment file. + * Index a given alignment file BAM/CRAM, e.g., create a .bai file from a .bam file. * @param data Alignment index params. * @param params Map containing any of the following optional parameters. * study: study. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalAnalysisClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalAnalysisClient.java index e4858049e0b..d1880093243 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalAnalysisClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalAnalysisClient.java @@ -54,7 +54,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-02-14 +* Autogenerated on: 2024-02-19 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/CohortClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/CohortClient.java index f136be11310..5353610096a 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/CohortClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/CohortClient.java @@ -37,7 +37,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-02-14 +* Autogenerated on: 2024-02-19 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/DiseasePanelClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/DiseasePanelClient.java index 497e02f5445..e53672f2de6 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/DiseasePanelClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/DiseasePanelClient.java @@ -35,7 +35,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-02-14 +* Autogenerated on: 2024-02-19 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FamilyClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FamilyClient.java index 437d6e65255..488f65ca967 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FamilyClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FamilyClient.java @@ -36,7 +36,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-02-14 +* Autogenerated on: 2024-02-19 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FileClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FileClient.java index 66ae008ab7c..3302dc2b444 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FileClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FileClient.java @@ -43,7 +43,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-02-14 +* Autogenerated on: 2024-02-19 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/GA4GHClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/GA4GHClient.java index 5d007b409bd..b60696ce807 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/GA4GHClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/GA4GHClient.java @@ -27,7 +27,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-02-14 +* Autogenerated on: 2024-02-19 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/IndividualClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/IndividualClient.java index 67bb7feb715..3e90a072c20 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/IndividualClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/IndividualClient.java @@ -36,7 +36,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-02-14 +* Autogenerated on: 2024-02-19 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java index c4fb407be93..24f479f78a1 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java @@ -37,7 +37,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-02-14 +* Autogenerated on: 2024-02-19 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/MetaClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/MetaClient.java index 4f1873c571c..1fa28045d78 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/MetaClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/MetaClient.java @@ -28,7 +28,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-02-14 +* Autogenerated on: 2024-02-19 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ProjectClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ProjectClient.java index 19b3144a06d..8bc22703b5a 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ProjectClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ProjectClient.java @@ -32,7 +32,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-02-14 +* Autogenerated on: 2024-02-19 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/SampleClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/SampleClient.java index f1e5b7ac798..4ce697cc525 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/SampleClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/SampleClient.java @@ -36,7 +36,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-02-14 +* Autogenerated on: 2024-02-19 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java index 4a9b4550de8..31c1656eb3c 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java @@ -45,7 +45,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-02-14 +* Autogenerated on: 2024-02-19 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/UserClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/UserClient.java index bbf582a2cc6..bcb01480219 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/UserClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/UserClient.java @@ -36,7 +36,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-02-14 +* Autogenerated on: 2024-02-19 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantClient.java index c7f91a4730a..7bac1c031ac 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantClient.java @@ -62,7 +62,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-02-14 +* Autogenerated on: 2024-02-19 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantOperationClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantOperationClient.java index 92f5dd6772d..607280de631 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantOperationClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantOperationClient.java @@ -50,7 +50,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-02-14 +* Autogenerated on: 2024-02-19 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Admin.js b/opencga-client/src/main/javascript/Admin.js index 30fe2d2b09b..75a59ef7fc5 100644 --- a/opencga-client/src/main/javascript/Admin.js +++ b/opencga-client/src/main/javascript/Admin.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-02-14 + * Autogenerated on: 2024-02-19 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Alignment.js b/opencga-client/src/main/javascript/Alignment.js index 764a51d164d..d21eda0ba06 100644 --- a/opencga-client/src/main/javascript/Alignment.js +++ b/opencga-client/src/main/javascript/Alignment.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-02-14 + * Autogenerated on: 2024-02-19 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -47,7 +47,7 @@ export default class Alignment extends OpenCGAParentClass { return this._post("analysis", null, "alignment/bwa", null, "run", data, params); } - /** Compute coverage for a list of alignment files + /** Compute the coverage from a given BAM alignment file, e.g., create a .bw file from a .bam file * @param {Object} data - Coverage computation parameters. * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.study] - study. @@ -157,7 +157,7 @@ export default class Alignment extends OpenCGAParentClass { return this._post("analysis", null, "alignment/fastqc", null, "run", data, params); } - /** Index alignment file + /** Index a given alignment file BAM/CRAM, e.g., create a .bai file from a .bam file * @param {Object} data - Alignment index params. * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.study] - study. diff --git a/opencga-client/src/main/javascript/ClinicalAnalysis.js b/opencga-client/src/main/javascript/ClinicalAnalysis.js index 1556652d3d3..df74e1b9ab5 100644 --- a/opencga-client/src/main/javascript/ClinicalAnalysis.js +++ b/opencga-client/src/main/javascript/ClinicalAnalysis.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-02-14 + * Autogenerated on: 2024-02-19 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Cohort.js b/opencga-client/src/main/javascript/Cohort.js index 97ef19dcbdf..d94234c929a 100644 --- a/opencga-client/src/main/javascript/Cohort.js +++ b/opencga-client/src/main/javascript/Cohort.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-02-14 + * Autogenerated on: 2024-02-19 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/DiseasePanel.js b/opencga-client/src/main/javascript/DiseasePanel.js index 52601203a30..d4aedcc7593 100644 --- a/opencga-client/src/main/javascript/DiseasePanel.js +++ b/opencga-client/src/main/javascript/DiseasePanel.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-02-14 + * Autogenerated on: 2024-02-19 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Family.js b/opencga-client/src/main/javascript/Family.js index 608c23c087d..b22fff636d3 100644 --- a/opencga-client/src/main/javascript/Family.js +++ b/opencga-client/src/main/javascript/Family.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-02-14 + * Autogenerated on: 2024-02-19 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/File.js b/opencga-client/src/main/javascript/File.js index f09d5c4562f..c9c141c6643 100644 --- a/opencga-client/src/main/javascript/File.js +++ b/opencga-client/src/main/javascript/File.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-02-14 + * Autogenerated on: 2024-02-19 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/GA4GH.js b/opencga-client/src/main/javascript/GA4GH.js index b656c0bd696..08b158d10cc 100644 --- a/opencga-client/src/main/javascript/GA4GH.js +++ b/opencga-client/src/main/javascript/GA4GH.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-02-14 + * Autogenerated on: 2024-02-19 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Individual.js b/opencga-client/src/main/javascript/Individual.js index d8bd896ae88..cf05cba4f63 100644 --- a/opencga-client/src/main/javascript/Individual.js +++ b/opencga-client/src/main/javascript/Individual.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-02-14 + * Autogenerated on: 2024-02-19 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Job.js b/opencga-client/src/main/javascript/Job.js index 9489c82bc7a..c880b14248b 100644 --- a/opencga-client/src/main/javascript/Job.js +++ b/opencga-client/src/main/javascript/Job.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-02-14 + * Autogenerated on: 2024-02-19 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Meta.js b/opencga-client/src/main/javascript/Meta.js index b5636ff6a5e..81da6d19dc9 100644 --- a/opencga-client/src/main/javascript/Meta.js +++ b/opencga-client/src/main/javascript/Meta.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-02-14 + * Autogenerated on: 2024-02-19 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Project.js b/opencga-client/src/main/javascript/Project.js index bde7382c84d..f74ca2c10c0 100644 --- a/opencga-client/src/main/javascript/Project.js +++ b/opencga-client/src/main/javascript/Project.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-02-14 + * Autogenerated on: 2024-02-19 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Sample.js b/opencga-client/src/main/javascript/Sample.js index 990f3b3b04e..cf7c00bb3a6 100644 --- a/opencga-client/src/main/javascript/Sample.js +++ b/opencga-client/src/main/javascript/Sample.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-02-14 + * Autogenerated on: 2024-02-19 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Study.js b/opencga-client/src/main/javascript/Study.js index 7d443653ad9..78447e0b2f7 100644 --- a/opencga-client/src/main/javascript/Study.js +++ b/opencga-client/src/main/javascript/Study.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-02-14 + * Autogenerated on: 2024-02-19 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/User.js b/opencga-client/src/main/javascript/User.js index 2093a3aadb4..9ae25a5c05b 100644 --- a/opencga-client/src/main/javascript/User.js +++ b/opencga-client/src/main/javascript/User.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-02-14 + * Autogenerated on: 2024-02-19 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Variant.js b/opencga-client/src/main/javascript/Variant.js index c98bccd6bf5..25ed20c87b9 100644 --- a/opencga-client/src/main/javascript/Variant.js +++ b/opencga-client/src/main/javascript/Variant.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-02-14 + * Autogenerated on: 2024-02-19 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/VariantOperation.js b/opencga-client/src/main/javascript/VariantOperation.js index ad54d4104cf..1e228bf7db0 100644 --- a/opencga-client/src/main/javascript/VariantOperation.js +++ b/opencga-client/src/main/javascript/VariantOperation.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-02-14 + * Autogenerated on: 2024-02-19 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/admin_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/admin_client.py index 4a60e9c7e2f..a0cfae24658 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/admin_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/admin_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-02-14 + Autogenerated on: 2024-02-19 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/alignment_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/alignment_client.py index 98b40a3f711..ea10373b0b6 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/alignment_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/alignment_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-02-14 + Autogenerated on: 2024-02-19 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -41,7 +41,8 @@ def run_bwa(self, data=None, **options): def run_coverage_index(self, data=None, **options): """ - Compute coverage for a list of alignment files. + Compute the coverage from a given BAM alignment file, e.g., create a + .bw file from a .bam file. PATH: /{apiVersion}/analysis/alignment/coverage/index/run :param dict data: Coverage computation parameters. (REQUIRED) @@ -189,7 +190,8 @@ def run_fastqc(self, data=None, **options): def run_index(self, data=None, **options): """ - Index alignment file. + Index a given alignment file BAM/CRAM, e.g., create a .bai file from a + .bam file. PATH: /{apiVersion}/analysis/alignment/index/run :param dict data: Alignment index params. (REQUIRED) diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/clinical_analysis_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/clinical_analysis_client.py index 1e4a73902cc..4fd0ca988e7 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/clinical_analysis_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/clinical_analysis_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-02-14 + Autogenerated on: 2024-02-19 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/cohort_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/cohort_client.py index 50af2a865a2..a0a55bbb6fb 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/cohort_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/cohort_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-02-14 + Autogenerated on: 2024-02-19 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/disease_panel_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/disease_panel_client.py index e234b2ff148..074a68209e4 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/disease_panel_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/disease_panel_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-02-14 + Autogenerated on: 2024-02-19 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/family_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/family_client.py index 4fd32087258..17415f3700a 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/family_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/family_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-02-14 + Autogenerated on: 2024-02-19 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/file_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/file_client.py index 22e58ba63be..c75893ad88b 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/file_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/file_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-02-14 + Autogenerated on: 2024-02-19 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/ga4gh_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/ga4gh_client.py index b15c6e8bd76..af0f87f3b18 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/ga4gh_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/ga4gh_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-02-14 + Autogenerated on: 2024-02-19 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/individual_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/individual_client.py index 3b393328993..d0eacbe900b 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/individual_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/individual_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-02-14 + Autogenerated on: 2024-02-19 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py index 1d5369c4639..2d24508ccc7 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-02-14 + Autogenerated on: 2024-02-19 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/meta_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/meta_client.py index 1506926345b..f5109a6a05a 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/meta_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/meta_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-02-14 + Autogenerated on: 2024-02-19 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/project_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/project_client.py index 338e65d466c..9e1fe1e15e6 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/project_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/project_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-02-14 + Autogenerated on: 2024-02-19 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/sample_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/sample_client.py index e83e98d3be2..c7a5b17d0b7 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/sample_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/sample_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-02-14 + Autogenerated on: 2024-02-19 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py index 17fda942ec7..1c39642a26d 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-02-14 + Autogenerated on: 2024-02-19 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/user_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/user_client.py index f991cf25dae..ba9777bbca9 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/user_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/user_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-02-14 + Autogenerated on: 2024-02-19 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/variant_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/variant_client.py index 57cd7a1a9cf..b5c6b37f774 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/variant_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/variant_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-02-14 + Autogenerated on: 2024-02-19 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/variant_operation_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/variant_operation_client.py index b48bb9d8ad4..d5072623fe7 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/variant_operation_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/variant_operation_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-02-14 + Autogenerated on: 2024-02-19 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java index d5476e10f60..d534f86fb69 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java @@ -1,5 +1,7 @@ package org.opencb.opencga.core.api; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.opencb.commons.annotations.DataField; import org.opencb.opencga.core.models.alignment.AlignmentQcParams; import org.opencb.opencga.core.models.variant.MutationalSignatureAnalysisParams; import org.opencb.opencga.core.models.variant.SampleQcAnalysisParams; @@ -485,7 +487,12 @@ public class FieldConstants { public static final String VARIANT_STATS_DESCRIPTION_DESCRIPTION = "Variant stats description."; public static final String VARIANT_STATS_QUERY_DESCRIPTION = "Variant stats query in JSON format."; - // Alignment QC analysis (asample-qc-run) + // Alignment index and coverage + public static final String ALIGNMENT_BAM_FILE_ID_DESCRIPTION = "Alignment file ID (in format BAM or CRAM)"; + public static final String ALIGNMENT_WINDOW_SIZE_DESCRIPTION = "Size of the bins, in bases, for the output of the BIGWIG file"; + public static final String ALIGNMENT_OVERWRITE_DESCRIPTION = "Overwrite file"; + + // Alignment QC analysis (alignment-qc-run) public static final String ALIGNMENT_QC_BAM_FILE_DESCRIPTION = "ID for the BAM file to process."; public static final String ALIGNMENT_QC_SKIP_DESCRIPTION = "To skip any alignment QC metrics use the following keywords (separated by" + " commas): " + AlignmentQcParams.STATS_SKIP_VALUE + ", " + AlignmentQcParams.FLAGSTATS_SKIP_VALUE + ", " diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java b/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java index ebcfa07153c..db500c868c9 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java @@ -580,9 +580,7 @@ public class ParamConstants { public static final String INDEX_AUXILIAR_COLLECTION_DESCRIPTION = "Index auxiliar collection to improve performance assuming RGA is " + "completely indexed."; public static final String INDEX_AUXILIAR_COLLECTION = "auxiliarIndex"; - public static final String ALIGNMENT_INDEX_DESCRIPTION = "Index alignment file"; public static final String ALIGNMENT_QUERY_DESCRIPTION = "Search over indexed alignments"; - public static final String ALIGNMENT_COVERAGE_DESCRIPTION = "Compute coverage for a given alignemnt file"; public static final String ALIGNMENT_COVERAGE_QUERY_DESCRIPTION = "Query the coverage of an alignment file for regions or genes"; public static final String ALIGNMENT_COVERAGE_RATIO_DESCRIPTION = "Compute coverage ratio from file #1 vs file #2, (e.g. somatic vs " + "germline)"; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/alignment/AlignmentIndexParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/alignment/AlignmentIndexParams.java index b4e4dc87607..c0b96832957 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/alignment/AlignmentIndexParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/alignment/AlignmentIndexParams.java @@ -1,36 +1,41 @@ package org.opencb.opencga.core.models.alignment; +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.FieldConstants; import org.opencb.opencga.core.tools.ToolParams; public class AlignmentIndexParams extends ToolParams { public static final String DESCRIPTION = "Alignment index params"; - private String file; + @DataField(id = "fileId", description = FieldConstants.ALIGNMENT_BAM_FILE_ID_DESCRIPTION) + private String fileId; + + @DataField(id = "overwrite", description = FieldConstants.ALIGNMENT_OVERWRITE_DESCRIPTION) private boolean overwrite; public AlignmentIndexParams() { } - public AlignmentIndexParams(String file, boolean overwrite) { - this.file = file; + public AlignmentIndexParams(String fileId, boolean overwrite) { + this.fileId = fileId; this.overwrite = overwrite; } @Override public String toString() { final StringBuilder sb = new StringBuilder("AlignmentIndexParams{"); - sb.append("file='").append(file).append('\''); + sb.append("fileId='").append(fileId).append('\''); sb.append(", overwrite=").append(overwrite); sb.append('}'); return sb.toString(); } - public String getFile() { - return file; + public String getFileId() { + return fileId; } - public AlignmentIndexParams setFile(String file) { - this.file = file; + public AlignmentIndexParams setFileId(String fileId) { + this.fileId = fileId; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/alignment/CoverageIndexParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/alignment/CoverageIndexParams.java index f655829625d..655e498db0a 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/alignment/CoverageIndexParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/alignment/CoverageIndexParams.java @@ -2,60 +2,66 @@ import com.fasterxml.jackson.annotation.JsonProperty; +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.FieldConstants; import org.opencb.opencga.core.tools.ToolParams; public class CoverageIndexParams extends ToolParams { public static final String DESCRIPTION = "Coverage computation parameters"; - private String bamFileId; - private String baiFileId; + @DataField(id = "fileId", description = FieldConstants.ALIGNMENT_BAM_FILE_ID_DESCRIPTION) + private String fileId; - @JsonProperty(defaultValue = "1") + @JsonProperty(defaultValue = "50") + @DataField(id = "windowSize", defaultValue = "50", description = FieldConstants.ALIGNMENT_WINDOW_SIZE_DESCRIPTION) private int windowSize; + @DataField(id = "overwrite", description = FieldConstants.ALIGNMENT_OVERWRITE_DESCRIPTION) + private boolean overwrite; + public CoverageIndexParams() { } - public CoverageIndexParams(String bamFileId, String baiFileId, int windowSize) { - this.bamFileId = bamFileId; - this.baiFileId = baiFileId; + public CoverageIndexParams(String fileId, int windowSize, boolean overwrite) { + this.fileId = fileId; this.windowSize = windowSize; + this.overwrite = overwrite; } @Override public String toString() { final StringBuilder sb = new StringBuilder("CoverageIndexParams{"); - sb.append("bamFileId='").append(bamFileId).append('\''); - sb.append(", baiFileId='").append(baiFileId).append('\''); + sb.append("fileId='").append(fileId).append('\''); sb.append(", windowSize=").append(windowSize); + sb.append(", overwrite=").append(overwrite); sb.append('}'); return sb.toString(); } - public String getBamFileId() { - return bamFileId; + public String getFileId() { + return fileId; } - public CoverageIndexParams setBamFileId(String bamFileId) { - this.bamFileId = bamFileId; + public CoverageIndexParams setFileId(String fileId) { + this.fileId = fileId; return this; } - public String getBaiFileId() { - return baiFileId; + public int getWindowSize() { + return windowSize; } - public CoverageIndexParams setBaiFileId(String baiFileId) { - this.baiFileId = baiFileId; + public CoverageIndexParams setWindowSize(int windowSize) { + this.windowSize = windowSize; return this; } - public int getWindowSize() { - return windowSize; + public boolean isOverwrite() { + return overwrite; } - public CoverageIndexParams setWindowSize(int windowSize) { - this.windowSize = windowSize; + public CoverageIndexParams setOverwrite(boolean overwrite) { + this.overwrite = overwrite; return this; } } diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/AlignmentWebService.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/AlignmentWebService.java index 99fa4402fc9..fc2dfa52ea2 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/AlignmentWebService.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/AlignmentWebService.java @@ -76,7 +76,7 @@ public AlignmentWebService(String apiVersion, @Context UriInfo uriInfo, @Context @POST @Path("/index/run") - @ApiOperation(value = ALIGNMENT_INDEX_DESCRIPTION, response = Job.class) + @ApiOperation(value = AlignmentIndexOperation.DESCRIPTION, response = Job.class) public Response indexRun( @ApiParam(value = ParamConstants.STUDY_PARAM) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = ParamConstants.JOB_ID_CREATION_DESCRIPTION) @QueryParam(ParamConstants.JOB_ID) String jobName, @@ -170,7 +170,7 @@ public Response query(@ApiParam(value = FILE_ID_DESCRIPTION, required = true) @Q @POST @Path("/coverage/index/run") - @ApiOperation(value = "Compute coverage for a list of alignment files", response = Job.class) + @ApiOperation(value = AlignmentCoverageAnalysis.DESCRIPTION, response = Job.class) public Response coverageRun( @ApiParam(value = ParamConstants.STUDY_PARAM) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = ParamConstants.JOB_ID_CREATION_DESCRIPTION) @QueryParam(ParamConstants.JOB_ID) String jobName, From 0f942568946688c61daebbae82d8a86db83ebb48 Mon Sep 17 00:00:00 2001 From: pfurio Date: Mon, 7 Oct 2024 12:53:58 +0200 Subject: [PATCH 04/15] catalog: add new type field in Note data model, #TASK-7046 --- .../opencga/catalog/db/api/NoteDBAdaptor.java | 1 + .../db/mongodb/NoteMongoDBAdaptor.java | 3 ++- .../catalog/migration/MigrationManager.java | 3 +++ .../src/main/resources/catalog-indexes.txt | 1 + .../opencga/core/api/FieldConstants.java | 2 ++ .../opencga/core/models/notes/Note.java | 20 ++++++++++++++++--- .../core/models/notes/NoteCreateParams.java | 20 +++++++++++++++---- .../opencga/core/models/notes/NoteType.java | 18 +++++++++++++++++ .../core/models/notes/NoteUpdateParams.java | 18 ++++++++++++++--- .../server/rest/OrganizationWSServer.java | 1 + .../opencga/server/rest/StudyWSServer.java | 1 + 11 files changed, 77 insertions(+), 11 deletions(-) create mode 100644 opencga-core/src/main/java/org/opencb/opencga/core/models/notes/NoteType.java diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/NoteDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/NoteDBAdaptor.java index 34d3495407d..fc12d8d9bb9 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/NoteDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/NoteDBAdaptor.java @@ -31,6 +31,7 @@ enum QueryParams implements QueryParam { UUID("uuid", STRING, ""), SCOPE("scope", STRING, ""), STUDY("study", STRING, ""), + TYPE("type", STRING, ""), TAGS("tags", TEXT_ARRAY, ""), USER_ID("userId", STRING, ""), VISIBILITY("visibility", STRING, ""), diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/NoteMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/NoteMongoDBAdaptor.java index c32644fee46..a937796a6d8 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/NoteMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/NoteMongoDBAdaptor.java @@ -208,7 +208,7 @@ private UpdateDocument parseAndValidateUpdateParams(Document note, ObjectMap par // document.getSet().put(PRIVATE_MODIFICATION_DATE, date); // } - final String[] acceptedStringParams = {QueryParams.USER_ID.key(), QueryParams.VISIBILITY.key()}; + final String[] acceptedStringParams = {QueryParams.USER_ID.key(), QueryParams.VISIBILITY.key(), QueryParams.TYPE.key()}; filterStringParams(parameters, document.getSet(), acceptedStringParams); Object value = parameters.get(QueryParams.VALUE.key()); @@ -399,6 +399,7 @@ private Bson parseQuery(Query query) throws CatalogDBException { break; case ID: case UUID: + case TYPE: case SCOPE: case USER_ID: case TAGS: diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationManager.java index fa8fd7b9ee0..d108c477d12 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationManager.java @@ -210,6 +210,9 @@ public void runMigration(String organizationId, String version, Collection migration : runnableMigrations) { run(organizationId, migration, appHomePath, params, token); } + + // 5. Execute install indexes just in case there are new indexes + catalogManager.installIndexes(organizationId, token); } public List> getPendingMigrations(String organizationId, String version, String token) diff --git a/opencga-catalog/src/main/resources/catalog-indexes.txt b/opencga-catalog/src/main/resources/catalog-indexes.txt index 557a660716a..637d8ee7054 100644 --- a/opencga-catalog/src/main/resources/catalog-indexes.txt +++ b/opencga-catalog/src/main/resources/catalog-indexes.txt @@ -2,6 +2,7 @@ {"collections": ["note", "note_archive"], "fields": {"uid": 1, "version": 1}, "options": {"unique": true}} {"collections": ["note", "note_archive"], "fields": {"uuid": 1, "version": 1}, "options": {"unique": true}} {"collections": ["note", "note_archive"], "fields": {"scope": 1, "studyUid": 1}, "options": {}} +{"collections": ["note", "note_archive"], "fields": {"type": 1, "studyUid": 1}, "options": {}} {"collections": ["note", "note_archive"], "fields": {"studyUid": 1}, "options": {}} {"collections": ["note", "note_archive"], "fields": {"visibility": 1, "studyUid": 1}, "options": {}} {"collections": ["note", "note_archive"], "fields": {"userId": 1, "studyUid": 1}, "options": {}} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java index 835f89a61a6..247490ad27a 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java @@ -54,6 +54,7 @@ public class FieldConstants { public static final String NOTES_ID_DESCRIPTION = "Note unique identifier."; public static final String NOTES_SCOPE_DESCRIPTION = "Scope of the Note."; public static final String NOTES_STUDY_DESCRIPTION = "Study FQN if the Note scope is STUDY."; + public static final String NOTES_TYPE_DESCRIPTION = "Note type."; public static final String NOTES_TAGS_DESCRIPTION = "Note tags."; public static final String NOTES_USER_ID_DESCRIPTION = "User that wrote that Note."; public static final String NOTES_VISIBILITY_DESCRIPTION = "Visibility of the Note."; @@ -62,6 +63,7 @@ public class FieldConstants { public static final String NOTES_ID_PARAM = "id"; public static final String NOTES_SCOPE_PARAM = "scope"; + public static final String NOTES_TYPE_PARAM = "type"; public static final String NOTES_STUDY_PARAM = "study"; public static final String NOTES_TAGS_PARAM = "tags"; public static final String NOTES_USER_ID_PARAM = "userId"; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/notes/Note.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/notes/Note.java index ac12c5a474e..789fdd79a3f 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/notes/Note.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/notes/Note.java @@ -26,6 +26,9 @@ public class Note extends PrivateStudyUid { @DataField(id = "study", indexed = true, immutable = true, description = FieldConstants.NOTES_STUDY_DESCRIPTION) private String study; + @DataField(id = "type", indexed = true, description = FieldConstants.NOTES_TYPE_DESCRIPTION) + private NoteType type; + @DataField(id = "tags", indexed = true, description = FieldConstants.NOTES_TAGS_DESCRIPTION) private List tags; @@ -73,12 +76,13 @@ public enum Type { public Note() { } - public Note(String id, String uuid, Scope scope, String study, List tags, String userId, Visibility visibility, int version, - String creationDate, String modificationDate, Type valueType, Object value) { + public Note(String id, String uuid, Scope scope, String study, NoteType type, List tags, String userId, Visibility visibility, + int version, String creationDate, String modificationDate, Type valueType, Object value) { this.id = id; this.uuid = uuid; this.scope = scope; this.study = study; + this.type = type; this.tags = tags; this.userId = userId; this.visibility = visibility; @@ -91,11 +95,12 @@ public Note(String id, String uuid, Scope scope, String study, List tags @Override public String toString() { - final StringBuilder sb = new StringBuilder("Notes{"); + final StringBuilder sb = new StringBuilder("Note{"); sb.append("id='").append(id).append('\''); sb.append(", uuid='").append(uuid).append('\''); sb.append(", scope=").append(scope); sb.append(", study='").append(study).append('\''); + sb.append(", type=").append(type); sb.append(", tags=").append(tags); sb.append(", userId='").append(userId).append('\''); sb.append(", visibility=").append(visibility); @@ -144,6 +149,15 @@ public Note setStudy(String study) { return this; } + public NoteType getType() { + return type; + } + + public Note setType(NoteType type) { + this.type = type; + return this; + } + public List getTags() { return tags; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/notes/NoteCreateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/notes/NoteCreateParams.java index dfc277ccce9..3924ce431cc 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/notes/NoteCreateParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/notes/NoteCreateParams.java @@ -8,6 +8,7 @@ public class NoteCreateParams { private String id; + private NoteType type; private List tags; private Note.Visibility visibility; private Note.Type valueType; @@ -16,8 +17,9 @@ public class NoteCreateParams { public NoteCreateParams() { } - public NoteCreateParams(String id, List tags, Note.Visibility visibility, Note.Type valueType, Object value) { + public NoteCreateParams(String id, NoteType type, List tags, Note.Visibility visibility, Note.Type valueType, Object value) { this.id = id; + this.type = type; this.tags = tags; this.visibility = visibility; this.valueType = valueType; @@ -26,8 +28,9 @@ public NoteCreateParams(String id, List tags, Note.Visibility visibility @Override public String toString() { - final StringBuilder sb = new StringBuilder("NotesCreateParams{"); + final StringBuilder sb = new StringBuilder("NoteCreateParams{"); sb.append("id='").append(id).append('\''); + sb.append(", type=").append(type); sb.append(", tags=").append(tags); sb.append(", visibility=").append(visibility); sb.append(", valueType=").append(valueType); @@ -37,8 +40,8 @@ public String toString() { } public Note toNote(Note.Scope scope, String userId) { - return new Note(id, null, scope, null, tags != null ? tags : Collections.emptyList(), userId, visibility, 1, TimeUtils.getTime(), - TimeUtils.getTime(), valueType, value != null ? value : Collections.emptyMap()); + return new Note(id, null, scope, null, type, tags != null ? tags : Collections.emptyList(), userId, visibility, 1, + TimeUtils.getTime(), TimeUtils.getTime(), valueType, value != null ? value : Collections.emptyMap()); } public String getId() { @@ -50,6 +53,15 @@ public NoteCreateParams setId(String id) { return this; } + public NoteType getType() { + return type; + } + + public NoteCreateParams setType(NoteType type) { + this.type = type; + return this; + } + public List getTags() { return tags; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/notes/NoteType.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/notes/NoteType.java new file mode 100644 index 00000000000..9fff8b0745f --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/notes/NoteType.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.core.models.notes; + +public enum NoteType { + VARIANT, + GENE, + TRANSCRIPT, + PROTEIN, + JOB, + FILE, + SAMPLE, + INDIVIDUAL, + FAMILY, + COHORT, + DISEASE_PANEL, + CLINICAL_ANALYSIS, + WORKFLOW, + OTHER +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/notes/NoteUpdateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/notes/NoteUpdateParams.java index ba8c792add0..128f4593c2c 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/notes/NoteUpdateParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/notes/NoteUpdateParams.java @@ -10,6 +10,7 @@ public class NoteUpdateParams { + private NoteType type; private List tags; private Note.Visibility visibility; private Object value; @@ -17,7 +18,8 @@ public class NoteUpdateParams { public NoteUpdateParams() { } - public NoteUpdateParams(List tags, Note.Visibility visibility, Object value) { + public NoteUpdateParams(NoteType type, List tags, Note.Visibility visibility, Object value) { + this.type = type; this.tags = tags; this.visibility = visibility; this.value = value; @@ -30,14 +32,24 @@ public ObjectMap getUpdateMap() throws JsonProcessingException { @Override public String toString() { - final StringBuilder sb = new StringBuilder("NotesUpdateParams{"); - sb.append("tags=").append(tags); + final StringBuilder sb = new StringBuilder("NoteUpdateParams{"); + sb.append("type=").append(type); + sb.append(", tags=").append(tags); sb.append(", visibility=").append(visibility); sb.append(", value=").append(value); sb.append('}'); return sb.toString(); } + public NoteType getType() { + return type; + } + + public NoteUpdateParams setType(NoteType type) { + this.type = type; + return this; + } + public List getTags() { return tags; } diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/OrganizationWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/OrganizationWSServer.java index 15085581bca..a69442f95dd 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/OrganizationWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/OrganizationWSServer.java @@ -160,6 +160,7 @@ public Response noteSearch( @ApiParam(value = ParamConstants.CREATION_DATE_DESCRIPTION) @QueryParam(ParamConstants.CREATION_DATE_PARAM) String creationDate, @ApiParam(value = ParamConstants.MODIFICATION_DATE_DESCRIPTION) @QueryParam(ParamConstants.MODIFICATION_DATE_PARAM) String modificationDate, @ApiParam(value = FieldConstants.NOTES_ID_DESCRIPTION) @QueryParam(FieldConstants.NOTES_ID_PARAM) String noteId, + @ApiParam(value = FieldConstants.NOTES_TYPE_DESCRIPTION) @QueryParam(FieldConstants.NOTES_TYPE_PARAM) String type, @ApiParam(value = FieldConstants.NOTES_SCOPE_DESCRIPTION) @QueryParam(FieldConstants.NOTES_SCOPE_PARAM) String scope, @ApiParam(value = FieldConstants.NOTES_VISIBILITY_DESCRIPTION) @QueryParam(FieldConstants.NOTES_VISIBILITY_PARAM) String visibility, @ApiParam(value = FieldConstants.GENERIC_UUID_DESCRIPTION) @QueryParam("uuid") String uuid, diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/StudyWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/StudyWSServer.java index f6cc3fe8882..6c57a76b52a 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/StudyWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/StudyWSServer.java @@ -568,6 +568,7 @@ public Response noteSearch( @ApiParam(value = ParamConstants.CREATION_DATE_DESCRIPTION) @QueryParam(ParamConstants.CREATION_DATE_PARAM) String creationDate, @ApiParam(value = ParamConstants.MODIFICATION_DATE_DESCRIPTION) @QueryParam(ParamConstants.MODIFICATION_DATE_PARAM) String modificationDate, @ApiParam(value = FieldConstants.NOTES_ID_DESCRIPTION) @QueryParam(FieldConstants.NOTES_ID_PARAM) String noteId, + @ApiParam(value = FieldConstants.NOTES_TYPE_DESCRIPTION) @QueryParam(FieldConstants.NOTES_TYPE_PARAM) String type, @ApiParam(value = FieldConstants.GENERIC_UUID_DESCRIPTION) @QueryParam("uuid") String uuid, @ApiParam(value = FieldConstants.NOTES_USER_ID_DESCRIPTION) @QueryParam(FieldConstants.NOTES_USER_ID_PARAM) String userId, @ApiParam(value = FieldConstants.NOTES_TAGS_DESCRIPTION) @QueryParam(FieldConstants.NOTES_TAGS_PARAM) String tags, From c97cade8f3e407b377516dc1765d97867879d1cd Mon Sep 17 00:00:00 2001 From: pfurio Date: Mon, 28 Oct 2024 11:01:49 +0100 Subject: [PATCH 05/15] catalog: implement bam,bai,bw,cram,crai automatic association,#TASK-5662 --- .../opencga/catalog/db/api/FileDBAdaptor.java | 1 + .../db/mongodb/FileMongoDBAdaptor.java | 201 +++++++++++++++++- .../opencga/catalog/managers/FileManager.java | 38 ++-- .../catalog/managers/FileManagerTest.java | 43 ++++ 4 files changed, 264 insertions(+), 19 deletions(-) diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java index 11e82e7f81e..75ac49f4933 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java @@ -96,6 +96,7 @@ enum QueryParams implements QueryParam { INTERNAL_VARIANT_SECONDARY_INDEX("internal.variant.secondaryIndex", TEXT_ARRAY, ""), @Deprecated // Deprecated filter. Should use INTERNAL_VARIANT_SECONDARY_ANNOTATION_INDEX_STATUS_ID once the migration is completed INTERNAL_VARIANT_SECONDARY_INDEX_STATUS_ID("internal.variant.secondaryIndex.status.id", TEXT_ARRAY, ""), + INTERNAL_ALIGNMENT("internal.alignment", OBJECT, ""), INTERNAL_ALIGNMENT_INDEX("internal.alignment.index", TEXT_ARRAY, ""), INTERNAL_ALIGNMENT_INDEX_STATUS_ID("internal.alignment.index.status.id", TEXT_ARRAY, ""), INTERNAL_COVERAGE_INDEX("internal.alignment.coverage", TEXT_ARRAY, ""), diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java index d749272fec4..e81da69a6ce 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java @@ -134,8 +134,8 @@ public OpenCGAResult insert(long studyId, File file, List existingSample logger.debug("Starting file insert transaction for file id '{}'", file.getId()); dbAdaptorFactory.getCatalogStudyDBAdaptor().checkId(clientSession, studyId); - insert(clientSession, studyId, file, existingSamples, nonExistingSamples, variableSetList); - return endWrite(tmpStartTime, 1, 1, 0, 0, null); + List events = insert(clientSession, studyId, file, existingSamples, nonExistingSamples, variableSetList); + return endWrite(tmpStartTime, 1, 1, 0, 0, events); }, (e) -> logger.error("Could not create file {}: {}", file.getId(), e.getMessage())); } @@ -151,7 +151,9 @@ public OpenCGAResult insertWithVirtualFile(long studyId, File file, File virtual virtualFile.getId()); dbAdaptorFactory.getCatalogStudyDBAdaptor().checkId(clientSession, studyId); - insert(clientSession, studyId, file, null, null, variableSetList); + List events = new ArrayList<>(); + List tmpEvents = insert(clientSession, studyId, file, null, null, variableSetList); + events.addAll(tmpEvents); Map actionMap = new HashMap<>(); actionMap.put(QueryParams.RELATED_FILES.key(), BasicUpdateAction.ADD); @@ -161,7 +163,8 @@ public OpenCGAResult insertWithVirtualFile(long studyId, File file, File virtual virtualFile.setRelatedFiles(Collections.singletonList( new FileRelatedFile(file, FileRelatedFile.Relation.MULTIPART)) ); - insert(clientSession, studyId, virtualFile, existingSamples, nonExistingSamples, variableSetList); + tmpEvents = insert(clientSession, studyId, virtualFile, existingSamples, nonExistingSamples, variableSetList); + events.addAll(tmpEvents); } else { // Add multipart file in virtual file ObjectMap params = new ObjectMap(QueryParams.RELATED_FILES.key(), Collections.singletonList( @@ -176,12 +179,12 @@ public OpenCGAResult insertWithVirtualFile(long studyId, File file, File virtual )); transactionalUpdate(clientSession, file, params, null, qOptions); - return endWrite(tmpStartTime, 1, 1, 0, 0, null); + return endWrite(tmpStartTime, 1, 1, 0, 0, events); }, (e) -> logger.error("Could not create file {}: {}", file.getId(), e.getMessage())); } - long insert(ClientSession clientSession, long studyId, File file, List existingSamples, List nonExistingSamples, + List insert(ClientSession clientSession, long studyId, File file, List existingSamples, List nonExistingSamples, List variableSetList) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { if (filePathExists(clientSession, studyId, file.getPath())) { throw CatalogDBException.alreadyExists("File", studyId, "path", file.getPath()); @@ -192,6 +195,7 @@ long insert(ClientSession clientSession, long studyId, File file, List e if (nonExistingSamples == null) { nonExistingSamples = Collections.emptyList(); } + List eventList = new LinkedList<>(); List samples = new ArrayList<>(existingSamples.size() + nonExistingSamples.size()); if (existingSamples.size() + nonExistingSamples.size() < fileSampleLinkThreshold) { @@ -253,6 +257,8 @@ long insert(ClientSession clientSession, long studyId, File file, List e file.setUuid(UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.FILE)); } + List events = adjustAlignmentRelatedFiles(clientSession, file); + eventList.addAll(events); Document fileDocument = fileConverter.convertToStorageType(file, samples, variableSetList); fileDocument.put(PERMISSION_RULES_APPLIED, Collections.emptyList()); @@ -268,9 +274,187 @@ long insert(ClientSession clientSession, long studyId, File file, List e dbAdaptorFactory.getCatalogStudyDBAdaptor().updateDiskUsage(clientSession, studyId, file.getSize()); } - return fileUid; + return eventList; } + private List adjustAlignmentRelatedFiles(ClientSession clientSession, File file) + throws CatalogParameterException, CatalogDBException, CatalogAuthorizationException { + if (!file.getBioformat().equals(File.Bioformat.ALIGNMENT)) { + return Collections.emptyList(); + } + + List path; + switch (file.getFormat()) { + case BIGWIG: + path = Collections.singletonList(file.getPath().replace(".bw", "")); + break; + case BAM: + path = Arrays.asList(file.getPath() + ".bai", file.getPath() + ".bw"); + break; + case BAI: + path = Collections.singletonList(file.getPath().replace(".bai", "")); + break; + case CRAM: + path = Collections.singletonList(file.getPath() + ".crai"); + break; + case CRAI: + path = Collections.singletonList(file.getPath().replace(".crai", "")); + break; + default: + return Collections.emptyList(); + } + + Query query = new Query() + .append(QueryParams.STUDY_UID.key(), file.getStudyUid()) + .append(QueryParams.PATH.key(), path); + QueryOptions fileOptions = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(FileDBAdaptor.QueryParams.ID.key(), + FileDBAdaptor.QueryParams.UID.key(), FileDBAdaptor.QueryParams.FORMAT.key(), FileDBAdaptor.QueryParams.BIOFORMAT.key(), + FileDBAdaptor.QueryParams.URI.key(), FileDBAdaptor.QueryParams.PATH.key(), QueryParams.RELATED_FILES.key(), + FileDBAdaptor.QueryParams.STUDY_UID.key(), QueryParams.INTERNAL_ALIGNMENT.key())); + OpenCGAResult result = get(clientSession, query, fileOptions); + List eventList = new ArrayList<>(); + if (result.getNumResults() > 0) { + switch (file.getFormat()) { + case BIGWIG: + associateBigWigToBamFile(clientSession, file, result.first(), eventList); + break; + case BAM: + for (File tmpResult : result.getResults()) { + if (tmpResult.getFormat().equals(File.Format.BAI)) { + associateAlignmentFileToIndexFile(clientSession, file, tmpResult, eventList); + } else if (tmpResult.getFormat().equals(File.Format.BIGWIG)) { + associateBamFileToBigWigFile(clientSession, file, tmpResult, eventList); + } + } + break; + case BAI: + case CRAI: + associateIndexFileToAlignmentFile(clientSession, file, result.first(), eventList); + break; + case CRAM: + associateAlignmentFileToIndexFile(clientSession, file, result.first(), eventList); + break; + default: + break; + } + } + return eventList; + } + + private void associateBamFileToBigWigFile(ClientSession clientSession, File bamFile, File bigWigFile, List eventList) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + eventList.add(createAssociationInfoEvent(bamFile, bigWigFile)); + + if (CollectionUtils.isNotEmpty(bigWigFile.getRelatedFiles())) { + for (FileRelatedFile relatedFile : bigWigFile.getRelatedFiles()) { + if (relatedFile.getRelation().equals(FileRelatedFile.Relation.ALIGNMENT)) { + eventList.add(createAssociationWarningEvent(bigWigFile, bamFile.getFormat(), bamFile.getPath())); + } + } + } + + // Add BIGWIG reference in BAM file + bamFile.getInternal().getAlignment().setCoverage(new FileInternalCoverageIndex( + new InternalStatus(InternalStatus.READY), bigWigFile.getId(), "", -1)); + + // Add BAM file to list of related files in BIGWIG file + addAlignmentReferenceToRelatedFiles(clientSession, bamFile, bigWigFile); + } + + private void addAlignmentReferenceToRelatedFiles(ClientSession clientSession, File bamFile, File targetFile) + throws CatalogParameterException, CatalogDBException, CatalogAuthorizationException { + List tmpRelatedFileList = Collections.singletonList(new FileRelatedFile(bamFile, + FileRelatedFile.Relation.ALIGNMENT)); + + Map tmpActionMap = new HashMap<>(); + tmpActionMap.put(QueryParams.RELATED_FILES.key(), BasicUpdateAction.ADD); + QueryOptions options = new QueryOptions(Constants.ACTIONS, tmpActionMap); + + ObjectMap params = new ObjectMap(QueryParams.RELATED_FILES.key(), tmpRelatedFileList); + transactionalUpdate(clientSession, targetFile, params, null, options); + } + + private void associateAlignmentFileToIndexFile(ClientSession clientSession, File alignFile, File indexFile, List eventList) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + eventList.add(createAssociationInfoEvent(alignFile, indexFile)); + + if (CollectionUtils.isNotEmpty(indexFile.getRelatedFiles())) { + for (FileRelatedFile relatedFile : indexFile.getRelatedFiles()) { + if (relatedFile.getRelation().equals(FileRelatedFile.Relation.ALIGNMENT)) { + eventList.add(createAssociationWarningEvent(indexFile, alignFile.getFormat(), alignFile.getPath())); + } + } + } + + // Add index reference to alignment file + alignFile.getInternal().getAlignment().setIndex(new FileInternalAlignmentIndex( + new InternalStatus(InternalStatus.READY), indexFile.getId(), "")); + + + // Add alignment file to list of related files in index file + addAlignmentReferenceToRelatedFiles(clientSession, alignFile, indexFile); + } + + private void associateBigWigToBamFile(ClientSession clientSession, File bigWig, File bamFile, List eventList) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + eventList.add(createAssociationInfoEvent(bigWig, bamFile)); + + if (bamFile.getInternal() != null && bamFile.getInternal().getAlignment() != null + && bamFile.getInternal().getAlignment().getCoverage() != null + && StringUtils.isNotEmpty(bamFile.getInternal().getAlignment().getCoverage().getFileId())) { + eventList.add(createAssociationWarningEvent(bamFile, bigWig.getFormat(), + bamFile.getInternal().getAlignment().getCoverage().getFileId())); + } + + List relatedFileList = bigWig.getRelatedFiles() != null ? new ArrayList<>(bigWig.getRelatedFiles()) + : new ArrayList<>(); + relatedFileList.add(new FileRelatedFile(bamFile, FileRelatedFile.Relation.ALIGNMENT)); + bigWig.setRelatedFiles(relatedFileList); + + FileInternalCoverageIndex coverage = new FileInternalCoverageIndex(new InternalStatus(InternalStatus.READY), + bigWig.getId(), "", -1); + bigWig.getInternal().getAlignment().setCoverage(coverage); + + ObjectMap params = new ObjectMap(QueryParams.INTERNAL_COVERAGE_INDEX.key(), coverage); + transactionalUpdate(clientSession, bamFile, params, null, null); + } + + private void associateIndexFileToAlignmentFile(ClientSession clientSession, File indexFile, File alignmentFile, List eventList) + throws CatalogParameterException, CatalogDBException, CatalogAuthorizationException { + eventList.add(createAssociationInfoEvent(indexFile, alignmentFile)); + + if (alignmentFile.getInternal() != null && alignmentFile.getInternal().getAlignment() != null + && alignmentFile.getInternal().getAlignment().getIndex() != null + && StringUtils.isNotEmpty(alignmentFile.getInternal().getAlignment().getIndex().getFileId())) { + eventList.add(createAssociationWarningEvent(alignmentFile, indexFile.getFormat(), + alignmentFile.getInternal().getAlignment().getIndex().getFileId())); + } + + List relatedFileList = indexFile.getRelatedFiles() != null ? new ArrayList<>(indexFile.getRelatedFiles()) + : new ArrayList<>(); + relatedFileList.add(new FileRelatedFile(alignmentFile, FileRelatedFile.Relation.ALIGNMENT)); + indexFile.setRelatedFiles(relatedFileList); + + FileInternalAlignmentIndex alignmentIndex = new FileInternalAlignmentIndex(new InternalStatus(InternalStatus.READY), + indexFile.getId(), ""); + indexFile.getInternal().getAlignment().setIndex(alignmentIndex); + + ObjectMap params = new ObjectMap(QueryParams.INTERNAL_ALIGNMENT_INDEX.key(), alignmentIndex); + transactionalUpdate(clientSession, alignmentFile, params, null, null); + } + + private Event createAssociationInfoEvent(File firstFile, File secondFile) { + return new Event(Event.Type.INFO, "The " + firstFile.getFormat() + " file '" + firstFile.getPath() + "' was automatically" + + " associated to the " + secondFile.getFormat() + " file '" + secondFile.getPath() + "'."); + } + + private Event createAssociationWarningEvent(File file, File.Format format, String fileIdFound) { + return new Event(Event.Type.WARNING, "Found another " + format + " file '" + fileIdFound + "' associated to the " + file.getFormat() + + " file '" + file.getPath() + "'. This relation was automatically removed."); + } + + + @Override public long getId(long studyId, String path) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { Query query = new Query(QueryParams.STUDY_UID.key(), studyId).append(QueryParams.PATH.key(), path); @@ -382,6 +566,9 @@ public OpenCGAResult update(Query query, ObjectMap parameters, List OpenCGAResult transactionalUpdate(ClientSession clientSession, File file, ObjectMap parameters, List variableSetList, QueryOptions queryOptions) throws CatalogParameterException, CatalogDBException, CatalogAuthorizationException { + variableSetList = ParamUtils.defaultObject(variableSetList, Collections::emptyList); + queryOptions = ParamUtils.defaultObject(queryOptions, QueryOptions::empty); + long tmpStartTime = startQuery(); long studyUid = file.getStudyUid(); long fileUid = file.getUid(); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java index da8c67adf03..4289e0f51ce 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java @@ -107,8 +107,8 @@ public class FileManager extends AnnotationSetManager { INCLUDE_FILE_URI_PATH = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(FileDBAdaptor.QueryParams.ID.key(), FileDBAdaptor.QueryParams.NAME.key(), FileDBAdaptor.QueryParams.UID.key(), FileDBAdaptor.QueryParams.UUID.key(), FileDBAdaptor.QueryParams.INTERNAL_STATUS.key(), FileDBAdaptor.QueryParams.FORMAT.key(), - FileDBAdaptor.QueryParams.URI.key(), FileDBAdaptor.QueryParams.PATH.key(), FileDBAdaptor.QueryParams.EXTERNAL.key(), - FileDBAdaptor.QueryParams.STUDY_UID.key(), FileDBAdaptor.QueryParams.TYPE.key())); + FileDBAdaptor.QueryParams.BIOFORMAT.key(), FileDBAdaptor.QueryParams.URI.key(), FileDBAdaptor.QueryParams.PATH.key(), + FileDBAdaptor.QueryParams.EXTERNAL.key(), FileDBAdaptor.QueryParams.STUDY_UID.key(), FileDBAdaptor.QueryParams.TYPE.key())); EXCLUDE_FILE_ATTRIBUTES = new QueryOptions(QueryOptions.EXCLUDE, Arrays.asList(FileDBAdaptor.QueryParams.ATTRIBUTES.key(), FileDBAdaptor.QueryParams.ANNOTATION_SETS.key(), FileDBAdaptor.QueryParams.STATS.key())); INCLUDE_STUDY_URI = new QueryOptions(QueryOptions.INCLUDE, StudyDBAdaptor.QueryParams.URI.key()); @@ -1577,6 +1577,10 @@ public OpenCGAResult count(String studyId, Query query, String token) thro } } + public OpenCGAResult delete(String studyStr, String fileId, QueryOptions options, String token) throws CatalogException { + return delete(studyStr, Collections.singletonList(fileId), options, false, token); + } + @Override public OpenCGAResult delete(String studyStr, List fileIds, QueryOptions options, String token) throws CatalogException { return delete(studyStr, fileIds, options, false, token); @@ -3534,6 +3538,7 @@ private OpenCGAResult privateLink(String organizationId, Study study, File } String finalExternalPathDestinyStr = externalPathDestinyStr; + OpenCGAResult result = OpenCGAResult.empty(File.class); // Link all the files and folders present in the uri ioManager.walkFileTree(normalizedUri, new SimpleFileVisitor() { @Override @@ -3577,8 +3582,9 @@ public FileVisitResult preVisitDirectory(URI dir, BasicFileAttributes attrs) thr FileInternal.init(), Collections.emptyMap()); folder.setUuid(UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.FILE)); checkHooks(folder, study.getFqn(), HookConfiguration.Stage.CREATE); - getFileDBAdaptor(organizationId).insert(study.getUid(), folder, Collections.emptyList(), Collections.emptyList(), - Collections.emptyList(), new QueryOptions()); + OpenCGAResult tmpResult = getFileDBAdaptor(organizationId).insert(study.getUid(), folder, Collections.emptyList(), + Collections.emptyList(), Collections.emptyList(), new QueryOptions()); + result.append(tmpResult); OpenCGAResult queryResult = getFile(organizationId, study.getUid(), folder.getUuid(), QueryOptions.empty()); // Propagate ACLs @@ -3667,7 +3673,7 @@ public FileVisitResult visitFile(URI fileUri, BasicFileAttributes attrs) throws if (vFileResult.getNumResults() == 1) { if (!vFileResult.first().getType().equals(File.Type.VIRTUAL)) { throw new IOException("A file with path '" + virtualFile.getPath() - + "' already existed which is not of " + "type " + File.Type.VIRTUAL); + + "' already exists which is not of " + File.Type.VIRTUAL); } virtualFile = vFileResult.first(); @@ -3694,11 +3700,13 @@ public FileVisitResult visitFile(URI fileUri, BasicFileAttributes attrs) throws } subfile.setSampleIds(null); - getFileDBAdaptor(organizationId).insertWithVirtualFile(study.getUid(), subfile, virtualFile, existingSamples, - nonExistingSamples, Collections.emptyList(), new QueryOptions()); + OpenCGAResult tmpResult = getFileDBAdaptor(organizationId).insertWithVirtualFile(study.getUid(), subfile, + virtualFile, existingSamples, nonExistingSamples, Collections.emptyList(), new QueryOptions()); + result.append(tmpResult); } else { - getFileDBAdaptor(organizationId).insert(study.getUid(), subfile, existingSamples, nonExistingSamples, - Collections.emptyList(), new QueryOptions()); + OpenCGAResult tmpResult = getFileDBAdaptor(organizationId).insert(study.getUid(), subfile, existingSamples, + nonExistingSamples, Collections.emptyList(), new QueryOptions()); + result.append(tmpResult); } subfile = getFile(organizationId, study.getUid(), subfile.getUuid(), QueryOptions.empty()).first(); @@ -3744,16 +3752,22 @@ public FileVisitResult postVisitDirectory(URI dir, IOException exc) throws IOExc logger.warn("Matching avro to variant file: {}", e.getMessage()); } - // Check if the uri was already linked to that same path query = new Query() - .append(FileDBAdaptor.QueryParams.URI.key(), "~^" + normalizedUri) .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()) .append(FileDBAdaptor.QueryParams.EXTERNAL.key(), true); + if (Paths.get(normalizedUri).toFile().isFile()) { + query.append(FileDBAdaptor.QueryParams.URI.key(), normalizedUri); + } else { + query.append(FileDBAdaptor.QueryParams.URI.key(), "~^" + normalizedUri); + } + // Limit the number of results and only some fields QueryOptions queryOptions = new QueryOptions() .append(QueryOptions.LIMIT, 100); - return getFileDBAdaptor(organizationId).get(query, queryOptions); + OpenCGAResult tmpResult = getFileDBAdaptor(organizationId).get(query, queryOptions); + result.append(tmpResult); + return result; } OpenCGAResult registerFile(String organizationId, Study study, String filePath, URI fileUri, String jobId, diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FileManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FileManagerTest.java index 5e655998abd..0f1f58a73f2 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FileManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FileManagerTest.java @@ -420,6 +420,49 @@ public void testLinkVCFandBAMPair() throws CatalogException { assertTrue(Arrays.asList("variant-test-file.vcf.gz", "NA19600.chrom20.small.bam").containsAll(sample.getFileIds())); } + @Test + public void associateAlignmentFilesTest() throws CatalogException { + // Link BAM file + String bamFileStr = getClass().getResource("/biofiles/NA19600.chrom20.small.bam").getFile(); + File bamFile = fileManager.link(studyFqn, new FileLinkParams(bamFileStr, "", "", "", null, null, null, null, null), false, ownerToken).first(); + assertTrue(StringUtils.isEmpty(bamFile.getInternal().getAlignment().getCoverage().getFileId())); + + // Link BAI file + String baiFileStr = getClass().getResource("/biofiles/NA19600.chrom20.small.bam.bai").getFile(); + OpenCGAResult result = fileManager.link(studyFqn, new FileLinkParams(baiFileStr, "", "", "", null, null, null, null, null), false, ownerToken); + assertEquals(1, result.getEvents().size()); + assertTrue(result.getEvents().get(0).getMessage().contains("BAM")); + + File baiFile = result.first(); + assertEquals(1, baiFile.getRelatedFiles().size()); + assertEquals(FileRelatedFile.Relation.ALIGNMENT, baiFile.getRelatedFiles().get(0).getRelation()); + assertEquals(bamFile.getId(), baiFile.getRelatedFiles().get(0).getFile().getId()); + + bamFile = fileManager.get(studyFqn, bamFile.getPath(), QueryOptions.empty(), ownerToken).first(); + assertNotNull(bamFile.getInternal().getAlignment().getIndex()); + assertEquals(baiFile.getId(), bamFile.getInternal().getAlignment().getIndex().getFileId()); + assertEquals(FileStatus.READY, bamFile.getInternal().getAlignment().getIndex().getStatus().getId()); + + // Unlink BAM file + Query query = new Query(FileDBAdaptor.QueryParams.UID.key(), bamFile.getUid()); + setToPendingDelete(studyFqn, query); + fileManager.unlink(studyFqn, bamFile.getPath(), ownerToken); + baiFile = fileManager.get(studyFqn, baiFile.getPath(), QueryOptions.empty(), ownerToken).first(); + assertEquals(0, baiFile.getRelatedFiles().size()); + + // Link BAM file back + bamFile = fileManager.link(studyFqn, new FileLinkParams(bamFileStr, "", "", "", null, null, null, null, null), false, ownerToken).first(); + assertEquals(File.Format.BAM, bamFile.getFormat()); + assertNotNull(bamFile.getInternal().getAlignment().getIndex()); + assertEquals(baiFile.getId(), bamFile.getInternal().getAlignment().getIndex().getFileId()); + assertEquals(FileStatus.READY, bamFile.getInternal().getAlignment().getIndex().getStatus().getId()); + + baiFile = fileManager.get(studyFqn, baiFile.getPath(), QueryOptions.empty(), ownerToken).first(); + assertEquals(1, baiFile.getRelatedFiles().size()); + assertEquals(FileRelatedFile.Relation.ALIGNMENT, baiFile.getRelatedFiles().get(0).getRelation()); + assertEquals(bamFile.getId(), baiFile.getRelatedFiles().get(0).getFile().getId()); + } + @Test public void testLinkVirtualWithDifferentSamples() throws CatalogException { String vcfFile = getClass().getResource("/biofiles/variant-test-file.vcf.gz").getFile(); From 9a6adbbc03ae50d610c42a953492f86965671d5d Mon Sep 17 00:00:00 2001 From: pfurio Date: Mon, 28 Oct 2024 16:24:25 +0100 Subject: [PATCH 06/15] catalog: add migration script to associate alignment files, #TASK-5662 --- .../file/AssociateAlignmentFiles.java | 18 +++++ .../catalog/AssociateAlignmentFiles.java | 33 +++++++++ .../opencga/catalog/db/api/FileDBAdaptor.java | 2 + .../db/mongodb/FileMongoDBAdaptor.java | 70 ++++++++++++++++++- .../opencga/catalog/managers/FileManager.java | 11 +++ 5 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/AssociateAlignmentFiles.java create mode 100644 opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/AssociateAlignmentFiles.java diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/AssociateAlignmentFiles.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/AssociateAlignmentFiles.java new file mode 100644 index 00000000000..02624c6f1e2 --- /dev/null +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/AssociateAlignmentFiles.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.analysis.file; + +import org.opencb.opencga.analysis.tools.OpenCgaToolScopeStudy; +import org.opencb.opencga.core.models.common.Enums; +import org.opencb.opencga.core.tools.annotations.Tool; + +@Tool(id = AssociateAlignmentFiles.ID, resource = Enums.Resource.FILE, type = Tool.Type.OPERATION, + description = "Automatically associate alignment files with its index and coverage files.", priority = Enums.Priority.LOW) +public class AssociateAlignmentFiles extends OpenCgaToolScopeStudy { + + public final static String ID = "associate-alignment-files"; + + @Override + protected void run() throws Exception { + catalogManager.getFileManager().associateAlignmentFiles(getStudyFqn(), token); + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/AssociateAlignmentFiles.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/AssociateAlignmentFiles.java new file mode 100644 index 00000000000..875e1eff894 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/AssociateAlignmentFiles.java @@ -0,0 +1,33 @@ +package org.opencb.opencga.app.migrations.v4.v4_0_0.catalog; + +import com.mongodb.client.model.Projections; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +import java.util.ArrayList; +import java.util.List; + +@Migration(id = "associate_alignment_files__task_5662", + description = "Associate BAM files with BAI and BIGWIG files and CRAM files with CRAI files #5662", version = "4.0.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20241028) +public class AssociateAlignmentFiles extends MigrationTool { + + @Override + protected void run() throws Exception { + Bson studyProjection = Projections.include(StudyDBAdaptor.QueryParams.FQN.key()); + List studyFqns = new ArrayList<>(); + queryMongo(OrganizationMongoDBAdaptorFactory.STUDY_COLLECTION, new Document(), studyProjection, document -> { + studyFqns.add(document.getString(StudyDBAdaptor.QueryParams.FQN.key())); + }); + + for (String studyFqn : studyFqns) { + logger.info("Checking alignment files from study '{}'...", studyFqn); + catalogManager.getFileManager().associateAlignmentFiles(studyFqn, token); + } + } + +} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java index 75ac49f4933..5f3714fd2b2 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java @@ -319,6 +319,8 @@ OpenCGAResult getAllInStudy(long studyId, QueryOptions options) */ OpenCGAResult delete(Query query, String status) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + void associateAlignmentFiles(long studyUid) throws CatalogException; + int getFileSampleLinkThreshold(); void setFileSampleLinkThreshold(int numSamples); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java index e81da69a6ce..713ba6cffca 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java @@ -185,7 +185,7 @@ public OpenCGAResult insertWithVirtualFile(long studyId, File file, File virtual } List insert(ClientSession clientSession, long studyId, File file, List existingSamples, List nonExistingSamples, - List variableSetList) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + List variableSetList) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { if (filePathExists(clientSession, studyId, file.getPath())) { throw CatalogDBException.alreadyExists("File", studyId, "path", file.getPath()); } @@ -341,6 +341,70 @@ private List adjustAlignmentRelatedFiles(ClientSession clientSession, Fil return eventList; } + @Override + public void associateAlignmentFiles(long studyUid) throws CatalogException { + Query query = new Query() + .append(QueryParams.STUDY_UID.key(), studyUid) + .append(QueryParams.FORMAT.key(), Arrays.asList(File.Format.BAM, File.Format.CRAM)); + QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(QueryParams.PATH.key(), QueryParams.FORMAT.key(), + QueryParams.RELATED_FILES.key(), QueryParams.INTERNAL.key())); + + try (DBIterator iterator = iterator(query, options)) { + while (iterator.hasNext()) { + File alignmentFile = iterator.next(); + List additionalFiles = new ArrayList<>(2); + if (alignmentFile.getInternal().getAlignment() == null || alignmentFile.getInternal().getAlignment().getIndex() == null + || StringUtils.isEmpty(alignmentFile.getInternal().getAlignment().getIndex().getFileId())) { + switch (alignmentFile.getFormat()) { + case BAM: + additionalFiles.add(alignmentFile.getPath() + ".bai"); + break; + case CRAM: + additionalFiles.add(alignmentFile.getPath() + ".crai"); + break; + default: + break; + } + } + if (alignmentFile.getFormat().equals(File.Format.BAM) + && (alignmentFile.getInternal().getAlignment() == null + || alignmentFile.getInternal().getAlignment().getCoverage() == null + || StringUtils.isEmpty(alignmentFile.getInternal().getAlignment().getCoverage().getFileId()))) { + additionalFiles.add(alignmentFile.getPath() + ".bw"); + } + + if (!additionalFiles.isEmpty()) { + runTransaction(session -> { + Query tmpQuery = new Query() + .append(QueryParams.STUDY_UID.key(), studyUid) + .append(QueryParams.PATH.key(), additionalFiles); + QueryOptions fileOptions = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(FileDBAdaptor.QueryParams.ID.key(), + FileDBAdaptor.QueryParams.UID.key(), FileDBAdaptor.QueryParams.FORMAT.key(), + FileDBAdaptor.QueryParams.BIOFORMAT.key(), FileDBAdaptor.QueryParams.URI.key(), + FileDBAdaptor.QueryParams.PATH.key(), QueryParams.RELATED_FILES.key(), + FileDBAdaptor.QueryParams.STUDY_UID.key(), QueryParams.INTERNAL_ALIGNMENT.key())); + try (DBIterator tmpIterator = iterator(session, tmpQuery, fileOptions)) { + while (tmpIterator.hasNext()) { + File secondFile = tmpIterator.next(); + switch (secondFile.getFormat()) { + case BAI: + case CRAI: + associateAlignmentFileToIndexFile(session, alignmentFile, secondFile, new LinkedList<>()); + break; + case BIGWIG: + associateBamFileToBigWigFile(session, alignmentFile, secondFile, new LinkedList<>()); + default: + break; + } + } + } + return null; + }); + } + } + } + } + private void associateBamFileToBigWigFile(ClientSession clientSession, File bamFile, File bigWigFile, List eventList) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { eventList.add(createAssociationInfoEvent(bamFile, bigWigFile)); @@ -444,11 +508,15 @@ private void associateIndexFileToAlignmentFile(ClientSession clientSession, File } private Event createAssociationInfoEvent(File firstFile, File secondFile) { + logger.info("The {} file '{}' was automatically associated to the {} file '{}'.", firstFile.getFormat(), firstFile.getPath(), + secondFile.getFormat(), secondFile.getPath()); return new Event(Event.Type.INFO, "The " + firstFile.getFormat() + " file '" + firstFile.getPath() + "' was automatically" + " associated to the " + secondFile.getFormat() + " file '" + secondFile.getPath() + "'."); } private Event createAssociationWarningEvent(File file, File.Format format, String fileIdFound) { + logger.warn("Found another {} file '{}' associated to the {} file '{}'. This relation was automatically removed.", format, + fileIdFound, file.getFormat(), file.getPath()); return new Event(Event.Type.WARNING, "Found another " + format + " file '" + fileIdFound + "' associated to the " + file.getFormat() + " file '" + file.getPath() + "'. This relation was automatically removed."); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java index 4289e0f51ce..7c1c7076042 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java @@ -3124,6 +3124,17 @@ public void checkCanDeleteFile(String studyStr, String fileId, boolean unlink, S checkCanDeleteFile(organizationId, study, fileId, unlink, Arrays.asList(FileStatus.READY, FileStatus.TRASHED), userId); } + public void associateAlignmentFiles(String studyStr, String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, userId, organizationId); + + authorizationManager.checkIsAtLeastStudyAdministrator(organizationId, study.getUid(), userId); + getFileDBAdaptor(organizationId).associateAlignmentFiles(study.getUid()); + } + /** * Method to check if a file or folder can be deleted. It will check for indexation, status, permissions and file system availability. * From 164b91bad6143edf18b97d4d62fa55f13b1d6077 Mon Sep 17 00:00:00 2001 From: pfurio Date: Tue, 29 Oct 2024 13:23:39 +0100 Subject: [PATCH 07/15] catalog: ensure migration updates references both ways, #TASK-5662 --- .../opencga/catalog/db/api/FileDBAdaptor.java | 4 +- .../db/mongodb/FileMongoDBAdaptor.java | 62 ++++++++++++++----- .../opencga/catalog/managers/FileManager.java | 2 +- 3 files changed, 50 insertions(+), 18 deletions(-) diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java index 5f3714fd2b2..1666884fbb8 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java @@ -99,8 +99,8 @@ enum QueryParams implements QueryParam { INTERNAL_ALIGNMENT("internal.alignment", OBJECT, ""), INTERNAL_ALIGNMENT_INDEX("internal.alignment.index", TEXT_ARRAY, ""), INTERNAL_ALIGNMENT_INDEX_STATUS_ID("internal.alignment.index.status.id", TEXT_ARRAY, ""), - INTERNAL_COVERAGE_INDEX("internal.alignment.coverage", TEXT_ARRAY, ""), - INTERNAL_COVERAGE_INDEX_STATUS_ID("internal.alignment.coverage.status.id", TEXT_ARRAY, ""), + INTERNAL_ALIGNMENT_COVERAGE("internal.alignment.coverage", TEXT_ARRAY, ""), + INTERNAL_ALIGNMENT_COVERAGE_STATUS_ID("internal.alignment.coverage.status.id", TEXT_ARRAY, ""), INTERNAL_MISSING_SAMPLES("internal.missingSamples", OBJECT, ""), INTERNAL_MISSING_SAMPLES_EXISTING("internal.missingSamples.existing", TEXT_ARRAY, ""), INTERNAL_MISSING_SAMPLES_NON_EXISTING("internal.missingSamples.nonExisting", TEXT_ARRAY, ""), diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java index 713ba6cffca..9bcd821405f 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java @@ -321,9 +321,9 @@ private List adjustAlignmentRelatedFiles(ClientSession clientSession, Fil case BAM: for (File tmpResult : result.getResults()) { if (tmpResult.getFormat().equals(File.Format.BAI)) { - associateAlignmentFileToIndexFile(clientSession, file, tmpResult, eventList); + associateAlignmentFileToIndexFile(clientSession, file, tmpResult, true, eventList); } else if (tmpResult.getFormat().equals(File.Format.BIGWIG)) { - associateBamFileToBigWigFile(clientSession, file, tmpResult, eventList); + associateBamFileToBigWigFile(clientSession, file, tmpResult, true, eventList); } } break; @@ -332,7 +332,7 @@ private List adjustAlignmentRelatedFiles(ClientSession clientSession, Fil associateIndexFileToAlignmentFile(clientSession, file, result.first(), eventList); break; case CRAM: - associateAlignmentFileToIndexFile(clientSession, file, result.first(), eventList); + associateAlignmentFileToIndexFile(clientSession, file, result.first(), true, eventList); break; default: break; @@ -407,6 +407,12 @@ public void associateAlignmentFiles(long studyUid) throws CatalogException { private void associateBamFileToBigWigFile(ClientSession clientSession, File bamFile, File bigWigFile, List eventList) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + associateBamFileToBigWigFile(clientSession, bamFile, bigWigFile, false, eventList); + } + + private void associateBamFileToBigWigFile(ClientSession clientSession, File bamFile, File bigWigFile, boolean bamFileNotYetInserted, + List eventList) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { eventList.add(createAssociationInfoEvent(bamFile, bigWigFile)); if (CollectionUtils.isNotEmpty(bigWigFile.getRelatedFiles())) { @@ -417,14 +423,32 @@ private void associateBamFileToBigWigFile(ClientSession clientSession, File bamF } } - // Add BIGWIG reference in BAM file - bamFile.getInternal().getAlignment().setCoverage(new FileInternalCoverageIndex( - new InternalStatus(InternalStatus.READY), bigWigFile.getId(), "", -1)); + FileInternalCoverageIndex coverage = new FileInternalCoverageIndex(new InternalStatus(InternalStatus.READY), bigWigFile.getId(), "", + -1); + if (bamFileNotYetInserted) { + // Add BIGWIG reference in BAM file + bamFile.getInternal().getAlignment().setCoverage(coverage); + } else { + // Add coverage reference in BAM document + addCoverageReferenceInBamFile(clientSession, bamFile, coverage); + } // Add BAM file to list of related files in BIGWIG file addAlignmentReferenceToRelatedFiles(clientSession, bamFile, bigWigFile); } + private void addCoverageReferenceInBamFile(ClientSession clientSession, File bamFile, FileInternalCoverageIndex coverage) + throws CatalogParameterException, CatalogDBException, CatalogAuthorizationException { + ObjectMap params = new ObjectMap(QueryParams.INTERNAL_ALIGNMENT_COVERAGE.key(), coverage); + transactionalUpdate(clientSession, bamFile, params, null, null); + } + + private void addIndexReferenceInAlignmentFile(ClientSession clientSession, File alignFile, FileInternalAlignmentIndex index) + throws CatalogParameterException, CatalogDBException, CatalogAuthorizationException { + ObjectMap params = new ObjectMap(QueryParams.INTERNAL_ALIGNMENT_INDEX.key(), index); + transactionalUpdate(clientSession, alignFile, params, null, null); + } + private void addAlignmentReferenceToRelatedFiles(ClientSession clientSession, File bamFile, File targetFile) throws CatalogParameterException, CatalogDBException, CatalogAuthorizationException { List tmpRelatedFileList = Collections.singletonList(new FileRelatedFile(bamFile, @@ -440,6 +464,12 @@ private void addAlignmentReferenceToRelatedFiles(ClientSession clientSession, Fi private void associateAlignmentFileToIndexFile(ClientSession clientSession, File alignFile, File indexFile, List eventList) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + associateAlignmentFileToIndexFile(clientSession, alignFile, indexFile, false, eventList); + } + + private void associateAlignmentFileToIndexFile(ClientSession clientSession, File alignFile, File indexFile, + boolean alignmentFileNotYetInserted, List eventList) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { eventList.add(createAssociationInfoEvent(alignFile, indexFile)); if (CollectionUtils.isNotEmpty(indexFile.getRelatedFiles())) { @@ -450,10 +480,13 @@ private void associateAlignmentFileToIndexFile(ClientSession clientSession, File } } - // Add index reference to alignment file - alignFile.getInternal().getAlignment().setIndex(new FileInternalAlignmentIndex( - new InternalStatus(InternalStatus.READY), indexFile.getId(), "")); - + FileInternalAlignmentIndex index = new FileInternalAlignmentIndex(new InternalStatus(InternalStatus.READY), indexFile.getId(), ""); + if (alignmentFileNotYetInserted) { + // Add index reference to alignment file + alignFile.getInternal().getAlignment().setIndex(index); + } else { + addIndexReferenceInAlignmentFile(clientSession, alignFile, index); + } // Add alignment file to list of related files in index file addAlignmentReferenceToRelatedFiles(clientSession, alignFile, indexFile); @@ -479,8 +512,8 @@ private void associateBigWigToBamFile(ClientSession clientSession, File bigWig, bigWig.getId(), "", -1); bigWig.getInternal().getAlignment().setCoverage(coverage); - ObjectMap params = new ObjectMap(QueryParams.INTERNAL_COVERAGE_INDEX.key(), coverage); - transactionalUpdate(clientSession, bamFile, params, null, null); + // Add coverage reference to bam file + addCoverageReferenceInBamFile(clientSession, bamFile, coverage); } private void associateIndexFileToAlignmentFile(ClientSession clientSession, File indexFile, File alignmentFile, List eventList) @@ -503,8 +536,7 @@ private void associateIndexFileToAlignmentFile(ClientSession clientSession, File indexFile.getId(), ""); indexFile.getInternal().getAlignment().setIndex(alignmentIndex); - ObjectMap params = new ObjectMap(QueryParams.INTERNAL_ALIGNMENT_INDEX.key(), alignmentIndex); - transactionalUpdate(clientSession, alignmentFile, params, null, null); + addIndexReferenceInAlignmentFile(clientSession, alignmentFile, alignmentIndex); } private Event createAssociationInfoEvent(File firstFile, File secondFile) { @@ -1114,7 +1146,7 @@ private UpdateDocument getValidatedUpdateParams(ClientSession clientSession, lon String[] acceptedObjectParams = {QueryParams.INTERNAL_VARIANT_INDEX.key(), QueryParams.INTERNAL_VARIANT_ANNOTATION_INDEX.key(), QueryParams.INTERNAL_VARIANT_SECONDARY_INDEX.key(), QueryParams.INTERNAL_VARIANT_SECONDARY_ANNOTATION_INDEX.key(), - QueryParams.INTERNAL_ALIGNMENT_INDEX.key(), QueryParams.INTERNAL_COVERAGE_INDEX.key(), QueryParams.SOFTWARE.key(), + QueryParams.INTERNAL_ALIGNMENT_INDEX.key(), QueryParams.INTERNAL_ALIGNMENT_COVERAGE.key(), QueryParams.SOFTWARE.key(), QueryParams.EXPERIMENT.key(), QueryParams.STATUS.key(), QueryParams.INTERNAL_MISSING_SAMPLES.key(), QueryParams.QUALITY_CONTROL.key(), QueryParams.INTERNAL_STATUS.key()}; filterObjectParams(parameters, document.getSet(), acceptedObjectParams); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java index 7c1c7076042..fcbb16c4e9d 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java @@ -471,7 +471,7 @@ public OpenCGAResult updateFileInternalAlignmentIndex(String studyFqn, File f public OpenCGAResult updateFileInternalCoverageIndex(String studyFqn, File file, FileInternalCoverageIndex index, String token) throws CatalogException { - return updateFileInternalField(studyFqn, file, index, FileDBAdaptor.QueryParams.INTERNAL_COVERAGE_INDEX.key(), token); + return updateFileInternalField(studyFqn, file, index, FileDBAdaptor.QueryParams.INTERNAL_ALIGNMENT_COVERAGE.key(), token); } private OpenCGAResult updateFileInternalField(String studyFqn, File file, Object value, String fieldKey, String token) From 3d44dc00c41b19127fa255d9866f8d455e281f16 Mon Sep 17 00:00:00 2001 From: pfurio Date: Tue, 29 Oct 2024 16:27:17 +0100 Subject: [PATCH 08/15] catalog: add missing includes for migration, #TASK-5662 --- .../opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java index 9bcd821405f..34983c60b7a 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java @@ -346,8 +346,8 @@ public void associateAlignmentFiles(long studyUid) throws CatalogException { Query query = new Query() .append(QueryParams.STUDY_UID.key(), studyUid) .append(QueryParams.FORMAT.key(), Arrays.asList(File.Format.BAM, File.Format.CRAM)); - QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(QueryParams.PATH.key(), QueryParams.FORMAT.key(), - QueryParams.RELATED_FILES.key(), QueryParams.INTERNAL.key())); + QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(QueryParams.UID.key(), QueryParams.PATH.key(), + QueryParams.FORMAT.key(), QueryParams.RELATED_FILES.key(), QueryParams.INTERNAL.key(), QueryParams.STUDY_UID.key())); try (DBIterator iterator = iterator(query, options)) { while (iterator.hasNext()) { From 6f087f5e21af1168bc95989333cd6293b0e3a70b Mon Sep 17 00:00:00 2001 From: pfurio Date: Wed, 30 Oct 2024 12:14:43 +0100 Subject: [PATCH 09/15] catalog: add default type value UNKNOWN, #TASK-7046 --- .../OrganizationsCommandExecutor.java | 4 +++ .../executors/StudiesCommandExecutor.java | 4 +++ .../options/OrganizationsCommandOptions.java | 9 ++++++ .../main/options/StudiesCommandOptions.java | 9 ++++++ .../catalog/AddNewNoteTypeMigration.java | 31 +++++++++++++++++++ .../db/mongodb/MongoDBAdaptorFactory.java | 3 +- .../opencga/catalog/managers/NoteManager.java | 2 ++ .../catalog/managers/NoteManagerTest.java | 8 ++++- .../src/main/R/R/Organization-methods.R | 3 +- opencga-client/src/main/R/R/Study-methods.R | 3 +- .../rest/clients/OrganizationClient.java | 1 + .../client/rest/clients/StudyClient.java | 1 + .../src/main/javascript/Organization.js | 1 + opencga-client/src/main/javascript/Study.js | 1 + .../rest_clients/organization_client.py | 1 + .../pyopencga/rest_clients/study_client.py | 1 + .../opencga/core/models/notes/NoteType.java | 4 ++- 17 files changed, 81 insertions(+), 5 deletions(-) create mode 100644 opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v4/v4_0_0/catalog/AddNewNoteTypeMigration.java diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OrganizationsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OrganizationsCommandExecutor.java index 1294b1397e7..caefda1d320 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OrganizationsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OrganizationsCommandExecutor.java @@ -19,6 +19,7 @@ import org.opencb.opencga.core.config.Optimizations; import org.opencb.opencga.core.models.notes.Note; import org.opencb.opencga.core.models.notes.NoteCreateParams; +import org.opencb.opencga.core.models.notes.NoteType; import org.opencb.opencga.core.models.notes.NoteUpdateParams; import org.opencb.opencga.core.models.organizations.Organization; import org.opencb.opencga.core.models.organizations.OrganizationConfiguration; @@ -164,6 +165,7 @@ private RestResponse createNotes() throws Exception { } else { ObjectMap beanParams = new ObjectMap(); putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotNull(beanParams, "type", commandOptions.type, true); putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); putNestedIfNotNull(beanParams, "visibility", commandOptions.visibility, true); putNestedIfNotNull(beanParams, "valueType", commandOptions.valueType, true); @@ -186,6 +188,7 @@ private RestResponse searchNotes() throws Exception { queryParams.putIfNotEmpty("creationDate", commandOptions.creationDate); queryParams.putIfNotEmpty("modificationDate", commandOptions.modificationDate); queryParams.putIfNotEmpty("id", commandOptions.id); + queryParams.putIfNotEmpty("type", commandOptions.type); queryParams.putIfNotEmpty("scope", commandOptions.scope); queryParams.putIfNotEmpty("visibility", commandOptions.visibility); queryParams.putIfNotEmpty("uuid", commandOptions.uuid); @@ -230,6 +233,7 @@ private RestResponse updateNotes() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), NoteUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); + putNestedIfNotNull(beanParams, "type", commandOptions.type, true); putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); putNestedIfNotNull(beanParams, "visibility", commandOptions.visibility, true); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/StudiesCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/StudiesCommandExecutor.java index 0c18639ed52..009be9f6b7a 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/StudiesCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/StudiesCommandExecutor.java @@ -33,6 +33,7 @@ import org.opencb.opencga.core.models.job.Job; import org.opencb.opencga.core.models.notes.Note; import org.opencb.opencga.core.models.notes.NoteCreateParams; +import org.opencb.opencga.core.models.notes.NoteType; import org.opencb.opencga.core.models.notes.NoteUpdateParams; import org.opencb.opencga.core.models.study.CustomGroup; import org.opencb.opencga.core.models.study.Group; @@ -398,6 +399,7 @@ private RestResponse createNotes() throws Exception { } else { ObjectMap beanParams = new ObjectMap(); putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotNull(beanParams, "type", commandOptions.type, true); putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); putNestedIfNotNull(beanParams, "visibility", commandOptions.visibility, true); putNestedIfNotNull(beanParams, "valueType", commandOptions.valueType, true); @@ -420,6 +422,7 @@ private RestResponse searchNotes() throws Exception { queryParams.putIfNotEmpty("creationDate", commandOptions.creationDate); queryParams.putIfNotEmpty("modificationDate", commandOptions.modificationDate); queryParams.putIfNotEmpty("id", commandOptions.id); + queryParams.putIfNotEmpty("type", commandOptions.type); queryParams.putIfNotEmpty("uuid", commandOptions.uuid); queryParams.putIfNotEmpty("userId", commandOptions.userId); queryParams.putIfNotEmpty("tags", commandOptions.tags); @@ -463,6 +466,7 @@ private RestResponse updateNotes() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), NoteUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); + putNestedIfNotNull(beanParams, "type", commandOptions.type, true); putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); putNestedIfNotNull(beanParams, "visibility", commandOptions.visibility, true); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OrganizationsCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OrganizationsCommandOptions.java index 0f9f2a18c49..6fc4e04c9c7 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OrganizationsCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OrganizationsCommandOptions.java @@ -127,6 +127,9 @@ public class CreateNotesCommandOptions { @Parameter(names = {"--id"}, description = "The body web service id parameter", required = false, arity = 1) public String id; + @Parameter(names = {"--type"}, description = "Enum param allowed values: VARIANT, GENE, TRANSCRIPT, PROTEIN, JOB, FILE, SAMPLE, INDIVIDUAL, FAMILY, COHORT, DISEASE_PANEL, CLINICAL_ANALYSIS, WORKFLOW, ORGANIZATION, OTHER, UNKNOWN", required = false, arity = 1) + public String type; + @Parameter(names = {"--tags"}, description = "The body web service tags parameter", required = false, arity = 1) public String tags; @@ -159,6 +162,9 @@ public class SearchNotesCommandOptions { @Parameter(names = {"--id"}, description = "Note unique identifier.", required = false, arity = 1) public String id; + @Parameter(names = {"--type"}, description = "Note type.", required = false, arity = 1) + public String type; + @Parameter(names = {"--scope"}, description = "Scope of the Note.", required = false, arity = 1) public String scope; @@ -220,6 +226,9 @@ public class UpdateNotesCommandOptions { @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) public boolean includeResult = false; + @Parameter(names = {"--type"}, description = "Enum param allowed values: VARIANT, GENE, TRANSCRIPT, PROTEIN, JOB, FILE, SAMPLE, INDIVIDUAL, FAMILY, COHORT, DISEASE_PANEL, CLINICAL_ANALYSIS, WORKFLOW, ORGANIZATION, OTHER, UNKNOWN", required = false, arity = 1) + public String type; + @Parameter(names = {"--tags"}, description = "The body web service tags parameter", required = false, arity = 1) public String tags; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/StudiesCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/StudiesCommandOptions.java index d75022de0b9..f317c8e4122 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/StudiesCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/StudiesCommandOptions.java @@ -412,6 +412,9 @@ public class CreateNotesCommandOptions { @Parameter(names = {"--id"}, description = "The body web service id parameter", required = false, arity = 1) public String id; + @Parameter(names = {"--type"}, description = "Enum param allowed values: VARIANT, GENE, TRANSCRIPT, PROTEIN, JOB, FILE, SAMPLE, INDIVIDUAL, FAMILY, COHORT, DISEASE_PANEL, CLINICAL_ANALYSIS, WORKFLOW, ORGANIZATION, OTHER, UNKNOWN", required = false, arity = 1) + public String type; + @Parameter(names = {"--tags"}, description = "The body web service tags parameter", required = false, arity = 1) public String tags; @@ -447,6 +450,9 @@ public class SearchNotesCommandOptions { @Parameter(names = {"--id"}, description = "Note unique identifier.", required = false, arity = 1) public String id; + @Parameter(names = {"--type"}, description = "Note type.", required = false, arity = 1) + public String type; + @Parameter(names = {"--uuid"}, description = "Unique 32-character identifier assigned automatically by OpenCGA.", required = false, arity = 1) public String uuid; @@ -511,6 +517,9 @@ public class UpdateNotesCommandOptions { @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) public boolean includeResult = false; + @Parameter(names = {"--type"}, description = "Enum param allowed values: VARIANT, GENE, TRANSCRIPT, PROTEIN, JOB, FILE, SAMPLE, INDIVIDUAL, FAMILY, COHORT, DISEASE_PANEL, CLINICAL_ANALYSIS, WORKFLOW, ORGANIZATION, OTHER, UNKNOWN", required = false, arity = 1) + public String type; + @Parameter(names = {"--tags"}, description = "The body web service tags parameter", required = false, arity = 1) public String tags; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v4/v4_0_0/catalog/AddNewNoteTypeMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v4/v4_0_0/catalog/AddNewNoteTypeMigration.java new file mode 100644 index 00000000000..6165cd2ff96 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v4/v4_0_0/catalog/AddNewNoteTypeMigration.java @@ -0,0 +1,31 @@ +package org.opencb.opencga.app.migrations.v3.v4.v4_0_0.catalog; + +import com.mongodb.client.MongoCollection; +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Updates; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; +import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.models.notes.NoteType; + +@Migration(id = "addNewNoteType__task_7046", + description = "Add new Note type #7046", version = "4.0.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20241030) +public class AddNewNoteTypeMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + NoteType type = NoteType.UNKNOWN; + if (ParamConstants.ADMIN_ORGANIZATION.equals(organizationId)) { + type = NoteType.ORGANIZATION; + } + MongoCollection collection = getMongoCollection(OrganizationMongoDBAdaptorFactory.NOTE_COLLECTION); + Bson query = Filters.exists("type", false); + Bson update = Updates.set("type", type); + logger.info("Setting all notes from organization '{}' to type '{}'", organizationId, type); + collection.updateMany(query, update); + } +} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java index e262c3d3595..2ff8ed5332e 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java @@ -37,6 +37,7 @@ import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.models.notes.Note; import org.opencb.opencga.core.models.notes.NoteCreateParams; +import org.opencb.opencga.core.models.notes.NoteType; import org.opencb.opencga.core.models.organizations.Organization; import org.opencb.opencga.core.models.organizations.OrganizationSummary; import org.opencb.opencga.core.response.OpenCGAResult; @@ -273,7 +274,7 @@ public OpenCGAResult createOrganization(Organization organization, OrganizationSummary organizationSummary = new OrganizationSummary(organization.getId(), organizationDBAdaptorFactory.getMongoDataStore().getDatabaseName(), OrganizationTag.ACTIVE.name(), null); - NoteCreateParams noteCreateParams = new NoteCreateParams(ORGANIZATION_PREFIX + organization.getId(), + NoteCreateParams noteCreateParams = new NoteCreateParams(ORGANIZATION_PREFIX + organization.getId(), NoteType.ORGANIZATION, Collections.singletonList(OrganizationTag.ACTIVE.name()), Note.Visibility.PRIVATE, Note.Type.OBJECT, null); try { String orgSummaryString = JacksonUtils.getDefaultObjectMapper().writeValueAsString(organizationSummary); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/NoteManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/NoteManager.java index f32c8422dec..33f0fa0bdee 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/NoteManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/NoteManager.java @@ -24,6 +24,7 @@ import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.notes.Note; import org.opencb.opencga.core.models.notes.NoteCreateParams; +import org.opencb.opencga.core.models.notes.NoteType; import org.opencb.opencga.core.models.notes.NoteUpdateParams; import org.opencb.opencga.core.models.study.Study; import org.opencb.opencga.core.response.OpenCGAResult; @@ -396,6 +397,7 @@ public static void validateNewNote(Note note, String userId) throws CatalogParam ParamUtils.checkObj(note.getValueType(), NoteDBAdaptor.QueryParams.VALUE_TYPE.key()); ParamUtils.checkObj(note.getValue(), NoteDBAdaptor.QueryParams.VALUE.key()); + note.setType(ParamUtils.defaultObject(note.getType(), NoteType.UNKNOWN)); note.setTags(CollectionUtils.isNotEmpty(note.getTags()) ? note.getTags() : Collections.emptyList()); note.setUserId(userId); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/NoteManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/NoteManagerTest.java index 9d1afcadb2b..843bb95c037 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/NoteManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/NoteManagerTest.java @@ -12,6 +12,7 @@ import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.models.notes.Note; import org.opencb.opencga.core.models.notes.NoteCreateParams; +import org.opencb.opencga.core.models.notes.NoteType; import org.opencb.opencga.core.models.notes.NoteUpdateParams; import org.opencb.opencga.core.models.organizations.Organization; import org.opencb.opencga.core.models.study.Study; @@ -37,11 +38,14 @@ public void createOrganizationNoteTest() throws CatalogException { Note note = catalogManager.getNotesManager().createOrganizationNote(noteCreateParams, INCLUDE_RESULT, ownerToken).first(); assertEquals(noteCreateParams.getId(), note.getId()); assertEquals(orgOwnerUserId, note.getUserId()); + assertEquals(NoteType.UNKNOWN, note.getType()); noteCreateParams.setId("note2"); + noteCreateParams.setType(NoteType.FAMILY); note = catalogManager.getNotesManager().createOrganizationNote(noteCreateParams, INCLUDE_RESULT, orgAdminToken1).first(); assertEquals(noteCreateParams.getId(), note.getId()); assertEquals(orgAdminUserId1, note.getUserId()); + assertEquals(NoteType.FAMILY, note.getType()); thrown.expect(CatalogAuthorizationException.class); thrown.expectMessage("denied"); @@ -59,10 +63,12 @@ public void updateOrganizationNoteTest() throws CatalogException { assertEquals(1, note.getVersion()); NoteUpdateParams noteUpdateParams = new NoteUpdateParams() - .setTags(Arrays.asList("tag1", "tag2")); + .setTags(Arrays.asList("tag1", "tag2")) + .setType(NoteType.GENE); note = catalogManager.getNotesManager().updateOrganizationNote(note.getId(), noteUpdateParams, INCLUDE_RESULT, ownerToken).first(); assertEquals(2, note.getVersion()); assertEquals(orgOwnerUserId, note.getUserId()); + assertEquals(NoteType.GENE, note.getType()); assertEquals(2, note.getTags().size()); assertArrayEquals(noteUpdateParams.getTags().toArray(), note.getTags().toArray()); diff --git a/opencga-client/src/main/R/R/Organization-methods.R b/opencga-client/src/main/R/R/Organization-methods.R index 14e44864ed9..70e53aa5144 100644 --- a/opencga-client/src/main/R/R/Organization-methods.R +++ b/opencga-client/src/main/R/R/Organization-methods.R @@ -21,7 +21,7 @@ #' | -- | :-- | --: | #' | create | /{apiVersion}/organizations/create | include, exclude, includeResult, body[*] | #' | createNotes | /{apiVersion}/organizations/notes/create | include, exclude, includeResult, body[*] | -#' | searchNotes | /{apiVersion}/organizations/notes/search | include, exclude, creationDate, modificationDate, id, scope, visibility, uuid, userId, tags, version | +#' | searchNotes | /{apiVersion}/organizations/notes/search | include, exclude, creationDate, modificationDate, id, type, scope, visibility, uuid, userId, tags, version | #' | deleteNotes | /{apiVersion}/organizations/notes/{id}/delete | id[*], includeResult | #' | updateNotes | /{apiVersion}/organizations/notes/{id}/update | include, exclude, id[*], tagsAction, includeResult, body[*] | #' | userUpdateStatus | /{apiVersion}/organizations/user/{user}/status/update | include, exclude, user[*], organization, includeResult, body[*] | @@ -64,6 +64,7 @@ setMethod("organizationClient", "OpencgaR", function(OpencgaR, id, organization, #' @param creationDate Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. #' @param modificationDate Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. #' @param id Note unique identifier. + #' @param type Note type. #' @param scope Scope of the Note. #' @param visibility Visibility of the Note. #' @param uuid Unique 32-character identifier assigned automatically by OpenCGA. diff --git a/opencga-client/src/main/R/R/Study-methods.R b/opencga-client/src/main/R/R/Study-methods.R index ee3f4509354..bd20437eb98 100644 --- a/opencga-client/src/main/R/R/Study-methods.R +++ b/opencga-client/src/main/R/R/Study-methods.R @@ -29,7 +29,7 @@ #' | updateGroups | /{apiVersion}/studies/{study}/groups/update | study[*], action, body[*] | #' | updateGroupsUsers | /{apiVersion}/studies/{study}/groups/{group}/users/update | study[*], group[*], action, body[*] | #' | createNotes | /{apiVersion}/studies/{study}/notes/create | include, exclude, study[*], includeResult, body[*] | -#' | searchNotes | /{apiVersion}/studies/{study}/notes/search | include, exclude, study[*], creationDate, modificationDate, id, uuid, userId, tags, visibility, version | +#' | searchNotes | /{apiVersion}/studies/{study}/notes/search | include, exclude, study[*], creationDate, modificationDate, id, type, uuid, userId, tags, visibility, version | #' | deleteNotes | /{apiVersion}/studies/{study}/notes/{id}/delete | study[*], id[*], includeResult | #' | updateNotes | /{apiVersion}/studies/{study}/notes/{id}/update | include, exclude, study[*], id[*], tagsAction, includeResult, body[*] | #' | permissionRules | /{apiVersion}/studies/{study}/permissionRules | study[*], entity[*] | @@ -169,6 +169,7 @@ setMethod("studyClient", "OpencgaR", function(OpencgaR, group, id, members, stud #' @param creationDate Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. #' @param modificationDate Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. #' @param id Note unique identifier. + #' @param type Note type. #' @param uuid Unique 32-character identifier assigned automatically by OpenCGA. #' @param userId User that wrote that Note. #' @param tags Note tags. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/OrganizationClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/OrganizationClient.java index b5d5a3b7fa1..f8fe3d8d7a4 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/OrganizationClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/OrganizationClient.java @@ -93,6 +93,7 @@ public RestResponse createNotes(NoteCreateParams data, ObjectMap params) t * creationDate: Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. * modificationDate: Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. * id: Note unique identifier. + * type: Note type. * scope: Scope of the Note. * visibility: Visibility of the Note. * uuid: Unique 32-character identifier assigned automatically by OpenCGA. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java index b1a8bce6959..a073df38553 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java @@ -252,6 +252,7 @@ public RestResponse createNotes(String study, NoteCreateParams data, Objec * creationDate: Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. * modificationDate: Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. * id: Note unique identifier. + * type: Note type. * uuid: Unique 32-character identifier assigned automatically by OpenCGA. * userId: User that wrote that Note. * tags: Note tags. diff --git a/opencga-client/src/main/javascript/Organization.js b/opencga-client/src/main/javascript/Organization.js index 09b79429246..59723bae9f7 100644 --- a/opencga-client/src/main/javascript/Organization.js +++ b/opencga-client/src/main/javascript/Organization.js @@ -67,6 +67,7 @@ export default class Organization extends OpenCGAParentClass { * @param {String} [params.creationDate] - Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. * @param {String} [params.modificationDate] - Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. * @param {String} [params.id] - Note unique identifier. + * @param {String} [params.type] - Note type. * @param {String} [params.scope] - Scope of the Note. * @param {String} [params.visibility] - Visibility of the Note. * @param {String} [params.uuid] - Unique 32-character identifier assigned automatically by OpenCGA. diff --git a/opencga-client/src/main/javascript/Study.js b/opencga-client/src/main/javascript/Study.js index bbe53e93df9..fe51a747d05 100644 --- a/opencga-client/src/main/javascript/Study.js +++ b/opencga-client/src/main/javascript/Study.js @@ -188,6 +188,7 @@ export default class Study extends OpenCGAParentClass { * @param {String} [params.creationDate] - Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. * @param {String} [params.modificationDate] - Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. * @param {String} [params.id] - Note unique identifier. + * @param {String} [params.type] - Note type. * @param {String} [params.uuid] - Unique 32-character identifier assigned automatically by OpenCGA. * @param {String} [params.userId] - User that wrote that Note. * @param {String} [params.tags] - Note tags. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/organization_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/organization_client.py index 0d7dc0ff159..ff7a5423da9 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/organization_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/organization_client.py @@ -66,6 +66,7 @@ def search_notes(self, **options): :param str modification_date: Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. :param str id: Note unique identifier. + :param str type: Note type. :param str scope: Scope of the Note. :param str visibility: Visibility of the Note. :param str uuid: Unique 32-character identifier assigned automatically diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py index b730dc06d5d..4aa1518fdf5 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py @@ -229,6 +229,7 @@ def search_notes(self, study, **options): :param str modification_date: Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. :param str id: Note unique identifier. + :param str type: Note type. :param str uuid: Unique 32-character identifier assigned automatically by OpenCGA. :param str user_id: User that wrote that Note. diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/notes/NoteType.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/notes/NoteType.java index 9fff8b0745f..a3acf18c6ab 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/notes/NoteType.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/notes/NoteType.java @@ -14,5 +14,7 @@ public enum NoteType { DISEASE_PANEL, CLINICAL_ANALYSIS, WORKFLOW, - OTHER + ORGANIZATION, + OTHER, + UNKNOWN } From eaa5f8fb9c0b842d8bd24252078f6614b4d3e786 Mon Sep 17 00:00:00 2001 From: pfurio Date: Wed, 30 Oct 2024 12:33:09 +0100 Subject: [PATCH 10/15] app: generate new index and drop old ones, #TASK-7046 --- .../v4/v4_0_0/catalog/AddNewNoteTypeMigration.java | 14 +++++++++++++- .../src/main/resources/catalog-indexes.txt | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) rename opencga-app/src/main/java/org/opencb/opencga/app/migrations/{v3 => }/v4/v4_0_0/catalog/AddNewNoteTypeMigration.java (72%) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v4/v4_0_0/catalog/AddNewNoteTypeMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/AddNewNoteTypeMigration.java similarity index 72% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v4/v4_0_0/catalog/AddNewNoteTypeMigration.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/AddNewNoteTypeMigration.java index 6165cd2ff96..b18c1350268 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v4/v4_0_0/catalog/AddNewNoteTypeMigration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/AddNewNoteTypeMigration.java @@ -1,4 +1,4 @@ -package org.opencb.opencga.app.migrations.v3.v4.v4_0_0.catalog; +package org.opencb.opencga.app.migrations.v4.v4_0_0.catalog; import com.mongodb.client.MongoCollection; import com.mongodb.client.model.Filters; @@ -11,6 +11,8 @@ import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.models.notes.NoteType; +import java.util.Arrays; + @Migration(id = "addNewNoteType__task_7046", description = "Add new Note type #7046", version = "4.0.0", language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20241030) @@ -27,5 +29,15 @@ protected void run() throws Exception { Bson update = Updates.set("type", type); logger.info("Setting all notes from organization '{}' to type '{}'", organizationId, type); collection.updateMany(query, update); + + // Drop old index + Document oldIndex = new Document() + .append("id", 1) + .append("studyUid", 1) + .append("version", 1); + dropIndex(Arrays.asList(OrganizationMongoDBAdaptorFactory.NOTE_COLLECTION, OrganizationMongoDBAdaptorFactory.NOTE_ARCHIVE_COLLECTION), + oldIndex); + // Generate new indexes + catalogManager.installIndexes(organizationId, token); } } diff --git a/opencga-catalog/src/main/resources/catalog-indexes.txt b/opencga-catalog/src/main/resources/catalog-indexes.txt index 09436684a31..6c8d34f487f 100644 --- a/opencga-catalog/src/main/resources/catalog-indexes.txt +++ b/opencga-catalog/src/main/resources/catalog-indexes.txt @@ -1,4 +1,4 @@ -{"collections": ["note", "note_archive"], "fields": {"id": 1, "studyUid": 1, "version": 1}, "options": {"unique": true}} +{"collections": ["note", "note_archive"], "fields": {"id": 1, "studyUid": 1, "type": 1, "version": 1}, "options": {"unique": true}} {"collections": ["note", "note_archive"], "fields": {"uid": 1, "version": 1}, "options": {"unique": true}} {"collections": ["note", "note_archive"], "fields": {"uuid": 1, "version": 1}, "options": {"unique": true}} {"collections": ["note", "note_archive"], "fields": {"scope": 1, "studyUid": 1}, "options": {}} From 096f16b1f80b8d4207f9d9a36e893f476ffb05ab Mon Sep 17 00:00:00 2001 From: halender <35806821+halender@users.noreply.github.com> Date: Wed, 27 Nov 2024 13:25:07 +0100 Subject: [PATCH 11/15] Update README.md Signed-off-by: halender <35806821+halender@users.noreply.github.com> --- README.md | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index c9da3fffcc1..5b239e3a303 100644 --- a/README.md +++ b/README.md @@ -6,19 +6,13 @@ OpenCGA constitutes the big data analysis component of [OpenCB](http://www.openc ### Documentation You can find OpenCGA documentation and tutorials at: https://github.com/opencb/opencga/wiki. -### Issues Tracking -You can report bugs or request new features at [GitHub issue tracking](https://github.com/opencb/opencga/issues). +### Feedback and Feature Requests +Found a bug or have an idea for a new feature? Let us know at zettagenomics.com/academic. ### Release Notes and Roadmap -Releases notes are available at [GitHub releases](https://github.com/opencb/opencga/releases). - -Roadmap is available at [GitHub milestones](https://github.com/opencb/opencga/milestones). You can report bugs or request new features at [GitHub issue tracking](https://github.com/opencb/opencga/issues). - -### Versioning -OpenCGA is versioned following the rules from [Semantic versioning](https://semver.org/). +Releases notes are available at https://zettagenomics.com/release-notes/ ### Maintainers -We recommend to contact OpenCGA developers by writing to OpenCB mailing list opencb@googlegroups.com. Current main developers and maintainers are: * Ignacio Medina (im411@cam.ac.uk) (_Founder and Project Leader_) * Jacobo Coll (jacobo.coll-moragon@genomicsengland.co.uk) * Pedro Furio (pedro.furio@genomicsengland.co.uk) @@ -33,7 +27,7 @@ We recommend to contact OpenCGA developers by writing to OpenCB mailing list ope * Franscisco Salavert (fsalavert@cipf.es) ##### Contributing -OpenCGA is an open-source and collaborative project. We appreciate any help and feedback from users, you can contribute in many different ways such as simple bug reporting and feature request. Dependending on your skills you are more than welcome to develop client tools, new features or even fixing bugs. +OpenCGA is an open-source and collaborative project. We appreciate any help and feedback from users, you can contribute in many different ways such as simple bug reporting and feature request. Dependending on your skills you are more than welcome to develop client tools, new features, or even fix bugs. # How to build From ecdc49716514ebd5f37cec7219fdf2fb5d637cdb Mon Sep 17 00:00:00 2001 From: pfurio Date: Mon, 9 Dec 2024 12:37:51 +0100 Subject: [PATCH 12/15] catalog: remove alignment index references when deleting file,#TASK-5662 --- .../opencga/catalog/db/api/FileDBAdaptor.java | 2 + .../db/mongodb/FileMongoDBAdaptor.java | 38 ++++++++++++++++++- .../src/main/resources/catalog-indexes.txt | 2 + .../catalog/managers/FileManagerTest.java | 9 +++++ 4 files changed, 49 insertions(+), 2 deletions(-) diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java index 1666884fbb8..811354e0a82 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java @@ -99,7 +99,9 @@ enum QueryParams implements QueryParam { INTERNAL_ALIGNMENT("internal.alignment", OBJECT, ""), INTERNAL_ALIGNMENT_INDEX("internal.alignment.index", TEXT_ARRAY, ""), INTERNAL_ALIGNMENT_INDEX_STATUS_ID("internal.alignment.index.status.id", TEXT_ARRAY, ""), + INTERNAL_ALIGNMENT_INDEX_FILE_ID("internal.alignment.index.fileId", STRING, ""), INTERNAL_ALIGNMENT_COVERAGE("internal.alignment.coverage", TEXT_ARRAY, ""), + INTERNAL_ALIGNMENT_COVERAGE_FILE_ID("internal.alignment.coverage.fileId", STRING, ""), INTERNAL_ALIGNMENT_COVERAGE_STATUS_ID("internal.alignment.coverage.status.id", TEXT_ARRAY, ""), INTERNAL_MISSING_SAMPLES("internal.missingSamples", OBJECT, ""), INTERNAL_MISSING_SAMPLES_EXISTING("internal.missingSamples.existing", TEXT_ARRAY, ""), diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java index 34983c60b7a..cfc41792fb9 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java @@ -738,7 +738,7 @@ OpenCGAResult transactionalUpdate(ClientSession clientSession, long studyU List events = new ArrayList<>(); logger.debug("Update file. Query: {}, Update: {}", query.toBsonDocument(), fileUpdate.toBsonDocument()); - DataResult result = fileCollection.update(clientSession, query, fileUpdate, null); + DataResult result = fileCollection.update(clientSession, query, fileUpdate, new QueryOptions(MongoDBCollection.MULTI, true)); logger.debug("{} file(s) successfully updated", result.getNumUpdated()); return endWrite(tmpStartTime, result.getNumMatches(), result.getNumUpdated(), events); @@ -1340,11 +1340,12 @@ OpenCGAResult privateDelete(ClientSession clientSession, Document fileDo void removeFileReferences(ClientSession clientSession, long studyUid, long fileUid, Document fileDoc) throws CatalogParameterException, CatalogDBException, CatalogAuthorizationException { File file = fileConverter.convertToDataModelType(fileDoc); + + // Remove file references from relatedFiles array FileRelatedFile relatedFile = new FileRelatedFile(file, null); ObjectMap parameters = new ObjectMap(QueryParams.RELATED_FILES.key(), Collections.singletonList(relatedFile)); ObjectMap actionMap = new ObjectMap(QueryParams.RELATED_FILES.key(), ParamUtils.BasicUpdateAction.REMOVE); QueryOptions options = new QueryOptions(Constants.ACTIONS, actionMap); - Query query = new Query() .append(QueryParams.STUDY_UID.key(), studyUid) .append(QueryParams.RELATED_FILES_FILE_UID.key(), fileUid); @@ -1357,6 +1358,37 @@ void removeFileReferences(ClientSession clientSession, long studyUid, long fileU logger.debug("File '{}' removed from related files array from {} files.", file.getPath(), result.getNumUpdated()); } } + + // Remove file references from alignment indexes + FileInternalAlignmentIndex alignmentIndex = new FileInternalAlignmentIndex(new InternalStatus(InternalStatus.DELETED), null, null); + parameters = new ObjectMap(QueryParams.INTERNAL_ALIGNMENT_INDEX.key(), alignmentIndex); + query = new Query() + .append(QueryParams.STUDY_UID.key(), studyUid) + .append(QueryParams.INTERNAL_ALIGNMENT_INDEX_FILE_ID.key(), file.getId()); + updateDocument = getValidatedUpdateParams(clientSession, studyUid, parameters, query, options); + updateDoc = updateDocument.toFinalUpdateDocument(); + if (!updateDoc.isEmpty()) { + Bson bsonQuery = parseQuery(query); + OpenCGAResult result = transactionalUpdate(clientSession, studyUid, bsonQuery, updateDocument); + if (result.getNumUpdated() > 0) { + logger.debug("File '{}' removed from internal.alignment.index object", file.getPath()); + } + } + + FileInternalCoverageIndex coverageIndex = new FileInternalCoverageIndex(new InternalStatus(InternalStatus.DELETED), null, null, 0); + parameters = new ObjectMap(QueryParams.INTERNAL_ALIGNMENT_COVERAGE.key(), coverageIndex); + query = new Query() + .append(QueryParams.STUDY_UID.key(), studyUid) + .append(QueryParams.INTERNAL_ALIGNMENT_COVERAGE_FILE_ID.key(), file.getId()); + updateDocument = getValidatedUpdateParams(clientSession, studyUid, parameters, query, options); + updateDoc = updateDocument.toFinalUpdateDocument(); + if (!updateDoc.isEmpty()) { + Bson bsonQuery = parseQuery(query); + OpenCGAResult result = transactionalUpdate(clientSession, studyUid, bsonQuery, updateDocument); + if (result.getNumUpdated() > 0) { + logger.debug("File '{}' removed from internal.alignment.coverage object", file.getPath()); + } + } } // OpenCGAResult privateDelete(ClientSession clientSession, File file, String status) throws CatalogDBException { @@ -1818,6 +1850,8 @@ private Bson parseQuery(Query query, Document extraQuery, String user) case SIZE: case SOFTWARE_NAME: case RELATED_FILES_FILE_UID: + case INTERNAL_ALIGNMENT_INDEX_FILE_ID: + case INTERNAL_ALIGNMENT_COVERAGE_FILE_ID: case JOB_ID: addAutoOrQuery(queryParam.key(), queryParam.key(), myQuery, queryParam.type(), andBsonList); break; diff --git a/opencga-catalog/src/main/resources/catalog-indexes.txt b/opencga-catalog/src/main/resources/catalog-indexes.txt index fe7f43506d5..5e9eb72df6f 100644 --- a/opencga-catalog/src/main/resources/catalog-indexes.txt +++ b/opencga-catalog/src/main/resources/catalog-indexes.txt @@ -68,6 +68,8 @@ {"collections": ["file"], "fields": {"status.id": 1, "studyUid": 1}, "options": {}} {"collections": ["file"], "fields": {"internal.status.id": 1, "studyUid": 1}, "options": {}} {"collections": ["file"], "fields": {"internal.variant.index.status.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["file"], "fields": {"internal.alignment.coverage.fileId": 1, "studyUid": 1}, "options": {}} +{"collections": ["file"], "fields": {"internal.alignment.index.fileId": 1, "studyUid": 1}, "options": {}} {"collections": ["file"], "fields": {"studyUid": 1, "_acl": 1}, "options": {}} {"collections": ["file"], "fields": {"studyUid": 1, "release": 1, "_acl": 1}, "options": {}} {"collections": ["file"], "fields": {"_as.as": 1, "studyUid": 1}, "options": {}} diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FileManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FileManagerTest.java index 0f1f58a73f2..81c28e9daa8 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FileManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FileManagerTest.java @@ -461,6 +461,15 @@ public void associateAlignmentFilesTest() throws CatalogException { assertEquals(1, baiFile.getRelatedFiles().size()); assertEquals(FileRelatedFile.Relation.ALIGNMENT, baiFile.getRelatedFiles().get(0).getRelation()); assertEquals(bamFile.getId(), baiFile.getRelatedFiles().get(0).getFile().getId()); + + // Unlink BAI file + query = new Query(FileDBAdaptor.QueryParams.UID.key(), baiFile.getUid()); + setToPendingDelete(studyFqn, query); + fileManager.unlink(studyFqn, baiFile.getPath(), ownerToken); + bamFile = fileManager.get(studyFqn, bamFile.getPath(), QueryOptions.empty(), ownerToken).first(); + assertNotNull(bamFile.getInternal().getAlignment().getIndex()); + assertNull(bamFile.getInternal().getAlignment().getIndex().getFileId()); + assertEquals(FileStatus.DELETED, bamFile.getInternal().getAlignment().getIndex().getStatus().getId()); } @Test From 69e143f2b431183e9bd135dc993c71e3c184716c Mon Sep 17 00:00:00 2001 From: pfurio Date: Mon, 9 Dec 2024 13:04:52 +0100 Subject: [PATCH 13/15] catalog: set fileId and indexer to empty string instead, #TASK-5662 --- .../opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java index cfc41792fb9..4bbd9e40995 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java @@ -1360,7 +1360,7 @@ void removeFileReferences(ClientSession clientSession, long studyUid, long fileU } // Remove file references from alignment indexes - FileInternalAlignmentIndex alignmentIndex = new FileInternalAlignmentIndex(new InternalStatus(InternalStatus.DELETED), null, null); + FileInternalAlignmentIndex alignmentIndex = new FileInternalAlignmentIndex(new InternalStatus(InternalStatus.DELETED), "", ""); parameters = new ObjectMap(QueryParams.INTERNAL_ALIGNMENT_INDEX.key(), alignmentIndex); query = new Query() .append(QueryParams.STUDY_UID.key(), studyUid) @@ -1375,7 +1375,7 @@ void removeFileReferences(ClientSession clientSession, long studyUid, long fileU } } - FileInternalCoverageIndex coverageIndex = new FileInternalCoverageIndex(new InternalStatus(InternalStatus.DELETED), null, null, 0); + FileInternalCoverageIndex coverageIndex = new FileInternalCoverageIndex(new InternalStatus(InternalStatus.DELETED), "", "", 0); parameters = new ObjectMap(QueryParams.INTERNAL_ALIGNMENT_COVERAGE.key(), coverageIndex); query = new Query() .append(QueryParams.STUDY_UID.key(), studyUid) From 98c9b198f1a681a64e8d9a2383cb88bf7318574f Mon Sep 17 00:00:00 2001 From: pfurio Date: Wed, 11 Dec 2024 09:43:57 +0100 Subject: [PATCH 14/15] catalog: fix style compilation issue --- .../opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java index 4bbd9e40995..896e20764a0 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java @@ -185,7 +185,8 @@ public OpenCGAResult insertWithVirtualFile(long studyId, File file, File virtual } List insert(ClientSession clientSession, long studyId, File file, List existingSamples, List nonExistingSamples, - List variableSetList) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + List variableSetList) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { if (filePathExists(clientSession, studyId, file.getPath())) { throw CatalogDBException.alreadyExists("File", studyId, "path", file.getPath()); } From 33d55c9f968af244e63ff6b7177d5cb1c1cd8f2c Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Wed, 18 Dec 2024 17:24:07 +0100 Subject: [PATCH 15/15] Added manual-deploy-python-notebook.yml #TASK-6757 --- .../manual-deploy-python-notebook.yml | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 .github/workflows/manual-deploy-python-notebook.yml diff --git a/.github/workflows/manual-deploy-python-notebook.yml b/.github/workflows/manual-deploy-python-notebook.yml new file mode 100644 index 00000000000..6d5b45eaacd --- /dev/null +++ b/.github/workflows/manual-deploy-python-notebook.yml @@ -0,0 +1,59 @@ +name: Reusable deploy Docker python-notebook +run-name: 'Run deploy Docker python-notebook ${{ inputs.branch }} by @${{ github.actor }}' +on: + workflow_dispatch: + inputs: + branch: + description: "The branch, tag or SHA of the source code to build docker." + type: string + required: true + tag: + description: "The tag for the new docker." + type: string + required: true + hadoop: + type: string + description: 'Hadoop flavour. Any of: [hdp3.1, hdi5.1, emr6.1, emr6.13]' + required: false + default: "hdp3.1" + pyopencga_version: + type: string + description: 'PyOpenCGA version.' + required: true + +jobs: + build: + name: Build Java app + runs-on: ubuntu-22.04 + outputs: + version: ${{ steps.get_project_version.outputs.version }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: '10' + ref: "${{ inputs.branch }}" + - name: Set up JDK 8 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '8' + cache: 'maven' + - name: Install dependencies branches + run: | + if [ -f "./.github/workflows/scripts/get_same_branch.sh" ]; then + chmod +x ./.github/workflows/scripts/get_same_branch.sh + ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }} ${{ inputs.hadoop }} + fi + - name: Maven Build (skip tests) + run: mvn -T 2 clean install -DskipTests + - uses: actions/upload-artifact@v4 + with: + name: build-folder + path: build + + deploy-docker-python-notebook: + uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop + needs: build + with: + cli: python3 ./build/cloud/docker/docker-build.py push --images python-notebook --tag ${{ inputs.tag }} --docker-build-args "-e VERSION=${{ inputs.pyopencga_version }}" + secrets: inherit \ No newline at end of file