Merge branch 'release-2.12.x' into TASK-5895

opencb · Jun 6, 2024 · 97c09c6 · 97c09c6
2 parents dd1dcec + 7fcd257
commit 97c09c6
Show file tree

Hide file tree

Showing 30 changed files with 165 additions and 92 deletions.
diff --git a/.github/workflows/check-junit-test.yml b/.github/workflows/check-junit-test.yml
@@ -27,7 +27,7 @@ jobs:
     outputs:
       profiles: ${{ steps.getter.outputs.profiles }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: '10'
       - name: Building string profiles to run

diff --git a/.github/workflows/manual-deploy-docker.yml b/.github/workflows/manual-deploy-docker.yml
@@ -18,12 +18,12 @@ jobs:
     outputs:
       version: ${{ steps.get_project_version.outputs.version }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: '10'
           ref: "${{ inputs.branch }}"
       - name: Set up JDK 8
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
           java-version: '8'
@@ -36,7 +36,7 @@ jobs:
           fi
       - name: Maven Build (skip tests)
         run: mvn -T 2 clean install -DskipTests -P storage-hadoop,hdp3.1,RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl '!:opencga-storage-hadoop-deps-emr6.1,!:opencga-storage-hadoop-deps-hdp2.6'
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: build-folder
           path: build

diff --git a/.github/workflows/manual-deploy-ext-tools.yml b/.github/workflows/manual-deploy-ext-tools.yml
@@ -18,12 +18,12 @@ jobs:
     outputs:
       version: ${{ steps.get_project_version.outputs.version }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: '10'
           ref: "${{ inputs.branch }}"
       - name: Set up JDK 8
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
           java-version: '8'
@@ -36,7 +36,7 @@ jobs:
           fi
       - name: Maven Build (skip tests)
         run: mvn -T 2 clean install -DskipTests
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: build-folder
           path: build

diff --git a/.github/workflows/test-analysis.yml b/.github/workflows/test-analysis.yml
@@ -20,11 +20,11 @@ jobs:
     name: Execute Sonar Analysis
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: '0'
       - name: Set up JDK 11
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
           java-version: '17'
@@ -47,11 +47,11 @@ jobs:
     name: Execute JUnit and Jacoco tests
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: '0'
       - name: Set up JDK 8
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
           java-version: '8'

diff --git a/opencga-analysis/pom.xml b/opencga-analysis/pom.xml
@@ -22,7 +22,7 @@
     <parent>
         <groupId>org.opencb.opencga</groupId>
         <artifactId>opencga</artifactId>
-        <version>2.12.5-SNAPSHOT</version>
+        <version>2.12.6-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/opencga-app/pom.xml b/opencga-app/pom.xml
@@ -22,7 +22,7 @@
     <parent>
         <groupId>org.opencb.opencga</groupId>
         <artifactId>opencga</artifactId>
-        <version>2.12.5-SNAPSHOT</version>
+        <version>2.12.6-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/opencga-catalog/pom.xml b/opencga-catalog/pom.xml
@@ -23,7 +23,7 @@
     <parent>
         <groupId>org.opencb.opencga</groupId>
         <artifactId>opencga</artifactId>
-        <version>2.12.5-SNAPSHOT</version>
+        <version>2.12.6-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/opencga-client/pom.xml b/opencga-client/pom.xml
@@ -22,7 +22,7 @@
     <parent>
         <groupId>org.opencb.opencga</groupId>
         <artifactId>opencga</artifactId>
-        <version>2.12.5-SNAPSHOT</version>
+        <version>2.12.6-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/opencga-clinical/pom.xml b/opencga-clinical/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>org.opencb.opencga</groupId>
         <artifactId>opencga</artifactId>
-        <version>2.12.5-SNAPSHOT</version>
+        <version>2.12.6-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

diff --git a/opencga-core/pom.xml b/opencga-core/pom.xml
@@ -22,7 +22,7 @@
     <parent>
         <groupId>org.opencb.opencga</groupId>
         <artifactId>opencga</artifactId>
-        <version>2.12.5-SNAPSHOT</version>
+        <version>2.12.6-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/opencga-master/pom.xml b/opencga-master/pom.xml
@@ -22,7 +22,7 @@
     <parent>
         <artifactId>opencga</artifactId>
         <groupId>org.opencb.opencga</groupId>
-        <version>2.12.5-SNAPSHOT</version>
+        <version>2.12.6-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/opencga-server/pom.xml b/opencga-server/pom.xml
@@ -22,7 +22,7 @@
     <parent>
         <groupId>org.opencb.opencga</groupId>
         <artifactId>opencga</artifactId>
-        <version>2.12.5-SNAPSHOT</version>
+        <version>2.12.6-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/opencga-storage/opencga-storage-app/pom.xml b/opencga-storage/opencga-storage-app/pom.xml
@@ -22,7 +22,7 @@
     <parent>
         <groupId>org.opencb.opencga</groupId>
         <artifactId>opencga-storage</artifactId>
-        <version>2.12.5-SNAPSHOT</version>
+        <version>2.12.6-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/opencga-storage/opencga-storage-benchmark/pom.xml b/opencga-storage/opencga-storage-benchmark/pom.xml
@@ -22,7 +22,7 @@
     <parent>
         <artifactId>opencga-storage</artifactId>
         <groupId>org.opencb.opencga</groupId>
-        <version>2.12.5-SNAPSHOT</version>
+        <version>2.12.6-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/opencga-storage/opencga-storage-core/pom.xml b/opencga-storage/opencga-storage-core/pom.xml
@@ -22,7 +22,7 @@
     <parent>
         <groupId>org.opencb.opencga</groupId>
         <artifactId>opencga-storage</artifactId>
-        <version>2.12.5-SNAPSHOT</version>
+        <version>2.12.6-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/...-core/src/test/java/org/opencb/opencga/storage/core/variant/search/VariantSearchTest.java b/...-core/src/test/java/org/opencb/opencga/storage/core/variant/search/VariantSearchTest.java
@@ -187,15 +187,16 @@ public void testSpecialCharacter() throws Exception {
         List<Variant> variants = getVariants(limit);
         List<Variant> annotatedVariants = annotatedVariants(variants);
 
-        String studyId = "abyu12";
-        String fileId = "a.vcf";
+        String study = "abyu12";
+        String file = "a.vcf";
 
-        variants.get(0).getStudies().get(0).getFiles().get(0).setFileId(fileId);
+        variants.get(0).getStudies().get(0).getFiles().get(0).setFileId(file);
         System.out.println(variants.get(0).getStudies().get(0).getFiles().get(0).getFileId());
         //System.exit(-1);
 
-        scm.createStudy(studyId);
-
+        int studyId = scm.createStudy(study).getId();
+        int fileId = scm.registerFile(studyId, file, Arrays.asList("A-A", "B", "C", "D"));
+        scm.addIndexedFiles(studyId, Collections.singletonList(fileId));
         String collection = solr.coreName;
         variantSearchManager.create(collection);
 
@@ -204,13 +205,13 @@ public void testSpecialCharacter() throws Exception {
         samplePosition.put("B", 1);
         samplePosition.put("C", 2);
         samplePosition.put("D", 3);
-        annotatedVariants.get(0).getStudies().get(0).setStudyId(studyId).setSortedSamplesPosition(samplePosition);
+        annotatedVariants.get(0).getStudies().get(0).setStudyId(study).setSortedSamplesPosition(samplePosition);
         variantSearchManager.insert(collection, annotatedVariants);
 
         Query query = new Query();
-        query.put(VariantQueryParam.STUDY.key(), studyId);
+        query.put(VariantQueryParam.STUDY.key(), study);
 //        query.put(VariantQueryParam.SAMPLE.key(), samplePosition.keySet().toArray()[0]);
-        query.put(VariantQueryParam.FILE.key(), fileId);
+        query.put(VariantQueryParam.FILE.key(), file);
         query.put(VariantQueryParam.FILTER.key(), "PASS");
         query.put(VariantQueryParam.ANNOT_CLINICAL_SIGNIFICANCE.key(), "benign");
         VariantQueryResult<Variant> results = variantSearchManager.query(collection, variantStorageEngine.parseQuery(query,

diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml
@@ -23,7 +23,7 @@
     <parent>
         <groupId>org.opencb.opencga</groupId>
         <artifactId>opencga-storage-hadoop</artifactId>
-        <version>2.12.5-SNAPSHOT</version>
+        <version>2.12.6-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/...n/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParser.java b/...n/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParser.java
@@ -1216,18 +1216,7 @@ protected SampleAnnotationIndexQuery parseAnnotationIndexQuery(SampleIndexSchema
         CtBtFtCombinationIndexSchema.Filter ctBtTfFilter = schema.getCtBtTfIndex().getField().noOpFilter();
         IndexFilter clinicalFilter = schema.getClinicalIndexSchema().noOpFilter();
 
-        Boolean intergenic = null;
-
-        ParsedVariantQuery.VariantQueryXref variantQueryXref = VariantQueryParser.parseXrefs(query);
-        if (!isValidParam(query, REGION)) {
-            if (!variantQueryXref.getGenes().isEmpty()
-                    && variantQueryXref.getIds().isEmpty()
-                    && variantQueryXref.getOtherXrefs().isEmpty()
-                    && variantQueryXref.getVariants().isEmpty()) {
-                // If only filtering by genes, is not intergenic.
-                intergenic = false;
-            }
-        }
+        final Boolean intergenic = isIntergenicQuery(query);
 
 //        BiotypeConsquenceTypeFlagCombination combination = BiotypeConsquenceTypeFlagCombination
 //                .fromQuery(query, Arrays.asList(schema.getTranscriptFlagIndexSchema().getField().getConfiguration().getValues()));
@@ -1237,18 +1226,10 @@ protected SampleAnnotationIndexQuery parseAnnotationIndexQuery(SampleIndexSchema
         boolean tfCovered = false;
 
         if (isValidParam(query, ANNOT_CONSEQUENCE_TYPE)) {
-            List<String> soNames = query.getAsStringList(VariantQueryParam.ANNOT_CONSEQUENCE_TYPE.key());
-            soNames = soNames.stream()
+            List<String> soNames = query.getAsStringList(VariantQueryParam.ANNOT_CONSEQUENCE_TYPE.key())
+                    .stream()
                     .map(ct -> ConsequenceTypeMappings.accessionToTerm.get(VariantQueryUtils.parseConsequenceType(ct)))
                     .collect(Collectors.toList());
-            if (!soNames.contains(VariantAnnotationConstants.INTERGENIC_VARIANT)
-                    && !soNames.contains(VariantAnnotationConstants.REGULATORY_REGION_VARIANT)
-                    && !soNames.contains(VariantAnnotationConstants.TF_BINDING_SITE_VARIANT)) {
-                // All ct values but "intergenic_variant" and "regulatory_region_variant" are in genes (i.e. non-intergenic)
-                intergenic = false;
-            } else if (soNames.size() == 1 && soNames.contains(VariantAnnotationConstants.INTERGENIC_VARIANT)) {
-                intergenic = true;
-            } // else, leave undefined : intergenic = null
             boolean ctFilterCoveredBySummary = false;
             boolean ctBtCombinationCoveredBySummary = false;
             if (SampleIndexSchema.CUSTOM_LOF.containsAll(soNames)) {
@@ -1295,14 +1276,17 @@ protected SampleAnnotationIndexQuery parseAnnotationIndexQuery(SampleIndexSchema
                 }
             }
 
-            // Do not use ctIndex if the CT filter is covered by the summary
-            // Use the ctIndex if:
+            // Do not use ctIndex for intergenic queries (intergenic == true)
+            // or queries that might return intergenic variants (intergenic == null)
+            //
+            // Use the ctIndex if any of:
             // - The CtFilter is not covered by the summary
             // - The query has the combination CT+BT , and it is not covered by the summary
             // - The query has the combination CT+TF
-            boolean useCtIndexFilter = !ctFilterCoveredBySummary
-                    || (!ctBtCombinationCoveredBySummary && combination.isBiotype())
-                    || combination.isFlag();
+            boolean useCtIndexFilter =
+                    intergenic == Boolean.FALSE && (!ctFilterCoveredBySummary
+                            || (!ctBtCombinationCoveredBySummary && combination.isBiotype())
+                            || combination.isFlag());
             if (useCtIndexFilter) {
                 ctCovered = completeIndex;
                 consequenceTypeFilter = schema.getCtIndex().getField().buildFilter(new OpValue<>("=", soNames));
@@ -1317,8 +1301,6 @@ protected SampleAnnotationIndexQuery parseAnnotationIndexQuery(SampleIndexSchema
         }
 
         if (isValidParam(query, ANNOT_BIOTYPE)) {
-            // All biotype values are in genes (i.e. non-intergenic)
-            intergenic = false;
             boolean biotypeFilterCoveredBySummary = false;
             List<String> biotypes = query.getAsStringList(VariantQueryParam.ANNOT_BIOTYPE.key());
             if (BIOTYPE_SET.containsAll(biotypes)) {
@@ -1350,8 +1332,6 @@ protected SampleAnnotationIndexQuery parseAnnotationIndexQuery(SampleIndexSchema
             List<String> transcriptFlags = query.getAsStringList(ANNOT_TRANSCRIPT_FLAG.key());
             tfFilter = schema.getTranscriptFlagIndexSchema().getField().buildFilter(new OpValue<>("=", transcriptFlags));
             tfCovered = completeIndex & tfFilter.isExactFilter();
-            // Transcript flags are in transcripts/genes. (i.e. non-intergenic)
-            intergenic = false;
             // TranscriptFlag filter is covered by index
             if (tfCovered) {
                 if (!isValidParam(query, GENE) && simpleCombination(combination)) {
@@ -1538,12 +1518,60 @@ protected SampleAnnotationIndexQuery parseAnnotationIndexQuery(SampleIndexSchema
             // If intergenic is undefined, or true, CT and BT filters can not be used.
             biotypeFilter = schema.getBiotypeIndex().getField().noOpFilter();
             consequenceTypeFilter = schema.getCtIndex().getField().noOpFilter();
+            if (!biotypeFilter.isNoOp()) {
+                throw new IllegalStateException("Unexpected BT filter for intergenic=" + intergenic);
+            }
+            if (!consequenceTypeFilter.isNoOp()) {
+                throw new IllegalStateException("Unexpected CT filter for intergenic=" + intergenic);
+            }
         }
 
         return new SampleAnnotationIndexQuery(new byte[]{annotationIndexMask, annotationIndex},
                 consequenceTypeFilter, biotypeFilter, tfFilter, ctBtTfFilter, clinicalFilter, populationFrequencyFilter);
     }
 
+    private Boolean isIntergenicQuery(Query query) {
+        ParsedVariantQuery.VariantQueryXref variantQueryXref = VariantQueryParser.parseXrefs(query);
+        if (!isValidParam(query, REGION)) {
+            if (!variantQueryXref.getGenes().isEmpty()
+                    && variantQueryXref.getIds().isEmpty()
+                    && variantQueryXref.getOtherXrefs().isEmpty()
+                    && variantQueryXref.getVariants().isEmpty()) {
+                // If only filtering by genes, is not intergenic.
+                return false;
+            }
+        }
+
+        if (isValidParam(query, ANNOT_BIOTYPE)) {
+            // All biotype values are in genes (i.e. non-intergenic)
+            return false;
+        }
+        if (isValidParam(query, ANNOT_BIOTYPE)) {
+            // All biotype values are in genes (i.e. non-intergenic)
+            return false;
+        }
+        if (isValidParam(query, ANNOT_TRANSCRIPT_FLAG)) {
+            // Transcript flags are in transcripts/genes. (i.e. non-intergenic)
+            return false;
+        }
+        if (isValidParam(query, ANNOT_CONSEQUENCE_TYPE)) {
+            List<String> soNames = query.getAsStringList(VariantQueryParam.ANNOT_CONSEQUENCE_TYPE.key());
+            soNames = soNames.stream()
+                    .map(ct -> ConsequenceTypeMappings.accessionToTerm.get(VariantQueryUtils.parseConsequenceType(ct)))
+                    .collect(Collectors.toList());
+            if (!soNames.contains(VariantAnnotationConstants.INTERGENIC_VARIANT)
+                    && !soNames.contains(VariantAnnotationConstants.REGULATORY_REGION_VARIANT)
+                    && !soNames.contains(VariantAnnotationConstants.TF_BINDING_SITE_VARIANT)) {
+                // All ct values but "intergenic_variant" and "regulatory_region_variant" are in genes (i.e. non-intergenic)
+                return false;
+            } else if (soNames.size() == 1 && soNames.contains(VariantAnnotationConstants.INTERGENIC_VARIANT)) {
+                return true;
+            } // else, leave undefined : intergenic = null
+        }
+        // Unable to determine if the query is intergenic or not. Return null for uncertain.
+        return null;
+    }
+
     private boolean simpleCombination(BiotypeConsquenceTypeFlagCombination combination) {
         return combination.numParams() == 1;
     }