Merge branch 'master' into 5516-awsbatch-error-500

.github/workflows/seqeradocs-changelog.yml

@@ -0,0 +1,62 @@
+name: Push release change log to Seqera Docs
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+    inputs:
+      release_name:
+        description: "Release version (e.g. 1.0.0)"
+        required: true
+      release_body:
+        description: "Release changelog content"
+        required: true
+
+jobs:
+  update-docs:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Clone seqeralabs/docs
+        run: |
+          git clone https://github.com/seqeralabs/docs.git seqeralabs-docs
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create changelog file
+        run: |
+          mkdir -p seqeralabs-docs/changelog/nextflow
+          cat << EOF > seqeralabs-docs/changelog/nextflow/${{ github.event.release.name || inputs.release_name }}.mdx
+          ---
+          title: Nextflow ${{ github.event.release.name || inputs.release_name }}
+          date: $(date +%Y-%m-%d)
+          tags: [nextflow]
+          ---
+
+          ${{ github.event.release.body || inputs.release_body }}
+          EOF
+
+      - uses: actions/create-github-app-token@v1
+        id: generate-token
+        with:
+          app-id: ${{ secrets.DOCS_BOT_APP_ID }}
+          private-key: ${{ secrets.DOCS_BOT_APP_PRIVATE_KEY }}
+          owner: seqeralabs
+          repositories: docs
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v7
+        with:
+          token: ${{ steps.generate-token.outputs.token }}
+          branch-token: ${{ steps.generate-token.outputs.token }}
+          path: seqeralabs-docs
+          commit-message: "Changelog: Nextflow ${{ github.event.release.name || inputs.release_name }}"
+          title: "Changelog: Nextflow ${{ github.event.release.name || inputs.release_name }}"
+          body: |
+            This PR adds the changelog for Nextflow ${{ github.event.release.name || inputs.release_name }} to the Seqera documentation.
+
+            This is an automated PR created from the Nextflow repository.
+          branch: changelog-nextflow-${{ github.event.release.name || inputs.release_name }}
+          base: master
+          delete-branch: true

VERSION

@@ -1 +1 @@
-24.10.0
+24.11.0-edge

build.gradle

@@ -109,8 +109,8 @@ allprojects {
 
         // Documentation required libraries
         groovyDoc 'org.fusesource.jansi:jansi:2.4.0'
-        groovyDoc "org.apache.groovy:groovy-groovydoc:4.0.23"
-        groovyDoc "org.apache.groovy:groovy-ant:4.0.23"
+        groovyDoc "org.apache.groovy:groovy-groovydoc:4.0.24"
+        groovyDoc "org.apache.groovy:groovy-ant:4.0.24"
     }
 
     test {

changelog.txt

@@ -1,5 +1,61 @@
 NEXTFLOW CHANGE-LOG
 ===================
+24.11.0-edge - 3 Dec 2024
+- Add GHA to submit dependencies to dependabot (#5440) [80395a6d]
+- Add NotFoundException to retry condition for Google Batch [aa4d19cc]
+- Add Rahel Hirsch to run name generator (#5442) [ff2bc6ae]
+- Add `env()` function (#5506) [fa0e8e0f]
+- Add more scientists to run name generator (#5447) [38d9eda0]
+- Add `singularity.libraryDir` to config page (#5498) [b5e31bb0]
+- Add RepositoryProvider.revision now public property (#5500) [f0a4c526]
+- Deprecate process `shell` block (#5508) [6f527551]
+- Detecting errors in data unstaging (#5345) [3c8e602d]
+- Disable virtual threads on CI tests [ci slip] [69d07dbc]
+- Fix Fargate warning on memory check (#5475) [bdf0ad00]
+- Fix `isContainerReady` when wave is disabled (#5509) [3215afa8]
+- Fix missing wave response (#5547) [ee252173]
+- Fix overlapping conda lock file (#5540) [9248c04d]
+- Fix overlapping conda lock exception (#5489) [eaaeb3de]
+- Fix possible deadlock in dynamic `maxRetry` resolution (#5474) [25bbb621]
+- Fix Wave GCP integration test (#5490) [ad56c89b]
+- Fixing bug when execution with stub and no stub defined (#5473) [f7fd56db]
+- Fix Incorrect CPU value in Azure example (#5549) [fc5e2c2a]
+- Improve docs for using the GPU accelerator directive (#5488) [4b908524]
+- Improve groupTuple docs with scatter/gather example (#5520) [b5c63a9f]
+- Prevent NPE with null AWS Batch response [12fc1d60]
+- Target Java 17 as minimal Java version (#5045) [0140f954]
+- Update 'nexus-staging' plugin to latest version (#5462) [07934513]
+- Update gradle 'shadow' plugin version to 8.3.5 (#5463) [2a5f15f0]
+- Update install docs to reflect change from 'all' to 'dist' (#5496) [c9115659]
+- Update process snippets to comply with strict syntax (#5526) [be1694bf]
+- Update Wave dependencies [09ccd295]
+- Bump aws sdk 1.12.777 (#5458) [8bad0b4b]
+- Bump bouncycastle to jdk18on:1.78.1  (#5467) [cd8c385f]
+- Bump com.azure:azure-identity from 1.11.3 to 1.12.2 (#5449) [cb70f1df]
+- Bump commons-io:2.15.1 [767e4a0a]
+- Bump groovy 4.0.24 [dd71ad31]
+- Bump netty-common to version 4.1.115.Final [d1bbd3d0]
+- Bump [email protected] [2b653b07]
+- Bump [email protected] [6af7198d]
+- Bump [email protected] [9494f970]
+- Bump [email protected] [7e2c8d82]
+- Bump protobuf-java:3.25.5 to nf-google [488b7906]
+
+24.10.2 - 27 Nov 2024
+- Prevent NPE with null AWS Batch response [3d491934]
+- Fix overlapping conda lock file (#5540) [df66deaa]
+- Fix missing wave response (#5547) [eb85cda8]
+- Bump [email protected] [93d09404]
+- Bump [email protected] [469a35dd]
+
+24.10.1 - 18 Nov 2024
+- Fix overlapping file lock exception (#5489) [a2566d54]
+- Fix isContainerReady when wave is disabled (#5509) [c69e3711]
+- Bump [email protected] [e7709a0f]
+- Bump [email protected] [54496ac4]
+- Bump [email protected] [fa227933]
+- Bump netty-common to version 4.1.115.Final [90623c1e]
+
 24.10.0 - 27 Oct 2024
 - Add `manifest.contributors` config option (#5322) [cf0f9690]
 - Add wave mirror and scan config [92e69776]

docs/_static/dag.mmd

@@ -1,3 +1,17 @@
+%%{
+    init: {
+        'theme': 'base',
+        'themeVariables': {
+            'primaryColor': '#B6ECE2',
+            'primaryTextColor': '#160F26',
+            'primaryBorderColor': '#065647',
+            'lineColor': '#545555',
+            'clusterBkg': '#BABCBD22',
+            'clusterBorder': '#DDDEDE',
+            'fontFamily': 'arial'
+        }
+    }
+}%%
 flowchart TB
     subgraph " "
     v0["Channel.fromFilePairs"]

docs/aws.md

@@ -539,7 +539,8 @@ See the AWS documentation for details how to create the required AWS Batch queue
 and the Batch Execution Role.
 
 :::{note}
-This feature requires the use {ref}`Wave <wave-page>` container provisioning service.
+Nextflow uses [s5cmd](https://github.com/peak/s5cmd) to download the task input data and upload the task outputs.
+To enable this capability, you need to enable the Wave service in the Nextflow configuration, as shown in the above example. See {ref}`Wave <wave-page>` documentation for more details.
 :::
 
 ## Advanced configuration

docs/azure.md

@@ -155,7 +155,7 @@ Replacing `<PIPELINE NAME>` with a pipeline name e.g. `nextflow-io/rnaseq-nf` an
 
 See the [Batch documentation](https://docs.microsoft.com/en-us/azure/batch/quick-create-portal) for further details about the configuration for Azure Batch.
 
-### Pools configuration
+### Autopools
 
 When using the `autoPoolMode` option, Nextflow automatically creates a `pool` of compute nodes appropriate for your pipeline.
 
@@ -167,12 +167,12 @@ To specify multiple Azure machine families, use a comma separated list with glob
 process.machineType = "Standard_D*d_v5,Standard_E*d_v5"
 ```
 
-For example, the following process will create a pool of `Standard_E4d_v5` machines based when using `autoPoolMode`:
+For example, the following process will create a pool of `Standard_E8d_v5` machines based when using `autoPoolMode`:
 
 ```nextflow
 process EXAMPLE_PROCESS {
     machineType "Standard_E*d_v5"
-    cpus 16
+    cpus 8
     memory 8.GB
 
     script:

modules/nextflow/build.gradle

@@ -21,11 +21,11 @@ dependencies {
     api(project(':nf-commons'))
     api(project(':nf-httpfs'))
     api "org.apache.groovy:groovy:4.0.24"
-    api "org.apache.groovy:groovy-nio:4.0.23"
-    api "org.apache.groovy:groovy-xml:4.0.23"
-    api "org.apache.groovy:groovy-json:4.0.23"
-    api "org.apache.groovy:groovy-templates:4.0.23"
-    api "org.apache.groovy:groovy-yaml:4.0.23"
+    api "org.apache.groovy:groovy-nio:4.0.24"
+    api "org.apache.groovy:groovy-xml:4.0.24"
+    api "org.apache.groovy:groovy-json:4.0.24"
+    api "org.apache.groovy:groovy-templates:4.0.24"
+    api "org.apache.groovy:groovy-yaml:4.0.24"
     api "org.slf4j:jcl-over-slf4j:2.0.7"
     api "org.slf4j:jul-to-slf4j:2.0.7"
     api "org.slf4j:log4j-over-slf4j:2.0.7"
@@ -39,7 +39,7 @@ dependencies {
     api "com.beust:jcommander:1.35"
     api("com.esotericsoftware.kryo:kryo:2.24.0") { exclude group: 'com.esotericsoftware.minlog', module: 'minlog' }
     api('org.iq80.leveldb:leveldb:0.12')
-    api('org.eclipse.jgit:org.eclipse.jgit:6.10.0.202406032230-r')
+    api('org.eclipse.jgit:org.eclipse.jgit:7.1.0.202411261347-r')
     api ('javax.activation:activation:1.1.1')
     api ('javax.mail:mail:1.4.7')
     api ('org.yaml:snakeyaml:2.2')
@@ -53,7 +53,7 @@ dependencies {
     testImplementation 'org.subethamail:subethasmtp:3.1.7'
 
     // test configuration
-    testFixturesApi ("org.apache.groovy:groovy-test:4.0.23") { exclude group: 'org.apache.groovy' }
+    testFixturesApi ("org.apache.groovy:groovy-test:4.0.24") { exclude group: 'org.apache.groovy' }
     testFixturesApi ("org.objenesis:objenesis:3.4")
     testFixturesApi ("net.bytebuddy:byte-buddy:1.14.17")
     testFixturesApi ("org.spockframework:spock-core:2.3-groovy-4.0") { exclude group: 'org.apache.groovy' }

modules/nextflow/src/main/groovy/nextflow/executor/SimpleFileCopyStrategy.groovy

@@ -183,7 +183,7 @@ class SimpleFileCopyStrategy implements ScriptFileCopyStrategy {
         return """\
             IFS=\$'\\n'
             for name in \$(eval "ls -1d ${escape.join(' ')}" | sort | uniq); do
-                ${stageOutCommand('$name', targetDir, mode)} || true
+                ${stageOutCommand('$name', targetDir, mode)}
             done
             unset IFS""".stripIndent(true)
     }

modules/nextflow/src/main/groovy/nextflow/scm/AssetManager.groovy

@@ -833,7 +833,7 @@ class AssetManager {
 
     protected Map refToMap(Ref ref, Map<String,Ref> remote) {
         final entry = new HashMap(2)
-        final peel = git.getRepository().peel(ref)
+        final peel = git.getRepository().getRefDatabase().peel(ref)
         final objId = peel.getPeeledObjectId() ?: peel.getObjectId()
         // the branch or tag name
         entry.name = shortenRefName(ref.name)
@@ -867,7 +867,7 @@ class AssetManager {
         result << (name == current ? '*' : ' ')
 
         if( level ) {
-            def peel = git.getRepository().peel(ref)
+            def peel = git.getRepository().getRefDatabase().peel(ref)
             def obj = peel.getPeeledObjectId() ?: peel.getObjectId()
             result << ' '
             result << formatObjectId(obj, level == 1)

modules/nextflow/src/main/resources/META-INF/build-info.properties

@@ -1,4 +1,4 @@
-build=5928
-version=24.10.0
-timestamp=1730054192154
-commitId=6524d8dc9
+build=5929
+version=24.11.0-edge
+timestamp=1733218258400
+commitId=7e2c8d82b

modules/nextflow/src/main/resources/META-INF/plugins-info.txt

@@ -1,8 +1,8 @@
-nf-amazon@2.9.0
-nf-azure@1.10.1
+nf-amazon@2.10.0
+nf-azure@1.11.0
 [email protected]
 [email protected]
 [email protected]
-nf-google@1.15.2
+nf-google@1.16.0
 [email protected]
-nf-wave@1.7.2
+nf-wave@1.8.0

modules/nextflow/src/main/resources/nextflow/dag/mermaid.dag.template.html

@@ -19,6 +19,20 @@
 </head>
 <body>
 <pre class="mermaid" style="text-align: center;">
+%%{
+  init: {
+    'theme': 'base',
+    'themeVariables': {
+      'primaryColor': '#B6ECE2',
+      'primaryTextColor': '#160F26',
+      'primaryBorderColor': '#065647',
+      'lineColor': '#545555',
+      'clusterBkg': '#BABCBD22',
+      'clusterBorder': '#DDDEDE',
+      'fontFamily': 'arial'
+    }
+  }
+}%%
 REPLACE_WITH_NETWORK_DATA
 </pre>
 <script type="module">

modules/nextflow/src/main/resources/nextflow/executor/command-run.txt

@@ -99,7 +99,13 @@ nxf_fs_fcp() {
 }
 
 on_exit() {
-    exit_status=${nxf_main_ret:=$?}
+    ## Capture possible errors.
+    ## Can be caused either by the task script, unstage script or after script if defined
+    local last_err=$?
+    ## capture the task error first or fallback to unstage error
+    local exit_status=${nxf_main_ret:=0}
+    [[ ${exit_status} -eq 0 && ${nxf_unstage_ret:=0} -ne 0 ]] && exit_status=${nxf_unstage_ret:=0}
+    [[ ${exit_status} -eq 0 && ${last_err} -ne 0 ]] && exit_status=${last_err}
     printf -- $exit_status {{exit_file}}
     set +u
     {{cleanup_cmd}}
@@ -121,13 +127,26 @@ nxf_stage() {
     {{stage_inputs}}
 }
 
-nxf_unstage() {
+nxf_unstage_outputs() {
     true
-    {{unstage_controls}}
-    [[ ${nxf_main_ret:=0} != 0 ]] && return
     {{unstage_outputs}}
 }
 
+nxf_unstage_controls() {
+    true
+    {{unstage_controls}}
+}
+
+nxf_unstage() {
+    ## Deactivate fast failure to allow uploading stdout and stderr files later
+    if [[ ${nxf_main_ret:=0} == 0 ]]; then
+        ## Data unstaging redirecting stdout and stderr with append mode
+        (set -e -o pipefail; (nxf_unstage_outputs | tee -a {{stdout_file}}) 3>&1 1>&2 2>&3 | tee -a {{stderr_file}})
+        nxf_unstage_ret=$?
+    fi
+    nxf_unstage_controls
+}
+
 nxf_main() {
     trap on_exit EXIT
     trap on_term TERM INT USR2