From fc5e2c2a8e4608c19d7072f622f3368024420c47 Mon Sep 17 00:00:00 2001 From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Date: Fri, 29 Nov 2024 18:47:38 +0000 Subject: [PATCH 01/18] Incorrect CPU value in Azure example (#5549) [ci skip] Signed-off-by: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Signed-off-by: adamrtalbot <12817534+adamrtalbot@users.noreply.github.com> --- docs/azure.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/azure.md b/docs/azure.md index 6087c18ec2..61269e43ac 100644 --- a/docs/azure.md +++ b/docs/azure.md @@ -167,12 +167,12 @@ To specify multiple Azure machine families, use a comma separated list with glob process.machineType = "Standard_D*d_v5,Standard_E*d_v5" ``` -For example, the following process will create a pool of `Standard_E4d_v5` machines based when using `autoPoolMode`: +For example, the following process will create a pool of `Standard_E8d_v5` machines based when using `autoPoolMode`: ```nextflow process EXAMPLE_PROCESS { machineType "Standard_E*d_v5" - cpus 16 + cpus 8 memory 8.GB script: From ab13ce58b2b176afd6cabbeb7149d8f0611a77f8 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Wed, 27 Nov 2024 22:30:30 +0100 Subject: [PATCH 02/18] Update changelog [ci skip] Signed-off-by: Paolo Di Tommaso --- changelog.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/changelog.txt b/changelog.txt index 3420a4041e..0be1d17adf 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,5 +1,20 @@ NEXTFLOW CHANGE-LOG =================== +24.10.2 - 27 Nov 2024 +- Prevent NPE with null AWS Batch response [3d491934] +- Fix overlapping conda lock file (#5540) [df66deaa] +- Fix missing wave response (#5547) [eb85cda8] +- Bump nf-wave@1.7.4 [93d09404] +- Bump nf-amazon@2.9.2 [469a35dd] + +24.10.1 - 18 Nov 2024 +- Fix overlapping file lock exception (#5489) [a2566d54] +- Fix isContainerReady when wave is disabled (#5509) [c69e3711] +- Bump nf-wave@1.7.3 [e7709a0f] +- Bump nf-azure@1.10.2 [54496ac4] +- Bump nf-amazon@2.9.1 [fa227933] +- Bump netty-common to version 4.1.115.Final [90623c1e] + 24.10.0 - 27 Oct 2024 - Add `manifest.contributors` config option (#5322) [cf0f9690] - Add wave mirror and scan config [92e69776] From 3c8e602d487a24654da8046ada3f7dc12e0fe39f Mon Sep 17 00:00:00 2001 From: Jorge Ejarque Date: Mon, 2 Dec 2024 18:42:00 +0100 Subject: [PATCH 03/18] Detecting errors in data unstaging (#5345) Signed-off-by: jorgee Signed-off-by: Paolo Di Tommaso Co-authored-by: Paolo Di Tommaso --- .../executor/SimpleFileCopyStrategy.groovy | 2 +- .../nextflow/executor/command-run.txt | 27 ++++++++++++++++--- .../executor/BashWrapperBuilderTest.groovy | 4 +-- .../SimpleFileCopyStrategyTest.groovy | 6 ++--- .../executor/test-bash-wrapper-with-trace.txt | 20 +++++++++++--- .../nextflow/executor/test-bash-wrapper.txt | 20 +++++++++++--- .../BashWrapperBuilderWithS3Test.groovy | 2 +- .../BashWrapperBuilderWithAzTest.groovy | 2 +- .../GoogleLifeSciencesHelper.groovy | 2 +- .../GoogleLifeSciencesHelperTest.groovy | 2 +- .../google/lifesciences/bash-wrapper-gcp.txt | 20 +++++++++++--- validation/awsbatch-unstage-fail.config | 12 +++++++++ validation/awsbatch.sh | 9 ++++++- .../Dockerfile | 11 ++++++++ .../fake_aws/bin/aws | 9 +++++++ validation/test-aws-unstage-fail.nf | 16 +++++++++++ 16 files changed, 140 insertions(+), 24 deletions(-) create mode 100644 validation/awsbatch-unstage-fail.config create mode 100644 validation/test-aws-unstage-fail-container/Dockerfile create mode 100755 validation/test-aws-unstage-fail-container/fake_aws/bin/aws create mode 100644 validation/test-aws-unstage-fail.nf diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/SimpleFileCopyStrategy.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/SimpleFileCopyStrategy.groovy index e1a92b172c..6d9dcbd538 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/SimpleFileCopyStrategy.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/SimpleFileCopyStrategy.groovy @@ -183,7 +183,7 @@ class SimpleFileCopyStrategy implements ScriptFileCopyStrategy { return """\ IFS=\$'\\n' for name in \$(eval "ls -1d ${escape.join(' ')}" | sort | uniq); do - ${stageOutCommand('$name', targetDir, mode)} || true + ${stageOutCommand('$name', targetDir, mode)} done unset IFS""".stripIndent(true) } diff --git a/modules/nextflow/src/main/resources/nextflow/executor/command-run.txt b/modules/nextflow/src/main/resources/nextflow/executor/command-run.txt index 2bf34b617a..26cbf6a829 100644 --- a/modules/nextflow/src/main/resources/nextflow/executor/command-run.txt +++ b/modules/nextflow/src/main/resources/nextflow/executor/command-run.txt @@ -99,7 +99,13 @@ nxf_fs_fcp() { } on_exit() { - exit_status=${nxf_main_ret:=$?} + ## Capture possible errors. + ## Can be caused either by the task script, unstage script or after script if defined + local last_err=$? + ## capture the task error first or fallback to unstage error + local exit_status=${nxf_main_ret:=0} + [[ ${exit_status} -eq 0 && ${nxf_unstage_ret:=0} -ne 0 ]] && exit_status=${nxf_unstage_ret:=0} + [[ ${exit_status} -eq 0 && ${last_err} -ne 0 ]] && exit_status=${last_err} printf -- $exit_status {{exit_file}} set +u {{cleanup_cmd}} @@ -121,13 +127,26 @@ nxf_stage() { {{stage_inputs}} } -nxf_unstage() { +nxf_unstage_outputs() { true - {{unstage_controls}} - [[ ${nxf_main_ret:=0} != 0 ]] && return {{unstage_outputs}} } +nxf_unstage_controls() { + true + {{unstage_controls}} +} + +nxf_unstage() { + ## Deactivate fast failure to allow uploading stdout and stderr files later + if [[ ${nxf_main_ret:=0} == 0 ]]; then + ## Data unstaging redirecting stdout and stderr with append mode + (set -e -o pipefail; (nxf_unstage_outputs | tee -a {{stdout_file}}) 3>&1 1>&2 2>&3 | tee -a {{stderr_file}}) + nxf_unstage_ret=$? + fi + nxf_unstage_controls +} + nxf_main() { trap on_exit EXIT trap on_term TERM INT USR2 diff --git a/modules/nextflow/src/test/groovy/nextflow/executor/BashWrapperBuilderTest.groovy b/modules/nextflow/src/test/groovy/nextflow/executor/BashWrapperBuilderTest.groovy index 5df66b6369..e54bd97d72 100644 --- a/modules/nextflow/src/test/groovy/nextflow/executor/BashWrapperBuilderTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/executor/BashWrapperBuilderTest.groovy @@ -559,7 +559,7 @@ class BashWrapperBuilderTest extends Specification { binding.unstage_outputs == '''\ IFS=$'\\n' for name in $(eval "ls -1d test.bam test.bai" | sort | uniq); do - nxf_fs_copy "$name" /work/dir || true + nxf_fs_copy "$name" /work/dir done unset IFS '''.stripIndent().rightTrim() @@ -576,7 +576,7 @@ class BashWrapperBuilderTest extends Specification { binding.unstage_outputs == '''\ IFS=$'\\n' for name in $(eval "ls -1d test.bam test.bai" | sort | uniq); do - nxf_fs_move "$name" /another/dir || true + nxf_fs_move "$name" /another/dir done unset IFS '''.stripIndent().rightTrim() diff --git a/modules/nextflow/src/test/groovy/nextflow/executor/SimpleFileCopyStrategyTest.groovy b/modules/nextflow/src/test/groovy/nextflow/executor/SimpleFileCopyStrategyTest.groovy index 29cbb35697..6361e4f394 100644 --- a/modules/nextflow/src/test/groovy/nextflow/executor/SimpleFileCopyStrategyTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/executor/SimpleFileCopyStrategyTest.groovy @@ -270,7 +270,7 @@ class SimpleFileCopyStrategyTest extends Specification { script == ''' IFS=$'\\n' for name in $(eval "ls -1d simple.txt my/path/file.bam" | sort | uniq); do - nxf_fs_copy "$name" /target/work\\ dir || true + nxf_fs_copy "$name" /target/work\\ dir done unset IFS ''' @@ -293,7 +293,7 @@ class SimpleFileCopyStrategyTest extends Specification { script == ''' IFS=$'\\n' for name in $(eval "ls -1d simple.txt my/path/file.bam" | sort | uniq); do - nxf_fs_move "$name" /target/store || true + nxf_fs_move "$name" /target/store done unset IFS ''' @@ -315,7 +315,7 @@ class SimpleFileCopyStrategyTest extends Specification { script == ''' IFS=$'\\n' for name in $(eval "ls -1d simple.txt my/path/file.bam" | sort | uniq); do - nxf_fs_rsync "$name" /target/work\\'s || true + nxf_fs_rsync "$name" /target/work\\'s done unset IFS ''' diff --git a/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper-with-trace.txt b/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper-with-trace.txt index ef1380e7cf..1de9614e11 100644 --- a/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper-with-trace.txt +++ b/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper-with-trace.txt @@ -270,7 +270,10 @@ nxf_fs_fcp() { } on_exit() { - exit_status=${nxf_main_ret:=$?} + local last_err=$? + local exit_status=${nxf_main_ret:=0} + [[ ${exit_status} -eq 0 && ${nxf_unstage_ret:=0} -ne 0 ]] && exit_status=${nxf_unstage_ret:=0} + [[ ${exit_status} -eq 0 && ${last_err} -ne 0 ]] && exit_status=${last_err} printf -- $exit_status > {{folder}}/.exitcode set +u exit $exit_status @@ -289,9 +292,20 @@ nxf_stage() { true } -nxf_unstage() { +nxf_unstage_outputs() { + true +} + +nxf_unstage_controls() { true - [[ ${nxf_main_ret:=0} != 0 ]] && return +} + +nxf_unstage() { + if [[ ${nxf_main_ret:=0} == 0 ]]; then + (set -e -o pipefail; (nxf_unstage_outputs | tee -a .command.out) 3>&1 1>&2 2>&3 | tee -a .command.err) + nxf_unstage_ret=$? + fi + nxf_unstage_controls } nxf_main() { diff --git a/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper.txt b/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper.txt index f465b5b9b4..3bb4f34fe5 100644 --- a/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper.txt +++ b/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper.txt @@ -81,7 +81,10 @@ nxf_fs_fcp() { } on_exit() { - exit_status=${nxf_main_ret:=$?} + local last_err=$? + local exit_status=${nxf_main_ret:=0} + [[ ${exit_status} -eq 0 && ${nxf_unstage_ret:=0} -ne 0 ]] && exit_status=${nxf_unstage_ret:=0} + [[ ${exit_status} -eq 0 && ${last_err} -ne 0 ]] && exit_status=${last_err} printf -- $exit_status > {{folder}}/.exitcode set +u exit $exit_status @@ -100,9 +103,20 @@ nxf_stage() { true } -nxf_unstage() { +nxf_unstage_outputs() { + true +} + +nxf_unstage_controls() { true - [[ ${nxf_main_ret:=0} != 0 ]] && return +} + +nxf_unstage() { + if [[ ${nxf_main_ret:=0} == 0 ]]; then + (set -e -o pipefail; (nxf_unstage_outputs | tee -a .command.out) 3>&1 1>&2 2>&3 | tee -a .command.err) + nxf_unstage_ret=$? + fi + nxf_unstage_controls } nxf_main() { diff --git a/plugins/nf-amazon/src/test/nextflow/executor/BashWrapperBuilderWithS3Test.groovy b/plugins/nf-amazon/src/test/nextflow/executor/BashWrapperBuilderWithS3Test.groovy index 3e213444cb..4f90e22aa2 100644 --- a/plugins/nf-amazon/src/test/nextflow/executor/BashWrapperBuilderWithS3Test.groovy +++ b/plugins/nf-amazon/src/test/nextflow/executor/BashWrapperBuilderWithS3Test.groovy @@ -58,7 +58,7 @@ class BashWrapperBuilderWithS3Test extends Specification { binding.unstage_outputs == '''\ IFS=$'\\n' for name in $(eval "ls -1d test.bam test.bai bla\\ nk.txt" | sort | uniq); do - nxf_s3_upload $name s3://some/buck\\ et || true + nxf_s3_upload $name s3://some/buck\\ et done unset IFS '''.stripIndent().rightTrim() diff --git a/plugins/nf-azure/src/test/nextflow/executor/BashWrapperBuilderWithAzTest.groovy b/plugins/nf-azure/src/test/nextflow/executor/BashWrapperBuilderWithAzTest.groovy index eb72d25163..1969345af5 100644 --- a/plugins/nf-azure/src/test/nextflow/executor/BashWrapperBuilderWithAzTest.groovy +++ b/plugins/nf-azure/src/test/nextflow/executor/BashWrapperBuilderWithAzTest.groovy @@ -47,7 +47,7 @@ class BashWrapperBuilderWithAzTest extends Specification { binding.unstage_outputs == """\ IFS=\$'\\n' for name in \$(eval "ls -1d test.bam test.bai" | sort | uniq); do - nxf_az_upload \$name '${AzHelper.toHttpUrl(target)}' || true + nxf_az_upload \$name '${AzHelper.toHttpUrl(target)}' done unset IFS """.stripIndent().rightTrim() diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/lifesciences/GoogleLifeSciencesHelper.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/lifesciences/GoogleLifeSciencesHelper.groovy index be08d069e3..7352770b32 100644 --- a/plugins/nf-google/src/main/nextflow/cloud/google/lifesciences/GoogleLifeSciencesHelper.groovy +++ b/plugins/nf-google/src/main/nextflow/cloud/google/lifesciences/GoogleLifeSciencesHelper.groovy @@ -365,7 +365,7 @@ class GoogleLifeSciencesHelper { final remoteTaskDir = getRemoteTaskDir(workDir) def result = 'set -x; ' result += "trap 'err=\$?; exec 1>&2; gsutil -m -q cp -R $localTaskDir/${TaskRun.CMD_LOG} ${remoteTaskDir}/${TaskRun.CMD_LOG} || true; [[ \$err -gt 0 || \$GOOGLE_LAST_EXIT_STATUS -gt 0 || \$NXF_DEBUG -gt 0 ]] && { ls -lah $localTaskDir || true; gsutil -m -q cp -R /google/ ${remoteTaskDir}; } || rm -rf $localTaskDir; exit \$err' EXIT; " - result += "{ cd $localTaskDir; bash ${TaskRun.CMD_RUN} nxf_unstage; } >> $localTaskDir/${TaskRun.CMD_LOG} 2>&1" + result += "{ cd $localTaskDir; bash ${TaskRun.CMD_RUN} nxf_unstage;} >> $localTaskDir/${TaskRun.CMD_LOG} 2>&1" return result } diff --git a/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesHelperTest.groovy b/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesHelperTest.groovy index 35cda62f0b..9db824a902 100644 --- a/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesHelperTest.groovy +++ b/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesHelperTest.groovy @@ -548,7 +548,7 @@ class GoogleLifeSciencesHelperTest extends GoogleSpecification { def unstage = helper.getUnstagingScript(dir) then: unstage == - 'set -x; trap \'err=$?; exec 1>&2; gsutil -m -q cp -R /work/dir/.command.log gs://my-bucket/work/dir/.command.log || true; [[ $err -gt 0 || $GOOGLE_LAST_EXIT_STATUS -gt 0 || $NXF_DEBUG -gt 0 ]] && { ls -lah /work/dir || true; gsutil -m -q cp -R /google/ gs://my-bucket/work/dir; } || rm -rf /work/dir; exit $err\' EXIT; { cd /work/dir; bash .command.run nxf_unstage; } >> /work/dir/.command.log 2>&1' + 'set -x; trap \'err=$?; exec 1>&2; gsutil -m -q cp -R /work/dir/.command.log gs://my-bucket/work/dir/.command.log || true; [[ $err -gt 0 || $GOOGLE_LAST_EXIT_STATUS -gt 0 || $NXF_DEBUG -gt 0 ]] && { ls -lah /work/dir || true; gsutil -m -q cp -R /google/ gs://my-bucket/work/dir; } || rm -rf /work/dir; exit $err\' EXIT; { cd /work/dir; bash .command.run nxf_unstage;} >> /work/dir/.command.log 2>&1' } @Unroll diff --git a/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/bash-wrapper-gcp.txt b/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/bash-wrapper-gcp.txt index 70a68452aa..c7382062a1 100644 --- a/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/bash-wrapper-gcp.txt +++ b/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/bash-wrapper-gcp.txt @@ -168,7 +168,10 @@ nxf_fs_fcp() { } on_exit() { - exit_status=${nxf_main_ret:=$?} + local last_err=$? + local exit_status=${nxf_main_ret:=0} + [[ ${exit_status} -eq 0 && ${nxf_unstage_ret:=0} -ne 0 ]] && exit_status=${nxf_unstage_ret:=0} + [[ ${exit_status} -eq 0 && ${last_err} -ne 0 ]] && exit_status=${last_err} printf -- $exit_status > {{folder}}/.exitcode set +u exit $exit_status @@ -192,12 +195,23 @@ nxf_stage() { nxf_parallel "${downloads[@]}" } -nxf_unstage() { +nxf_unstage_outputs() { + true +} + +nxf_unstage_controls() { true gsutil -m -q cp -R .command.out gs://bucket/work/dir/.command.out || true gsutil -m -q cp -R .command.err gs://bucket/work/dir/.command.err || true gsutil -m -q cp -R .exitcode gs://bucket/work/dir/.exitcode || true - [[ ${nxf_main_ret:=0} != 0 ]] && return +} + +nxf_unstage() { + if [[ ${nxf_main_ret:=0} == 0 ]]; then + (set -e -o pipefail; (nxf_unstage_outputs | tee -a .command.out) 3>&1 1>&2 2>&3 | tee -a .command.err) + nxf_unstage_ret=$? + fi + nxf_unstage_controls } nxf_main() { diff --git a/validation/awsbatch-unstage-fail.config b/validation/awsbatch-unstage-fail.config new file mode 100644 index 0000000000..81b96579d7 --- /dev/null +++ b/validation/awsbatch-unstage-fail.config @@ -0,0 +1,12 @@ +/* + * do not include plugin requirements otherwise latest + * published version will be downloaded instead of using local build + */ + +workDir = 's3://nextflow-ci/work' +process.executor = 'awsbatch' +process.queue = 'nextflow-ci' +process.container = 'quay.io/nextflow/test-aws-unstage-fail:1.0' +aws.region = 'eu-west-1' +aws.batch.maxTransferAttempts = 3 +aws.batch.delayBetweenAttempts = '5 sec' diff --git a/validation/awsbatch.sh b/validation/awsbatch.sh index d58727e7e8..b73571cbd6 100644 --- a/validation/awsbatch.sh +++ b/validation/awsbatch.sh @@ -7,6 +7,13 @@ get_abs_filename() { export NXF_CMD=${NXF_CMD:-$(get_abs_filename ../launch.sh)} +# Execution should fail ignoring +$NXF_CMD run test-aws-unstage-fail.nf -c awsbatch-unstage-fail.config || true +[[ `grep -c "Error executing process > 'test (1)'" .nextflow.log` == 1 ]] || false +[[ `grep -c " Essential container in task exited" .nextflow.log` == 1 ]] || false +[[ `grep -cozP "Command exit status:\n 1" .nextflow.log` == 1 ]] || false +[[ `grep -c "Producing a failure in aws" .nextflow.log` == 2 ]] || false + $NXF_CMD run test-complexpaths.nf -c awsbatch.config [[ -d foo ]] || false [[ -e 'foo/.alpha' ]] || false @@ -73,4 +80,4 @@ $NXF_CMD run nextflow-io/hello \ -process.array 10 \ -with-wave \ -with-fusion \ - -c awsbatch.config \ No newline at end of file + -c awsbatch.config diff --git a/validation/test-aws-unstage-fail-container/Dockerfile b/validation/test-aws-unstage-fail-container/Dockerfile new file mode 100644 index 0000000000..0dd281ba58 --- /dev/null +++ b/validation/test-aws-unstage-fail-container/Dockerfile @@ -0,0 +1,11 @@ +FROM ubuntu + +RUN apt-get update && apt-get -y install curl unzip && apt-get clean + + +RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \ + unzip awscliv2.zip && ./aws/install && rm -rf aws* + +ADD fake_aws /fake_aws + +ENV PATH=/fake_aws/bin/:$PATH diff --git a/validation/test-aws-unstage-fail-container/fake_aws/bin/aws b/validation/test-aws-unstage-fail-container/fake_aws/bin/aws new file mode 100755 index 0000000000..80985d9e08 --- /dev/null +++ b/validation/test-aws-unstage-fail-container/fake_aws/bin/aws @@ -0,0 +1,9 @@ +#!/bin/bash + +if [[ "$*" == *".command."* ]] || [[ "$*" == *".exitcode"* ]]; then + /usr/local/bin/aws $@ +else + >&2 echo "Producing a failure in aws $@" + exit 2 +fi + diff --git a/validation/test-aws-unstage-fail.nf b/validation/test-aws-unstage-fail.nf new file mode 100644 index 0000000000..96bcb9af1e --- /dev/null +++ b/validation/test-aws-unstage-fail.nf @@ -0,0 +1,16 @@ +process test { + input: + val i + output: + file("test${i}") + file("test_2_${i}") + script: + """ + dd if=/dev/urandom of=test${i} bs=1K count=90 + dd if=/dev/urandom of=test_2_${i} bs=1K count=90 + """ +} + +workflow { + Channel.of(1) | test +} From 2b653b0742cd998b23f6dd85daba5cc3c7150b55 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 3 Dec 2024 07:14:09 +0000 Subject: [PATCH 04/18] Bump nf-amazon@2.10.0 Signed-off-by: Paolo Di Tommaso --- plugins/nf-amazon/changelog.txt | 8 ++++++++ plugins/nf-amazon/src/resources/META-INF/MANIFEST.MF | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/plugins/nf-amazon/changelog.txt b/plugins/nf-amazon/changelog.txt index 387d69476d..d9fc1213e9 100644 --- a/plugins/nf-amazon/changelog.txt +++ b/plugins/nf-amazon/changelog.txt @@ -1,5 +1,13 @@ nf-amazon changelog =================== +2.10.0 - 3 Dec 2024 +- Detecting errors in data unstaging (#5345) [3c8e602d] +- Prevent NPE with null AWS Batch response [12fc1d60] +- Fix Fargate warning on memory check (#5475) [bdf0ad00] +- Bump groovy 4.0.24 [dd71ad31] +- Bump aws sdk 1.12.777 (#5458) [8bad0b4b] +- Bump netty-common to version 4.1.115.Final [d1bbd3d0] + 2.9.0 - 2 Oct 2024 - Add Platform workflow prefix in AWS Batch job names (#5318) [e2e test] [42dd4ba8] - Fix AWS spot attempts with zero value (#5331) [ci fast] [bac2da12] diff --git a/plugins/nf-amazon/src/resources/META-INF/MANIFEST.MF b/plugins/nf-amazon/src/resources/META-INF/MANIFEST.MF index 41ac624f51..7772ceb96b 100644 --- a/plugins/nf-amazon/src/resources/META-INF/MANIFEST.MF +++ b/plugins/nf-amazon/src/resources/META-INF/MANIFEST.MF @@ -1,6 +1,6 @@ Manifest-Version: 1.0 Plugin-Class: nextflow.cloud.aws.AmazonPlugin Plugin-Id: nf-amazon -Plugin-Version: 2.9.0 +Plugin-Version: 2.10.0 Plugin-Provider: Seqera Labs Plugin-Requires: >=24.04.4 From 6af7198db82b36dc92eb4ed57f4a9428e6c65bcf Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 3 Dec 2024 07:15:13 +0000 Subject: [PATCH 05/18] Bump nf-azure@1.11.0 Signed-off-by: Paolo Di Tommaso --- plugins/nf-azure/changelog.txt | 7 +++++++ plugins/nf-azure/src/resources/META-INF/MANIFEST.MF | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/plugins/nf-azure/changelog.txt b/plugins/nf-azure/changelog.txt index 91afbac22d..780fda1082 100644 --- a/plugins/nf-azure/changelog.txt +++ b/plugins/nf-azure/changelog.txt @@ -1,5 +1,12 @@ nf-azure changelog =================== +1.11.0 - 3 Dec 2024 +- Detecting errors in data unstaging (#5345) [3c8e602d] +- Bump netty-common to version 4.1.115.Final [d1bbd3d0] +- Bump groovy 4.0.24 [dd71ad31] +- Bump com.azure:azure-identity from 1.11.3 to 1.12.2 (#5449) [cb70f1df] +- Target Java 17 as minimal Java version (#5045) [0140f954] + 1.10.1 - 27 Oct 2024 - Demote azure batch task status log level to trace (#5416) [ci skip] [d6c684bb] diff --git a/plugins/nf-azure/src/resources/META-INF/MANIFEST.MF b/plugins/nf-azure/src/resources/META-INF/MANIFEST.MF index 2918b09d26..1ebcbf274f 100644 --- a/plugins/nf-azure/src/resources/META-INF/MANIFEST.MF +++ b/plugins/nf-azure/src/resources/META-INF/MANIFEST.MF @@ -1,6 +1,6 @@ Manifest-Version: 1.0 Plugin-Class: nextflow.cloud.azure.AzurePlugin Plugin-Id: nf-azure -Plugin-Version: 1.10.1 +Plugin-Version: 1.11.0 Plugin-Provider: Seqera Labs Plugin-Requires: >=24.04.4 From 9494f970a5fa80e885a6323e64663d3bdaedd2f3 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 3 Dec 2024 07:16:50 +0000 Subject: [PATCH 06/18] Bump nf-google@1.16.0 Signed-off-by: Paolo Di Tommaso --- plugins/nf-google/changelog.txt | 7 +++++++ plugins/nf-google/src/resources/META-INF/MANIFEST.MF | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/plugins/nf-google/changelog.txt b/plugins/nf-google/changelog.txt index 9d4cade5e7..f7d23811e6 100644 --- a/plugins/nf-google/changelog.txt +++ b/plugins/nf-google/changelog.txt @@ -1,5 +1,12 @@ nf-google changelog =================== +1.16.0 - 3 Dec 2024 +- Detecting errors in data unstaging (#5345) [3c8e602d] +- Bump bouncycastle to jdk18on:1.78.1 (#5467) [cd8c385f] +- Bump groovy 4.0.24 [dd71ad31] +- Bump protobuf-java:3.25.5 to nf-google [488b7906] +- Add NotFoundException to retry condition for Google Batch [aa4d19cc] + 1.15.2 - 14 Oct 2024 - Add Google LS deprecation notice (#5400) [0ee1d9bc] diff --git a/plugins/nf-google/src/resources/META-INF/MANIFEST.MF b/plugins/nf-google/src/resources/META-INF/MANIFEST.MF index 65849a3ba7..fb9f7deae5 100644 --- a/plugins/nf-google/src/resources/META-INF/MANIFEST.MF +++ b/plugins/nf-google/src/resources/META-INF/MANIFEST.MF @@ -1,6 +1,6 @@ Manifest-Version: 1.0 Plugin-Class: nextflow.cloud.google.GoogleCloudPlugin Plugin-Id: nf-google -Plugin-Version: 1.15.2 +Plugin-Version: 1.16.0 Plugin-Provider: Seqera Labs Plugin-Requires: >=24.04.4 From 7e2c8d82b4a757bb164e7b8c7bdc10cdde2c2ee4 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 3 Dec 2024 07:17:20 +0000 Subject: [PATCH 07/18] Bump nf-google@1.8.0 [ci fast] Signed-off-by: Paolo Di Tommaso --- plugins/nf-wave/changelog.txt | 6 ++++++ plugins/nf-wave/src/resources/META-INF/MANIFEST.MF | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/plugins/nf-wave/changelog.txt b/plugins/nf-wave/changelog.txt index 6c9c6abf0f..1a13763b31 100644 --- a/plugins/nf-wave/changelog.txt +++ b/plugins/nf-wave/changelog.txt @@ -1,5 +1,11 @@ nf-wave changelog ================== +1.8.0 - 3 Dec 2024 +- Fix missing wave response (#5547) [ci fast] [ee252173] +- Update wave deps [09ccd295] +- Fix isContainerReady when wave is disabled (#5509) [ci fast] [3215afa8] +- Bump groovy 4.0.24 [dd71ad31] + 1.7.2 - 27 Oct 2024 - Add wave mirror vs module bundles conflicts warning [b37a8a5b] diff --git a/plugins/nf-wave/src/resources/META-INF/MANIFEST.MF b/plugins/nf-wave/src/resources/META-INF/MANIFEST.MF index 1eb76aa0db..bb42c60b97 100644 --- a/plugins/nf-wave/src/resources/META-INF/MANIFEST.MF +++ b/plugins/nf-wave/src/resources/META-INF/MANIFEST.MF @@ -1,6 +1,6 @@ Manifest-Version: 1.0 Plugin-Class: io.seqera.wave.plugin.WavePlugin Plugin-Id: nf-wave -Plugin-Version: 1.7.2 +Plugin-Version: 1.8.0 Plugin-Provider: Seqera Labs Plugin-Requires: >=24.04.4 From 6e231be115c31f4e29e6deab45f668ae7e75917d Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 3 Dec 2024 09:37:57 +0000 Subject: [PATCH 08/18] [release 24.11.0-edge] Update timestamp and build number Signed-off-by: Paolo Di Tommaso --- VERSION | 2 +- .../src/main/resources/META-INF/build-info.properties | 8 ++++---- .../nextflow/src/main/resources/META-INF/plugins-info.txt | 8 ++++---- nextflow | 2 +- nextflow.md5 | 2 +- nextflow.sha1 | 2 +- nextflow.sha256 | 2 +- 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/VERSION b/VERSION index 21651351e2..2b70d664e4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -24.10.0 +24.11.0-edge diff --git a/modules/nextflow/src/main/resources/META-INF/build-info.properties b/modules/nextflow/src/main/resources/META-INF/build-info.properties index d6072256a9..2053248f03 100644 --- a/modules/nextflow/src/main/resources/META-INF/build-info.properties +++ b/modules/nextflow/src/main/resources/META-INF/build-info.properties @@ -1,4 +1,4 @@ -build=5928 -version=24.10.0 -timestamp=1730054192154 -commitId=6524d8dc9 +build=5929 +version=24.11.0-edge +timestamp=1733218258400 +commitId=7e2c8d82b diff --git a/modules/nextflow/src/main/resources/META-INF/plugins-info.txt b/modules/nextflow/src/main/resources/META-INF/plugins-info.txt index 39d506b59d..c650c71066 100644 --- a/modules/nextflow/src/main/resources/META-INF/plugins-info.txt +++ b/modules/nextflow/src/main/resources/META-INF/plugins-info.txt @@ -1,8 +1,8 @@ -nf-amazon@2.9.0 -nf-azure@1.10.1 +nf-amazon@2.10.0 +nf-azure@1.11.0 nf-cloudcache@0.4.2 nf-codecommit@0.2.2 nf-console@1.1.4 -nf-google@1.15.2 +nf-google@1.16.0 nf-tower@1.9.3 -nf-wave@1.7.2 \ No newline at end of file +nf-wave@1.8.0 \ No newline at end of file diff --git a/nextflow b/nextflow index 5dbb589bdf..c89ca7617d 100755 --- a/nextflow +++ b/nextflow @@ -15,7 +15,7 @@ # limitations under the License. [[ "$NXF_DEBUG" == 'x' ]] && set -x -NXF_VER=${NXF_VER:-'24.10.0'} +NXF_VER=${NXF_VER:-'24.11.0-edge'} NXF_ORG=${NXF_ORG:-'nextflow-io'} NXF_HOME=${NXF_HOME:-$HOME/.nextflow} NXF_PROT=${NXF_PROT:-'https'} diff --git a/nextflow.md5 b/nextflow.md5 index c51c7ce709..deb7f7a40a 100644 --- a/nextflow.md5 +++ b/nextflow.md5 @@ -1 +1 @@ -7dfd8066370310bff610aa209c988b3e +66995c4139ebcd17bf99f17d9dd030d1 diff --git a/nextflow.sha1 b/nextflow.sha1 index 80cc138bfd..ccadbdc25f 100644 --- a/nextflow.sha1 +++ b/nextflow.sha1 @@ -1 +1 @@ -c142828a82fa6678a5af978d3545bb7f56072be6 +cdbb67bdb21c0e63fb48aabb8b168c12a31fa5b3 diff --git a/nextflow.sha256 b/nextflow.sha256 index e9a09bcc10..f4e9ccb3bd 100644 --- a/nextflow.sha256 +++ b/nextflow.sha256 @@ -1 +1 @@ -e848918fb9b85762822c078435d9ff71979a88cccff81ce5babd75d5eee52fe6 +69a86852c52dcfa7662407c46d16f05bd3dec16e0c505c2a2f71ccc56219d631 From 51aea8f389adbaff8e820d226337c47587ad3c57 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 3 Dec 2024 10:46:44 +0000 Subject: [PATCH 09/18] Update changelog [ci skip] Signed-off-by: Paolo Di Tommaso --- changelog.txt | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/changelog.txt b/changelog.txt index 0be1d17adf..54fba39ad4 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,5 +1,46 @@ NEXTFLOW CHANGE-LOG =================== +24.11.0-edge - 3 Dec 2024 +- Add GHA to submit dependencies to dependabot (#5440) [80395a6d] +- Add NotFoundException to retry condition for Google Batch [aa4d19cc] +- Add Rahel Hirsch to run name generator (#5442) [ff2bc6ae] +- Add `env()` function (#5506) [fa0e8e0f] +- Add more scientists to run name generator (#5447) [38d9eda0] +- Add `singularity.libraryDir` to config page (#5498) [b5e31bb0] +- Add RepositoryProvider.revision now public property (#5500) [f0a4c526] +- Deprecate process `shell` block (#5508) [6f527551] +- Detecting errors in data unstaging (#5345) [3c8e602d] +- Disable virtual threads on CI tests [ci slip] [69d07dbc] +- Fix Fargate warning on memory check (#5475) [bdf0ad00] +- Fix `isContainerReady` when wave is disabled (#5509) [3215afa8] +- Fix missing wave response (#5547) [ee252173] +- Fix overlapping conda lock file (#5540) [9248c04d] +- Fix overlapping conda lock exception (#5489) [eaaeb3de] +- Fix possible deadlock in dynamic `maxRetry` resolution (#5474) [25bbb621] +- Fix Wave GCP integration test (#5490) [ad56c89b] +- Fixing bug when execution with stub and no stub defined (#5473) [f7fd56db] +- Fix Incorrect CPU value in Azure example (#5549) [fc5e2c2a] +- Improve docs for using the GPU accelerator directive (#5488) [4b908524] +- Improve groupTuple docs with scatter/gather example (#5520) [b5c63a9f] +- Prevent NPE with null AWS Batch response [12fc1d60] +- Target Java 17 as minimal Java version (#5045) [0140f954] +- Update 'nexus-staging' plugin to latest version (#5462) [07934513] +- Update gradle 'shadow' plugin version to 8.3.5 (#5463) [2a5f15f0] +- Update install docs to reflect change from 'all' to 'dist' (#5496) [c9115659] +- Update process snippets to comply with strict syntax (#5526) [be1694bf] +- Update Wave dependencies [09ccd295] +- Bump aws sdk 1.12.777 (#5458) [8bad0b4b] +- Bump bouncycastle to jdk18on:1.78.1 (#5467) [cd8c385f] +- Bump com.azure:azure-identity from 1.11.3 to 1.12.2 (#5449) [cb70f1df] +- Bump commons-io:2.15.1 [767e4a0a] +- Bump groovy 4.0.24 [dd71ad31] +- Bump netty-common to version 4.1.115.Final [d1bbd3d0] +- Bump nf-amazon@2.10.0 [2b653b07] +- Bump nf-azure@1.11.0 [6af7198d] +- Bump nf-google@1.16.0 [9494f970] +- Bump nf-google@1.8.0 [7e2c8d82] +- Bump protobuf-java:3.25.5 to nf-google [488b7906] + 24.10.2 - 27 Nov 2024 - Prevent NPE with null AWS Batch response [3d491934] - Fix overlapping conda lock file (#5540) [df66deaa] From 12be47efa0cce264e0a8d82eb4f4bdcdb837362a Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 4 Dec 2024 11:11:06 +0100 Subject: [PATCH 10/18] Improve Fargate + Wave docs (#5562) [ci skip] Signed-off-by: Phil Ewels Signed-off-by: Paolo Di Tommaso Co-authored-by: Paolo Di Tommaso Co-authored-by: Christopher Hakkaart --- docs/aws.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/aws.md b/docs/aws.md index ba518ad7ae..84ab5c448f 100644 --- a/docs/aws.md +++ b/docs/aws.md @@ -539,7 +539,8 @@ See the AWS documentation for details how to create the required AWS Batch queue and the Batch Execution Role. :::{note} -This feature requires the use {ref}`Wave ` container provisioning service. +Nextflow uses [s5cmd](https://github.com/peak/s5cmd) to download the task input data and upload the task outputs. +To enable this capability, you need to enable the Wave service in the Nextflow configuration, as shown in the above example. See {ref}Wave documentation for more details. ::: ## Advanced configuration From 54713a16a006161f453f3215440253e57b08064d Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 4 Dec 2024 11:12:52 +0100 Subject: [PATCH 11/18] Customise mermaid diagram theme in DAG template (#5553) [ci skip] Signed-off-by: Phil Ewels --- docs/_static/dag.mmd | 14 ++++++++++++++ .../nextflow/dag/mermaid.dag.template.html | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/docs/_static/dag.mmd b/docs/_static/dag.mmd index 0963e2c106..989a536ae9 100644 --- a/docs/_static/dag.mmd +++ b/docs/_static/dag.mmd @@ -1,3 +1,17 @@ +%%{ + init: { + 'theme': 'base', + 'themeVariables': { + 'primaryColor': '#B6ECE2', + 'primaryTextColor': '#160F26', + 'primaryBorderColor': '#065647', + 'lineColor': '#545555', + 'clusterBkg': '#BABCBD22', + 'clusterBorder': '#DDDEDE', + 'fontFamily': 'arial' + } + } +}%% flowchart TB subgraph " " v0["Channel.fromFilePairs"] diff --git a/modules/nextflow/src/main/resources/nextflow/dag/mermaid.dag.template.html b/modules/nextflow/src/main/resources/nextflow/dag/mermaid.dag.template.html index 47062c1218..0ab1d9475e 100644 --- a/modules/nextflow/src/main/resources/nextflow/dag/mermaid.dag.template.html +++ b/modules/nextflow/src/main/resources/nextflow/dag/mermaid.dag.template.html @@ -19,6 +19,20 @@ 
+%%{
+  init: {
+    'theme': 'base',
+    'themeVariables': {
+      'primaryColor': '#B6ECE2',
+      'primaryTextColor': '#160F26',
+      'primaryBorderColor': '#065647',
+      'lineColor': '#545555',
+      'clusterBkg': '#BABCBD22',
+      'clusterBorder': '#DDDEDE',
+      'fontFamily': 'arial'
+    }
+  }
+}%%
 REPLACE_WITH_NETWORK_DATA