From 40e0b55a528bbc5fd1d2bd6e308d46b1ff13c5fe Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Thu, 9 Jan 2025 08:17:03 -0600 Subject: [PATCH 1/5] Allow COPY_ATTRIBUTES on S3 copy Signed-off-by: Ben Sherman --- .../src/main/nextflow/cloud/aws/nio/S3FileSystemProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3FileSystemProvider.java b/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3FileSystemProvider.java index c7c4f74164..d5810b1c81 100644 --- a/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3FileSystemProvider.java +++ b/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3FileSystemProvider.java @@ -535,7 +535,7 @@ public void copy(Path source, Path target, CopyOption... options) * "copying directories is not yet supported: %s", target); // TODO */ ImmutableSet actualOptions = ImmutableSet.copyOf(options); - verifySupportedOptions(EnumSet.of(StandardCopyOption.REPLACE_EXISTING), + verifySupportedOptions(EnumSet.of(StandardCopyOption.COPY_ATTRIBUTES, StandardCopyOption.REPLACE_EXISTING), actualOptions); if (!actualOptions.contains(StandardCopyOption.REPLACE_EXISTING)) { From d484182f6f136c571ce59ac35f9368b8d381e2b7 Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Thu, 9 Jan 2025 08:35:11 -0600 Subject: [PATCH 2/5] Add changes from #4522 Signed-off-by: Ben Sherman --- .../src/main/groovy/nextflow/processor/PublishDir.groovy | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy index 4ca5764519..97aec045a8 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy @@ -26,6 +26,7 @@ import java.nio.file.LinkOption import java.nio.file.NoSuchFileException import java.nio.file.Path import java.nio.file.PathMatcher +import java.nio.file.StandardCopyOption import java.time.temporal.ChronoUnit import java.util.concurrent.ExecutorService @@ -509,13 +510,13 @@ class PublishDir { FilesEx.mklink(source, [hard:true], destination) } else if( mode == Mode.MOVE ) { - FileHelper.movePath(source, destination) + FileHelper.movePath(source, destination, StandardCopyOption.COPY_ATTRIBUTES) } else if( mode == Mode.COPY ) { - FileHelper.copyPath(source, destination) + FileHelper.copyPath(source, destination, StandardCopyOption.COPY_ATTRIBUTES) } else if( mode == Mode.COPY_NO_FOLLOW ) { - FileHelper.copyPath(source, destination, LinkOption.NOFOLLOW_LINKS) + FileHelper.copyPath(source, destination, StandardCopyOption.COPY_ATTRIBUTES, LinkOption.NOFOLLOW_LINKS) } else { throw new IllegalArgumentException("Unknown file publish mode: ${mode}") From 11058433b2333a98cb0079ded84304654c824be2 Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Tue, 21 Jan 2025 11:45:31 -0600 Subject: [PATCH 3/5] Add config option to copy file attributes when publishing Signed-off-by: Ben Sherman --- docs/reference/config.md | 5 +++++ .../groovy/nextflow/processor/PublishDir.groovy | 13 ++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/docs/reference/config.md b/docs/reference/config.md index 0efb1ce6ae..3466e0d5af 100644 --- a/docs/reference/config.md +++ b/docs/reference/config.md @@ -1651,6 +1651,11 @@ The `workflow` scope provides workflow execution options. : *Currently only supported for S3.* : Specify the media type, also known as [MIME type](https://developer.mozilla.org/en-US/docs/Web/HTTP/MIME_types), of published files (default: `false`). Can be a string (e.g. `'text/html'`), or `true` to infer the content type from the file extension. +`workflow.output.copyAttributes` +: :::{versionadded} 25.01.0-edge + ::: +: Copy file attributes (such as the last modified timestamp) to the published file (default: `false`). + `workflow.output.enabled` : Enable or disable publishing (default: `true`). diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy index 97aec045a8..efd9c90dbe 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy @@ -18,6 +18,7 @@ package nextflow.processor import static nextflow.util.CacheHelper.* +import java.nio.file.CopyOption import java.nio.file.FileAlreadyExistsException import java.nio.file.FileSystem import java.nio.file.FileSystems @@ -499,6 +500,11 @@ class PublishDir { protected void processFileImpl( Path source, Path destination ) { log.trace "publishing file: $source -[$mode]-> $destination" + final options = new ArrayList<>(2) + final copyAttributes = session.config.navigate('workflow.output.copyAttributes') as Boolean + if( copyAttributes ) + options.add(StandardCopyOption.COPY_ATTRIBUTES) + if( !mode || mode == Mode.SYMLINK ) { Files.createSymbolicLink(destination, source) } @@ -510,13 +516,14 @@ class PublishDir { FilesEx.mklink(source, [hard:true], destination) } else if( mode == Mode.MOVE ) { - FileHelper.movePath(source, destination, StandardCopyOption.COPY_ATTRIBUTES) + FileHelper.movePath(source, destination, options as CopyOption[]) } else if( mode == Mode.COPY ) { - FileHelper.copyPath(source, destination, StandardCopyOption.COPY_ATTRIBUTES) + FileHelper.copyPath(source, destination, options as CopyOption[]) } else if( mode == Mode.COPY_NO_FOLLOW ) { - FileHelper.copyPath(source, destination, StandardCopyOption.COPY_ATTRIBUTES, LinkOption.NOFOLLOW_LINKS) + options.add(LinkOption.NOFOLLOW_LINKS) + FileHelper.copyPath(source, destination, options as CopyOption[]) } else { throw new IllegalArgumentException("Unknown file publish mode: ${mode}") From 49dca8bd7d2d432a229d857a77cead527da1644d Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Tue, 21 Jan 2025 15:21:10 -0600 Subject: [PATCH 4/5] Apply suggestions for review Signed-off-by: Ben Sherman --- docs/reference/config.md | 1 + .../src/main/groovy/nextflow/processor/PublishDir.groovy | 2 +- .../src/main/nextflow/cloud/aws/nio/S3FileSystemProvider.java | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/reference/config.md b/docs/reference/config.md index 3466e0d5af..cc5045345a 100644 --- a/docs/reference/config.md +++ b/docs/reference/config.md @@ -1654,6 +1654,7 @@ The `workflow` scope provides workflow execution options. `workflow.output.copyAttributes` : :::{versionadded} 25.01.0-edge ::: +: *Currently only supported for local and shared filesystems.* : Copy file attributes (such as the last modified timestamp) to the published file (default: `false`). `workflow.output.enabled` diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy index efd9c90dbe..e20825c74d 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy @@ -501,7 +501,7 @@ class PublishDir { log.trace "publishing file: $source -[$mode]-> $destination" final options = new ArrayList<>(2) - final copyAttributes = session.config.navigate('workflow.output.copyAttributes') as Boolean + final copyAttributes = session.config.navigate('workflow.output.copyAttributes', false) if( copyAttributes ) options.add(StandardCopyOption.COPY_ATTRIBUTES) diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3FileSystemProvider.java b/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3FileSystemProvider.java index d5810b1c81..c7c4f74164 100644 --- a/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3FileSystemProvider.java +++ b/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3FileSystemProvider.java @@ -535,7 +535,7 @@ public void copy(Path source, Path target, CopyOption... options) * "copying directories is not yet supported: %s", target); // TODO */ ImmutableSet actualOptions = ImmutableSet.copyOf(options); - verifySupportedOptions(EnumSet.of(StandardCopyOption.COPY_ATTRIBUTES, StandardCopyOption.REPLACE_EXISTING), + verifySupportedOptions(EnumSet.of(StandardCopyOption.REPLACE_EXISTING), actualOptions); if (!actualOptions.contains(StandardCopyOption.REPLACE_EXISTING)) { From 5ffbcdb04a0303e951121d2482ee034100ee8699 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Fri, 24 Jan 2025 10:45:39 +0100 Subject: [PATCH 5/5] Simplified + tests Signed-off-by: Paolo Di Tommaso --- .../nextflow/processor/PublishDir.groovy | 21 +++++++++++-------- .../nextflow/processor/PublishDirTest.groovy | 21 +++++++++++++++++++ 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy index e20825c74d..6d0335f9be 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy @@ -38,6 +38,7 @@ import dev.failsafe.event.ExecutionAttemptedEvent import groovy.transform.CompileDynamic import groovy.transform.CompileStatic import groovy.transform.EqualsAndHashCode +import groovy.transform.Memoized import groovy.transform.PackageScope import groovy.transform.ToString import groovy.util.logging.Slf4j @@ -500,11 +501,6 @@ class PublishDir { protected void processFileImpl( Path source, Path destination ) { log.trace "publishing file: $source -[$mode]-> $destination" - final options = new ArrayList<>(2) - final copyAttributes = session.config.navigate('workflow.output.copyAttributes', false) - if( copyAttributes ) - options.add(StandardCopyOption.COPY_ATTRIBUTES) - if( !mode || mode == Mode.SYMLINK ) { Files.createSymbolicLink(destination, source) } @@ -516,20 +512,27 @@ class PublishDir { FilesEx.mklink(source, [hard:true], destination) } else if( mode == Mode.MOVE ) { - FileHelper.movePath(source, destination, options as CopyOption[]) + FileHelper.movePath(source, destination, copyOpts()) } else if( mode == Mode.COPY ) { - FileHelper.copyPath(source, destination, options as CopyOption[]) + FileHelper.copyPath(source, destination, copyOpts()) } else if( mode == Mode.COPY_NO_FOLLOW ) { - options.add(LinkOption.NOFOLLOW_LINKS) - FileHelper.copyPath(source, destination, options as CopyOption[]) + FileHelper.copyPath(source, destination, copyOpts(LinkOption.NOFOLLOW_LINKS)) } else { throw new IllegalArgumentException("Unknown file publish mode: ${mode}") } } + @Memoized + protected CopyOption[] copyOpts(CopyOption... opts) { + final copyAttributes = session.config.navigate('workflow.output.copyAttributes', false) + return copyAttributes + ? opts + StandardCopyOption.COPY_ATTRIBUTES + : opts + } + protected void createPublishDir() { try { makeDirs(path) diff --git a/modules/nextflow/src/test/groovy/nextflow/processor/PublishDirTest.groovy b/modules/nextflow/src/test/groovy/nextflow/processor/PublishDirTest.groovy index 4147a65120..1dac96d29d 100644 --- a/modules/nextflow/src/test/groovy/nextflow/processor/PublishDirTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/processor/PublishDirTest.groovy @@ -16,9 +16,12 @@ package nextflow.processor +import java.nio.file.CopyOption import java.nio.file.FileSystems import java.nio.file.Files +import java.nio.file.LinkOption import java.nio.file.Paths +import java.nio.file.StandardCopyOption import nextflow.Global import nextflow.Session @@ -417,4 +420,22 @@ class PublishDirTest extends Specification { [NXF_PUBLISH_FAIL_ON_ERROR: 'true'] | true [NXF_PUBLISH_FAIL_ON_ERROR: 'false'] | false } + + def 'should return copy attributes' () { + expect: + new PublishDir().copyOpts() == [] as CopyOption[] + and: + new PublishDir().copyOpts(LinkOption.NOFOLLOW_LINKS) == [LinkOption.NOFOLLOW_LINKS] as CopyOption[] + + when: + Global.session = Mock(Session) { getConfig()>>[workflow:[output:[copyAttributes: true]]] } + then: + new PublishDir().copyOpts() == [StandardCopyOption.COPY_ATTRIBUTES] as CopyOption[] + and: + new PublishDir().copyOpts(LinkOption.NOFOLLOW_LINKS) == [LinkOption.NOFOLLOW_LINKS,StandardCopyOption.COPY_ATTRIBUTES] as CopyOption[] + + cleanup: + Global.session = null + } + }