From 48cea185ac76de8028d2db34a91ba2f0705622de Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Fri, 4 Oct 2024 12:05:21 +0200 Subject: [PATCH 01/43] prepare for mypy 1.12 --- cwltool/context.py | 3 +- mypy-stubs/black/__init__.pyi | 26 ----- mypy-stubs/mistune.pyi | 197 ---------------------------------- setup.py | 2 +- 4 files changed, 3 insertions(+), 225 deletions(-) delete mode 100644 mypy-stubs/black/__init__.pyi delete mode 100644 mypy-stubs/mistune.pyi diff --git a/cwltool/context.py b/cwltool/context.py index 1e82ecc4a..283936d61 100644 --- a/cwltool/context.py +++ b/cwltool/context.py @@ -31,6 +31,7 @@ from .utils import DEFAULT_TMP_PREFIX, CWLObjectType, HasReqsHints, ResolverType if TYPE_CHECKING: + from _typeshed import SupportsWrite from cwl_utils.parser.cwl_v1_2 import LoadingOptions from .builder import Builder @@ -199,7 +200,7 @@ def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None: self.default_stdout: Optional[Union[IO[bytes], TextIO]] = None self.default_stderr: Optional[Union[IO[bytes], TextIO]] = None self.validate_only: bool = False - self.validate_stdout: Optional[Union[IO[bytes], TextIO, IO[str]]] = None + self.validate_stdout: Optional["SupportsWrite[str]"] = None super().__init__(kwargs) if self.tmp_outdir_prefix == "": self.tmp_outdir_prefix = self.tmpdir_prefix diff --git a/mypy-stubs/black/__init__.pyi b/mypy-stubs/black/__init__.pyi deleted file mode 100644 index f741ef771..000000000 --- a/mypy-stubs/black/__init__.pyi +++ /dev/null @@ -1,26 +0,0 @@ -import asyncio -from concurrent.futures import Executor -from enum import Enum -from pathlib import Path -from typing import ( - Any, - Iterator, - List, - MutableMapping, - Optional, - Pattern, - Set, - Sized, - Tuple, - Union, -) - -from black.mode import Mode as Mode -from black.mode import TargetVersion as TargetVersion - -FileContent = str -Encoding = str -NewLine = str -FileMode = Mode - -def format_str(src_contents: str, mode: Mode) -> FileContent: ... diff --git a/mypy-stubs/mistune.pyi b/mypy-stubs/mistune.pyi deleted file mode 100644 index 3778c9195..000000000 --- a/mypy-stubs/mistune.pyi +++ /dev/null @@ -1,197 +0,0 @@ -__author__ = "Aleksandr Slepchenkov" -__email__ = "Sl.aleksandr28@gmail.com" - -from typing import ( - Any, - Dict, - Iterable, - List, - Match, - Optional, - Pattern, - Sequence, - Tuple, - Type, -) - -Tokens = List[Dict[str, Any]] -# There are too much levels of optional unions of lists of text in cell and align 385 and 396 lines in mistune - -def escape(text: str, quote: bool = ..., smart_amp: bool = ...) -> str: ... - -class BlockGrammar: - def_links: Pattern[str] - def_footnotes: Pattern[str] - newline: Pattern[str] - block_code: Pattern[str] - fences: Pattern[str] - hrule: Pattern[str] - heading: Pattern[str] - lheading: Pattern[str] - block_quote: Pattern[str] - list_block: Pattern[str] - list_item: Pattern[str] - list_bullet: Pattern[str] - paragraph: Pattern[str] - block_html: Pattern[str] - table: Pattern[str] - nptable: Pattern[str] - text: Pattern[str] - -class BlockLexer: - grammar_class: Type[BlockGrammar] - default_rules: List[str] - list_rules: Tuple[str] - footnote_rules: Tuple[str] - tokens: Tokens - def_links: Dict[str, Dict[str, str]] - def_footnotes: Dict[str, int] - rules = ... # type: BlockGrammar - def __init__(self, rules: Optional[BlockGrammar] = ..., **kwargs: Any) -> None: ... - def __call__(self, text: str, rules: Optional[Sequence[str]] = ...) -> Tokens: ... - def parse(self, text: str, rules: Optional[Sequence[str]] = ...) -> Tokens: ... - def parse_newline(self, m: Match[str]) -> None: ... - def parse_block_code(self, m: Match[str]) -> None: ... - def parse_fences(self, m: Match[str]) -> None: ... - def parse_heading(self, m: Match[str]) -> None: ... - def parse_lheading(self, m: Match[str]) -> None: ... - def parse_hrule(self, m: Match[str]) -> None: ... - def parse_list_block(self, m: Match[str]) -> None: ... - def parse_block_quote(self, m: Match[str]) -> None: ... - def parse_def_links(self, m: Match[str]) -> None: ... - def parse_def_footnotes(self, m: Match[str]) -> None: ... - def parse_table(self, m: Match[str]) -> None: ... - def parse_nptable(self, m: Match[str]) -> None: ... - def parse_block_html(self, m: Match[str]) -> None: ... - def parse_paragraph(self, m: Match[str]) -> None: ... - def parse_text(self, m: Match[str]) -> None: ... - -class InlineGrammar: - escape: Pattern[str] - inline_html: Pattern[str] - autolink: Pattern[str] - link: Pattern[str] - reflink: Pattern[str] - nolink: Pattern[str] - url: Pattern[str] - double_emphasis: Pattern[str] - emphasis: Pattern[str] - code: Pattern[str] - linebreak: Pattern[str] - strikethrough: Pattern[str] - footnote: Pattern[str] - text: Pattern[str] - def hard_wrap(self) -> None: ... - -class InlineLexer: - grammar_class: Type[InlineGrammar] - default_rules: List[str] - inline_html_rules: List[str] - renderer: Renderer - links: Dict[str, Dict[str, str]] - footnotes: Dict[str, int] - footnote_index: int - _in_link: bool - _in_footnote: bool - _parse_inline_html: bool - rules: InlineGrammar - def __init__( - self, renderer: Renderer, rules: Optional[InlineGrammar] = ..., **kwargs: Any - ) -> None: ... - def __call__(self, text: str, rules: Optional[Sequence[str]] = ...) -> str: ... - def setup( - self, - links: Optional[Dict[str, Dict[str, str]]], - footnotes: Optional[Dict[str, int]], - ) -> None: ... - line_match: Match[str] - line_started: bool - def output(self, text: str, rules: Optional[Sequence[str]] = ...) -> str: ... - def output_escape(self, m: Match[str]) -> str: ... - def output_autolink(self, m: Match[str]) -> str: ... - def output_url(self, m: Match[str]) -> str: ... - def output_inline_html(self, m: Match[str]) -> str: ... - def output_footnote(self, m: Match[str]) -> Optional[str]: ... - def output_link(self, m: Match[str]) -> str: ... - def output_reflink(self, m: Match[str]) -> Optional[str]: ... - def output_nolink(self, m: Match[str]) -> Optional[str]: ... - def output_double_emphasis(self, m: Match[str]) -> str: ... - def output_emphasis(self, m: Match[str]) -> str: ... - def output_code(self, m: Match[str]) -> str: ... - def output_linebreak(self, m: Match[str]) -> str: ... - def output_strikethrough(self, m: Match[str]) -> str: ... - def output_text(self, m: Match[str]) -> str: ... - -class Renderer: - options: Dict[str, str] - def __init__(self, **kwargs: Any) -> None: ... - def placeholder(self) -> str: ... - def block_code( - self, code: str, lang: Any = ... - ) -> str: ... # It seems that lang should be string, however other types are valid as well - def block_quote(self, text: str) -> str: ... - def block_html(self, html: str) -> str: ... - def header(self, text: str, level: int, raw: Optional[str] = ...) -> str: ... - def hrule(self) -> str: ... - def list( - self, body: Any, ordered: bool = ... - ) -> str: ... # body - same reason as for lang above, and for other Any in this class - def list_item(self, text: Any) -> str: ... - def paragraph(self, text: str) -> str: ... - def table(self, header: Any, body: Any) -> str: ... - def table_row(self, content: Any) -> str: ... - def table_cell(self, content: Any, **flags: Dict[str, Any]) -> str: ... - def double_emphasis(self, text: Any) -> str: ... - def emphasis(self, text: Any) -> str: ... - def codespan(self, text: str) -> str: ... - def linebreak(self) -> str: ... - def strikethrough(self, text: Any) -> str: ... - def text(self, text: Any) -> str: ... - def escape(self, text: Any) -> str: ... - def autolink(self, link: Any, is_email: bool = ...) -> str: ... - def link(self, link: Any, title: Any, text: Any) -> str: ... - def image(self, src: Any, title: Any, text: Any) -> str: ... - def inline_html(self, html: Any) -> str: ... - def newline(self) -> str: ... - def footnote_ref(self, key: Any, index: int) -> str: ... - def footnote_item(self, key: Any, text: str) -> str: ... - def footnotes(self, text: Any) -> str: ... - -class Markdown: - renderer = ... # type: Renderer - inline = ... # type: InlineLexer - block = ... # type: BlockLexer - footnotes = ... # type: List[Dict[str, Any]] - tokens = ... # type: Tokens - def __init__( - self, - renderer: Optional[Renderer] = ..., - inline: Optional[InlineLexer] = ..., - block: Optional[BlockLexer] = ..., - **kwargs: Any, - ) -> None: ... - def __call__(self, text: str) -> str: ... - def render(self, text: str) -> str: ... - def parse(self, text: str) -> str: ... - token = ... # type: Dict[str, Any] - def pop(self) -> Optional[Dict[str, Any]]: ... - def peek(self) -> Optional[Dict[str, Any]]: ... - def output(self, text: str, rules: Optional[Sequence[str]] = ...) -> str: ... - def tok(self) -> str: ... - def tok_text(self) -> str: ... - def output_newline(self) -> str: ... - def output_hrule(self) -> str: ... - def output_heading(self) -> str: ... - def output_code(self) -> str: ... - def output_table(self) -> str: ... - def output_block_quote(self) -> str: ... - def output_list(self) -> str: ... - def output_list_item(self) -> str: ... - def output_loose_item(self) -> str: ... - def output_footnote(self) -> str: ... - def output_close_html(self) -> str: ... - def output_open_html(self) -> str: ... - def output_paragraph(self) -> str: ... - def output_text(self) -> str: ... - -def markdown(text: str, escape: bool = ..., **kwargs: Any) -> str: ... diff --git a/setup.py b/setup.py index 9980276e5..5ddb526c1 100644 --- a/setup.py +++ b/setup.py @@ -134,7 +134,7 @@ "importlib_resources>=1.4;python_version<'3.9'", "coloredlogs", "pydot >= 1.4.1, <3", - "argcomplete", + "argcomplete >= 1.12.0", "pyparsing != 3.0.2", # breaks --print-dot (pydot) https://github.com/pyparsing/pyparsing/issues/319 "cwl-utils >= 0.32", "spython >= 0.3.0", From 75e073cf7f54660cf5e0f5fec3b9054ce4da8dfa Mon Sep 17 00:00:00 2001 From: Alexandre Detiste Date: Fri, 4 Oct 2024 14:47:53 +0200 Subject: [PATCH 02/43] remove usage of ancient shellescape backport of Stdlib's shlex (#2041) --- cwltool/command_line_tool.py | 4 ++-- cwltool/job.py | 4 ++-- mypy-stubs/shellescape/__init__.pyi | 5 ----- mypy-stubs/shellescape/main.pyi | 5 ----- requirements.txt | 1 - setup.py | 1 - 6 files changed, 4 insertions(+), 16 deletions(-) delete mode 100644 mypy-stubs/shellescape/__init__.pyi delete mode 100644 mypy-stubs/shellescape/main.pyi diff --git a/cwltool/command_line_tool.py b/cwltool/command_line_tool.py index 7a4e8ff71..eb0b1a4f5 100644 --- a/cwltool/command_line_tool.py +++ b/cwltool/command_line_tool.py @@ -7,6 +7,7 @@ import logging import os import re +import shlex import shutil import threading import urllib @@ -31,7 +32,6 @@ cast, ) -import shellescape from mypy_extensions import mypyc_attr from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.avro.schema import Schema @@ -1164,7 +1164,7 @@ def register_reader(f: CWLObjectType) -> None: for b in builder.bindings: arg = builder.generate_arg(b) if b.get("shellQuote", True): - arg = [shellescape.quote(a) for a in aslist(arg)] + arg = [shlex.quote(a) for a in aslist(arg)] cmd.extend(aslist(arg)) j.command_line = ["/bin/sh", "-c", " ".join(cmd)] else: diff --git a/cwltool/job.py b/cwltool/job.py index 817cb04c0..1731a5350 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -5,6 +5,7 @@ import math import os import re +import shlex import shutil import signal import stat @@ -35,7 +36,6 @@ ) import psutil -import shellescape from prov.model import PROV from schema_salad.sourceline import SourceLine from schema_salad.utils import json_dump, json_dumps @@ -271,7 +271,7 @@ def _execute( self.outdir, " \\\n ".join( [ - shellescape.quote(str(arg)) if shouldquote(str(arg)) else str(arg) + shlex.quote(str(arg)) if shouldquote(str(arg)) else str(arg) for arg in (runtime + self.command_line) ] ), diff --git a/mypy-stubs/shellescape/__init__.pyi b/mypy-stubs/shellescape/__init__.pyi deleted file mode 100644 index 621241e5e..000000000 --- a/mypy-stubs/shellescape/__init__.pyi +++ /dev/null @@ -1,5 +0,0 @@ -# Stubs for shellescape (Python 2) -# -# NOTE: This dynamically typed stub was automatically generated by stubgen. - -from .main import quote as quote diff --git a/mypy-stubs/shellescape/main.pyi b/mypy-stubs/shellescape/main.pyi deleted file mode 100644 index 69eade63e..000000000 --- a/mypy-stubs/shellescape/main.pyi +++ /dev/null @@ -1,5 +0,0 @@ -# Stubs for shellescape.main (Python 2) - -from typing import AnyStr - -def quote(s: AnyStr) -> AnyStr: ... diff --git a/requirements.txt b/requirements.txt index df82a6f43..b1fc2207d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,6 @@ requests>=2.6.1 ruamel.yaml>=0.16.0,<0.19 rdflib>=4.2.2,<7.1 -shellescape>=3.4.1,<3.9 schema-salad>=8.7,<9 prov==1.5.1 mypy-extensions diff --git a/setup.py b/setup.py index 5ddb526c1..9ab4f240e 100644 --- a/setup.py +++ b/setup.py @@ -126,7 +126,6 @@ # https://github.com/ionrock/cachecontrol/issues/137 "ruamel.yaml >= 0.16, < 0.19", "rdflib >= 4.2.2, < 7.1.0", - "shellescape >= 3.4.1, < 3.9", "schema-salad >= 8.7, < 9", "prov == 1.5.1", "mypy-extensions", From 04fd4264493fb6689d832967e72430188e60de00 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Fri, 4 Oct 2024 17:00:54 +0200 Subject: [PATCH 03/43] setuptools is not a install requirement --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 9ab4f240e..fa39a378b 100644 --- a/setup.py +++ b/setup.py @@ -121,7 +121,6 @@ package_dir={"cwltool.tests": "tests"}, include_package_data=True, install_requires=[ - "setuptools", "requests >= 2.6.1", # >= 2.6.1 to workaround # https://github.com/ionrock/cachecontrol/issues/137 "ruamel.yaml >= 0.16, < 0.19", From 0056ca52742cf7fa0c9556f511e59e778d445bd8 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Fri, 4 Oct 2024 16:35:37 +0200 Subject: [PATCH 04/43] gh-actions: fail-fast false --- .github/workflows/ci-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 1530ab92b..7de5ad329 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -165,6 +165,7 @@ jobs: runs-on: ubuntu-22.04 strategy: + fail-fast: false matrix: cwl-version: [v1.0, v1.1, v1.2] container: [docker, singularity, podman] From 18b8fdf7d425d8e7d8986e08904ef09492798cf6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Oct 2024 07:23:48 +0000 Subject: [PATCH 05/43] Bump sphinx-rtd-theme from 2.0.0 to 3.0.0 Bumps [sphinx-rtd-theme](https://github.com/readthedocs/sphinx_rtd_theme) from 2.0.0 to 3.0.0. - [Changelog](https://github.com/readthedocs/sphinx_rtd_theme/blob/master/docs/changelog.rst) - [Commits](https://github.com/readthedocs/sphinx_rtd_theme/compare/2.0.0...3.0.0) --- updated-dependencies: - dependency-name: sphinx-rtd-theme dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 40a2eefbf..875c44720 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,5 @@ sphinx >= 2.2 -sphinx-rtd-theme==2.0.0 +sphinx-rtd-theme==3.0.0 sphinx-autoapi sphinx-autodoc-typehints sphinxcontrib-autoprogram From 6970186d3f47a3998f4f1b88caf89a4b05f3d302 Mon Sep 17 00:00:00 2001 From: Iacopo Colonnelli Date: Tue, 8 Oct 2024 15:16:43 +0200 Subject: [PATCH 06/43] Handle spurious `ReceiveScatterOutput` callbacks (#2051) This commit fixes #2003 by handling spurious, repeated callbacks of the `receive_scatter_output` method of the `ReceiveScatterOutput` class. The reason of multiple awakenings has not been investigated deeply, though. In the future, a thorough examination of the `MultithreadedJobExecutor` logic may be necessary. --------- Co-authored-by: Michael R. Crusoe <1330696+mr-c@users.noreply.github.com> --- cwltool/workflow_job.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/cwltool/workflow_job.py b/cwltool/workflow_job.py index 2e69ca70c..d144128e6 100644 --- a/cwltool/workflow_job.py +++ b/cwltool/workflow_job.py @@ -10,6 +10,7 @@ MutableMapping, MutableSequence, Optional, + Set, Sized, Tuple, Union, @@ -88,12 +89,17 @@ def __init__( ) -> None: """Initialize.""" self.dest = dest - self.completed = 0 + self._completed: Set[int] = set() self.processStatus = "success" self.total = total self.output_callback = output_callback self.steps: List[Optional[JobsGeneratorType]] = [] + @property + def completed(self) -> int: + """The number of completed internal jobs.""" + return len(self._completed) + def receive_scatter_output(self, index: int, jobout: CWLObjectType, processStatus: str) -> None: """Record the results of a scatter operation.""" for key, val in jobout.items(): @@ -108,10 +114,11 @@ def receive_scatter_output(self, index: int, jobout: CWLObjectType, processStatu if self.processStatus != "permanentFail": self.processStatus = processStatus - self.completed += 1 + if index not in self._completed: + self._completed.add(index) - if self.completed == self.total: - self.output_callback(self.dest, self.processStatus) + if self.completed == self.total: + self.output_callback(self.dest, self.processStatus) def setTotal( self, From 83000ae041c4e516ed326274d8cf315861d49aa6 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Tue, 8 Oct 2024 17:58:30 +0200 Subject: [PATCH 07/43] prepare for future mypy release and enable --local-partial-types now --- mypy.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/mypy.ini b/mypy.ini index bac992869..02545dce5 100644 --- a/mypy.ini +++ b/mypy.ini @@ -5,6 +5,7 @@ show_column_numbers = true show_error_codes = true pretty = true warn_unreachable = True +local_partial_types = true [mypy-galaxy.tool_util.*] ignore_missing_imports = True From c6ad93a7003c4dc05cefe8d5667f2d9830002c21 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 10 Oct 2024 07:47:41 +0000 Subject: [PATCH 08/43] Bump sphinx-rtd-theme from 3.0.0 to 3.0.1 Bumps [sphinx-rtd-theme](https://github.com/readthedocs/sphinx_rtd_theme) from 3.0.0 to 3.0.1. - [Changelog](https://github.com/readthedocs/sphinx_rtd_theme/blob/master/docs/changelog.rst) - [Commits](https://github.com/readthedocs/sphinx_rtd_theme/compare/3.0.0...3.0.1) --- updated-dependencies: - dependency-name: sphinx-rtd-theme dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 875c44720..d614584fc 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,5 @@ sphinx >= 2.2 -sphinx-rtd-theme==3.0.0 +sphinx-rtd-theme==3.0.1 sphinx-autoapi sphinx-autodoc-typehints sphinxcontrib-autoprogram From 82f37d5bed7587690356ae90e71ddca8c653ff78 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Oct 2024 07:51:43 +0000 Subject: [PATCH 09/43] Update black requirement from ~=24.8 to ~=24.10 Updates the requirements on [black](https://github.com/psf/black) to permit the latest version. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/compare/24.8.0...24.10.0) --- updated-dependencies: - dependency-name: black dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- lint-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lint-requirements.txt b/lint-requirements.txt index b0d7944eb..3dc9139cb 100644 --- a/lint-requirements.txt +++ b/lint-requirements.txt @@ -1,3 +1,3 @@ flake8-bugbear<24.9 -black~=24.8 +black~=24.10 codespell From 1a2c939865e9bc67a4b6217d649dc3255ca11261 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" <1330696+mr-c@users.noreply.github.com> Date: Tue, 8 Oct 2024 10:05:28 +0200 Subject: [PATCH 10/43] Update lint-requirements.txt --- lint-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lint-requirements.txt b/lint-requirements.txt index 3dc9139cb..50dc69060 100644 --- a/lint-requirements.txt +++ b/lint-requirements.txt @@ -1,3 +1,3 @@ flake8-bugbear<24.9 -black~=24.10 +black=24.* codespell From 83def6a48c0489cfbc13706d44fc60bcb9621533 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" <1330696+mr-c@users.noreply.github.com> Date: Tue, 8 Oct 2024 10:06:49 +0200 Subject: [PATCH 11/43] Update lint-requirements.txt --- lint-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lint-requirements.txt b/lint-requirements.txt index 50dc69060..5af76cd93 100644 --- a/lint-requirements.txt +++ b/lint-requirements.txt @@ -1,3 +1,3 @@ flake8-bugbear<24.9 -black=24.* +black==24.* codespell From 05af6c1357c327b3146e9f5da40e7c0aa3e6d976 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Fri, 4 Oct 2024 14:46:47 +0200 Subject: [PATCH 12/43] Drop Python 3.8 --- .github/workflows/ci-tests.yml | 2 +- Makefile | 4 +- cwltool/argparser.py | 33 ++++------ cwltool/builder.py | 60 +++++++---------- cwltool/checker.py | 72 +++++++++----------- cwltool/command_line_tool.py | 80 ++++++++++------------- cwltool/context.py | 34 ++++------ cwltool/cuda.py | 3 +- cwltool/cwlprov/__init__.py | 14 ++-- cwltool/cwlprov/provenance_profile.py | 37 ++++------- cwltool/cwlprov/ro.py | 53 ++++++--------- cwltool/cwlprov/writablebagfile.py | 5 +- cwltool/cwlrdf.py | 5 +- cwltool/cwlviewer.py | 5 +- cwltool/docker.py | 39 +++++------ cwltool/docker_id.py | 14 ++-- cwltool/env_to_stdout.py | 3 +- cwltool/executors.py | 39 +++++------ cwltool/factory.py | 4 +- cwltool/flatten.py | 4 +- cwltool/job.py | 54 ++++++--------- cwltool/load_tool.py | 39 +++++------ cwltool/main.py | 65 ++++++++---------- cwltool/mpi.py | 11 ++-- cwltool/mutation.py | 4 +- cwltool/pack.py | 47 ++++++------- cwltool/pathmapper.py | 27 +++----- cwltool/process.py | 78 +++++++++------------- cwltool/procgenerator.py | 6 +- cwltool/run_job.py | 6 +- cwltool/secrets.py | 7 +- cwltool/singularity.py | 31 ++++----- cwltool/software_requirements.py | 19 ++---- cwltool/stdfsaccess.py | 6 +- cwltool/subgraph.py | 37 ++++------- cwltool/udocker.py | 4 +- cwltool/update.py | 40 +++++------- cwltool/utils.py | 49 ++++++-------- cwltool/validate_js.py | 25 +++---- cwltool/workflow.py | 22 ++----- cwltool/workflow_job.py | 43 +++++------- mypy-requirements.txt | 1 + pyproject.toml | 2 +- setup.py | 4 +- tests/cwl-conformance/cwltool-conftest.py | 6 +- tests/test_dependencies.py | 6 +- tests/test_environment.py | 7 +- tests/test_examples.py | 22 +++---- tests/test_fetch.py | 4 +- tests/test_http_input.py | 8 +-- tests/test_js_sandbox.py | 6 +- tests/test_loop.py | 2 +- tests/test_loop_ext.py | 2 +- tests/test_mpi.py | 17 ++--- tests/test_override.py | 5 +- tests/test_pack.py | 3 +- tests/test_path_checks.py | 4 +- tests/test_pathmapper.py | 6 +- tests/test_provenance.py | 3 +- tests/test_relocate.py | 7 +- tests/test_secrets.py | 10 +-- tests/test_tmpdir.py | 4 +- tests/test_toolargparse.py | 4 +- tests/util.py | 11 ++-- tox.ini | 45 +++++++------ 65 files changed, 541 insertions(+), 778 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 7de5ad329..2095b53b6 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -32,7 +32,7 @@ jobs: strategy: matrix: py-ver-major: [3] - py-ver-minor: [8, 9, 10, 11, 12, 13] + py-ver-minor: [9, 10, 11, 12, 13] step: [lint, unit, bandit, mypy] env: diff --git a/Makefile b/Makefile index 1b08f4290..5b9dd214e 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ MODULE=cwltool # `SHELL=bash` doesn't work for some, so don't use BASH-isms like # `[[` conditional expressions. -PYSOURCES=$(wildcard ${MODULE}/**.py cwltool/cwlprov/*.py tests/*.py) setup.py +PYSOURCES=$(wildcard ${MODULE}/**.py cwltool/cwlprov/*.py tests/*.py tests/cwl-conformance/*.py) setup.py DEVPKGS=diff_cover pylint pep257 pydocstyle 'tox<4' tox-pyenv auto-walrus \ isort wheel autoflake pyupgrade bandit -rlint-requirements.txt\ -rtest-requirements.txt -rmypy-requirements.txt -rdocs/requirements.txt @@ -190,7 +190,7 @@ shellcheck: FORCE cwltool-in-docker.sh pyupgrade: $(PYSOURCES) - pyupgrade --exit-zero-even-if-changed --py38-plus $^ + pyupgrade --exit-zero-even-if-changed --py39-plus $^ auto-walrus $^ release-test: FORCE diff --git a/cwltool/argparser.py b/cwltool/argparser.py index efced5386..7b3125d94 100644 --- a/cwltool/argparser.py +++ b/cwltool/argparser.py @@ -3,19 +3,8 @@ import argparse import os import urllib -from typing import ( - Any, - Callable, - Dict, - List, - MutableMapping, - MutableSequence, - Optional, - Sequence, - Type, - Union, - cast, -) +from collections.abc import MutableMapping, MutableSequence, Sequence +from typing import Any, Callable, Optional, Union, cast from .loghandler import _logger from .process import Process, shortname @@ -718,7 +707,7 @@ def arg_parser() -> argparse.ArgumentParser: return parser -def get_default_args() -> Dict[str, Any]: +def get_default_args() -> dict[str, Any]: """Get default values of cwltool's command line options.""" ap = arg_parser() args = ap.parse_args([]) @@ -732,7 +721,7 @@ class FSAction(argparse.Action): def __init__( self, - option_strings: List[str], + option_strings: list[str], dest: str, nargs: Any = None, urljoin: Callable[[str, str], str] = urllib.parse.urljoin, @@ -770,7 +759,7 @@ class FSAppendAction(argparse.Action): def __init__( self, - option_strings: List[str], + option_strings: list[str], dest: str, nargs: Any = None, urljoin: Callable[[str, str], str] = urllib.parse.urljoin, @@ -827,7 +816,7 @@ class AppendAction(argparse.Action): def __init__( self, - option_strings: List[str], + option_strings: list[str], dest: str, nargs: Any = None, **kwargs: Any, @@ -859,7 +848,7 @@ def add_argument( toolparser: argparse.ArgumentParser, name: str, inptype: Any, - records: List[str], + records: list[str], description: str = "", default: Any = None, input_required: bool = True, @@ -888,9 +877,9 @@ def add_argument( return None ahelp = description.replace("%", "%%") - action: Optional[Union[Type[argparse.Action], str]] = None + action: Optional[Union[type[argparse.Action], str]] = None atype: Optional[Any] = None - typekw: Dict[str, Any] = {} + typekw: dict[str, Any] = {} if inptype == "File": action = FileAction @@ -962,8 +951,8 @@ def add_argument( def generate_parser( toolparser: argparse.ArgumentParser, tool: Process, - namemap: Dict[str, str], - records: List[str], + namemap: dict[str, str], + records: list[str], input_required: bool = True, urljoin: Callable[[str, str], str] = urllib.parse.urljoin, base_uri: str = "", diff --git a/cwltool/builder.py b/cwltool/builder.py index 2ba1e6543..066a77f86 100644 --- a/cwltool/builder.py +++ b/cwltool/builder.py @@ -3,21 +3,9 @@ import copy import logging import math +from collections.abc import MutableMapping, MutableSequence from decimal import Decimal -from typing import ( - IO, - TYPE_CHECKING, - Any, - Callable, - Dict, - List, - MutableMapping, - MutableSequence, - Optional, - Type, - Union, - cast, -) +from typing import IO, TYPE_CHECKING, Any, Callable, Optional, Union, cast from cwl_utils import expression from cwl_utils.file_formats import check_format @@ -55,7 +43,7 @@ ) from .pathmapper import PathMapper -INPUT_OBJ_VOCAB: Dict[str, str] = { +INPUT_OBJ_VOCAB: dict[str, str] = { "Any": "https://w3id.org/cwl/salad#Any", "File": "https://w3id.org/cwl/cwl#File", "Directory": "https://w3id.org/cwl/cwl#Directory", @@ -107,16 +95,16 @@ class Builder(HasReqsHints): def __init__( self, job: CWLObjectType, - files: List[CWLObjectType], - bindings: List[CWLObjectType], + files: list[CWLObjectType], + bindings: list[CWLObjectType], schemaDefs: MutableMapping[str, CWLObjectType], names: Names, - requirements: List[CWLObjectType], - hints: List[CWLObjectType], - resources: Dict[str, Union[int, float]], + requirements: list[CWLObjectType], + hints: list[CWLObjectType], + resources: dict[str, Union[int, float]], mutation_manager: Optional[MutationManager], formatgraph: Optional[Graph], - make_fs_access: Type[StdFsAccess], + make_fs_access: type[StdFsAccess], fs_access: StdFsAccess, job_script_provider: Optional[DependenciesConfiguration], timeout: float, @@ -172,7 +160,7 @@ def __init__( self.find_default_container: Optional[Callable[[], str]] = None self.container_engine = container_engine - def build_job_script(self, commands: List[str]) -> Optional[str]: + def build_job_script(self, commands: list[str]) -> Optional[str]: if self.job_script_provider is not None: return self.job_script_provider.build_job_script(self, commands) return None @@ -180,11 +168,11 @@ def build_job_script(self, commands: List[str]) -> Optional[str]: def bind_input( self, schema: CWLObjectType, - datum: Union[CWLObjectType, List[CWLObjectType]], + datum: Union[CWLObjectType, list[CWLObjectType]], discover_secondaryFiles: bool, - lead_pos: Optional[Union[int, List[int]]] = None, - tail_pos: Optional[Union[str, List[int]]] = None, - ) -> List[MutableMapping[str, Union[str, List[int]]]]: + lead_pos: Optional[Union[int, list[int]]] = None, + tail_pos: Optional[Union[str, list[int]]] = None, + ) -> list[MutableMapping[str, Union[str, list[int]]]]: """ Bind an input object to the command line. @@ -200,8 +188,8 @@ def bind_input( if lead_pos is None: lead_pos = [] - bindings: List[MutableMapping[str, Union[str, List[int]]]] = [] - binding: Union[MutableMapping[str, Union[str, List[int]]], CommentedMap] = {} + bindings: list[MutableMapping[str, Union[str, list[int]]]] = [] + binding: Union[MutableMapping[str, Union[str, list[int]]], CommentedMap] = {} value_from_expression = False if "inputBinding" in schema and isinstance(schema["inputBinding"], MutableMapping): binding = CommentedMap(schema["inputBinding"].items()) @@ -324,7 +312,7 @@ def bind_input( if schema["type"] == "record": datum = cast(CWLObjectType, datum) - for f in cast(List[CWLObjectType], schema["fields"]): + for f in cast(list[CWLObjectType], schema["fields"]): name = cast(str, f["name"]) if name in datum and datum[name] is not None: bindings.extend( @@ -372,7 +360,7 @@ def _capture_files(f: CWLObjectType) -> CWLObjectType: self.files.append(datum) loadContents_sourceline: Union[ - None, MutableMapping[str, Union[str, List[int]]], CWLObjectType + None, MutableMapping[str, Union[str, list[int]]], CWLObjectType ] = None if binding and binding.get("loadContents"): loadContents_sourceline = binding @@ -513,7 +501,7 @@ def addsf( if "format" in schema: eval_format: Any = self.do_eval(schema["format"]) if isinstance(eval_format, str): - evaluated_format: Union[str, List[str]] = eval_format + evaluated_format: Union[str, list[str]] = eval_format elif isinstance(eval_format, MutableSequence): for index, entry in enumerate(eval_format): message = None @@ -541,7 +529,7 @@ def addsf( raise SourceLine( schema["format"], index, WorkflowException, debug ).makeError(message) - evaluated_format = cast(List[str], eval_format) + evaluated_format = cast(list[str], eval_format) else: raise SourceLine(schema, "format", WorkflowException, debug).makeError( "An expression in the 'format' field must " @@ -586,8 +574,8 @@ def addsf( # Position to front of the sort key if binding: for bi in bindings: - bi["position"] = cast(List[int], binding["position"]) + cast( - List[int], bi["position"] + bi["position"] = cast(list[int], binding["position"]) + cast( + list[int], bi["position"] ) bindings.append(binding) @@ -618,7 +606,7 @@ def tostr(self, value: Union[MutableMapping[str, str], Any]) -> str: else: return str(value) - def generate_arg(self, binding: CWLObjectType) -> List[str]: + def generate_arg(self, binding: CWLObjectType) -> list[str]: value = binding.get("datum") debug = _logger.isEnabledFor(logging.DEBUG) if "valueFrom" in binding: @@ -648,7 +636,7 @@ def generate_arg(self, binding: CWLObjectType) -> List[str]: argl = [itemSeparator.join([self.tostr(v) for v in value])] elif binding.get("valueFrom"): value = [self.tostr(v) for v in value] - return cast(List[str], ([prefix] if prefix else [])) + cast(List[str], value) + return cast(list[str], ([prefix] if prefix else [])) + cast(list[str], value) elif prefix and value: return [prefix] else: diff --git a/cwltool/checker.py b/cwltool/checker.py index 676245698..7742adf5c 100644 --- a/cwltool/checker.py +++ b/cwltool/checker.py @@ -1,19 +1,8 @@ """Static checking of CWL workflow connectivity.""" from collections import namedtuple -from typing import ( - Any, - Dict, - Iterator, - List, - Literal, - MutableMapping, - MutableSequence, - Optional, - Sized, - Union, - cast, -) +from collections.abc import Iterator, MutableMapping, MutableSequence, Sized +from typing import Any, Literal, Optional, Union, cast from schema_salad.exceptions import ValidationException from schema_salad.sourceline import SourceLine, bullets, strip_dup_lineno @@ -25,8 +14,7 @@ from .utils import CWLObjectType, CWLOutputType, SinkType, aslist -def _get_type(tp): - # type: (Any) -> Any +def _get_type(tp: Any) -> Any: if isinstance(tp, MutableMapping): if tp.get("type") not in ("array", "record", "enum"): return tp["type"] @@ -98,10 +86,10 @@ def can_assign_src_to_sink(src: SinkType, sink: Optional[SinkType], strict: bool if src["type"] == "record" and sink["type"] == "record": return _compare_records(src, sink, strict) if src["type"] == "File" and sink["type"] == "File": - for sinksf in cast(List[CWLObjectType], sink.get("secondaryFiles", [])): + for sinksf in cast(list[CWLObjectType], sink.get("secondaryFiles", [])): if not [ 1 - for srcsf in cast(List[CWLObjectType], src.get("secondaryFiles", [])) + for srcsf in cast(list[CWLObjectType], src.get("secondaryFiles", [])) if sinksf == srcsf ]: if strict: @@ -167,7 +155,7 @@ def _rec_fields(rec: MutableMapping[str, Any]) -> MutableMapping[str, Any]: return True -def missing_subset(fullset: List[Any], subset: List[Any]) -> List[Any]: +def missing_subset(fullset: list[Any], subset: list[Any]) -> list[Any]: missing = [] for i in subset: if i not in fullset: @@ -176,11 +164,11 @@ def missing_subset(fullset: List[Any], subset: List[Any]) -> List[Any]: def static_checker( - workflow_inputs: List[CWLObjectType], + workflow_inputs: list[CWLObjectType], workflow_outputs: MutableSequence[CWLObjectType], step_inputs: MutableSequence[CWLObjectType], - step_outputs: List[CWLObjectType], - param_to_step: Dict[str, CWLObjectType], + step_outputs: list[CWLObjectType], + param_to_step: dict[str, CWLObjectType], ) -> None: """ Check if all source and sink types of a workflow are compatible before run time. @@ -191,7 +179,7 @@ def static_checker( # sink parameters: step_inputs and workflow_outputs # make a dictionary of source parameters, indexed by the "id" field - src_dict: Dict[str, CWLObjectType] = {} + src_dict: dict[str, CWLObjectType] = {} for param in workflow_inputs + step_outputs: src_dict[cast(str, param["id"])] = param @@ -245,7 +233,7 @@ def static_checker( ", ".join( shortname(cast(str, s["id"])) for s in cast( - List[Dict[str, Union[str, bool]]], + list[dict[str, Union[str, bool]]], param_to_step[sink["id"]]["inputs"], ) if not s.get("not_connected") @@ -328,11 +316,11 @@ def static_checker( def check_all_types( - src_dict: Dict[str, CWLObjectType], + src_dict: dict[str, CWLObjectType], sinks: MutableSequence[CWLObjectType], sourceField: Union[Literal["source"], Literal["outputSource"]], - param_to_step: Dict[str, CWLObjectType], -) -> Dict[str, List[SrcSink]]: + param_to_step: dict[str, CWLObjectType], +) -> dict[str, list[SrcSink]]: """ Given a list of sinks, check if their types match with the types of their sources. @@ -340,7 +328,7 @@ def check_all_types( (from :py:func:`check_types`) :raises ValidationException: if a sourceField is missing """ - validation = {"warning": [], "exception": []} # type: Dict[str, List[SrcSink]] + validation: dict[str, list[SrcSink]] = {"warning": [], "exception": []} for sink in sinks: if sourceField in sink: valueFrom = cast(Optional[str], sink.get("valueFrom")) @@ -351,18 +339,18 @@ def check_all_types( extra_message = "pickValue is: %s" % pickValue if isinstance(sink[sourceField], MutableSequence): - linkMerge = cast( + linkMerge: Optional[str] = cast( Optional[str], sink.get( "linkMerge", ("merge_nested" if len(cast(Sized, sink[sourceField])) > 1 else None), ), - ) # type: Optional[str] + ) if pickValue in ["first_non_null", "the_only_non_null"]: linkMerge = None - srcs_of_sink = [] # type: List[CWLObjectType] + srcs_of_sink: list[CWLObjectType] = [] for parm_id in cast(MutableSequence[str], sink[sourceField]): srcs_of_sink += [src_dict[parm_id]] if is_conditional_step(param_to_step, parm_id) and pickValue is None: @@ -404,10 +392,10 @@ def check_all_types( snk_typ = sink["type"] if "null" not in src_typ: - src_typ = ["null"] + cast(List[Any], src_typ) + src_typ = ["null"] + cast(list[Any], src_typ) if "null" not in cast( - Union[List[str], CWLObjectType], snk_typ + Union[list[str], CWLObjectType], snk_typ ): # Given our type names this works even if not a list validation["warning"].append( SrcSink( @@ -440,7 +428,7 @@ def check_all_types( return validation -def circular_dependency_checker(step_inputs: List[CWLObjectType]) -> None: +def circular_dependency_checker(step_inputs: list[CWLObjectType]) -> None: """ Check if a workflow has circular dependency. @@ -448,8 +436,8 @@ def circular_dependency_checker(step_inputs: List[CWLObjectType]) -> None: """ adjacency = get_dependency_tree(step_inputs) vertices = adjacency.keys() - processed: List[str] = [] - cycles: List[List[str]] = [] + processed: list[str] = [] + cycles: list[list[str]] = [] for vertex in vertices: if vertex not in processed: traversal_path = [vertex] @@ -461,7 +449,7 @@ def circular_dependency_checker(step_inputs: List[CWLObjectType]) -> None: raise ValidationException(exception_msg) -def get_dependency_tree(step_inputs: List[CWLObjectType]) -> Dict[str, List[str]]: +def get_dependency_tree(step_inputs: list[CWLObjectType]) -> dict[str, list[str]]: """Get the dependency tree in the form of adjacency list.""" adjacency = {} # adjacency list of the dependency tree for step_input in step_inputs: @@ -482,10 +470,10 @@ def get_dependency_tree(step_inputs: List[CWLObjectType]) -> Dict[str, List[str] def processDFS( - adjacency: Dict[str, List[str]], - traversal_path: List[str], - processed: List[str], - cycles: List[List[str]], + adjacency: dict[str, list[str]], + traversal_path: list[str], + processed: list[str], + cycles: list[list[str]], ) -> None: """Perform depth first search.""" tip = traversal_path[-1] @@ -509,14 +497,14 @@ def get_step_id(field_id: str) -> str: return step_id -def is_conditional_step(param_to_step: Dict[str, CWLObjectType], parm_id: str) -> bool: +def is_conditional_step(param_to_step: dict[str, CWLObjectType], parm_id: str) -> bool: if (source_step := param_to_step.get(parm_id)) is not None: if source_step.get("when") is not None: return True return False -def is_all_output_method_loop_step(param_to_step: Dict[str, CWLObjectType], parm_id: str) -> bool: +def is_all_output_method_loop_step(param_to_step: dict[str, CWLObjectType], parm_id: str) -> bool: """Check if a step contains a `loop` directive with `all_iterations` outputMethod.""" source_step: Optional[MutableMapping[str, Any]] = param_to_step.get(parm_id) if source_step is not None: diff --git a/cwltool/command_line_tool.py b/cwltool/command_line_tool.py index eb0b1a4f5..de1878593 100644 --- a/cwltool/command_line_tool.py +++ b/cwltool/command_line_tool.py @@ -12,25 +12,11 @@ import threading import urllib import urllib.parse +from collections.abc import Generator, Mapping, MutableMapping, MutableSequence from enum import Enum from functools import cmp_to_key, partial -from typing import ( - TYPE_CHECKING, - Any, - Dict, - Generator, - List, - Mapping, - MutableMapping, - MutableSequence, - Optional, - Pattern, - Set, - TextIO, - Type, - Union, - cast, -) +from re import Pattern +from typing import TYPE_CHECKING, Any, Optional, TextIO, Union, cast from mypy_extensions import mypyc_attr from ruamel.yaml.comments import CommentedMap, CommentedSeq @@ -163,8 +149,8 @@ def __init__( builder: Builder, script: str, output_callback: Optional[OutputCallbackType], - requirements: List[CWLObjectType], - hints: List[CWLObjectType], + requirements: list[CWLObjectType], + hints: list[CWLObjectType], outdir: Optional[str] = None, tmpdir: Optional[str] = None, ) -> None: @@ -241,7 +227,7 @@ def job( raise WorkflowException("Abstract operation cannot be executed.") -def remove_path(f): # type: (CWLObjectType) -> None +def remove_path(f: CWLObjectType) -> None: if "path" in f: del f["path"] @@ -334,7 +320,7 @@ def __init__( self.output_callback = output_callback self.cachebuilder = cachebuilder self.outdir = jobcache - self.prov_obj = None # type: Optional[ProvenanceProfile] + self.prov_obj: Optional[ProvenanceProfile] = None def run( self, @@ -392,7 +378,7 @@ def check_valid_locations(fs_access: StdFsAccess, ob: CWLObjectType) -> None: raise ValidationException("Does not exist or is not a Directory: '%s'" % location) -OutputPortsType = Dict[str, Optional[CWLOutputType]] +OutputPortsType = dict[str, Optional[CWLOutputType]] class ParameterOutputWorkflowException(WorkflowException): @@ -411,13 +397,13 @@ def __init__(self, toolpath_object: CommentedMap, loadingContext: LoadingContext """Initialize this CommandLineTool.""" super().__init__(toolpath_object, loadingContext) self.prov_obj = loadingContext.prov_obj - self.path_check_mode = ( + self.path_check_mode: PathCheckingMode = ( PathCheckingMode.RELAXED if loadingContext.relax_path_checks else PathCheckingMode.STRICT - ) # type: PathCheckingMode + ) - def make_job_runner(self, runtimeContext: RuntimeContext) -> Type[JobBase]: + def make_job_runner(self, runtimeContext: RuntimeContext) -> type[JobBase]: dockerReq, dockerRequired = self.get_requirement("DockerRequirement") mpiReq, mpiRequired = self.get_requirement(MPIRequirementName) @@ -477,7 +463,7 @@ def make_job_runner(self, runtimeContext: RuntimeContext) -> Type[JobBase]: @staticmethod def make_path_mapper( - reffiles: List[CWLObjectType], + reffiles: list[CWLObjectType], stagedir: str, runtimeContext: RuntimeContext, separateDirs: bool, @@ -499,9 +485,9 @@ def updatePathmap(self, outdir: str, pathmap: PathMapper, fn: CWLObjectType) -> ("Writable" if fn.get("writable") else "") + cast(str, fn["class"]), False, ) - for sf in cast(List[CWLObjectType], fn.get("secondaryFiles", [])): + for sf in cast(list[CWLObjectType], fn.get("secondaryFiles", [])): self.updatePathmap(outdir, pathmap, sf) - for ls in cast(List[CWLObjectType], fn.get("listing", [])): + for ls in cast(list[CWLObjectType], fn.get("listing", [])): self.updatePathmap(os.path.join(outdir, cast(str, fn["basename"])), pathmap, ls) def _initialworkdir(self, j: JobBase, builder: Builder) -> None: @@ -517,7 +503,7 @@ def _initialworkdir(self, j: JobBase, builder: Builder) -> None: cwl_version ) < ORDERED_VERSIONS.index("v1.1.0-dev1") - ls = [] # type: List[CWLObjectType] + ls: list[CWLObjectType] = [] if isinstance(initialWorkdir["listing"], str): # "listing" is just a string (must be an expression) so # just evaluate it and use the result as if it was in @@ -587,7 +573,7 @@ def _initialworkdir(self, j: JobBase, builder: Builder) -> None: raise SourceLine(initialWorkdir, "listing", WorkflowException, debug).makeError( message ) - ls = cast(List[CWLObjectType], ls_evaluated) + ls = cast(list[CWLObjectType], ls_evaluated) else: # "listing" is an array of either expressions or Dirent so # evaluate each item @@ -634,10 +620,10 @@ def _initialworkdir(self, j: JobBase, builder: Builder) -> None: for e in entry: ec = cast(CWLObjectType, e) ec["writable"] = t.get("writable", False) - ls.extend(cast(List[CWLObjectType], entry)) + ls.extend(cast(list[CWLObjectType], entry)) continue - et = {} # type: CWLObjectType + et: CWLObjectType = {} if isinstance(entry, Mapping) and entry.get("class") in ( "File", "Directory", @@ -686,7 +672,7 @@ def _initialworkdir(self, j: JobBase, builder: Builder) -> None: if not initwd_item: continue if isinstance(initwd_item, MutableSequence): - ls.extend(cast(List[CWLObjectType], initwd_item)) + ls.extend(cast(list[CWLObjectType], initwd_item)) else: ls.append(cast(CWLObjectType, initwd_item)) @@ -850,9 +836,9 @@ def job( cmdline = ["docker", "run", dockerimg] + cmdline # not really run using docker, just for hashing purposes - keydict = { + keydict: dict[str, Union[MutableSequence[Union[str, int]], CWLObjectType]] = { "cmdline": cmdline - } # type: Dict[str, Union[MutableSequence[Union[str, int]], CWLObjectType]] + } for shortcut in ["stdin", "stdout", "stderr"]: if shortcut in self.tool: @@ -1071,8 +1057,8 @@ def update_status_output_callback( j.inplace_update = cast(bool, inplaceUpdateReq["inplaceUpdate"]) normalizeFilesDirs(j.generatefiles) - readers = {} # type: Dict[str, CWLObjectType] - muts = set() # type: Set[str] + readers: dict[str, CWLObjectType] = {} + muts: set[str] = set() if builder.mutation_manager is not None: @@ -1103,7 +1089,7 @@ def register_reader(f: CWLObjectType) -> None: timelimit, _ = self.get_requirement("ToolTimeLimit") if timelimit is not None: with SourceLine(timelimit, "timelimit", ValidationException, debug): - limit_field = cast(Dict[str, Union[str, int]], timelimit)["timelimit"] + limit_field = cast(dict[str, Union[str, int]], timelimit)["timelimit"] if isinstance(limit_field, str): timelimit_eval = builder.do_eval(limit_field) if timelimit_eval and not isinstance(timelimit_eval, int): @@ -1142,7 +1128,7 @@ def register_reader(f: CWLObjectType) -> None: required_env = {} evr, _ = self.get_requirement("EnvVarRequirement") if evr is not None: - for eindex, t3 in enumerate(cast(List[Dict[str, str]], evr["envDef"])): + for eindex, t3 in enumerate(cast(list[dict[str, str]], evr["envDef"])): env_value_field = t3["envValue"] if "${" in env_value_field or "$(" in env_value_field: env_value_eval = builder.do_eval(env_value_field) @@ -1160,7 +1146,7 @@ def register_reader(f: CWLObjectType) -> None: shellcmd, _ = self.get_requirement("ShellCommandRequirement") if shellcmd is not None: - cmd = [] # type: List[str] + cmd: list[str] = [] for b in builder.bindings: arg = builder.generate_arg(b) if b.get("shellQuote", True): @@ -1201,7 +1187,7 @@ def register_reader(f: CWLObjectType) -> None: def collect_output_ports( self, - ports: Union[CommentedSeq, Set[CWLObjectType]], + ports: Union[CommentedSeq, set[CWLObjectType]], builder: Builder, outdir: str, rcode: int, @@ -1209,7 +1195,7 @@ def collect_output_ports( jobname: str = "", readers: Optional[MutableMapping[str, CWLObjectType]] = None, ) -> OutputPortsType: - ret = {} # type: OutputPortsType + ret: OutputPortsType = {} debug = _logger.isEnabledFor(logging.DEBUG) cwl_version = self.metadata.get(ORIGINAL_CWLVERSION, None) if cwl_version != "v1.0": @@ -1284,16 +1270,16 @@ def collect_output( fs_access: StdFsAccess, compute_checksum: bool = True, ) -> Optional[CWLOutputType]: - r = [] # type: List[CWLOutputType] + r: list[CWLOutputType] = [] empty_and_optional = False debug = _logger.isEnabledFor(logging.DEBUG) result: Optional[CWLOutputType] = None if "outputBinding" in schema: binding = cast( - MutableMapping[str, Union[bool, str, List[str]]], + MutableMapping[str, Union[bool, str, list[str]]], schema["outputBinding"], ) - globpatterns = [] # type: List[str] + globpatterns: list[str] = [] revmap = partial(revmap_file, builder, outdir) @@ -1359,7 +1345,7 @@ def collect_output( _logger.error("Unexpected error from fs_access", exc_info=True) raise - for files in cast(List[Dict[str, Optional[CWLOutputType]]], r): + for files in cast(list[dict[str, Optional[CWLOutputType]]], r): rfile = files.copy() revmap(rfile) if files["class"] == "Directory": @@ -1515,7 +1501,7 @@ def collect_output( and schema["type"]["type"] == "record" ): out = {} - for field in cast(List[CWLObjectType], schema["type"]["fields"]): + for field in cast(list[CWLObjectType], schema["type"]["fields"]): out[shortname(cast(str, field["name"]))] = self.collect_output( field, builder, outdir, fs_access, compute_checksum=compute_checksum ) diff --git a/cwltool/context.py b/cwltool/context.py index 283936d61..237a90968 100644 --- a/cwltool/context.py +++ b/cwltool/context.py @@ -5,20 +5,8 @@ import shutil import tempfile import threading -from typing import ( - IO, - TYPE_CHECKING, - Any, - Callable, - Dict, - Iterable, - List, - Literal, - Optional, - TextIO, - Tuple, - Union, -) +from collections.abc import Iterable +from typing import IO, TYPE_CHECKING, Any, Callable, Literal, Optional, TextIO, Union from ruamel.yaml.comments import CommentedMap from schema_salad.avro.schema import Names @@ -46,7 +34,7 @@ class ContextBase: """Shared kwargs based initializer for :py:class:`RuntimeContext` and :py:class:`LoadingContext`.""" - def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None: + def __init__(self, kwargs: Optional[dict[str, Any]] = None) -> None: """Initialize.""" if kwargs: for k, v in kwargs.items(): @@ -87,13 +75,13 @@ def set_log_dir(outdir: str, log_dir: str, subdir_name: str) -> str: class LoadingContext(ContextBase): - def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None: + def __init__(self, kwargs: Optional[dict[str, Any]] = None) -> None: """Initialize the LoadingContext from the kwargs.""" self.debug: bool = False self.metadata: CWLObjectType = {} - self.requirements: Optional[List[CWLObjectType]] = None - self.hints: Optional[List[CWLObjectType]] = None - self.overrides_list: List[CWLObjectType] = [] + self.requirements: Optional[list[CWLObjectType]] = None + self.hints: Optional[list[CWLObjectType]] = None + self.overrides_list: list[CWLObjectType] = [] self.loader: Optional[Loader] = None self.avsc_names: Optional[Names] = None self.disable_js_validation: bool = False @@ -117,7 +105,7 @@ def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None: self.singularity: bool = False self.podman: bool = False self.eval_timeout: float = 60 - self.codegen_idx: Dict[str, Tuple[Any, "LoadingOptions"]] = {} + self.codegen_idx: dict[str, tuple[Any, "LoadingOptions"]] = {} self.fast_parser = False self.skip_resolve_all = False self.skip_schemas = False @@ -136,11 +124,11 @@ class RuntimeContext(ContextBase): tmp_outdir_prefix: str = "" stagedir: str = "" - def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None: + def __init__(self, kwargs: Optional[dict[str, Any]] = None) -> None: """Initialize the RuntimeContext from the kwargs.""" select_resources_callable = Callable[ - [Dict[str, Union[int, float]], RuntimeContext], - Dict[str, Union[int, float]], + [dict[str, Union[int, float]], RuntimeContext], + dict[str, Union[int, float]], ] self.user_space_docker_cmd: Optional[str] = None self.secret_store: Optional["SecretStore"] = None diff --git a/cwltool/cuda.py b/cwltool/cuda.py index 719bfd867..1394ec239 100644 --- a/cwltool/cuda.py +++ b/cwltool/cuda.py @@ -2,13 +2,12 @@ import subprocess # nosec import xml.dom.minidom # nosec -from typing import Tuple from .loghandler import _logger from .utils import CWLObjectType -def cuda_version_and_device_count() -> Tuple[str, int]: +def cuda_version_and_device_count() -> tuple[str, int]: """Determine the CUDA version and number of attached CUDA GPUs.""" try: out = subprocess.check_output(["nvidia-smi", "-q", "-x"]) # nosec diff --git a/cwltool/cwlprov/__init__.py b/cwltool/cwlprov/__init__.py index b8ff8d14d..a09a57c34 100644 --- a/cwltool/cwlprov/__init__.py +++ b/cwltool/cwlprov/__init__.py @@ -6,10 +6,10 @@ import re import uuid from getpass import getuser -from typing import IO, Any, Callable, Dict, List, Optional, Tuple, TypedDict, Union +from typing import IO, Any, Callable, Optional, TypedDict, Union -def _whoami() -> Tuple[str, str]: +def _whoami() -> tuple[str, str]: """Return the current operating system account as (username, fullname).""" username = getuser() try: @@ -106,8 +106,8 @@ def _valid_orcid(orcid: Optional[str]) -> str: { "uri": str, "about": str, - "content": Optional[Union[str, List[str]]], - "oa:motivatedBy": Dict[str, str], + "content": Optional[Union[str, list[str]]], + "oa:motivatedBy": dict[str, str], }, ) @@ -116,11 +116,11 @@ class Aggregate(TypedDict, total=False): """RO Aggregate class.""" uri: Optional[str] - bundledAs: Optional[Dict[str, Any]] + bundledAs: Optional[dict[str, Any]] mediatype: Optional[str] - conformsTo: Optional[Union[str, List[str]]] + conformsTo: Optional[Union[str, list[str]]] createdOn: Optional[str] - createdBy: Optional[Dict[str, str]] + createdBy: Optional[dict[str, str]] # Aggregate.bundledAs is actually type Aggregate, but cyclic definitions are not supported diff --git a/cwltool/cwlprov/provenance_profile.py b/cwltool/cwlprov/provenance_profile.py index ce8d63ad4..59d835fff 100644 --- a/cwltool/cwlprov/provenance_profile.py +++ b/cwltool/cwlprov/provenance_profile.py @@ -3,22 +3,11 @@ import logging import urllib import uuid +from collections.abc import MutableMapping, MutableSequence, Sequence from io import BytesIO from pathlib import PurePath, PurePosixPath from socket import getfqdn -from typing import ( - TYPE_CHECKING, - Any, - Dict, - List, - MutableMapping, - MutableSequence, - Optional, - Sequence, - Tuple, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, Union, cast from prov.identifier import Identifier, QualifiedName from prov.model import PROV, PROV_LABEL, PROV_TYPE, PROV_VALUE, ProvDocument, ProvEntity @@ -117,7 +106,7 @@ def __str__(self) -> str: """Represent this Provenvance profile as a string.""" return f"ProvenanceProfile <{self.workflow_run_uri}> in <{self.research_object}>" - def generate_prov_doc(self) -> Tuple[str, ProvDocument]: + def generate_prov_doc(self) -> tuple[str, ProvDocument]: """Add basic namespaces.""" def host_provenance(document: ProvDocument) -> None: @@ -177,7 +166,7 @@ def host_provenance(document: ProvDocument) -> None: # by a user account, as cwltool is a command line tool account = self.document.agent(ACCOUNT_UUID) if self.orcid or self.full_name: - person: Dict[Union[str, Identifier], Any] = { + person: dict[Union[str, Identifier], Any] = { PROV_TYPE: PROV["Person"], "prov:type": SCHEMA["Person"], } @@ -291,7 +280,7 @@ def record_process_end( self.generate_output_prov(outputs, process_run_id, process_name) self.document.wasEndedBy(process_run_id, None, self.workflow_run_uri, when) - def declare_file(self, value: CWLObjectType) -> Tuple[ProvEntity, ProvEntity, str]: + def declare_file(self, value: CWLObjectType) -> tuple[ProvEntity, ProvEntity, str]: if value["class"] != "File": raise ValueError("Must have class:File: %s" % value) # Need to determine file hash aka RO filename @@ -399,10 +388,10 @@ def declare_directory(self, value: CWLObjectType) -> ProvEntity: # dir_bundle.identifier, {PROV["type"]: ORE["ResourceMap"], # ORE["describes"]: coll_b.identifier}) - coll_attribs: List[Tuple[Union[str, Identifier], Any]] = [ + coll_attribs: list[tuple[Union[str, Identifier], Any]] = [ (ORE["isDescribedBy"], dir_bundle.identifier) ] - coll_b_attribs: List[Tuple[Union[str, Identifier], Any]] = [] + coll_b_attribs: list[tuple[Union[str, Identifier], Any]] = [] # FIXME: .listing might not be populated yet - hopefully # a later call to this method will sort that @@ -469,7 +458,7 @@ def declare_directory(self, value: CWLObjectType) -> ProvEntity: self.research_object.add_uri(coll.identifier.uri) return coll - def declare_string(self, value: str) -> Tuple[ProvEntity, str]: + def declare_string(self, value: str) -> tuple[ProvEntity, str]: """Save as string in UTF-8.""" byte_s = BytesIO(str(value).encode(ENCODING)) data_file = self.research_object.add_data_file(byte_s, content_type=TEXT_PLAIN) @@ -518,7 +507,7 @@ def declare_artefact(self, value: Any) -> ProvEntity: # Already processed this value, but it might not be in this PROV entities = self.document.get_record(value["@id"]) if entities: - return cast(List[ProvEntity], entities)[0] + return cast(list[ProvEntity], entities)[0] # else, unknown in PROV, re-add below as if it's fresh # Base case - we found a File we need to update @@ -549,7 +538,7 @@ def declare_artefact(self, value: Any) -> ProvEntity: coll.add_asserted_type(CWLPROV[value["class"]]) # Let's iterate and recurse - coll_attribs: List[Tuple[Union[str, Identifier], Any]] = [] + coll_attribs: list[tuple[Union[str, Identifier], Any]] = [] for key, val in value.items(): v_ent = self.declare_artefact(val) self.document.membership(coll, v_ent) @@ -601,7 +590,7 @@ def declare_artefact(self, value: Any) -> ProvEntity: def used_artefacts( self, - job_order: Union[CWLObjectType, List[CWLObjectType]], + job_order: Union[CWLObjectType, list[CWLObjectType]], process_run_id: str, name: Optional[str] = None, ) -> None: @@ -704,7 +693,7 @@ def activity_has_provenance(self, activity: str, prov_ids: Sequence[Identifier]) """Add http://www.w3.org/TR/prov-aq/ relations to nested PROV files.""" # NOTE: The below will only work if the corresponding metadata/provenance arcp URI # is a pre-registered namespace in the PROV Document - attribs: List[Tuple[Union[str, Identifier], Any]] = [ + attribs: list[tuple[Union[str, Identifier], Any]] = [ (PROV["has_provenance"], prov_id) for prov_id in prov_ids ] self.document.activity(activity, other_attributes=attribs) @@ -713,7 +702,7 @@ def activity_has_provenance(self, activity: str, prov_ids: Sequence[Identifier]) uris = [i.uri for i in prov_ids] self.research_object.add_annotation(activity, uris, PROV["has_provenance"].uri) - def finalize_prov_profile(self, name: Optional[str]) -> List[QualifiedName]: + def finalize_prov_profile(self, name: Optional[str]) -> list[QualifiedName]: """Transfer the provenance related files to the RO.""" # NOTE: Relative posix path if name is None: diff --git a/cwltool/cwlprov/ro.py b/cwltool/cwlprov/ro.py index 7c6eaf5d6..ac60afc92 100644 --- a/cwltool/cwlprov/ro.py +++ b/cwltool/cwlprov/ro.py @@ -7,20 +7,9 @@ import tempfile import urllib import uuid +from collections.abc import MutableMapping, MutableSequence from pathlib import Path, PurePosixPath -from typing import ( - IO, - Any, - Dict, - List, - MutableMapping, - MutableSequence, - Optional, - Set, - Tuple, - Union, - cast, -) +from typing import IO, Any, Optional, Union, cast import prov.model as provM from prov.model import PROV, ProvDocument @@ -75,12 +64,12 @@ def __init__( self.folder = create_tmp_dir(temp_prefix_ro) self.closed = False # map of filename "data/de/alsdklkas": 12398123 bytes - self.bagged_size: Dict[str, int] = {} - self.tagfiles: Set[str] = set() - self._file_provenance: Dict[str, Aggregate] = {} - self._external_aggregates: List[Aggregate] = [] - self.annotations: List[Annotation] = [] - self._content_types: Dict[str, str] = {} + self.bagged_size: dict[str, int] = {} + self.tagfiles: set[str] = set() + self._file_provenance: dict[str, Aggregate] = {} + self._external_aggregates: list[Aggregate] = [] + self.annotations: list[Annotation] = [] + self._content_types: dict[str, str] = {} self.fsaccess = fsaccess # These should be replaced by generate_prov_doc when workflow/run IDs are known: self.engine_uuid = f"urn:uuid:{uuid.uuid4()}" @@ -202,14 +191,14 @@ def add_tagfile(self, path: str, timestamp: Optional[datetime.datetime] = None) "conformsTo": None, } - def _ro_aggregates(self) -> List[Aggregate]: + def _ro_aggregates(self) -> list[Aggregate]: """Gather dictionary of files to be added to the manifest.""" def guess_mediatype( rel_path: str, - ) -> Tuple[Optional[str], Optional[Union[str, List[str]]]]: + ) -> tuple[Optional[str], Optional[Union[str, list[str]]]]: """Return the mediatypes.""" - media_types: Dict[Union[str, None], str] = { + media_types: dict[Union[str, None], str] = { # Adapted from # https://w3id.org/bundle/2014-11-05/#media-types "txt": TEXT_PLAIN, @@ -223,12 +212,12 @@ def guess_mediatype( "provn": 'text/provenance-notation; charset="UTF-8"', "nt": "application/n-triples", } - conforms_to: Dict[Union[str, None], str] = { + conforms_to: dict[Union[str, None], str] = { "provn": "http://www.w3.org/TR/2013/REC-prov-n-20130430/", "cwl": "https://w3id.org/cwl/", } - prov_conforms_to: Dict[str, str] = { + prov_conforms_to: dict[str, str] = { "provn": "http://www.w3.org/TR/2013/REC-prov-n-20130430/", "rdf": "http://www.w3.org/TR/2013/REC-prov-o-20130430/", "ttl": "http://www.w3.org/TR/2013/REC-prov-o-20130430/", @@ -244,7 +233,7 @@ def guess_mediatype( extension = None mediatype: Optional[str] = media_types.get(extension, None) - conformsTo: Optional[Union[str, List[str]]] = conforms_to.get(extension, None) + conformsTo: Optional[Union[str, list[str]]] = conforms_to.get(extension, None) # TODO: Open CWL file to read its declared "cwlVersion", e.g. # cwlVersion = "v1.0" @@ -261,7 +250,7 @@ def guess_mediatype( conformsTo = prov_conforms_to[extension] return (mediatype, conformsTo) - aggregates: List[Aggregate] = [] + aggregates: list[Aggregate] = [] for path in self.bagged_size.keys(): temp_path = PurePosixPath(path) folder = temp_path.parent @@ -291,7 +280,7 @@ def guess_mediatype( bundledAs.update(self._file_provenance[path]) else: aggregate_dict["bundledAs"] = cast( - Optional[Dict[str, Any]], self._file_provenance[path] + Optional[dict[str, Any]], self._file_provenance[path] ) else: # Probably made outside wf run, part of job object? @@ -343,7 +332,7 @@ def add_uri(self, uri: str, timestamp: Optional[datetime.datetime] = None) -> Ag return aggr def add_annotation( - self, about: str, content: List[str], motivated_by: str = "oa:describing" + self, about: str, content: list[str], motivated_by: str = "oa:describing" ) -> str: """Cheap URI relativize for current directory and /.""" self.self_check() @@ -359,9 +348,9 @@ def add_annotation( self.annotations.append(ann) return uri - def _ro_annotations(self) -> List[Annotation]: + def _ro_annotations(self) -> list[Annotation]: """Append base RO and provenance annotations to the list of annotations.""" - annotations: List[Annotation] = [] + annotations: list[Annotation] = [] annotations.append( { "uri": uuid.uuid4().urn, @@ -511,7 +500,7 @@ def add_data_file( def _self_made( self, timestamp: Optional[datetime.datetime] = None - ) -> Tuple[str, Dict[str, str]]: # createdOn, createdBy + ) -> tuple[str, dict[str, str]]: # createdOn, createdBy if timestamp is None: timestamp = datetime.datetime.now() return ( @@ -519,7 +508,7 @@ def _self_made( {"uri": self.engine_uuid, "name": self.cwltool_version}, ) - def add_to_manifest(self, rel_path: str, checksums: Dict[str, str]) -> None: + def add_to_manifest(self, rel_path: str, checksums: dict[str, str]) -> None: """Add files to the research object manifest.""" self.self_check() if PurePosixPath(rel_path).is_absolute(): diff --git a/cwltool/cwlprov/writablebagfile.py b/cwltool/cwlprov/writablebagfile.py index d5ff3c731..06d7d0bf7 100644 --- a/cwltool/cwlprov/writablebagfile.py +++ b/cwltool/cwlprov/writablebagfile.py @@ -8,10 +8,11 @@ import uuid from array import array from collections import OrderedDict +from collections.abc import MutableMapping from io import FileIO, TextIOWrapper from mmap import mmap from pathlib import Path, PurePosixPath -from typing import Any, BinaryIO, Dict, MutableMapping, Optional, Union, cast +from typing import Any, BinaryIO, Optional, Union, cast from schema_salad.utils import json_dumps @@ -246,7 +247,7 @@ def create_job( relativised_input_objecttemp: CWLObjectType = {} research_object._relativise_files(copied) - def jdefault(o: Any) -> Dict[Any, Any]: + def jdefault(o: Any) -> dict[Any, Any]: return dict(o) if is_output: diff --git a/cwltool/cwlrdf.py b/cwltool/cwlrdf.py index dbe9e2f97..126f0c780 100644 --- a/cwltool/cwlrdf.py +++ b/cwltool/cwlrdf.py @@ -1,6 +1,7 @@ import urllib from codecs import StreamWriter -from typing import IO, Any, Dict, Iterator, Optional, TextIO, Union, cast +from collections.abc import Iterator +from typing import IO, Any, Optional, TextIO, Union, cast from rdflib import Graph from rdflib.query import ResultRow @@ -117,7 +118,7 @@ def dot_with_parameters(g: Graph, stdout: Union[TextIO, StreamWriter]) -> None: def dot_without_parameters(g: Graph, stdout: Union[TextIO, StreamWriter]) -> None: - dotname: Dict[str, str] = {} + dotname: dict[str, str] = {} clusternode = {} stdout.write("compound=true\n") diff --git a/cwltool/cwlviewer.py b/cwltool/cwlviewer.py index e544a568e..769343964 100644 --- a/cwltool/cwlviewer.py +++ b/cwltool/cwlviewer.py @@ -1,7 +1,8 @@ """Visualize a CWL workflow.""" +from collections.abc import Iterator from pathlib import Path -from typing import Iterator, List, cast +from typing import cast from urllib.parse import urlparse import pydot @@ -154,7 +155,7 @@ def _get_root_graph_uri(self) -> rdflib.term.Identifier: with open(_get_root_query_path) as f: get_root_query = f.read() root = cast( - List[rdflib.query.ResultRow], + list[rdflib.query.ResultRow], list( self._rdf_graph.query( get_root_query, diff --git a/cwltool/docker.py b/cwltool/docker.py index d0f628b15..b03ae635c 100644 --- a/cwltool/docker.py +++ b/cwltool/docker.py @@ -9,8 +9,9 @@ import subprocess # nosec import sys import threading +from collections.abc import MutableMapping from io import StringIO # pylint: disable=redefined-builtin -from typing import Callable, Dict, List, MutableMapping, Optional, Set, Tuple, cast +from typing import Callable, Optional, cast import requests @@ -23,13 +24,13 @@ from .pathmapper import MapperEnt, PathMapper from .utils import CWLObjectType, create_tmp_dir, ensure_writable -_IMAGES: Set[str] = set() +_IMAGES: set[str] = set() _IMAGES_LOCK = threading.Lock() -__docker_machine_mounts: Optional[List[str]] = None +__docker_machine_mounts: Optional[list[str]] = None __docker_machine_mounts_lock = threading.Lock() -def _get_docker_machine_mounts() -> List[str]: +def _get_docker_machine_mounts() -> list[str]: global __docker_machine_mounts if __docker_machine_mounts is None: with __docker_machine_mounts_lock: @@ -83,9 +84,9 @@ def __init__( self, builder: Builder, joborder: CWLObjectType, - make_path_mapper: Callable[[List[CWLObjectType], str, RuntimeContext, bool], PathMapper], - requirements: List[CWLObjectType], - hints: List[CWLObjectType], + make_path_mapper: Callable[[list[CWLObjectType], str, RuntimeContext, bool], PathMapper], + requirements: list[CWLObjectType], + hints: list[CWLObjectType], name: str, ) -> None: """Initialize a command line builder using the Docker software container engine.""" @@ -94,7 +95,7 @@ def __init__( def get_image( self, - docker_requirement: Dict[str, str], + docker_requirement: dict[str, str], pull_image: bool, force_pull: bool, tmp_outdir_prefix: str, @@ -127,7 +128,7 @@ def get_image( except (OSError, subprocess.CalledProcessError, UnicodeError): pass - cmd: List[str] = [] + cmd: list[str] = [] if "dockerFile" in docker_requirement: dockerfile_dir = create_tmp_dir(tmp_outdir_prefix) with open(os.path.join(dockerfile_dir, "Dockerfile"), "w") as dfile: @@ -204,13 +205,13 @@ def get_from_requirements( if not shutil.which(self.docker_exec): raise WorkflowException(f"{self.docker_exec} executable is not available") - if self.get_image(cast(Dict[str, str], r), pull_image, force_pull, tmp_outdir_prefix): + if self.get_image(cast(dict[str, str], r), pull_image, force_pull, tmp_outdir_prefix): return cast(Optional[str], r["dockerImageId"]) raise WorkflowException("Docker image %s not found" % r["dockerImageId"]) @staticmethod def append_volume( - runtime: List[str], + runtime: list[str], source: str, target: str, writable: bool = False, @@ -233,7 +234,7 @@ def append_volume( os.makedirs(source) def add_file_or_directory_volume( - self, runtime: List[str], volume: MapperEnt, host_outdir_tgt: Optional[str] + self, runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str] ) -> None: """Append volume a file/dir mapping to the runtime option list.""" if not volume.resolved.startswith("_:"): @@ -242,7 +243,7 @@ def add_file_or_directory_volume( def add_writable_file_volume( self, - runtime: List[str], + runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str], tmpdir_prefix: str, @@ -266,7 +267,7 @@ def add_writable_file_volume( def add_writable_directory_volume( self, - runtime: List[str], + runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str], tmpdir_prefix: str, @@ -295,7 +296,7 @@ def add_writable_directory_volume( shutil.copytree(volume.resolved, host_outdir_tgt) ensure_writable(host_outdir_tgt or new_dir) - def _required_env(self) -> Dict[str, str]: + def _required_env(self) -> dict[str, str]: # spec currently says "HOME must be set to the designated output # directory." but spec might change to designated temp directory. # runtime.append("--env=HOME=/tmp") @@ -306,7 +307,7 @@ def _required_env(self) -> Dict[str, str]: def create_runtime( self, env: MutableMapping[str, str], runtimeContext: RuntimeContext - ) -> Tuple[List[str], Optional[str]]: + ) -> tuple[list[str], Optional[str]]: any_path_okay = self.builder.get_requirement("DockerRequirement")[1] or False user_space_docker_cmd = runtimeContext.user_space_docker_cmd if user_space_docker_cmd: @@ -445,9 +446,9 @@ def __init__( self, builder: Builder, joborder: CWLObjectType, - make_path_mapper: Callable[[List[CWLObjectType], str, RuntimeContext, bool], PathMapper], - requirements: List[CWLObjectType], - hints: List[CWLObjectType], + make_path_mapper: Callable[[list[CWLObjectType], str, RuntimeContext, bool], PathMapper], + requirements: list[CWLObjectType], + hints: list[CWLObjectType], name: str, ) -> None: """Initialize a command line builder using the Podman software container engine.""" diff --git a/cwltool/docker_id.py b/cwltool/docker_id.py index bb436b2cb..90484b686 100644 --- a/cwltool/docker_id.py +++ b/cwltool/docker_id.py @@ -1,10 +1,10 @@ """Helper functions for docker.""" import subprocess # nosec -from typing import List, Optional, Tuple +from typing import Optional -def docker_vm_id() -> Tuple[Optional[int], Optional[int]]: +def docker_vm_id() -> tuple[Optional[int], Optional[int]]: """ Return the User ID and Group ID of the default docker user inside the VM. @@ -21,7 +21,7 @@ def docker_vm_id() -> Tuple[Optional[int], Optional[int]]: return (None, None) -def check_output_and_strip(cmd: List[str]) -> Optional[str]: +def check_output_and_strip(cmd: list[str]) -> Optional[str]: """ Pass a command list to :py:func:`subprocess.check_output`. @@ -48,7 +48,7 @@ def docker_machine_name() -> Optional[str]: return check_output_and_strip(["docker-machine", "active"]) -def cmd_output_matches(check_cmd: List[str], expected_status: str) -> bool: +def cmd_output_matches(check_cmd: list[str], expected_status: str) -> bool: """ Run a command and compares output to expected. @@ -80,7 +80,7 @@ def docker_machine_running() -> bool: return cmd_output_matches(["docker-machine", "status", machine_name], "Running") -def cmd_output_to_int(cmd: List[str]) -> Optional[int]: +def cmd_output_to_int(cmd: list[str]) -> Optional[int]: """ Run the provided command and returns the integer value of the result. @@ -97,7 +97,7 @@ def cmd_output_to_int(cmd: List[str]) -> Optional[int]: return None -def boot2docker_id() -> Tuple[Optional[int], Optional[int]]: +def boot2docker_id() -> tuple[Optional[int], Optional[int]]: """ Get the UID and GID of the docker user inside a running boot2docker vm. @@ -108,7 +108,7 @@ def boot2docker_id() -> Tuple[Optional[int], Optional[int]]: return (uid, gid) -def docker_machine_id() -> Tuple[Optional[int], Optional[int]]: +def docker_machine_id() -> tuple[Optional[int], Optional[int]]: """ Ask docker-machine for active machine and gets the UID of the docker user. diff --git a/cwltool/env_to_stdout.py b/cwltool/env_to_stdout.py index 33b832479..0309fe08f 100644 --- a/cwltool/env_to_stdout.py +++ b/cwltool/env_to_stdout.py @@ -11,10 +11,9 @@ """ import os -from typing import Dict -def deserialize_env(data: str) -> Dict[str, str]: +def deserialize_env(data: str) -> dict[str, str]: """Deserialize the output of `env -0` to dictionary.""" result = {} for item in data.strip("\0").split("\0"): diff --git a/cwltool/executors.py b/cwltool/executors.py index bfc87f9c7..6070462ab 100644 --- a/cwltool/executors.py +++ b/cwltool/executors.py @@ -7,18 +7,9 @@ import os import threading from abc import ABCMeta, abstractmethod +from collections.abc import Iterable, MutableSequence from threading import Lock -from typing import ( - Dict, - Iterable, - List, - MutableSequence, - Optional, - Set, - Tuple, - Union, - cast, -) +from typing import Optional, Union, cast import psutil from mypy_extensions import mypyc_attr @@ -50,8 +41,8 @@ class JobExecutor(metaclass=ABCMeta): def __init__(self) -> None: """Initialize.""" self.final_output: MutableSequence[Optional[CWLObjectType]] = [] - self.final_status: List[str] = [] - self.output_dirs: Set[str] = set() + self.final_status: list[str] = [] + self.output_dirs: set[str] = set() def __call__( self, @@ -59,7 +50,7 @@ def __call__( job_order_object: CWLObjectType, runtime_context: RuntimeContext, logger: logging.Logger = _logger, - ) -> Tuple[Optional[CWLObjectType], str]: + ) -> tuple[Optional[CWLObjectType], str]: return self.execute(process, job_order_object, runtime_context, logger) def output_callback(self, out: Optional[CWLObjectType], process_status: str) -> None: @@ -83,7 +74,7 @@ def execute( job_order_object: CWLObjectType, runtime_context: RuntimeContext, logger: logging.Logger = _logger, - ) -> Tuple[Union[Optional[CWLObjectType]], str]: + ) -> tuple[Union[Optional[CWLObjectType]], str]: """Execute the process.""" self.final_output = [] @@ -112,7 +103,7 @@ def check_for_abstract_op(tool: CWLObjectType) -> None: runtime_context.toplevel = True runtime_context.workflow_eval_lock = threading.Condition(threading.RLock()) - job_reqs: Optional[List[CWLObjectType]] = None + job_reqs: Optional[list[CWLObjectType]] = None if "https://w3id.org/cwl/cwl#requirements" in job_order_object: if process.metadata.get(ORIGINAL_CWLVERSION) == "v1.0": raise WorkflowException( @@ -121,7 +112,7 @@ def check_for_abstract_op(tool: CWLObjectType) -> None: "can set the cwlVersion to v1.1" ) job_reqs = cast( - List[CWLObjectType], + list[CWLObjectType], job_order_object["https://w3id.org/cwl/cwl#requirements"], ) elif "cwl:defaults" in process.metadata and "https://w3id.org/cwl/cwl#requirements" in cast( @@ -134,7 +125,7 @@ def check_for_abstract_op(tool: CWLObjectType) -> None: "can set the cwlVersion to v1.1" ) job_reqs = cast( - Optional[List[CWLObjectType]], + Optional[list[CWLObjectType]], cast(CWLObjectType, process.metadata["cwl:defaults"])[ "https://w3id.org/cwl/cwl#requirements" ], @@ -277,8 +268,8 @@ class MultithreadedJobExecutor(JobExecutor): def __init__(self) -> None: """Initialize.""" super().__init__() - self.exceptions: List[WorkflowException] = [] - self.pending_jobs: List[JobsType] = [] + self.exceptions: list[WorkflowException] = [] + self.pending_jobs: list[JobsType] = [] self.pending_jobs_lock = threading.Lock() self.max_ram = int(psutil.virtual_memory().available / 2**20) @@ -289,10 +280,10 @@ def __init__(self) -> None: self.allocated_cuda: int = 0 def select_resources( - self, request: Dict[str, Union[int, float]], runtime_context: RuntimeContext - ) -> Dict[str, Union[int, float]]: # pylint: disable=unused-argument + self, request: dict[str, Union[int, float]], runtime_context: RuntimeContext + ) -> dict[str, Union[int, float]]: # pylint: disable=unused-argument """Naïve check for available cpu cores and memory.""" - result: Dict[str, Union[int, float]] = {} + result: dict[str, Union[int, float]] = {} maxrsc = {"cores": self.max_cores, "ram": self.max_ram} resources_types = {"cores", "ram"} if "cudaDeviceCountMin" in request or "cudaDeviceCountMax" in request: @@ -491,5 +482,5 @@ def execute( job_order_object: CWLObjectType, runtime_context: RuntimeContext, logger: Optional[logging.Logger] = None, - ) -> Tuple[Optional[CWLObjectType], str]: + ) -> tuple[Optional[CWLObjectType], str]: return {}, "success" diff --git a/cwltool/factory.py b/cwltool/factory.py index 85d7344e6..eaf98e3cf 100644 --- a/cwltool/factory.py +++ b/cwltool/factory.py @@ -1,5 +1,5 @@ import os -from typing import Any, Dict, Optional, Union +from typing import Any, Optional, Union from . import load_tool from .context import LoadingContext, RuntimeContext @@ -62,7 +62,7 @@ def __init__( else: self.loading_context = loading_context - def make(self, cwl: Union[str, Dict[str, Any]]) -> Callable: + def make(self, cwl: Union[str, dict[str, Any]]) -> Callable: """Instantiate a CWL object from a CWl document.""" load = load_tool.load_tool(cwl, self.loading_context) if isinstance(load, int): diff --git a/cwltool/flatten.py b/cwltool/flatten.py index 420d90d04..5c9738cbf 100644 --- a/cwltool/flatten.py +++ b/cwltool/flatten.py @@ -1,4 +1,4 @@ -from typing import Any, Callable, List, cast +from typing import Any, Callable, cast # http://rightfootin.blogspot.com/2006/09/more-on-python-flatten.html @@ -22,4 +22,4 @@ def flatten(thing, ltypes=(list, tuple)): else: thing_list[i : i + 1] = thing_list[i] i += 1 - return cast(Callable[[Any], List[Any]], ltype)(thing_list) + return cast(Callable[[Any], list[Any]], ltype)(thing_list) diff --git a/cwltool/job.py b/cwltool/job.py index 1731a5350..b360be25f 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -16,24 +16,10 @@ import time import uuid from abc import ABCMeta, abstractmethod +from collections.abc import Iterable, Mapping, MutableMapping, MutableSequence +from re import Match from threading import Timer -from typing import ( - IO, - TYPE_CHECKING, - Callable, - Dict, - Iterable, - List, - Mapping, - Match, - MutableMapping, - MutableSequence, - Optional, - TextIO, - Tuple, - Union, - cast, -) +from typing import IO, TYPE_CHECKING, Callable, Optional, TextIO, Union, cast import psutil from prov.model import PROV @@ -122,9 +108,9 @@ def __init__( self, builder: Builder, joborder: CWLObjectType, - make_path_mapper: Callable[[List[CWLObjectType], str, RuntimeContext, bool], PathMapper], - requirements: List[CWLObjectType], - hints: List[CWLObjectType], + make_path_mapper: Callable[[list[CWLObjectType], str, RuntimeContext, bool], PathMapper], + requirements: list[CWLObjectType], + hints: list[CWLObjectType], name: str, ) -> None: """Initialize the job object.""" @@ -140,7 +126,7 @@ def __init__( self.requirements = requirements self.hints = hints self.name = name - self.command_line: List[str] = [] + self.command_line: list[str] = [] self.pathmapper = PathMapper([], "", "") self.make_path_mapper = make_path_mapper self.generatemapper: Optional[PathMapper] = None @@ -228,7 +214,7 @@ def is_streamable(file: str) -> bool: def _execute( self, - runtime: List[str], + runtime: list[str], env: MutableMapping[str, str], runtimeContext: RuntimeContext, monitor_function: Optional[Callable[["subprocess.Popen[str]"], None]] = None, @@ -321,7 +307,7 @@ def stderr_stdout_log_path( commands = [str(x) for x in runtime + self.command_line] if runtimeContext.secret_store is not None: commands = cast( - List[str], + list[str], runtimeContext.secret_store.retrieve(cast(CWLOutputType, commands)), ) env = cast( @@ -456,7 +442,7 @@ def stderr_stdout_log_path( shutil.rmtree(self.tmpdir, True) @abstractmethod - def _required_env(self) -> Dict[str, str]: + def _required_env(self) -> dict[str, str]: """Variables required by the CWL spec (HOME, TMPDIR, etc). Note that with containers, the paths will (likely) be those from @@ -481,7 +467,7 @@ def prepare_environment( applied (in that order). """ # Start empty - env: Dict[str, str] = {} + env: dict[str, str] = {} # Preserve any env vars if runtimeContext.preserve_entire_environment: @@ -589,7 +575,7 @@ def run( self._execute([], self.environment, runtimeContext, monitor_function) - def _required_env(self) -> Dict[str, str]: + def _required_env(self) -> dict[str, str]: env = {} env["HOME"] = self.outdir env["TMPDIR"] = self.tmpdir @@ -623,24 +609,24 @@ def create_runtime( self, env: MutableMapping[str, str], runtime_context: RuntimeContext, - ) -> Tuple[List[str], Optional[str]]: + ) -> tuple[list[str], Optional[str]]: """Return the list of commands to run the selected container engine.""" @staticmethod @abstractmethod - def append_volume(runtime: List[str], source: str, target: str, writable: bool = False) -> None: + def append_volume(runtime: list[str], source: str, target: str, writable: bool = False) -> None: """Add binding arguments to the runtime list.""" @abstractmethod def add_file_or_directory_volume( - self, runtime: List[str], volume: MapperEnt, host_outdir_tgt: Optional[str] + self, runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str] ) -> None: """Append volume a file/dir mapping to the runtime option list.""" @abstractmethod def add_writable_file_volume( self, - runtime: List[str], + runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str], tmpdir_prefix: str, @@ -650,7 +636,7 @@ def add_writable_file_volume( @abstractmethod def add_writable_directory_volume( self, - runtime: List[str], + runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str], tmpdir_prefix: str, @@ -674,7 +660,7 @@ def _preserve_environment_on_containers_warning( def create_file_and_add_volume( self, - runtime: List[str], + runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str], secret_store: Optional[SecretStore], @@ -706,7 +692,7 @@ def create_file_and_add_volume( def add_volumes( self, pathmapper: PathMapper, - runtime: List[str], + runtime: list[str], tmpdir_prefix: str, secret_store: Optional[SecretStore] = None, any_path_okay: bool = False, @@ -918,7 +904,7 @@ def docker_monitor( def _job_popen( - commands: List[str], + commands: list[str], stdin_path: Optional[str], stdout_path: Optional[str], stderr_path: Optional[str], diff --git a/cwltool/load_tool.py b/cwltool/load_tool.py index d6352f918..7a58a8330 100644 --- a/cwltool/load_tool.py +++ b/cwltool/load_tool.py @@ -7,18 +7,9 @@ import re import urllib import uuid +from collections.abc import MutableMapping, MutableSequence from functools import partial -from typing import ( - Any, - Dict, - List, - MutableMapping, - MutableSequence, - Optional, - Tuple, - Union, - cast, -) +from typing import Any, Optional, Union, cast from cwl_utils.parser import cwl_v1_2, cwl_v1_2_utils from ruamel.yaml.comments import CommentedMap, CommentedSeq @@ -93,7 +84,7 @@ def resolve_tool_uri( resolver: Optional[ResolverType] = None, fetcher_constructor: Optional[FetcherCallableType] = None, document_loader: Optional[Loader] = None, -) -> Tuple[str, str]: +) -> tuple[str, str]: uri = None # type: Optional[str] split = urllib.parse.urlsplit(argsworkflow) # In case of Windows path, urlsplit misjudge Drive letters as scheme, here we are skipping that @@ -117,7 +108,7 @@ def resolve_tool_uri( def fetch_document( argsworkflow: Union[str, CWLObjectType], loadingContext: Optional[LoadingContext] = None, -) -> Tuple[LoadingContext, CommentedMap, str]: +) -> tuple[LoadingContext, CommentedMap, str]: """Retrieve a CWL document.""" if loadingContext is None: loadingContext = LoadingContext() @@ -144,7 +135,7 @@ def fetch_document( return loadingContext, workflowobj, uri if isinstance(argsworkflow, MutableMapping): uri = cast(str, argsworkflow["id"]) if argsworkflow.get("id") else "_:" + str(uuid.uuid4()) - workflowobj = cast(CommentedMap, cmap(cast(Dict[str, Any], argsworkflow), fn=uri)) + workflowobj = cast(CommentedMap, cmap(cast(dict[str, Any], argsworkflow), fn=uri)) loadingContext.loader.idx[uri] = workflowobj return loadingContext, workflowobj, uri raise ValidationException("Must be URI or object: '%s'" % argsworkflow) @@ -306,7 +297,7 @@ def fast_parser( uri: str, loadingContext: LoadingContext, fetcher: Fetcher, -) -> Tuple[Union[CommentedMap, CommentedSeq], CommentedMap]: +) -> tuple[Union[CommentedMap, CommentedSeq], CommentedMap]: lopt = cwl_v1_2.LoadingOptions(idx=loadingContext.codegen_idx, fileuri=fileuri, fetcher=fetcher) if uri not in loadingContext.codegen_idx: @@ -326,7 +317,7 @@ def fast_parser( processobj = cwl_v1_2.save(objects, relative_uris=False) - metadata: Dict[str, Any] = {} + metadata: dict[str, Any] = {} metadata["id"] = loadopt.fileuri if loadopt.namespaces: @@ -353,7 +344,7 @@ def fast_parser( objects, loadopt = loadingContext.codegen_idx[nofrag] fileobj = cmap( cast( - Union[int, float, str, Dict[str, Any], List[Any], None], + Union[int, float, str, dict[str, Any], list[Any], None], cwl_v1_2.save(objects, relative_uris=False), ) ) @@ -370,7 +361,7 @@ def fast_parser( return cast( Union[CommentedMap, CommentedSeq], - cmap(cast(Union[Dict[str, Any], List[Any]], processobj)), + cmap(cast(Union[dict[str, Any], list[Any]], processobj)), ), cast(CommentedMap, cmap(metadata)) @@ -379,7 +370,7 @@ def resolve_and_validate_document( workflowobj: Union[CommentedMap, CommentedSeq], uri: str, preprocess_only: bool = False, -) -> Tuple[LoadingContext, str]: +) -> tuple[LoadingContext, str]: """Validate a CWL document.""" if not loadingContext.loader: raise ValueError("loadingContext must have a loader.") @@ -394,7 +385,7 @@ def resolve_and_validate_document( if "cwl:tool" in workflowobj: jobobj, _ = loader.resolve_all(workflowobj, uri) uri = urllib.parse.urljoin(uri, workflowobj["https://w3id.org/cwl/cwl#tool"]) - del cast(Dict[str, Any], jobobj)["https://w3id.org/cwl/cwl#tool"] + del cast(dict[str, Any], jobobj)["https://w3id.org/cwl/cwl#tool"] workflowobj = fetch_document(uri, loadingContext)[1] @@ -624,17 +615,17 @@ def resolve_overrides( ov: IdxResultType, ov_uri: str, baseurl: str, -) -> List[CWLObjectType]: +) -> list[CWLObjectType]: ovloader = Loader(overrides_ctx) ret, _ = ovloader.resolve_all(ov, baseurl) if not isinstance(ret, CommentedMap): raise Exception("Expected CommentedMap, got %s" % type(ret)) cwl_docloader = get_schema("v1.0")[0] cwl_docloader.resolve_all(ret, ov_uri) - return cast(List[CWLObjectType], ret["http://commonwl.org/cwltool#overrides"]) + return cast(list[CWLObjectType], ret["http://commonwl.org/cwltool#overrides"]) -def load_overrides(ov: str, base_url: str) -> List[CWLObjectType]: +def load_overrides(ov: str, base_url: str) -> list[CWLObjectType]: ovloader = Loader(overrides_ctx) return resolve_overrides(ovloader.fetch(ov), ov, base_url) @@ -644,7 +635,7 @@ def recursive_resolve_and_validate_document( workflowobj: Union[CommentedMap, CommentedSeq], uri: str, preprocess_only: bool = False, -) -> Tuple[LoadingContext, str, Process]: +) -> tuple[LoadingContext, str, Process]: """Validate a CWL document, checking that a tool object can be built.""" loadingContext, uri = resolve_and_validate_document( loadingContext, diff --git a/cwltool/main.py b/cwltool/main.py index 30f299f09..9477cb1a2 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -15,21 +15,8 @@ import urllib import warnings from codecs import getwriter -from typing import ( - IO, - Any, - Callable, - Dict, - List, - Mapping, - MutableMapping, - MutableSequence, - Optional, - Sized, - Tuple, - Union, - cast, -) +from collections.abc import Mapping, MutableMapping, MutableSequence, Sized +from typing import IO, Any, Callable, Optional, Union, cast import argcomplete import coloredlogs @@ -185,11 +172,11 @@ def append_word_to_default_user_agent(word: str) -> None: def generate_example_input( inptype: Optional[CWLOutputType], default: Optional[CWLOutputType], -) -> Tuple[Any, str]: +) -> tuple[Any, str]: """Convert a single input schema into an example.""" example = None comment = "" - defaults = { + defaults: CWLObjectType = { "null": "null", "Any": "null", "boolean": False, @@ -202,7 +189,7 @@ def generate_example_input( "Directory": ruamel.yaml.comments.CommentedMap( [("class", "Directory"), ("path", "a/directory/path")] ), - } # type: CWLObjectType + } if isinstance(inptype, MutableSequence): optional = False if "null" in inptype: @@ -244,7 +231,7 @@ def generate_example_input( if default is not None: example = default elif inptype["type"] == "enum": - symbols = cast(List[str], inptype["symbols"]) + symbols = cast(list[str], inptype["symbols"]) if default is not None: example = default elif "default" in inptype: @@ -260,7 +247,7 @@ def generate_example_input( comment = '"{}" record type.'.format(inptype["name"]) else: comment = "Anonymous record type." - for field in cast(List[CWLObjectType], inptype["fields"]): + for field in cast(list[CWLObjectType], inptype["fields"]): value, f_comment = generate_example_input(field["type"], None) example.insert(0, shortname(cast(str, field["name"])), value, f_comment) elif "default" in inptype: @@ -343,7 +330,7 @@ def generate_input_template(tool: Process) -> CWLObjectType: """Generate an example input object for the given CWL process.""" template = ruamel.yaml.comments.CommentedMap() for inp in cast( - List[MutableMapping[str, str]], + list[MutableMapping[str, str]], realize_input_schema(tool.tool["inputs"], tool.schemaDefs), ): name = shortname(inp["id"]) @@ -356,9 +343,9 @@ def load_job_order( args: argparse.Namespace, stdin: IO[Any], fetcher_constructor: Optional[FetcherCallableType], - overrides_list: List[CWLObjectType], + overrides_list: list[CWLObjectType], tool_file_uri: str, -) -> Tuple[Optional[CWLObjectType], str, Loader]: +) -> tuple[Optional[CWLObjectType], str, Loader]: job_order_object = None job_order_file = None @@ -423,8 +410,8 @@ def init_job_order( ) -> CWLObjectType: secrets_req, _ = process.get_requirement("http://commonwl.org/cwltool#Secrets") if job_order_object is None: - namemap = {} # type: Dict[str, str] - records = [] # type: List[str] + namemap: dict[str, str] = {} + records: list[str] = [] toolparser = generate_parser( argparse.ArgumentParser(prog=args.workflow), process, @@ -463,7 +450,7 @@ def init_job_order( if secret_store and secrets_req: secret_store.store( - [shortname(sc) for sc in cast(List[str], secrets_req["secrets"])], + [shortname(sc) for sc in cast(list[str], secrets_req["secrets"])], job_order_object, ) @@ -486,7 +473,7 @@ def path_to_loc(p: CWLObjectType) -> None: p["location"] = p["path"] del p["path"] - ns = {} # type: ContextType + ns: ContextType = {} ns.update(cast(ContextType, job_order_object.get("$namespaces", {}))) ns.update(cast(ContextType, process.metadata.get("$namespaces", {}))) ld = Loader(ns) @@ -532,7 +519,7 @@ def expand_formats(p: CWLObjectType) -> None: if secret_store and secrets_req: secret_store.store( - [shortname(sc) for sc in cast(List[str], secrets_req["secrets"])], + [shortname(sc) for sc in cast(list[str], secrets_req["secrets"])], job_order_object, ) @@ -583,7 +570,7 @@ def prov_deps( def remove_non_cwl(deps: CWLObjectType) -> None: if "secondaryFiles" in deps: - sec_files = cast(List[CWLObjectType], deps["secondaryFiles"]) + sec_files = cast(list[CWLObjectType], deps["secondaryFiles"]) for index, entry in enumerate(sec_files): if not ("format" in entry and entry["format"] == CWL_IANA): del sec_files[index] @@ -602,11 +589,11 @@ def find_deps( nestdirs: bool = True, ) -> CWLObjectType: """Find the dependencies of the CWL document.""" - deps = { + deps: CWLObjectType = { "class": "File", "location": uri, "format": CWL_IANA, - } # type: CWLObjectType + } def loadref(base: str, uri: str) -> Union[CommentedMap, CommentedSeq, str, None]: return document_loader.fetch(document_loader.fetcher.urljoin(base, uri)) @@ -638,7 +625,7 @@ def print_pack( return json_dumps(target, indent=4, default=str) -def supported_cwl_versions(enable_dev: bool) -> List[str]: +def supported_cwl_versions(enable_dev: bool) -> list[str]: # ALLUPDATES and UPDATES are dicts if enable_dev: versions = list(ALLUPDATES) @@ -692,8 +679,8 @@ def formatTime(self, record: logging.LogRecord, datefmt: Optional[str] = None) - def setup_provenance( args: argparse.Namespace, runtimeContext: RuntimeContext, - argsl: Optional[List[str]] = None, -) -> Tuple[ProvOut, "logging.StreamHandler[ProvOut]"]: + argsl: Optional[list[str]] = None, +) -> tuple[ProvOut, "logging.StreamHandler[ProvOut]"]: if not args.compute_checksum: _logger.error("--provenance incompatible with --no-compute-checksum") raise ArgumentException() @@ -940,7 +927,7 @@ def print_targets( _logger.info("%s steps targets:", prefix[:-1]) for t in tool.tool["steps"]: print(f" {prefix}{shortname(t['id'])}", file=stdout) - run: Union[str, Process, Dict[str, Any]] = t["run"] + run: Union[str, Process, dict[str, Any]] = t["run"] if isinstance(run, str): process = make_tool(run, loading_context) elif isinstance(run, dict): @@ -951,7 +938,7 @@ def print_targets( def main( - argsl: Optional[List[str]] = None, + argsl: Optional[list[str]] = None, args: Optional[argparse.Namespace] = None, job_order_object: Optional[CWLObjectType] = None, stdin: IO[Any] = sys.stdin, @@ -998,7 +985,7 @@ def main( if args is None: if argsl is None: argsl = sys.argv[1:] - addl = [] # type: List[str] + addl: list[str] = [] if "CWLTOOL_OPTIONS" in os.environ: c_opts = os.environ["CWLTOOL_OPTIONS"].split(" ") addl = [x for x in c_opts if x != ""] @@ -1250,7 +1237,7 @@ def main( if args.parallel: temp_executor = MultithreadedJobExecutor() runtimeContext.select_resources = temp_executor.select_resources - real_executor = temp_executor # type: JobExecutor + real_executor: JobExecutor = temp_executor else: real_executor = SingleJobExecutor() else: @@ -1260,7 +1247,7 @@ def main( runtimeContext.basedir = input_basedir if isinstance(tool, ProcessGenerator): - tfjob_order = {} # type: CWLObjectType + tfjob_order: CWLObjectType = {} if loadingContext.jobdefaults: tfjob_order.update(loadingContext.jobdefaults) if job_order_object: diff --git a/cwltool/mpi.py b/cwltool/mpi.py index 2cc1122c6..a7bdcbe03 100644 --- a/cwltool/mpi.py +++ b/cwltool/mpi.py @@ -3,7 +3,8 @@ import inspect import os import re -from typing import List, Mapping, MutableMapping, Optional, Type, TypeVar, Union +from collections.abc import Mapping, MutableMapping +from typing import Optional, TypeVar, Union from schema_salad.utils import yaml_no_ts @@ -18,9 +19,9 @@ def __init__( runner: str = "mpirun", nproc_flag: str = "-n", default_nproc: Union[int, str] = 1, - extra_flags: Optional[List[str]] = None, - env_pass: Optional[List[str]] = None, - env_pass_regex: Optional[List[str]] = None, + extra_flags: Optional[list[str]] = None, + env_pass: Optional[list[str]] = None, + env_pass_regex: Optional[list[str]] = None, env_set: Optional[Mapping[str, str]] = None, ) -> None: """ @@ -46,7 +47,7 @@ def __init__( self.env_set = env_set or {} @classmethod - def load(cls: Type[MpiConfigT], config_file_name: str) -> MpiConfigT: + def load(cls: type[MpiConfigT], config_file_name: str) -> MpiConfigT: """Create the MpiConfig object from the contents of a YAML file. The file must contain exactly one object, whose attributes must diff --git a/cwltool/mutation.py b/cwltool/mutation.py index 077b92cb7..9f58a86cf 100644 --- a/cwltool/mutation.py +++ b/cwltool/mutation.py @@ -1,5 +1,5 @@ from collections import namedtuple -from typing import Dict, cast +from typing import cast from .errors import WorkflowException from .utils import CWLObjectType @@ -20,7 +20,7 @@ class MutationManager: def __init__(self) -> None: """Initialize.""" - self.generations: Dict[str, MutationState] = {} + self.generations: dict[str, MutationState] = {} def register_reader(self, stepname: str, obj: CWLObjectType) -> None: loc = cast(str, obj["location"]) diff --git a/cwltool/pack.py b/cwltool/pack.py index c9fbc4e04..99684e003 100644 --- a/cwltool/pack.py +++ b/cwltool/pack.py @@ -2,17 +2,8 @@ import copy import urllib -from typing import ( - Any, - Callable, - Dict, - MutableMapping, - MutableSequence, - Optional, - Set, - Union, - cast, -) +from collections.abc import MutableMapping, MutableSequence +from typing import Any, Callable, Optional, Union, cast from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.ref_resolver import Loader, SubLoader @@ -30,7 +21,7 @@ def find_run( d: Union[CWLObjectType, ResolveType], loadref: LoadRefType, - runs: Set[str], + runs: set[str], ) -> None: if isinstance(d, MutableSequence): for s in d: @@ -46,7 +37,7 @@ def find_run( def find_ids( d: Union[CWLObjectType, CWLOutputType, MutableSequence[CWLObjectType], None], - ids: Set[str], + ids: set[str], ) -> None: if isinstance(d, MutableSequence): for s in d: @@ -59,7 +50,7 @@ def find_ids( find_ids(cast(CWLOutputType, s2), ids) -def replace_refs(d: Any, rewrite: Dict[str, str], stem: str, newstem: str) -> None: +def replace_refs(d: Any, rewrite: dict[str, str], stem: str, newstem: str) -> None: if isinstance(d, MutableSequence): for s, v in enumerate(d): if isinstance(v, str): @@ -88,7 +79,7 @@ def replace_refs(d: Any, rewrite: Dict[str, str], stem: str, newstem: str) -> No def import_embed( d: Union[MutableSequence[CWLObjectType], CWLObjectType, CWLOutputType], - seen: Set[str], + seen: set[str], ) -> None: if isinstance(d, MutableSequence): for v in d: @@ -114,7 +105,7 @@ def import_embed( def pack( loadingContext: LoadingContext, uri: str, - rewrite_out: Optional[Dict[str, str]] = None, + rewrite_out: Optional[dict[str, str]] = None, loader: Optional[Loader] = None, ) -> CWLObjectType: # The workflow document we have in memory right now may have been @@ -153,7 +144,7 @@ def pack( document_loader.idx[po["id"]] = CommentedMap(po.items()) document_loader.idx[metadata["id"]] = CommentedMap(metadata.items()) - found_versions = {cast(str, loadingContext.metadata["cwlVersion"])} # type: Set[str] + found_versions: set[str] = {cast(str, loadingContext.metadata["cwlVersion"])} def loadref(base: Optional[str], lr_uri: str) -> ResolveType: lr_loadingContext = loadingContext.copy() @@ -167,15 +158,15 @@ def loadref(base: Optional[str], lr_uri: str) -> ResolveType: raise Exception("loader should not be None") return lr_loadingContext.loader.resolve_ref(lr_uri, base_url=base)[0] - input_ids: Set[str] = set() - output_ids: Set[str] = set() + input_ids: set[str] = set() + output_ids: set[str] = set() if isinstance(processobj, MutableSequence): mainobj = processobj[0] else: mainobj = processobj - find_ids(cast(Dict[str, Any], mainobj)["inputs"], input_ids) - find_ids(cast(Dict[str, Any], mainobj)["outputs"], output_ids) + find_ids(cast(dict[str, Any], mainobj)["inputs"], input_ids) + find_ids(cast(dict[str, Any], mainobj)["outputs"], output_ids) runs = {uri} find_run(processobj, loadref, runs) @@ -190,15 +181,15 @@ def loadref(base: Optional[str], lr_uri: str) -> ResolveType: for f in runs: find_ids(document_loader.resolve_ref(f)[0], input_ids) - input_names: Set[str] = set() - output_names: Set[str] = set() + input_names: set[str] = set() + output_names: set[str] = set() - rewrite_inputs: Dict[str, str] = {} - rewrite_outputs: Dict[str, str] = {} + rewrite_inputs: dict[str, str] = {} + rewrite_outputs: dict[str, str] = {} mainpath, _ = urllib.parse.urldefrag(uri) - def rewrite_id(r: str, mainuri: str, rewrite: Dict[str, str], names: Set[str]) -> None: + def rewrite_id(r: str, mainuri: str, rewrite: dict[str, str], names: set[str]) -> None: if r == mainuri: rewrite[r] = "#main" elif r.startswith(mainuri) and r[len(mainuri)] in ("#", "/"): @@ -225,7 +216,7 @@ def rewrite_id(r: str, mainuri: str, rewrite: Dict[str, str], names: Set[str]) - packed = CommentedMap((("$graph", CommentedSeq()), ("cwlVersion", update_to_version))) namespaces = metadata.get("$namespaces", None) - schemas: Set[str] = set() + schemas: set[str] = set() if "$schemas" in metadata: for each_schema in metadata["$schemas"]: schemas.add(each_schema) @@ -261,7 +252,7 @@ def rewrite_id(r: str, mainuri: str, rewrite: Dict[str, str], names: Set[str]) - "Operation", ): continue - dc = cast(Dict[str, Any], copy.deepcopy(dcr)) + dc = cast(dict[str, Any], copy.deepcopy(dcr)) v = rewrite_inputs[r] dc["id"] = v for n in ("name", "cwlVersion", "$namespaces", "$schemas"): diff --git a/cwltool/pathmapper.py b/cwltool/pathmapper.py index 0a06eb47b..86fd9ae82 100644 --- a/cwltool/pathmapper.py +++ b/cwltool/pathmapper.py @@ -4,17 +4,8 @@ import stat import urllib import uuid -from typing import ( - Dict, - ItemsView, - Iterable, - Iterator, - KeysView, - List, - Optional, - Tuple, - cast, -) +from collections.abc import ItemsView, Iterable, Iterator, KeysView +from typing import Optional, cast from mypy_extensions import mypyc_attr from schema_salad.exceptions import ValidationException @@ -92,20 +83,20 @@ class PathMapper: def __init__( self, - referenced_files: List[CWLObjectType], + referenced_files: list[CWLObjectType], basedir: str, stagedir: str, separateDirs: bool = True, ) -> None: """Initialize the PathMapper.""" - self._pathmap: Dict[str, MapperEnt] = {} + self._pathmap: dict[str, MapperEnt] = {} self.stagedir = stagedir self.separateDirs = separateDirs self.setup(dedup(referenced_files), basedir) def visitlisting( self, - listing: List[CWLObjectType], + listing: list[CWLObjectType], stagedir: str, basedir: str, copy: bool = False, @@ -147,7 +138,7 @@ def visit( if location.startswith("file://"): staged = False self.visitlisting( - cast(List[CWLObjectType], obj.get("listing", [])), + cast(list[CWLObjectType], obj.get("listing", [])), tgt, basedir, copy=copy, @@ -189,14 +180,14 @@ def visit( deref, tgt, "WritableFile" if copy else "File", staged ) self.visitlisting( - cast(List[CWLObjectType], obj.get("secondaryFiles", [])), + cast(list[CWLObjectType], obj.get("secondaryFiles", [])), stagedir, basedir, copy=copy, staged=staged, ) - def setup(self, referenced_files: List[CWLObjectType], basedir: str) -> None: + def setup(self, referenced_files: list[CWLObjectType], basedir: str) -> None: # Go through each file and set the target to its own directory along # with any secondary files. stagedir = self.stagedir @@ -246,7 +237,7 @@ def parents(path: str) -> Iterable[str]: def reversemap( self, target: str, - ) -> Optional[Tuple[str, str]]: + ) -> Optional[tuple[str, str]]: """Find the (source, resolved_path) for the given target, if any.""" for k, v in self._pathmap.items(): if v[1] == target: diff --git a/cwltool/process.py b/cwltool/process.py index bde035118..ff96985c5 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -13,25 +13,9 @@ import textwrap import urllib.parse import uuid +from collections.abc import Iterable, Iterator, MutableMapping, MutableSequence, Sized from os import scandir -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Dict, - Iterable, - Iterator, - List, - MutableMapping, - MutableSequence, - Optional, - Set, - Sized, - Tuple, - Type, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Callable, Optional, Union, cast from cwl_utils import expression from mypy_extensions import mypyc_attr @@ -161,14 +145,14 @@ def filter(self, record: logging.LogRecord) -> bool: "vocab_res_proc.yml", ) -SCHEMA_CACHE: Dict[ - str, Tuple[Loader, Union[Names, SchemaParseException], CWLObjectType, Loader] +SCHEMA_CACHE: dict[ + str, tuple[Loader, Union[Names, SchemaParseException], CWLObjectType, Loader] ] = {} SCHEMA_FILE: Optional[CWLObjectType] = None SCHEMA_DIR: Optional[CWLObjectType] = None SCHEMA_ANY: Optional[CWLObjectType] = None -custom_schemas: Dict[str, Tuple[str, str]] = {} +custom_schemas: dict[str, tuple[str, str]] = {} def use_standard_schema(version: str) -> None: @@ -186,11 +170,11 @@ def use_custom_schema(version: str, name: str, text: str) -> None: def get_schema( version: str, -) -> Tuple[Loader, Union[Names, SchemaParseException], CWLObjectType, Loader]: +) -> tuple[Loader, Union[Names, SchemaParseException], CWLObjectType, Loader]: if version in SCHEMA_CACHE: return SCHEMA_CACHE[version] - cache: Dict[str, Union[str, Graph, bool]] = {} + cache: dict[str, Union[str, Graph, bool]] = {} version = version.split("#")[-1] if ".dev" in version: version = ".".join(version.split(".")[:-1]) @@ -244,7 +228,7 @@ def stage_files( :raises WorkflowException: if there is a file staging conflict """ items = pathmapper.items() if not symlink else pathmapper.items_exclude_children() - targets: Dict[str, MapperEnt] = {} + targets: dict[str, MapperEnt] = {} for key, entry in list(items): if "File" not in entry.type: continue @@ -309,11 +293,11 @@ def stage_files( def relocateOutputs( outputObj: CWLObjectType, destination_path: str, - source_directories: Set[str], + source_directories: set[str], action: str, fs_access: StdFsAccess, compute_checksum: bool = True, - path_mapper: Type[PathMapper] = PathMapper, + path_mapper: type[PathMapper] = PathMapper, ) -> CWLObjectType: adjustDirObjs(outputObj, functools.partial(get_listing, fs_access, recursive=True)) @@ -414,7 +398,7 @@ def add_sizes(fsaccess: StdFsAccess, obj: CWLObjectType) -> None: def fill_in_defaults( - inputs: List[CWLObjectType], + inputs: list[CWLObjectType], job: CWLObjectType, fsaccess: StdFsAccess, ) -> None: @@ -578,7 +562,7 @@ def __init__(self, toolpath_object: CommentedMap, loadingContext: LoadingContext self.tool["id"] = "_:" + str(uuid.uuid4()) self.requirements.extend( cast( - List[CWLObjectType], + list[CWLObjectType], get_overrides(getdefault(loadingContext.overrides_list, []), self.tool["id"]).get( "requirements", [] ), @@ -617,7 +601,7 @@ def __init__(self, toolpath_object: CommentedMap, loadingContext: LoadingContext avroize_type(cast(MutableSequence[CWLOutputType], sdtypes)) av = make_valid_avro( sdtypes, - {cast(str, t["name"]): cast(Dict[str, Any], t) for t in sdtypes}, + {cast(str, t["name"]): cast(dict[str, Any], t) for t in sdtypes}, set(), vocab=INPUT_OBJ_VOCAB, ) @@ -655,9 +639,9 @@ def __init__(self, toolpath_object: CommentedMap, loadingContext: LoadingContext c["type"] = avroize_type(c["type"], c["name"]) if key == "inputs": - cast(List[CWLObjectType], self.inputs_record_schema["fields"]).append(c) + cast(list[CWLObjectType], self.inputs_record_schema["fields"]).append(c) elif key == "outputs": - cast(List[CWLObjectType], self.outputs_record_schema["fields"]).append(c) + cast(list[CWLObjectType], self.outputs_record_schema["fields"]).append(c) with SourceLine(toolpath_object, "inputs", ValidationException, debug): self.inputs_record_schema = cast( @@ -681,7 +665,7 @@ def __init__(self, toolpath_object: CommentedMap, loadingContext: LoadingContext if toolpath_object.get("class") is not None and not getdefault( loadingContext.disable_js_validation, False ): - validate_js_options: Optional[Dict[str, Union[List[str], str, int]]] = None + validate_js_options: Optional[dict[str, Union[list[str], str, int]]] = None if loadingContext.js_hint_options_file is not None: try: with open(loadingContext.js_hint_options_file) as options_file: @@ -784,7 +768,7 @@ def _init_job(self, joborder: CWLObjectType, runtime_context: RuntimeContext) -> v = job[k] dircount = [0] - def inc(d: List[int]) -> None: + def inc(d: list[int]) -> None: d[0] += 1 visit_class(v, ("Directory",), lambda x: inc(dircount)) # noqa: B023 @@ -820,7 +804,7 @@ def inc(d: List[int]) -> None: except (ValidationException, WorkflowException) as err: raise WorkflowException("Invalid job input record:\n" + str(err)) from err - files: List[CWLObjectType] = [] + files: list[CWLObjectType] = [] bindings = CommentedSeq() outdir = "" tmpdir = "" @@ -947,7 +931,7 @@ def inc(d: List[int]) -> None: def evalResources( self, builder: Builder, runtimeContext: RuntimeContext - ) -> Dict[str, Union[int, float]]: + ) -> dict[str, Union[int, float]]: resourceReq, _ = self.get_requirement("ResourceRequirement") if resourceReq is None: resourceReq = {} @@ -957,7 +941,7 @@ def evalResources( ram = 1024 else: ram = 256 - request: Dict[str, Union[int, float, str]] = { + request: dict[str, Union[int, float, str]] = { "coresMin": 1, "coresMax": 1, "ramMin": ram, @@ -1005,7 +989,7 @@ def evalResources( request[a + "Min"] = mn request[a + "Max"] = cast(Union[int, float], mx) - request_evaluated = cast(Dict[str, Union[int, float]], request) + request_evaluated = cast(dict[str, Union[int, float]], request) if runtimeContext.select_resources is not None: # Call select resources hook return runtimeContext.select_resources(request_evaluated, runtimeContext) @@ -1038,7 +1022,7 @@ def checkRequirements( f"Unsupported requirement {entry['class']}." ) - def validate_hints(self, avsc_names: Names, hints: List[CWLObjectType], strict: bool) -> None: + def validate_hints(self, avsc_names: Names, hints: list[CWLObjectType], strict: bool) -> None: """Process the hints field.""" if self.doc_loader is None: return @@ -1085,10 +1069,10 @@ def __str__(self) -> str: return f"{type(self).__name__}: {self.tool['id']}" -_names: Set[str] = set() +_names: set[str] = set() -def uniquename(stem: str, names: Optional[Set[str]] = None) -> str: +def uniquename(stem: str, names: Optional[set[str]] = None) -> str: global _names if names is None: names = _names @@ -1123,8 +1107,8 @@ def nestdir(base: str, deps: CWLObjectType) -> CWLObjectType: def mergedirs( listing: MutableSequence[CWLObjectType], ) -> MutableSequence[CWLObjectType]: - r: List[CWLObjectType] = [] - ents: Dict[str, CWLObjectType] = {} + r: list[CWLObjectType] = [] + ents: dict[str, CWLObjectType] = {} for e in listing: basename = cast(str, e["basename"]) if basename not in ents: @@ -1138,14 +1122,14 @@ def mergedirs( if e.get("listing"): # name already in entries # merge it into the existing listing - cast(List[CWLObjectType], ents[basename].setdefault("listing", [])).extend( - cast(List[CWLObjectType], e["listing"]) + cast(list[CWLObjectType], ents[basename].setdefault("listing", [])).extend( + cast(list[CWLObjectType], e["listing"]) ) for e in ents.values(): if e["class"] == "Directory" and "listing" in e: e["listing"] = cast( MutableSequence[CWLOutputType], - mergedirs(cast(List[CWLObjectType], e["listing"])), + mergedirs(cast(list[CWLObjectType], e["listing"])), ) r.extend(ents.values()) return r @@ -1157,8 +1141,8 @@ def mergedirs( def scandeps( base: str, doc: Union[CWLObjectType, MutableSequence[CWLObjectType]], - reffields: Set[str], - urlfields: Set[str], + reffields: set[str], + urlfields: set[str], loadref: Callable[[str, str], Union[CommentedMap, CommentedSeq, str, None]], urljoin: Callable[[str, str], str] = urllib.parse.urljoin, nestdirs: bool = True, diff --git a/cwltool/procgenerator.py b/cwltool/procgenerator.py index 34c1e650f..9839ce5d4 100644 --- a/cwltool/procgenerator.py +++ b/cwltool/procgenerator.py @@ -1,5 +1,5 @@ import copy -from typing import Dict, Optional, Tuple, cast +from typing import Optional, cast from ruamel.yaml.comments import CommentedMap from schema_salad.exceptions import ValidationException @@ -99,12 +99,12 @@ def result( job_order: CWLObjectType, jobout: CWLObjectType, runtimeContext: RuntimeContext, - ) -> Tuple[Process, CWLObjectType]: + ) -> tuple[Process, CWLObjectType]: try: loadingContext = self.loadingContext.copy() loadingContext.metadata = {} embedded_tool = load_tool( - cast(Dict[str, str], jobout["runProcess"])["location"], loadingContext + cast(dict[str, str], jobout["runProcess"])["location"], loadingContext ) except ValidationException as vexc: if runtimeContext.debug: diff --git a/cwltool/run_job.py b/cwltool/run_job.py index 307872f7a..5a81ce20c 100644 --- a/cwltool/run_job.py +++ b/cwltool/run_job.py @@ -4,10 +4,10 @@ import os import subprocess # nosec import sys -from typing import BinaryIO, Dict, List, Optional, TextIO, Union +from typing import BinaryIO, Optional, TextIO, Union -def handle_software_environment(cwl_env: Dict[str, str], script: str) -> Dict[str, str]: +def handle_software_environment(cwl_env: dict[str, str], script: str) -> dict[str, str]: """Update the provided environment dict by running the script.""" exec_env = cwl_env.copy() exec_env["_CWLTOOL"] = "1" @@ -29,7 +29,7 @@ def handle_software_environment(cwl_env: Dict[str, str], script: str) -> Dict[st return env -def main(argv: List[str]) -> int: +def main(argv: list[str]) -> int: """ Read in the configuration JSON and execute the commands. diff --git a/cwltool/secrets.py b/cwltool/secrets.py index f35f24c37..c73e0108c 100644 --- a/cwltool/secrets.py +++ b/cwltool/secrets.py @@ -1,7 +1,8 @@ """Minimal in memory storage of secrets.""" import uuid -from typing import Dict, List, MutableMapping, MutableSequence, Optional, cast +from collections.abc import MutableMapping, MutableSequence +from typing import Optional, cast from .utils import CWLObjectType, CWLOutputType @@ -11,7 +12,7 @@ class SecretStore: def __init__(self) -> None: """Initialize the secret store.""" - self.secrets: Dict[str, str] = {} + self.secrets: dict[str, str] = {} def add(self, value: Optional[CWLOutputType]) -> Optional[CWLOutputType]: """ @@ -28,7 +29,7 @@ def add(self, value: Optional[CWLOutputType]) -> Optional[CWLOutputType]: return placeholder return value - def store(self, secrets: List[str], job: CWLObjectType) -> None: + def store(self, secrets: list[str], job: CWLObjectType) -> None: """Sanitize the job object of any of the given secrets.""" for j in job: if j in secrets: diff --git a/cwltool/singularity.py b/cwltool/singularity.py index c43183ac7..0029d3950 100644 --- a/cwltool/singularity.py +++ b/cwltool/singularity.py @@ -6,8 +6,9 @@ import re import shutil import sys +from collections.abc import MutableMapping from subprocess import check_call, check_output # nosec -from typing import Callable, Dict, List, MutableMapping, Optional, Tuple, cast +from typing import Callable, Optional, cast from schema_salad.sourceline import SourceLine from spython.main import Client @@ -28,13 +29,13 @@ # This is a list containing major and minor versions as integer. # (The number of minor version digits can vary among different distributions, # therefore we need a list here.) -_SINGULARITY_VERSION: Optional[List[int]] = None +_SINGULARITY_VERSION: Optional[list[int]] = None # Cached flavor / distribution of singularity # Can be singularity, singularity-ce or apptainer _SINGULARITY_FLAVOR: str = "" -def get_version() -> Tuple[List[int], str]: +def get_version() -> tuple[list[int], str]: """ Parse the output of 'singularity --version' to determine the flavor and version. @@ -131,9 +132,9 @@ def __init__( self, builder: Builder, joborder: CWLObjectType, - make_path_mapper: Callable[[List[CWLObjectType], str, RuntimeContext, bool], PathMapper], - requirements: List[CWLObjectType], - hints: List[CWLObjectType], + make_path_mapper: Callable[[list[CWLObjectType], str, RuntimeContext, bool], PathMapper], + requirements: list[CWLObjectType], + hints: list[CWLObjectType], name: str, ) -> None: """Builder for invoking the Singularty software container engine.""" @@ -141,7 +142,7 @@ def __init__( @staticmethod def get_image( - dockerRequirement: Dict[str, str], + dockerRequirement: dict[str, str], pull_image: bool, tmp_outdir_prefix: str, force_pull: bool = False, @@ -247,7 +248,7 @@ def get_image( dockerRequirement["dockerImageId"] = path found = True if (force_pull or not found) and pull_image: - cmd = [] # type: List[str] + cmd: list[str] = [] if "dockerPull" in dockerRequirement: if cache_folder: env = os.environ.copy() @@ -338,7 +339,7 @@ def get_from_requirements( if not bool(shutil.which("singularity")): raise WorkflowException("singularity executable is not available") - if not self.get_image(cast(Dict[str, str], r), pull_image, tmp_outdir_prefix, force_pull): + if not self.get_image(cast(dict[str, str], r), pull_image, tmp_outdir_prefix, force_pull): raise WorkflowException("Container image {} not found".format(r["dockerImageId"])) if "CWL_SINGULARITY_CACHE" in os.environ: @@ -350,7 +351,7 @@ def get_from_requirements( return os.path.abspath(img_path) @staticmethod - def append_volume(runtime: List[str], source: str, target: str, writable: bool = False) -> None: + def append_volume(runtime: list[str], source: str, target: str, writable: bool = False) -> None: """Add binding arguments to the runtime list.""" if is_version_3_9_or_newer(): DockerCommandLineJob.append_volume(runtime, source, target, writable, skip_mkdirs=True) @@ -364,7 +365,7 @@ def append_volume(runtime: List[str], source: str, target: str, writable: bool = runtime.append(vol) def add_file_or_directory_volume( - self, runtime: List[str], volume: MapperEnt, host_outdir_tgt: Optional[str] + self, runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str] ) -> None: if not volume.resolved.startswith("_:"): if host_outdir_tgt is not None and not is_version_3_4_or_newer(): @@ -380,7 +381,7 @@ def add_file_or_directory_volume( def add_writable_file_volume( self, - runtime: List[str], + runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str], tmpdir_prefix: str, @@ -417,7 +418,7 @@ def add_writable_file_volume( def add_writable_directory_volume( self, - runtime: List[str], + runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str], tmpdir_prefix: str, @@ -452,7 +453,7 @@ def add_writable_directory_volume( shutil.copytree(volume.resolved, host_outdir_tgt) ensure_writable(host_outdir_tgt or new_dir) - def _required_env(self) -> Dict[str, str]: + def _required_env(self) -> dict[str, str]: return { "TMPDIR": self.CONTAINER_TMPDIR, "HOME": self.builder.outdir, @@ -460,7 +461,7 @@ def _required_env(self) -> Dict[str, str]: def create_runtime( self, env: MutableMapping[str, str], runtime_context: RuntimeContext - ) -> Tuple[List[str], Optional[str]]: + ) -> tuple[list[str], Optional[str]]: """Return the Singularity runtime list of commands and options.""" any_path_okay = self.builder.get_requirement("DockerRequirement")[1] or False runtime = [ diff --git a/cwltool/software_requirements.py b/cwltool/software_requirements.py index ec99bda05..de34f8f6d 100644 --- a/cwltool/software_requirements.py +++ b/cwltool/software_requirements.py @@ -10,17 +10,8 @@ import argparse import os import string -from typing import ( - TYPE_CHECKING, - Any, - Dict, - List, - MutableMapping, - MutableSequence, - Optional, - Union, - cast, -) +from collections.abc import MutableMapping, MutableSequence +from typing import TYPE_CHECKING, Any, Optional, Union, cast from .utils import HasReqsHints @@ -79,7 +70,7 @@ def __init__(self, args: argparse.Namespace) -> None: if self.tool_dependency_dir and not os.path.exists(self.tool_dependency_dir): os.makedirs(self.tool_dependency_dir) - def build_job_script(self, builder: "Builder", command: List[str]) -> str: + def build_job_script(self, builder: "Builder", command: list[str]) -> str: ensure_galaxy_lib_available() resolution_config_dict = { "use": self.use_tool_dependencies, @@ -103,14 +94,14 @@ def build_job_script(self, builder: "Builder", command: List[str]) -> str: ) ) - template_kwds: Dict[str, str] = dict(handle_dependencies=handle_dependencies) + template_kwds: dict[str, str] = dict(handle_dependencies=handle_dependencies) job_script = COMMAND_WITH_DEPENDENCIES_TEMPLATE.substitute(template_kwds) return job_script def get_dependencies(builder: HasReqsHints) -> ToolRequirements: (software_requirement, _) = builder.get_requirement("SoftwareRequirement") - dependencies: List[Union["ToolRequirement", Dict[str, Any]]] = [] + dependencies: list[Union["ToolRequirement", dict[str, Any]]] = [] if software_requirement and software_requirement.get("packages"): packages = cast( MutableSequence[MutableMapping[str, Union[str, MutableSequence[str]]]], diff --git a/cwltool/stdfsaccess.py b/cwltool/stdfsaccess.py index 069289111..056b4b912 100644 --- a/cwltool/stdfsaccess.py +++ b/cwltool/stdfsaccess.py @@ -3,7 +3,7 @@ import glob import os import urllib -from typing import IO, Any, List +from typing import IO, Any from schema_salad.ref_resolver import file_uri, uri_file_path @@ -31,7 +31,7 @@ def __init__(self, basedir: str) -> None: def _abs(self, p: str) -> str: return abspath(p, self.basedir) - def glob(self, pattern: str) -> List[str]: + def glob(self, pattern: str) -> list[str]: return [file_uri(str(self._abs(line))) for line in glob.glob(self._abs(pattern))] def open(self, fn: str, mode: str) -> IO[Any]: @@ -49,7 +49,7 @@ def isfile(self, fn: str) -> bool: def isdir(self, fn: str) -> bool: return os.path.isdir(self._abs(fn)) - def listdir(self, fn: str) -> List[str]: + def listdir(self, fn: str) -> list[str]: return [abspath(urllib.parse.quote(entry), fn) for entry in os.listdir(self._abs(fn))] def join(self, path, *paths): # type: (str, *str) -> str diff --git a/cwltool/subgraph.py b/cwltool/subgraph.py index f6df7e69f..550dc7838 100644 --- a/cwltool/subgraph.py +++ b/cwltool/subgraph.py @@ -1,18 +1,7 @@ import urllib from collections import namedtuple -from typing import ( - Any, - Dict, - List, - Mapping, - MutableMapping, - MutableSequence, - Optional, - Set, - Tuple, - Union, - cast, -) +from collections.abc import Mapping, MutableMapping, MutableSequence +from typing import Any, Optional, Union, cast from ruamel.yaml.comments import CommentedMap, CommentedSeq @@ -33,7 +22,7 @@ def subgraph_visit( current: str, nodes: MutableMapping[str, Node], - visited: Set[str], + visited: set[str], direction: str, ) -> None: if current in visited: @@ -48,7 +37,7 @@ def subgraph_visit( subgraph_visit(c, nodes, visited, direction) -def declare_node(nodes: Dict[str, Node], nodeid: str, tp: Optional[str]) -> Node: +def declare_node(nodes: dict[str, Node], nodeid: str, tp: Optional[str]) -> Node: if nodeid in nodes: n = nodes[nodeid] if n.type is None: @@ -59,8 +48,8 @@ def declare_node(nodes: Dict[str, Node], nodeid: str, tp: Optional[str]) -> Node def find_step( - steps: List[WorkflowStep], stepid: str, loading_context: LoadingContext -) -> Tuple[Optional[CWLObjectType], Optional[WorkflowStep]]: + steps: list[WorkflowStep], stepid: str, loading_context: LoadingContext +) -> tuple[Optional[CWLObjectType], Optional[WorkflowStep]]: """Find the step (raw dictionary and WorkflowStep) for a given step id.""" for st in steps: st_tool_id = st.tool["id"] @@ -114,7 +103,7 @@ def get_subgraph( if tool.tool["class"] != "Workflow": raise Exception("Can only extract subgraph from workflow") - nodes: Dict[str, Node] = {} + nodes: dict[str, Node] = {} for inp in tool.tool["inputs"]: declare_node(nodes, inp["id"], INPUT) @@ -149,7 +138,7 @@ def get_subgraph( nodes[out].up.append(st["id"]) # Find all the downstream nodes from the starting points - visited_down: Set[str] = set() + visited_down: set[str] = set() for r in roots: if nodes[r].type == OUTPUT: subgraph_visit(r, nodes, visited_down, UP) @@ -157,8 +146,8 @@ def get_subgraph( subgraph_visit(r, nodes, visited_down, DOWN) # Now make sure all the nodes are connected to upstream inputs - visited: Set[str] = set() - rewire: Dict[str, Tuple[str, CWLObjectType]] = {} + visited: set[str] = set() + rewire: dict[str, tuple[str, CWLObjectType]] = {} for v in visited_down: visited.add(v) if nodes[v].type in (STEP, OUTPUT): @@ -221,7 +210,7 @@ def get_step(tool: Workflow, step_id: str, loading_context: LoadingContext) -> C extracted["inputs"] = CommentedSeq() extracted["outputs"] = CommentedSeq() - for in_port in cast(List[CWLObjectType], step["in"]): + for in_port in cast(list[CWLObjectType], step["in"]): name = "#" + cast(str, in_port["id"]).split("#")[-1].split("/")[-1] inp: CWLObjectType = {"id": name, "type": "Any"} if "default" in in_port: @@ -231,7 +220,7 @@ def get_step(tool: Workflow, step_id: str, loading_context: LoadingContext) -> C if "linkMerge" in in_port: del in_port["linkMerge"] - for outport in cast(List[Union[str, Mapping[str, Any]]], step["out"]): + for outport in cast(list[Union[str, Mapping[str, Any]]], step["out"]): if isinstance(outport, Mapping): outport_id = cast(str, outport["id"]) else: @@ -256,7 +245,7 @@ def get_step(tool: Workflow, step_id: str, loading_context: LoadingContext) -> C def get_process( tool: Workflow, step_id: str, loading_context: LoadingContext -) -> Tuple[Any, WorkflowStep]: +) -> tuple[Any, WorkflowStep]: """Find the underlying Process for a given Workflow step id.""" if loading_context.loader is None: raise Exception("loading_context.loader cannot be None") diff --git a/cwltool/udocker.py b/cwltool/udocker.py index 6598d6a7c..ea3fc78ca 100644 --- a/cwltool/udocker.py +++ b/cwltool/udocker.py @@ -1,7 +1,5 @@ """Enables Docker software containers via the udocker runtime.""" -from typing import List - from .docker import DockerCommandLineJob @@ -10,7 +8,7 @@ class UDockerCommandLineJob(DockerCommandLineJob): @staticmethod def append_volume( - runtime: List[str], + runtime: list[str], source: str, target: str, writable: bool = False, diff --git a/cwltool/update.py b/cwltool/update.py index 4fd66b37a..67e1f4257 100644 --- a/cwltool/update.py +++ b/cwltool/update.py @@ -1,15 +1,7 @@ import copy +from collections.abc import MutableMapping, MutableSequence from functools import partial -from typing import ( - Callable, - Dict, - MutableMapping, - MutableSequence, - Optional, - Tuple, - Union, - cast, -) +from typing import Callable, Optional, Union, cast from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.exceptions import ValidationException @@ -20,7 +12,7 @@ from .utils import CWLObjectType, CWLOutputType, aslist, visit_class, visit_field -def v1_2to1_3dev1(doc: CommentedMap, loader: Loader, baseuri: str) -> Tuple[CommentedMap, str]: +def v1_2to1_3dev1(doc: CommentedMap, loader: Loader, baseuri: str) -> tuple[CommentedMap, str]: """Public updater for v1.2 to v1.3.0-dev1.""" doc = copy.deepcopy(doc) @@ -78,7 +70,7 @@ def rewrite_loop_requirements(t: CWLObjectType) -> None: def v1_1to1_2( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Public updater for v1.1 to v1.2.""" doc = copy.deepcopy(doc) @@ -94,7 +86,7 @@ def v1_1to1_2( def v1_0to1_1( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Public updater for v1.0 to v1.1.""" doc = copy.deepcopy(doc) @@ -195,21 +187,21 @@ def fix_inputBinding(t: CWLObjectType) -> None: def v1_1_0dev1to1_1( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Public updater for v1.1.0-dev1 to v1.1.""" return (doc, "v1.1") def v1_2_0dev1todev2( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Public updater for v1.2.0-dev1 to v1.2.0-dev2.""" return (doc, "v1.2.0-dev2") def v1_2_0dev2todev3( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Public updater for v1.2.0-dev2 to v1.2.0-dev3.""" doc = copy.deepcopy(doc) @@ -232,21 +224,21 @@ def update_pickvalue(t: CWLObjectType) -> None: def v1_2_0dev3todev4( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Public updater for v1.2.0-dev3 to v1.2.0-dev4.""" return (doc, "v1.2.0-dev4") def v1_2_0dev4todev5( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Public updater for v1.2.0-dev4 to v1.2.0-dev5.""" return (doc, "v1.2.0-dev5") def v1_2_0dev5to1_2( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Public updater for v1.2.0-dev5 to v1.2.""" return (doc, "v1.2") @@ -264,13 +256,13 @@ def v1_2_0dev5to1_2( "v1.3.0-dev1", ] -UPDATES: Dict[str, Optional[Callable[[CommentedMap, Loader, str], Tuple[CommentedMap, str]]]] = { +UPDATES: dict[str, Optional[Callable[[CommentedMap, Loader, str], tuple[CommentedMap, str]]]] = { "v1.0": v1_0to1_1, "v1.1": v1_1to1_2, "v1.2": v1_2to1_3dev1, } -DEVUPDATES: Dict[str, Optional[Callable[[CommentedMap, Loader, str], Tuple[CommentedMap, str]]]] = { +DEVUPDATES: dict[str, Optional[Callable[[CommentedMap, Loader, str], tuple[CommentedMap, str]]]] = { "v1.1.0-dev1": v1_1_0dev1to1_1, "v1.2.0-dev1": v1_2_0dev1todev2, "v1.2.0-dev2": v1_2_0dev2todev3, @@ -291,7 +283,7 @@ def v1_2_0dev5to1_2( def identity( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Do-nothing, CWL document upgrade function.""" return (doc, cast(str, doc["cwlVersion"])) @@ -300,7 +292,7 @@ def checkversion( doc: Union[CommentedSeq, CommentedMap], metadata: CommentedMap, enable_dev: bool, -) -> Tuple[CommentedMap, str]: +) -> tuple[CommentedMap, str]: """Check the validity of the version of the give CWL document. Returns the document and the validated version string. @@ -365,7 +357,7 @@ def update( (cdoc, version) = checkversion(doc, metadata, enable_dev) originalversion = copy.copy(version) - nextupdate: Optional[Callable[[CommentedMap, Loader, str], Tuple[CommentedMap, str]]] = identity + nextupdate: Optional[Callable[[CommentedMap, Loader, str], tuple[CommentedMap, str]]] = identity while version != update_to and nextupdate: (cdoc, version) = nextupdate(cdoc, loader, baseuri) diff --git a/cwltool/utils.py b/cwltool/utils.py index c8620994a..e460842a9 100644 --- a/cwltool/utils.py +++ b/cwltool/utils.py @@ -19,9 +19,11 @@ import tempfile import urllib import uuid +from collections.abc import Generator, Iterable, MutableMapping, MutableSequence from datetime import datetime from email.utils import parsedate_to_datetime from functools import partial +from importlib.resources import as_file, files from itertools import zip_longest from pathlib import Path, PurePosixPath from tempfile import NamedTemporaryFile @@ -31,17 +33,9 @@ Any, Callable, Deque, - Dict, - Generator, - Iterable, - List, Literal, - MutableMapping, - MutableSequence, NamedTuple, Optional, - Set, - Tuple, TypedDict, Union, cast, @@ -54,11 +48,6 @@ from schema_salad.exceptions import ValidationException from schema_salad.ref_resolver import Loader -if sys.version_info >= (3, 9): - from importlib.resources import as_file, files -else: - from importlib_resources import as_file, files - if TYPE_CHECKING: from .command_line_tool import CallbackJob, ExpressionJob from .job import CommandLineJob, JobBase @@ -92,13 +81,13 @@ OutputCallbackType = Callable[[Optional[CWLObjectType], str], None] ResolverType = Callable[["Loader", str], Optional[str]] DestinationsType = MutableMapping[str, Optional[CWLOutputType]] -ScatterDestinationsType = MutableMapping[str, List[Optional[CWLOutputType]]] +ScatterDestinationsType = MutableMapping[str, list[Optional[CWLOutputType]]] ScatterOutputCallbackType = Callable[[Optional[ScatterDestinationsType], str], None] SinkType = Union[CWLOutputType, CWLObjectType] DirectoryType = TypedDict( - "DirectoryType", {"class": str, "listing": List[CWLObjectType], "basename": str} + "DirectoryType", {"class": str, "listing": list[CWLObjectType], "basename": str} ) -JSONType = Union[Dict[str, "JSONType"], List["JSONType"], str, int, float, bool, None] +JSONType = Union[dict[str, "JSONType"], list["JSONType"], str, int, float, bool, None] class WorkflowStateItem(NamedTuple): @@ -109,7 +98,7 @@ class WorkflowStateItem(NamedTuple): success: str -ParametersType = List[CWLObjectType] +ParametersType = list[CWLObjectType] StepType = CWLObjectType # WorkflowStep LoadListingType = Union[Literal["no_listing"], Literal["shallow_listing"], Literal["deep_listing"]] @@ -143,7 +132,7 @@ def copytree_with_merge(src: str, dst: str) -> None: shutil.copy2(spath, dpath) -def cmp_like_py2(dict1: Dict[str, Any], dict2: Dict[str, Any]) -> int: +def cmp_like_py2(dict1: dict[str, Any], dict2: dict[str, Any]) -> int: """ Compare in the same manner as Python2. @@ -259,20 +248,20 @@ def adjustDirObjs(rec: Any, op: Union[Callable[..., Any], "partial[Any]"]) -> No visit_class(rec, ("Directory",), op) -def dedup(listing: List[CWLObjectType]) -> List[CWLObjectType]: +def dedup(listing: list[CWLObjectType]) -> list[CWLObjectType]: marksub = set() - def mark(d: Dict[str, str]) -> None: + def mark(d: dict[str, str]) -> None: marksub.add(d["location"]) for entry in listing: if entry["class"] == "Directory": - for e in cast(List[CWLObjectType], entry.get("listing", [])): + for e in cast(list[CWLObjectType], entry.get("listing", [])): adjustFileObjs(e, mark) adjustDirObjs(e, mark) dd = [] - markdup: Set[str] = set() + markdup: set[str] = set() for r in listing: if r["location"] not in marksub and r["location"] not in markdup: dd.append(r) @@ -284,14 +273,14 @@ def mark(d: Dict[str, str]) -> None: def get_listing(fs_access: "StdFsAccess", rec: CWLObjectType, recursive: bool = True) -> None: """Expand, recursively, any 'listing' fields in a Directory.""" if rec.get("class") != "Directory": - finddirs: List[CWLObjectType] = [] + finddirs: list[CWLObjectType] = [] visit_class(rec, ("Directory",), finddirs.append) for f in finddirs: get_listing(fs_access, f, recursive=recursive) return if "listing" in rec: return - listing: List[CWLOutputType] = [] + listing: list[CWLOutputType] = [] loc = cast(str, rec["location"]) for ld in fs_access.listdir(loc): parse = urllib.parse.urlparse(ld) @@ -310,7 +299,7 @@ def get_listing(fs_access: "StdFsAccess", rec: CWLObjectType, recursive: bool = rec["listing"] = listing -def trim_listing(obj: Dict[str, Any]) -> None: +def trim_listing(obj: dict[str, Any]) -> None: """ Remove 'listing' field from Directory objects that are file references. @@ -322,7 +311,7 @@ def trim_listing(obj: Dict[str, Any]) -> None: del obj["listing"] -def downloadHttpFile(httpurl: str) -> Tuple[str, Optional[datetime]]: +def downloadHttpFile(httpurl: str) -> tuple[str, Optional[datetime]]: """ Download a remote file, possibly using a locally cached copy. @@ -414,7 +403,7 @@ def normalizeFilesDirs( ] ] ) -> None: - def addLocation(d: Dict[str, Any]) -> None: + def addLocation(d: dict[str, Any]) -> None: if "location" not in d: if d["class"] == "File" and ("contents" not in d): raise ValidationException( @@ -484,10 +473,10 @@ class HasReqsHints: def __init__(self) -> None: """Initialize this reqs decorator.""" - self.requirements: List[CWLObjectType] = [] - self.hints: List[CWLObjectType] = [] + self.requirements: list[CWLObjectType] = [] + self.hints: list[CWLObjectType] = [] - def get_requirement(self, feature: str) -> Tuple[Optional[CWLObjectType], Optional[bool]]: + def get_requirement(self, feature: str) -> tuple[Optional[CWLObjectType], Optional[bool]]: """Retrieve the named feature from the requirements field, or the hints field.""" for item in reversed(self.requirements): if item["class"] == feature: diff --git a/cwltool/validate_js.py b/cwltool/validate_js.py index de4adaa14..b43b7ef0d 100644 --- a/cwltool/validate_js.py +++ b/cwltool/validate_js.py @@ -3,17 +3,8 @@ import json import logging from collections import namedtuple -from typing import ( - Any, - Dict, - List, - MutableMapping, - MutableSequence, - Optional, - Tuple, - Union, - cast, -) +from collections.abc import MutableMapping, MutableSequence +from typing import Any, Optional, Union, cast from cwl_utils.errors import SubstitutionError from cwl_utils.expression import scanner as scan_expression @@ -63,7 +54,7 @@ def get_expressions( tool: Union[CommentedMap, str, CommentedSeq], schema: Optional[Union[Schema, ArraySchema]], source_line: Optional[SourceLine] = None, -) -> List[Tuple[str, Optional[SourceLine]]]: +) -> list[tuple[str, Optional[SourceLine]]]: debug = _logger.isEnabledFor(logging.DEBUG) if is_expression(tool, schema): return [(cast(str, tool), source_line)] @@ -124,8 +115,8 @@ def get_expressions( def jshint_js( js_text: str, - globals: Optional[List[str]] = None, - options: Optional[Dict[str, Union[List[str], str, int]]] = None, + globals: Optional[list[str]] = None, + options: Optional[dict[str, Union[list[str], str, int]]] = None, container_engine: str = "docker", eval_timeout: float = 60, ) -> JSHintJSReturn: @@ -177,7 +168,7 @@ def dump_jshint_error() -> None: except ValueError: dump_jshint_error() - jshint_errors: List[str] = [] + jshint_errors: list[str] = [] js_text_lines = js_text.split("\n") @@ -193,7 +184,7 @@ def dump_jshint_error() -> None: return JSHintJSReturn(jshint_errors, jshint_json.get("globals", [])) -def print_js_hint_messages(js_hint_messages: List[str], source_line: Optional[SourceLine]) -> None: +def print_js_hint_messages(js_hint_messages: list[str], source_line: Optional[SourceLine]) -> None: """Log the message from JSHint, using the line number.""" if source_line is not None: for js_hint_message in js_hint_messages: @@ -203,7 +194,7 @@ def print_js_hint_messages(js_hint_messages: List[str], source_line: Optional[So def validate_js_expressions( tool: CommentedMap, schema: Schema, - jshint_options: Optional[Dict[str, Union[List[str], str, int]]] = None, + jshint_options: Optional[dict[str, Union[list[str], str, int]]] = None, container_engine: str = "docker", eval_timeout: float = 60, ) -> None: diff --git a/cwltool/workflow.py b/cwltool/workflow.py index 982ec7e70..3bf32251f 100644 --- a/cwltool/workflow.py +++ b/cwltool/workflow.py @@ -3,16 +3,8 @@ import functools import logging import random -from typing import ( - Callable, - Dict, - List, - Mapping, - MutableMapping, - MutableSequence, - Optional, - cast, -) +from collections.abc import Mapping, MutableMapping, MutableSequence +from typing import Callable, Optional, cast from uuid import UUID from mypy_extensions import mypyc_attr @@ -98,7 +90,7 @@ def __init__( loadingContext.requirements = self.requirements loadingContext.hints = self.hints - self.steps: List[WorkflowStep] = [] + self.steps: list[WorkflowStep] = [] validation_errors = [] for index, step in enumerate(self.tool.get("steps", [])): try: @@ -119,9 +111,9 @@ def __init__( workflow_inputs = self.tool["inputs"] workflow_outputs = self.tool["outputs"] - step_inputs: List[CWLObjectType] = [] - step_outputs: List[CWLObjectType] = [] - param_to_step: Dict[str, CWLObjectType] = {} + step_inputs: list[CWLObjectType] = [] + step_outputs: list[CWLObjectType] = [] + param_to_step: dict[str, CWLObjectType] = {} for step in self.steps: step_inputs.extend(step.tool["inputs"]) step_outputs.extend(step.tool["outputs"]) @@ -220,7 +212,7 @@ def __init__( loadingContext.requirements.append(parent_req) loadingContext.requirements.extend( cast( - List[CWLObjectType], + list[CWLObjectType], get_overrides(getdefault(loadingContext.overrides_list, []), self.id).get( "requirements", [] ), diff --git a/cwltool/workflow_job.py b/cwltool/workflow_job.py index d144128e6..6cd0b2e7c 100644 --- a/cwltool/workflow_job.py +++ b/cwltool/workflow_job.py @@ -3,19 +3,8 @@ import functools import logging import threading -from typing import ( - TYPE_CHECKING, - Dict, - List, - MutableMapping, - MutableSequence, - Optional, - Set, - Sized, - Tuple, - Union, - cast, -) +from collections.abc import MutableMapping, MutableSequence, Sized +from typing import TYPE_CHECKING, Optional, Union, cast from cwl_utils import expression from schema_salad.sourceline import SourceLine @@ -89,11 +78,11 @@ def __init__( ) -> None: """Initialize.""" self.dest = dest - self._completed: Set[int] = set() + self._completed: set[int] = set() self.processStatus = "success" self.total = total self.output_callback = output_callback - self.steps: List[Optional[JobsGeneratorType]] = [] + self.steps: list[Optional[JobsGeneratorType]] = [] @property def completed(self) -> int: @@ -123,7 +112,7 @@ def receive_scatter_output(self, index: int, jobout: CWLObjectType, processStatu def setTotal( self, total: int, - steps: List[Optional[JobsGeneratorType]], + steps: list[Optional[JobsGeneratorType]], ) -> None: """ Set the total number of expected outputs along with the steps. @@ -137,7 +126,7 @@ def setTotal( def parallel_steps( - steps: List[Optional[JobsGeneratorType]], + steps: list[Optional[JobsGeneratorType]], rc: ReceiveScatterOutput, runtimeContext: RuntimeContext, ) -> JobsGeneratorType: @@ -187,7 +176,7 @@ def nested_crossproduct_scatter( rc = ReceiveScatterOutput(output_callback, output, jobl) - steps: List[Optional[JobsGeneratorType]] = [] + steps: list[Optional[JobsGeneratorType]] = [] for index in range(0, jobl): sjob: Optional[CWLObjectType] = copy.copy(joborder) assert sjob is not None # nosec @@ -254,11 +243,11 @@ def _flat_crossproduct_scatter( callback: ReceiveScatterOutput, startindex: int, runtimeContext: RuntimeContext, -) -> Tuple[List[Optional[JobsGeneratorType]], int]: +) -> tuple[list[Optional[JobsGeneratorType]], int]: """Inner loop.""" scatter_key = scatter_keys[0] jobl = len(cast(Sized, joborder[scatter_key])) - steps: List[Optional[JobsGeneratorType]] = [] + steps: list[Optional[JobsGeneratorType]] = [] put = startindex for index in range(0, jobl): sjob: Optional[CWLObjectType] = copy.copy(joborder) @@ -309,7 +298,7 @@ def dotproduct_scatter( rc = ReceiveScatterOutput(output_callback, output, jobl) - steps: List[Optional[JobsGeneratorType]] = [] + steps: list[Optional[JobsGeneratorType]] = [] for index in range(0, jobl): sjobo: Optional[CWLObjectType] = copy.copy(joborder) assert sjobo is not None # nosec @@ -357,7 +346,7 @@ def match_types( elif linkMerge: if iid not in inputobj: inputobj[iid] = [] - sourceTypes = cast(List[Optional[CWLOutputType]], inputobj[iid]) + sourceTypes = cast(list[Optional[CWLOutputType]], inputobj[iid]) if linkMerge == "merge_nested": sourceTypes.append(src.value) elif linkMerge == "merge_flattened": @@ -380,7 +369,7 @@ def match_types( def object_from_state( - state: Dict[str, Optional[WorkflowStateItem]], + state: dict[str, Optional[WorkflowStateItem]], params: ParametersType, frag_only: bool, supportsMultipleInput: bool, @@ -487,7 +476,7 @@ def __init__(self, workflow: "Workflow", runtimeContext: RuntimeContext) -> None self.prov_obj = workflow.provenance_object self.parent_wf = workflow.parent_wf self.steps = [WorkflowJobStep(s) for s in workflow.steps] - self.state: Dict[str, Optional[WorkflowStateItem]] = {} + self.state: dict[str, Optional[WorkflowStateItem]] = {} self.processStatus = "" self.did_callback = False self.made_progress: Optional[bool] = None @@ -554,7 +543,7 @@ def do_output_callback(self, final_output_callback: OutputCallbackType) -> None: def receive_output( self, step: WorkflowJobStep, - outputparms: List[CWLObjectType], + outputparms: list[CWLObjectType], final_output_callback: OutputCallbackType, jobout: CWLObjectType, processStatus: str, @@ -701,7 +690,7 @@ def valueFromFunc(k: str, v: Optional[CWLOutputType]) -> Optional[CWLOutputType] return psio if "scatter" in step.tool: - scatter = cast(List[str], aslist(step.tool["scatter"])) + scatter = cast(list[str], aslist(step.tool["scatter"])) method = step.tool.get("scatterMethod") if method is None and len(scatter) != 1: raise WorkflowException( @@ -961,7 +950,7 @@ def loop_callback( try: loop = cast(MutableSequence[CWLObjectType], self.step.tool.get("loop", [])) outputMethod = self.step.tool.get("outputMethod", "last_iteration") - state: Dict[str, Optional[WorkflowStateItem]] = {} + state: dict[str, Optional[WorkflowStateItem]] = {} for i in self.step.tool["outputs"]: if "id" in i: iid = cast(str, i["id"]) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index a7d0dacb8..ccd7737f4 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -1,6 +1,7 @@ mypy==1.11.2 # also update pyproject.toml ruamel.yaml>=0.16.0,<0.19 cwl-utils>=0.32 +cwltest types-requests types-setuptools types-psutil diff --git a/pyproject.toml b/pyproject.toml index 05a2b82f7..4f3f91c31 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,4 +19,4 @@ write_to = "cwltool/_version.py" [tool.black] line-length = 100 -target-version = [ "py38" ] +target-version = [ "py39" ] diff --git a/setup.py b/setup.py index fa39a378b..40c3fd8d4 100644 --- a/setup.py +++ b/setup.py @@ -129,7 +129,6 @@ "prov == 1.5.1", "mypy-extensions", "psutil >= 5.6.6", - "importlib_resources>=1.4;python_version<'3.9'", "coloredlogs", "pydot >= 1.4.1, <3", "argcomplete >= 1.12.0", @@ -143,7 +142,7 @@ "galaxy-util <24.2", ], }, - python_requires=">=3.8, <3.14", + python_requires=">=3.9, <3.14", use_scm_version=True, setup_requires=PYTEST_RUNNER + ["setuptools_scm>=8.0.4,<9"], test_suite="tests", @@ -169,7 +168,6 @@ "Operating System :: POSIX", "Operating System :: POSIX :: Linux", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", diff --git a/tests/cwl-conformance/cwltool-conftest.py b/tests/cwl-conformance/cwltool-conftest.py index 3e2b83990..c87cf0ef7 100644 --- a/tests/cwl-conformance/cwltool-conftest.py +++ b/tests/cwl-conformance/cwltool-conftest.py @@ -6,20 +6,20 @@ import json from io import StringIO -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Optional from cwltest import utils def pytest_cwl_execute_test( config: utils.CWLTestConfig, processfile: str, jobfile: Optional[str] -) -> Tuple[int, Optional[Dict[str, Any]]]: +) -> tuple[int, Optional[dict[str, Any]]]: """Use the CWL reference runner (cwltool) to execute tests.""" from cwltool import main from cwltool.errors import WorkflowException stdout = StringIO() - argsl: List[str] = [f"--outdir={config.outdir}"] + argsl: list[str] = [f"--outdir={config.outdir}"] if config.runner_quiet: argsl.append("--quiet") elif config.verbose: diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index ae18a41ae..b903c04d6 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -6,7 +6,7 @@ from pathlib import Path from shutil import which from types import ModuleType -from typing import Optional, Tuple +from typing import Optional import pytest @@ -56,7 +56,7 @@ def test_biocontainers_resolution(tmp_path: Path) -> None: @pytest.fixture(scope="session") -def bioconda_setup(request: pytest.FixtureRequest) -> Tuple[Optional[int], str]: +def bioconda_setup(request: pytest.FixtureRequest) -> tuple[Optional[int], str]: """ Caches the conda environment created for seqtk_seq.cwl. @@ -108,7 +108,7 @@ def bioconda_setup(request: pytest.FixtureRequest) -> Tuple[Optional[int], str]: @pytest.mark.skipif(not deps, reason="galaxy-tool-util is not installed") -def test_bioconda(bioconda_setup: Tuple[Optional[int], str]) -> None: +def test_bioconda(bioconda_setup: tuple[Optional[int], str]) -> None: error_code, stderr = bioconda_setup assert error_code == 0, stderr diff --git a/tests/test_environment.py b/tests/test_environment.py index ba87041b3..a4bfd1ac3 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -2,8 +2,9 @@ import os from abc import ABC, abstractmethod +from collections.abc import Mapping from pathlib import Path -from typing import Any, Callable, Dict, List, Mapping, Union +from typing import Any, Callable, Union import pytest @@ -17,7 +18,7 @@ # TODO: maybe add regex? Env = Mapping[str, str] CheckerTypes = Union[None, str, Callable[[str, Env], bool]] -EnvChecks = Dict[str, CheckerTypes] +EnvChecks = dict[str, CheckerTypes] def assert_envvar_matches(check: CheckerTypes, k: str, env: Mapping[str, str]) -> None: @@ -66,7 +67,7 @@ def checks(tmp_prefix: str) -> EnvChecks: """Return a mapping from environment variable names to how to check for correctness.""" # Any flags to pass to cwltool to force use of the correct container - flags: List[str] + flags: list[str] # Does the env tool (maybe in our container) accept a `-0` flag? env_accepts_null: bool diff --git a/tests/test_examples.py b/tests/test_examples.py index 4d479e313..23d17dcb2 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -9,7 +9,7 @@ import urllib.parse from io import StringIO from pathlib import Path -from typing import Any, Dict, List, Union, cast +from typing import Any, Union, cast import cwl_utils.expression as expr import pydot @@ -69,7 +69,7 @@ def test_expression_match(expression: str, expected: bool) -> None: assert (match is not None) == expected -interpolate_input = { +interpolate_input: dict[str, Any] = { "foo": { "bar": {"baz": "zab1"}, "b ar": {"baz": 2}, @@ -77,7 +77,7 @@ def test_expression_match(expression: str, expected: bool) -> None: 'b"ar': {"baz": None}, }, "lst": ["A", "B"], -} # type: Dict[str, Any] +} interpolate_parameters = [ ("$(foo)", interpolate_input["foo"]), @@ -410,7 +410,7 @@ def loadref( raise Exception("test case can't load things") scanned_deps = cast( - List[Dict[str, Any]], + list[dict[str, Any]], cwltool.process.scandeps( cast(str, obj["id"]), obj, @@ -473,7 +473,7 @@ def loadref( assert scanned_deps == expected_deps scanned_deps2 = cast( - List[Dict[str, Any]], + list[dict[str, Any]], cwltool.process.scandeps( cast(str, obj["id"]), obj, @@ -515,7 +515,7 @@ def loadref( raise Exception("test case can't load things") scanned_deps = cast( - List[Dict[str, Any]], + list[dict[str, Any]], cwltool.process.scandeps( "", obj, @@ -576,7 +576,7 @@ def test_scandeps_defaults_with_secondaryfiles() -> None: def test_dedupe() -> None: - not_deduped = [ + not_deduped: list[CWLObjectType] = [ {"class": "File", "location": "file:///example/a"}, {"class": "File", "location": "file:///example/a"}, {"class": "File", "location": "file:///example/d"}, @@ -585,7 +585,7 @@ def test_dedupe() -> None: "location": "file:///example/c", "listing": [{"class": "File", "location": "file:///example/d"}], }, - ] # type: List[CWLObjectType] + ] expected = [ {"class": "File", "location": "file:///example/a"}, @@ -649,7 +649,7 @@ def test_dedupe() -> None: @pytest.mark.parametrize("name, source, sink, expected", source_to_sink) def test_compare_types( - name: str, source: Dict[str, Any], sink: Dict[str, Any], expected: bool + name: str, source: dict[str, Any], sink: dict[str, Any], expected: bool ) -> None: assert can_assign_src_to_sink(source, sink) == expected, name @@ -675,7 +675,7 @@ def test_compare_types( @pytest.mark.parametrize("name, source, sink, expected", source_to_sink_strict) def test_compare_types_strict( - name: str, source: Dict[str, Any], sink: Dict[str, Any], expected: bool + name: str, source: dict[str, Any], sink: dict[str, Any], expected: bool ) -> None: assert can_assign_src_to_sink(source, sink, strict=True) == expected, name @@ -1682,7 +1682,7 @@ def test_arguments_self() -> None: else: factory.runtime_context.use_container = False check = factory.make(get_data("tests/wf/paramref_arguments_self.cwl")) - outputs = cast(Dict[str, Any], check()) + outputs = cast(dict[str, Any], check()) assert "self_review" in outputs assert len(outputs) == 1 assert outputs["self_review"]["checksum"] == "sha1$724ba28f4a9a1b472057ff99511ed393a45552e1" diff --git a/tests/test_fetch.py b/tests/test_fetch.py index e55491d90..962b7d7e5 100644 --- a/tests/test_fetch.py +++ b/tests/test_fetch.py @@ -1,6 +1,6 @@ import os from pathlib import Path -from typing import Any, List, Optional +from typing import Any, Optional from urllib.parse import urljoin, urlsplit import pytest @@ -25,7 +25,7 @@ def __init__( ) -> None: """Create a Fetcher that provides a fixed result for testing purposes.""" - def fetch_text(self, url: str, content_types: Optional[List[str]] = None) -> str: + def fetch_text(self, url: str, content_types: Optional[list[str]] = None) -> str: if url == "baz:bar/foo.cwl": return """ cwlVersion: v1.0 diff --git a/tests/test_http_input.py b/tests/test_http_input.py index 6b4d9b479..e80260ff9 100644 --- a/tests/test_http_input.py +++ b/tests/test_http_input.py @@ -1,10 +1,7 @@ import os -import sys from datetime import datetime from pathlib import Path -from typing import List -import pytest from pytest_httpserver import HTTPServer from cwltool.pathmapper import PathMapper @@ -15,7 +12,7 @@ def test_http_path_mapping(tmp_path: Path) -> None: input_file_path = ( "https://raw.githubusercontent.com/common-workflow-language/cwltool/main/tests/2.fasta" ) - base_file: List[CWLObjectType] = [ + base_file: list[CWLObjectType] = [ { "class": "File", "location": "https://raw.githubusercontent.com/common-workflow-language/" @@ -34,7 +31,6 @@ def test_http_path_mapping(tmp_path: Path) -> None: assert ">Sequence 561 BP; 135 A; 106 C; 98 G; 222 T; 0 other;" in contents -@pytest.mark.skipif(sys.version_info < (3, 7), reason="timesout on CI") def test_modification_date(tmp_path: Path) -> None: """Local copies of remote files should preserve last modification date.""" # Initialize the server @@ -58,7 +54,7 @@ def test_modification_date(tmp_path: Path) -> None: ) location = httpserver.url_for(f"/{remote_file_name}") - base_file: List[CWLObjectType] = [ + base_file: list[CWLObjectType] = [ { "class": "File", "location": location, diff --git a/tests/test_js_sandbox.py b/tests/test_js_sandbox.py index f4839e8a0..2c5df6339 100644 --- a/tests/test_js_sandbox.py +++ b/tests/test_js_sandbox.py @@ -5,7 +5,7 @@ import shutil import threading from pathlib import Path -from typing import Any, List +from typing import Any import pytest from cwl_utils import sandboxjs @@ -48,8 +48,8 @@ def test_value_from_two_concatenated_expressions() -> None: def hide_nodejs(temp_dir: Path) -> str: """Generate a new PATH that hides node{js,}.""" - paths: List[str] = os.environ.get("PATH", "").split(":") - names: List[str] = [] + paths: list[str] = os.environ.get("PATH", "").split(":") + names: list[str] = [] for name in ("nodejs", "node"): path = shutil.which(name) if path: diff --git a/tests/test_loop.py b/tests/test_loop.py index bf908196d..e8a043611 100644 --- a/tests/test_loop.py +++ b/tests/test_loop.py @@ -1,8 +1,8 @@ """Test the 1.3 loop feature.""" import json +from collections.abc import MutableMapping, MutableSequence from io import StringIO -from typing import MutableMapping, MutableSequence from cwltool.main import main diff --git a/tests/test_loop_ext.py b/tests/test_loop_ext.py index 499dd17b4..1769d64ad 100644 --- a/tests/test_loop_ext.py +++ b/tests/test_loop_ext.py @@ -1,8 +1,8 @@ """Test the prototype cwltool:Loop extension.""" import json +from collections.abc import MutableMapping, MutableSequence from io import StringIO -from typing import MutableMapping, MutableSequence from cwltool.main import main diff --git a/tests/test_mpi.py b/tests/test_mpi.py index 643907a39..92f0e353c 100644 --- a/tests/test_mpi.py +++ b/tests/test_mpi.py @@ -3,9 +3,10 @@ import json import os.path import sys +from collections.abc import Generator, MutableMapping from io import StringIO from pathlib import Path -from typing import Any, Generator, List, MutableMapping, Optional, Tuple +from typing import Any, Optional import pytest from ruamel.yaml.comments import CommentedMap, CommentedSeq @@ -75,12 +76,12 @@ def __init__(self): else: self.indata = sys.stdin.read().encode(sys.stdin.encoding) - def run_once(self, args: List[str]): + def run_once(self, args: list[str]): subprocess.run( args, input=self.indata, stdout=sys.stdout, stderr=sys.stderr ).check_returncode() - def run_many(self, n: int, args: List[str]): + def run_many(self, n: int, args: list[str]): for i in range(n): self.run_once(args) @@ -122,7 +123,7 @@ def make_processes_input(np: int, tmp_path: Path) -> Path: return input_file -def cwltool_args(fake_mpi_conf: str) -> List[str]: +def cwltool_args(fake_mpi_conf: str) -> list[str]: return ["--enable-ext", "--enable-dev", "--mpi-config-file", fake_mpi_conf] @@ -296,10 +297,10 @@ def schema_ext11() -> Generator[Names, None, None]: def mk_tool( schema: Names, - opts: List[str], - reqs: Optional[List[CommentedMap]] = None, - hints: Optional[List[CommentedMap]] = None, -) -> Tuple[LoadingContext, RuntimeContext, CommentedMap]: + opts: list[str], + reqs: Optional[list[CommentedMap]] = None, + hints: Optional[list[CommentedMap]] = None, +) -> tuple[LoadingContext, RuntimeContext, CommentedMap]: tool = basetool.copy() if reqs is not None: diff --git a/tests/test_override.py b/tests/test_override.py index 980c853bb..93c836c84 100644 --- a/tests/test_override.py +++ b/tests/test_override.py @@ -1,6 +1,5 @@ import json from io import StringIO -from typing import Dict, List import pytest @@ -76,7 +75,7 @@ @needs_docker @pytest.mark.parametrize("parameters,result", override_parameters) -def test_overrides(parameters: List[str], result: Dict[str, str]) -> None: +def test_overrides(parameters: list[str], result: dict[str, str]) -> None: sio = StringIO() assert main(parameters, stdout=sio) == 0 @@ -119,7 +118,7 @@ def test_overrides(parameters: List[str], result: Dict[str, str]) -> None: @needs_docker @pytest.mark.parametrize("parameters,expected_error", failing_override_parameters) -def test_overrides_fails(parameters: List[str], expected_error: str) -> None: +def test_overrides_fails(parameters: list[str], expected_error: str) -> None: sio = StringIO() assert main(parameters, stderr=sio) == 1 diff --git a/tests/test_pack.py b/tests/test_pack.py index 1d38e35e8..a65996f8f 100644 --- a/tests/test_pack.py +++ b/tests/test_pack.py @@ -5,7 +5,6 @@ from functools import partial from io import StringIO from pathlib import Path -from typing import Dict import pytest from schema_salad.utils import yaml_no_ts @@ -95,7 +94,7 @@ def test_pack_fragment() -> None: def test_pack_rewrites() -> None: - rewrites: Dict[str, str] = {} + rewrites: dict[str, str] = {} loadingContext, workflowobj, uri = fetch_document(get_data("tests/wf/default-wf5.cwl")) loadingContext.do_update = False diff --git a/tests/test_path_checks.py b/tests/test_path_checks.py index 01ab7fe17..096de9942 100644 --- a/tests/test_path_checks.py +++ b/tests/test_path_checks.py @@ -1,7 +1,7 @@ import urllib.parse from io import BytesIO from pathlib import Path -from typing import IO, Any, List, cast +from typing import IO, Any, cast import pytest from ruamel.yaml.comments import CommentedMap @@ -112,7 +112,7 @@ def test_unicode_in_output_files(tmp_path: Path, filename: str) -> None: class StubFsAccess(StdFsAccess): """Stub fs access object that doesn't rely on the filesystem.""" - def glob(self, pattern: str) -> List[str]: + def glob(self, pattern: str) -> list[str]: """glob.""" return [pattern] diff --git a/tests/test_pathmapper.py b/tests/test_pathmapper.py index b7cf2f6a1..4ffac24bd 100644 --- a/tests/test_pathmapper.py +++ b/tests/test_pathmapper.py @@ -1,5 +1,3 @@ -from typing import List, Tuple - import pytest from cwltool.pathmapper import PathMapper @@ -10,7 +8,7 @@ def test_subclass() -> None: class SubPathMapper(PathMapper): def __init__( self, - referenced_files: List[CWLObjectType], + referenced_files: list[CWLObjectType], basedir: str, stagedir: str, new: str, @@ -81,7 +79,7 @@ def test_normalizeFilesDirs(name: str, file_dir: CWLObjectType, expected: CWLObj @pytest.mark.parametrize("filename,expected", basename_generation_parameters) -def test_basename_field_generation(filename: str, expected: Tuple[str, str]) -> None: +def test_basename_field_generation(filename: str, expected: tuple[str, str]) -> None: nameroot, nameext = expected expected2 = { "class": "File", diff --git a/tests/test_provenance.py b/tests/test_provenance.py index 83eb61c22..e8d8416be 100644 --- a/tests/test_provenance.py +++ b/tests/test_provenance.py @@ -3,8 +3,9 @@ import pickle import sys import urllib +from collections.abc import Generator from pathlib import Path -from typing import IO, Any, Generator, cast +from typing import IO, Any, cast import arcp import bagit diff --git a/tests/test_relocate.py b/tests/test_relocate.py index 81877c776..692e995fa 100644 --- a/tests/test_relocate.py +++ b/tests/test_relocate.py @@ -1,18 +1,13 @@ import json import os import shutil -import sys +from io import StringIO from pathlib import Path from cwltool.main import main from .util import get_data, needs_docker -if sys.version_info[0] < 3: - from StringIO import StringIO -else: - from io import StringIO - @needs_docker def test_for_910(tmp_path: Path) -> None: diff --git a/tests/test_secrets.py b/tests/test_secrets.py index bd90bee78..a8c0b67af 100644 --- a/tests/test_secrets.py +++ b/tests/test_secrets.py @@ -1,7 +1,7 @@ import shutil import tempfile from io import StringIO -from typing import Callable, Dict, List, Tuple, Union +from typing import Callable, Union import pytest @@ -13,7 +13,7 @@ @pytest.fixture -def secrets() -> Tuple[SecretStore, CWLObjectType]: +def secrets() -> tuple[SecretStore, CWLObjectType]: """Fixture to return a secret store.""" sec_store = SecretStore() job: CWLObjectType = {"foo": "bar", "baz": "quux"} @@ -22,7 +22,7 @@ def secrets() -> Tuple[SecretStore, CWLObjectType]: return sec_store, job -def test_obscuring(secrets: Tuple[SecretStore, CWLObjectType]) -> None: +def test_obscuring(secrets: tuple[SecretStore, CWLObjectType]) -> None: """Basic test of secret store.""" storage, obscured = secrets assert obscured["foo"] != "bar" @@ -41,8 +41,8 @@ def test_obscuring(secrets: Tuple[SecretStore, CWLObjectType]) -> None: @pytest.mark.parametrize("factory,expected", obscured_factories_expected) def test_secrets( factory: Callable[[str], CWLObjectType], - expected: Union[str, List[str], Dict[str, str]], - secrets: Tuple[SecretStore, CWLObjectType], + expected: Union[str, list[str], dict[str, str]], + secrets: tuple[SecretStore, CWLObjectType], ) -> None: storage, obscured = secrets obs = obscured["foo"] diff --git a/tests/test_tmpdir.py b/tests/test_tmpdir.py index 73fe240d0..18a588cf8 100644 --- a/tests/test_tmpdir.py +++ b/tests/test_tmpdir.py @@ -6,7 +6,7 @@ import subprocess import sys from pathlib import Path -from typing import List, cast +from typing import cast import pytest from ruamel.yaml.comments import CommentedMap @@ -318,7 +318,7 @@ def test_docker_tmpdir_prefix(tmp_path: Path) -> None: "docker", ) job = DockerCommandLineJob(builder, {}, CommandLineTool.make_path_mapper, [], [], "") - runtime: List[str] = [] + runtime: list[str] = [] volume_writable_file = MapperEnt( resolved=get_data("tests/2.fastq"), target="foo", type=None, staged=None diff --git a/tests/test_toolargparse.py b/tests/test_toolargparse.py index 11ce5e3db..2e50fe722 100644 --- a/tests/test_toolargparse.py +++ b/tests/test_toolargparse.py @@ -1,7 +1,7 @@ import argparse from io import StringIO from pathlib import Path -from typing import Callable, List +from typing import Callable import pytest @@ -296,7 +296,7 @@ def test_argparser_without_doc() -> None: ), ], ) -def test_argparse_append_with_default(job_order: List[str], expected_values: List[str]) -> None: +def test_argparse_append_with_default(job_order: list[str], expected_values: list[str]) -> None: """ Confirm that the appended arguments must not include the default. diff --git a/tests/util.py b/tests/util.py index 0547cfa9a..44d2f108c 100644 --- a/tests/util.py +++ b/tests/util.py @@ -8,9 +8,10 @@ import shutil import subprocess import sys +from collections.abc import Generator, Mapping from contextlib import ExitStack from pathlib import Path -from typing import Dict, Generator, List, Mapping, Optional, Tuple, Union +from typing import Optional, Union import pytest @@ -83,11 +84,11 @@ def env_accepts_null() -> bool: def get_main_output( - args: List[str], + args: list[str], replacement_env: Optional[Mapping[str, str]] = None, extra_env: Optional[Mapping[str, str]] = None, monkeypatch: Optional[pytest.MonkeyPatch] = None, -) -> Tuple[Optional[int], str, str]: +) -> tuple[Optional[int], str, str]: """Run cwltool main. args: the command line args to call it with @@ -127,13 +128,13 @@ def get_main_output( def get_tool_env( tmp_path: Path, - flag_args: List[str], + flag_args: list[str], inputs_file: Optional[str] = None, replacement_env: Optional[Mapping[str, str]] = None, extra_env: Optional[Mapping[str, str]] = None, monkeypatch: Optional[pytest.MonkeyPatch] = None, runtime_env_accepts_null: Optional[bool] = None, -) -> Dict[str, str]: +) -> dict[str, str]: """Get the env vars for a tool's invocation.""" # GNU env accepts the -0 option to end each variable's # printing with "\0". No such luck on BSD-ish. diff --git a/tox.ini b/tox.ini index c75d0cc47..2a5a431b9 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,9 @@ [tox] envlist = - py3{8,9,10,11,12,13}-lint - py3{8,9,10,11,12,13}-unit - py3{8,9,10,11,12,13}-bandit - py3{8,9,10,11,12,13}-mypy + py3{9,10,11,12,13}-lint + py3{9,10,11,12,13}-unit + py3{9,10,11,12,13}-bandit + py3{9,10,11,12,13}-mypy py312-lintreadme py312-shellcheck py312-pydocstyle @@ -16,7 +16,6 @@ testpaths = tests [gh-actions] python = - 3.8: py38 3.9: py39 3.10: py310 3.11: py311 @@ -25,13 +24,13 @@ python = [testenv] skipsdist = - py3{8,9,10,11,12,13}-!{unit,mypy,lintreadme} = True + py3{9,10,11,12,13}-!{unit,mypy,lintreadme} = True description = - py3{8,9,10,11,12,13}-unit: Run the unit tests - py3{8,9,10,11,12,13}-lint: Lint the Python code - py3{8,9,10,11,12,13}-bandit: Search for common security issues - py3{8,9,10,11,12,13}-mypy: Check for type safety + py3{9,10,11,12,13}-unit: Run the unit tests + py3{9,10,11,12,13}-lint: Lint the Python code + py3{9,10,11,12,13}-bandit: Search for common security issues + py3{9,10,11,12,13}-mypy: Check for type safety py312-pydocstyle: docstring style checker py312-shellcheck: syntax check for shell scripts py312-lintreadme: Lint the README.rst→.md conversion @@ -44,14 +43,14 @@ passenv = SINGULARITY_FAKEROOT extras = - py3{8,9,10,11,12,13}-unit: deps + py3{9,10,11,12,13}-unit: deps deps = - py3{8,9,10,11,12,13}-{unit,lint,bandit,mypy}: -rrequirements.txt - py3{8,9,10,11,12,13}-{unit,mypy}: -rtest-requirements.txt - py3{8,9,10,11,12,13}-lint: -rlint-requirements.txt - py3{8,9,10,11,12,13}-bandit: bandit - py3{8,9,10,11,12,13}-mypy: -rmypy-requirements.txt + py3{9,10,11,12,13}-{unit,lint,bandit,mypy}: -rrequirements.txt + py3{9,10,11,12,13}-{unit,mypy}: -rtest-requirements.txt + py3{9,10,11,12,13}-lint: -rlint-requirements.txt + py3{9,10,11,12,13}-bandit: bandit + py3{9,10,11,12,13}-mypy: -rmypy-requirements.txt py312-pydocstyle: pydocstyle py312-pydocstyle: diff-cover py312-lintreadme: twine @@ -63,20 +62,20 @@ setenv = HOME = {envtmpdir} commands_pre = - py3{8,9,10,11,12,13}-unit: python -m pip install -U pip setuptools wheel + py3{9,10,11,12,13}-unit: python -m pip install -U pip setuptools wheel py312-lintreadme: python -m build --outdir {distdir} commands = - py3{8,9,10,11,12,13}-unit: make coverage-report coverage.xml PYTEST_EXTRA={posargs} - py3{8,9,10,11,12,13}-bandit: bandit -r cwltool - py3{8,9,10,11,12,13}-lint: make flake8 format-check codespell-check - py3{8,9,10,11,12,13}-mypy: make mypy PYTEST_EXTRA={posargs} - py3{8,9,10,11,12}-mypy: make mypyc PYTEST_EXTRA={posargs} + py3{9,10,11,12,13}-unit: make coverage-report coverage.xml PYTEST_EXTRA={posargs} + py3{9,10,11,12,13}-bandit: bandit -r cwltool + py3{9,10,11,12,13}-lint: make flake8 format-check codespell-check + py3{9,10,11,12,13}-mypy: make mypy PYTEST_EXTRA={posargs} + py3{9,10,11,12}-mypy: make mypyc PYTEST_EXTRA={posargs} py312-shellcheck: make shellcheck py312-pydocstyle: make diff_pydocstyle_report py312-lintreadme: twine check {distdir}/* skip_install = - py3{8,9,10,11,12,13}-{bandit,lint,mypy,shellcheck,pydocstyle,lintreadme}: true + py3{9,10,11,12,13}-{bandit,lint,mypy,shellcheck,pydocstyle,lintreadme}: true allowlist_externals = make From a9567667af28688042a5369bf35213d7932c3d9c Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Fri, 4 Oct 2024 14:56:29 +0200 Subject: [PATCH 13/43] add more docs --- cwltool/builder.py | 2 ++ cwltool/checker.py | 2 ++ cwltool/command_line_tool.py | 2 ++ cwltool/cwlprov/provenance_profile.py | 1 + cwltool/flatten.py | 13 +++++++++---- cwltool/load_tool.py | 1 + cwltool/main.py | 1 + cwltool/pack.py | 1 + cwltool/pathmapper.py | 7 +++++-- cwltool/process.py | 1 + cwltool/software_requirements.py | 1 + cwltool/stdfsaccess.py | 2 ++ cwltool/subgraph.py | 6 ++++++ 13 files changed, 34 insertions(+), 6 deletions(-) diff --git a/cwltool/builder.py b/cwltool/builder.py index 066a77f86..e1de5b857 100644 --- a/cwltool/builder.py +++ b/cwltool/builder.py @@ -161,6 +161,7 @@ def __init__( self.container_engine = container_engine def build_job_script(self, commands: list[str]) -> Optional[str]: + """Use the job_script_provider to turn the commands into a job script.""" if self.job_script_provider is not None: return self.job_script_provider.build_job_script(self, commands) return None @@ -607,6 +608,7 @@ def tostr(self, value: Union[MutableMapping[str, str], Any]) -> str: return str(value) def generate_arg(self, binding: CWLObjectType) -> list[str]: + """Convert an input binding to a list of command line arguments.""" value = binding.get("datum") debug = _logger.isEnabledFor(logging.DEBUG) if "valueFrom" in binding: diff --git a/cwltool/checker.py b/cwltool/checker.py index 7742adf5c..17cba77ba 100644 --- a/cwltool/checker.py +++ b/cwltool/checker.py @@ -156,6 +156,7 @@ def _rec_fields(rec: MutableMapping[str, Any]) -> MutableMapping[str, Any]: def missing_subset(fullset: list[Any], subset: list[Any]) -> list[Any]: + """Calculate the items missing from the fullset given the subset.""" missing = [] for i in subset: if i not in fullset: @@ -498,6 +499,7 @@ def get_step_id(field_id: str) -> str: def is_conditional_step(param_to_step: dict[str, CWLObjectType], parm_id: str) -> bool: + """Return True if the step given by the parm_id is a conditional step.""" if (source_step := param_to_step.get(parm_id)) is not None: if source_step.get("when") is not None: return True diff --git a/cwltool/command_line_tool.py b/cwltool/command_line_tool.py index de1878593..e201fb12b 100644 --- a/cwltool/command_line_tool.py +++ b/cwltool/command_line_tool.py @@ -228,6 +228,7 @@ def job( def remove_path(f: CWLObjectType) -> None: + """Remove any 'path' property, if present.""" if "path" in f: del f["path"] @@ -404,6 +405,7 @@ def __init__(self, toolpath_object: CommentedMap, loadingContext: LoadingContext ) def make_job_runner(self, runtimeContext: RuntimeContext) -> type[JobBase]: + """Return the correct CommandLineJob class given the container settings.""" dockerReq, dockerRequired = self.get_requirement("DockerRequirement") mpiReq, mpiRequired = self.get_requirement(MPIRequirementName) diff --git a/cwltool/cwlprov/provenance_profile.py b/cwltool/cwlprov/provenance_profile.py index 59d835fff..d4dfd6cb4 100644 --- a/cwltool/cwlprov/provenance_profile.py +++ b/cwltool/cwlprov/provenance_profile.py @@ -281,6 +281,7 @@ def record_process_end( self.document.wasEndedBy(process_run_id, None, self.workflow_run_uri, when) def declare_file(self, value: CWLObjectType) -> tuple[ProvEntity, ProvEntity, str]: + """Construct a FileEntity for the given CWL File object.""" if value["class"] != "File": raise ValueError("Must have class:File: %s" % value) # Need to determine file hash aka RO filename diff --git a/cwltool/flatten.py b/cwltool/flatten.py index 5c9738cbf..3c057ebbe 100644 --- a/cwltool/flatten.py +++ b/cwltool/flatten.py @@ -1,12 +1,17 @@ -from typing import Any, Callable, cast +""" +Our version of the popular flatten() method. + +http://rightfootin.blogspot.com/2006/09/more-on-python-flatten.html +""" -# http://rightfootin.blogspot.com/2006/09/more-on-python-flatten.html +from typing import Any, Callable, cast -def flatten(thing, ltypes=(list, tuple)): - # type: (Any, Any) -> List[Any] +def flatten(thing: Any) -> list[Any]: + """Flatten a list without recursion problems.""" if thing is None: return [] + ltypes = (list, tuple) if not isinstance(thing, ltypes): return [thing] diff --git a/cwltool/load_tool.py b/cwltool/load_tool.py index 7a58a8330..4d7f3a930 100644 --- a/cwltool/load_tool.py +++ b/cwltool/load_tool.py @@ -626,6 +626,7 @@ def resolve_overrides( def load_overrides(ov: str, base_url: str) -> list[CWLObjectType]: + """Load and resolve any overrides.""" ovloader = Loader(overrides_ctx) return resolve_overrides(ovloader.fetch(ov), ov, base_url) diff --git a/cwltool/main.py b/cwltool/main.py index 9477cb1a2..99928d0bd 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -626,6 +626,7 @@ def print_pack( def supported_cwl_versions(enable_dev: bool) -> list[str]: + """Return a list of currently supported CWL versions.""" # ALLUPDATES and UPDATES are dicts if enable_dev: versions = list(ALLUPDATES) diff --git a/cwltool/pack.py b/cwltool/pack.py index 99684e003..d3705d5e4 100644 --- a/cwltool/pack.py +++ b/cwltool/pack.py @@ -51,6 +51,7 @@ def find_ids( def replace_refs(d: Any, rewrite: dict[str, str], stem: str, newstem: str) -> None: + """Replace references with the actual value.""" if isinstance(d, MutableSequence): for s, v in enumerate(d): if isinstance(v, str): diff --git a/cwltool/pathmapper.py b/cwltool/pathmapper.py index 86fd9ae82..10cb7a733 100644 --- a/cwltool/pathmapper.py +++ b/cwltool/pathmapper.py @@ -188,8 +188,11 @@ def visit( ) def setup(self, referenced_files: list[CWLObjectType], basedir: str) -> None: - # Go through each file and set the target to its own directory along - # with any secondary files. + """ + For each file, set the target to its own directory. + + Also processes secondary files into that same directory. + """ stagedir = self.stagedir for fob in referenced_files: if self.separateDirs: diff --git a/cwltool/process.py b/cwltool/process.py index ff96985c5..fe5f84764 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -1073,6 +1073,7 @@ def __str__(self) -> str: def uniquename(stem: str, names: Optional[set[str]] = None) -> str: + """Construct a thread-unique name using the given stem as a prefix.""" global _names if names is None: names = _names diff --git a/cwltool/software_requirements.py b/cwltool/software_requirements.py index de34f8f6d..6ad84da4b 100644 --- a/cwltool/software_requirements.py +++ b/cwltool/software_requirements.py @@ -71,6 +71,7 @@ def __init__(self, args: argparse.Namespace) -> None: os.makedirs(self.tool_dependency_dir) def build_job_script(self, builder: "Builder", command: list[str]) -> str: + """Use the galaxy-tool-util library to construct a build script.""" ensure_galaxy_lib_available() resolution_config_dict = { "use": self.use_tool_dependencies, diff --git a/cwltool/stdfsaccess.py b/cwltool/stdfsaccess.py index 056b4b912..c58257f63 100644 --- a/cwltool/stdfsaccess.py +++ b/cwltool/stdfsaccess.py @@ -32,6 +32,7 @@ def _abs(self, p: str) -> str: return abspath(p, self.basedir) def glob(self, pattern: str) -> list[str]: + """Return a possibly empty list of absolute URI paths that match pathname.""" return [file_uri(str(self._abs(line))) for line in glob.glob(self._abs(pattern))] def open(self, fn: str, mode: str) -> IO[Any]: @@ -50,6 +51,7 @@ def isdir(self, fn: str) -> bool: return os.path.isdir(self._abs(fn)) def listdir(self, fn: str) -> list[str]: + """Return a list containing the absolute path URLs of the entries in the directory given by path.""" return [abspath(urllib.parse.quote(entry), fn) for entry in os.listdir(self._abs(fn))] def join(self, path, *paths): # type: (str, *str) -> str diff --git a/cwltool/subgraph.py b/cwltool/subgraph.py index 550dc7838..204e987a8 100644 --- a/cwltool/subgraph.py +++ b/cwltool/subgraph.py @@ -38,6 +38,12 @@ def subgraph_visit( def declare_node(nodes: dict[str, Node], nodeid: str, tp: Optional[str]) -> Node: + """ + Record the given nodeid in the graph. + + If the nodeid is already present, but its type is unset, set it. + :returns: The Node tuple (even if already present in the graph). + """ if nodeid in nodes: n = nodes[nodeid] if n.type is None: From 606ec26c754aadcfbec4f2898a05ca98e1539e4c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Oct 2024 16:26:53 +0000 Subject: [PATCH 14/43] Bump mypy from 1.11.2 to 1.12.0 Bumps [mypy](https://github.com/python/mypy) from 1.11.2 to 1.12.0. - [Changelog](https://github.com/python/mypy/blob/master/CHANGELOG.md) - [Commits](https://github.com/python/mypy/compare/v1.11.2...v1.12.0) --- updated-dependencies: - dependency-name: mypy dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- mypy-requirements.txt | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index ccd7737f4..4aa7f0a2d 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -1,4 +1,4 @@ -mypy==1.11.2 # also update pyproject.toml +mypy==1.12.0 # also update pyproject.toml ruamel.yaml>=0.16.0,<0.19 cwl-utils>=0.32 cwltest diff --git a/pyproject.toml b/pyproject.toml index 4f3f91c31..4d93750ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ requires = [ "setuptools>=45", "setuptools_scm[toml]>=8.0.4,<9", - "mypy==1.11.2", # also update mypy-requirements.txt + "mypy==1.12.0", # also update mypy-requirements.txt "types-requests", "types-psutil", "importlib_resources>=1.4;python_version<'3.9'", From 74a08ca0e90a223f98fdec73c88f36d783a4dde0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 17 Oct 2024 07:26:21 +0000 Subject: [PATCH 15/43] Update rdflib requirement from <7.1,>=4.2.2 to >=4.2.2,<7.2 Updates the requirements on [rdflib](https://github.com/RDFLib/rdflib) to permit the latest version. - [Release notes](https://github.com/RDFLib/rdflib/releases) - [Changelog](https://github.com/RDFLib/rdflib/blob/main/CHANGELOG.md) - [Commits](https://github.com/RDFLib/rdflib/compare/4.2.2...7.1.0) --- updated-dependencies: - dependency-name: rdflib dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index b1fc2207d..3ac631838 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ requests>=2.6.1 ruamel.yaml>=0.16.0,<0.19 -rdflib>=4.2.2,<7.1 +rdflib>=4.2.2,<7.2 schema-salad>=8.7,<9 prov==1.5.1 mypy-extensions diff --git a/setup.py b/setup.py index 40c3fd8d4..9bbee10f1 100644 --- a/setup.py +++ b/setup.py @@ -124,7 +124,7 @@ "requests >= 2.6.1", # >= 2.6.1 to workaround # https://github.com/ionrock/cachecontrol/issues/137 "ruamel.yaml >= 0.16, < 0.19", - "rdflib >= 4.2.2, < 7.1.0", + "rdflib >= 4.2.2, < 7.2.0", "schema-salad >= 8.7, < 9", "prov == 1.5.1", "mypy-extensions", From e3e6bf9d51946579081ee3b41c28f1da194aa895 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 22 Oct 2024 13:55:55 +0000 Subject: [PATCH 16/43] Bump mypy from 1.12.0 to 1.12.1 (#2057) * Bump mypy from 1.12.0 to 1.12.1 Bumps [mypy](https://github.com/python/mypy) from 1.12.0 to 1.12.1. - [Changelog](https://github.com/python/mypy/blob/master/CHANGELOG.md) - [Commits](https://github.com/python/mypy/compare/v1.12.0...v1.12.1) --- updated-dependencies: - dependency-name: mypy dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Update pyproject.toml --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Michael R. Crusoe <1330696+mr-c@users.noreply.github.com> --- mypy-requirements.txt | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index 4aa7f0a2d..d090f1943 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -1,4 +1,4 @@ -mypy==1.12.0 # also update pyproject.toml +mypy==1.12.1 # also update pyproject.toml ruamel.yaml>=0.16.0,<0.19 cwl-utils>=0.32 cwltest diff --git a/pyproject.toml b/pyproject.toml index 4d93750ff..c2546d266 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ requires = [ "setuptools>=45", "setuptools_scm[toml]>=8.0.4,<9", - "mypy==1.12.0", # also update mypy-requirements.txt + "mypy==1.12.1", # also update mypy-requirements.txt "types-requests", "types-psutil", "importlib_resources>=1.4;python_version<'3.9'", From 8dee8e9b4227b30a8a0c1f57e69d158e0c56f4ad Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 23 Oct 2024 08:52:24 +0000 Subject: [PATCH 17/43] Bump mypy from 1.12.1 to 1.13.0 (#2058) * Bump mypy from 1.12.1 to 1.13.0 Bumps [mypy](https://github.com/python/mypy) from 1.12.1 to 1.13.0. - [Changelog](https://github.com/python/mypy/blob/master/CHANGELOG.md) - [Commits](https://github.com/python/mypy/compare/v1.12.1...v1.13.0) --- updated-dependencies: - dependency-name: mypy dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Update pyproject.toml --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Michael R. Crusoe <1330696+mr-c@users.noreply.github.com> --- mypy-requirements.txt | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index d090f1943..5f18fa03a 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -1,4 +1,4 @@ -mypy==1.12.1 # also update pyproject.toml +mypy==1.13.0 # also update pyproject.toml ruamel.yaml>=0.16.0,<0.19 cwl-utils>=0.32 cwltest diff --git a/pyproject.toml b/pyproject.toml index c2546d266..cec213f52 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ requires = [ "setuptools>=45", "setuptools_scm[toml]>=8.0.4,<9", - "mypy==1.12.1", # also update mypy-requirements.txt + "mypy==1.13.0", # also update mypy-requirements.txt "types-requests", "types-psutil", "importlib_resources>=1.4;python_version<'3.9'", From e3f6cf77f2e81bd97db96234b79c968227429143 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Wed, 23 Oct 2024 12:01:04 +0200 Subject: [PATCH 18/43] build binary wheels But don't publish them yet --- .circleci/config.yml | 104 +++++++++++++++++++++++++ .github/workflows/ci-tests.yml | 16 ++-- .github/workflows/wheels.yml | 130 +++++++++++++++++++++++++++++++ MANIFEST.in | 1 + cibw-requirements.txt | 1 + cwltool/software_requirements.py | 2 + pyproject.toml | 13 ++++ test-requirements.txt | 2 +- tests/test_dependencies.py | 10 +-- 9 files changed, 265 insertions(+), 14 deletions(-) create mode 100644 .circleci/config.yml create mode 100644 .github/workflows/wheels.yml create mode 100644 cibw-requirements.txt diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000..2127e18be --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,104 @@ +version: 2.1 + +parameters: + REF: + type: string + default: '' + description: Optional tag to build + +jobs: + arm-wheels: + parameters: + build: + type: string + image: + type: string + + machine: + image: ubuntu-2204:current + resource_class: arm.medium # two vCPUs + + environment: + CIBW_ARCHS: "aarch64" + CIBW_MANYLINUX_AARCH64_IMAGE: "<< parameters.image >>" + CIBW_MUSLLINUX_AARCH64_IMAGE: "<< parameters.image >>" + CIBW_BUILD: "<< parameters.build >>" + + steps: + - checkout + - when: + condition: << pipeline.parameters.REF >> + steps: + - run: + name: Checkout branch/tag << pipeline.parameters.REF >> + command: | + echo "Switching to branch/tag << pipeline.parameters.REF >> if it exists" + git checkout << pipeline.parameters.REF >> || true + git pull origin << pipeline.parameters.REF >> || true + - run: + name: install cibuildwheel and other build reqs + command: | + python3 -m pip install --upgrade pip setuptools setuptools_scm[toml] + python3 -m pip install -rcibw-requirements.txt + + - run: + name: pip freeze + command: | + python3 -m pip freeze + + - run: + name: list wheels + command: | + python3 -m cibuildwheel . --print-build-identifiers + + - run: + name: cibuildwheel + command: | + python3 -m cibuildwheel . + + - store_test_results: + path: test-results/ + + - store_artifacts: + path: wheelhouse/ + + # - when: + # condition: + # or: + # - matches: + # pattern: ".+" + # value: "<< pipeline.git.tag >>" + # - << pipeline.parameters.REF >> + # steps: + # - run: + # environment: + # TWINE_NONINTERACTIVE: "1" + # command: | + # python3 -m pip install twine + # python3 -m twine upload --verbose --skip-existing wheelhouse/* + +workflows: + wheels: # This is the name of the workflow, feel free to change it to better match your workflow. + # Inside the workflow, you define the jobs you want to run. + jobs: + - arm-wheels: + name: arm-wheels-manylinux_2_28 + filters: + tags: + only: /.*/ + build: "*manylinux*" + image: quay.io/pypa/manylinux_2_28_aarch64 + - arm-wheels: + name: arm-wheels-musllinux_1_1 + filters: + tags: + only: /.*/ + build: "*musllinux*" + image: quay.io/pypa/musllinux_1_1_aarch64 + - arm-wheels: + name: arm-wheels-musllinux_1_2 + filters: + tags: + only: /.*/ + build: "*musllinux*" + image: quay.io/pypa/musllinux_1_2_aarch64 diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 2095b53b6..e5d9a7e83 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -44,11 +44,11 @@ jobs: with: fetch-depth: 0 - - name: Set up Singularity + - name: Set up Singularity and environment-modules if: ${{ matrix.step == 'unit' || matrix.step == 'mypy' }} run: | wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-focal_amd64.deb - sudo apt-get install -y ./singularity-ce_3.10.4-focal_amd64.deb + sudo apt-get install -y ./singularity-ce_3.10.4-focal_amd64.deb environment-modules - name: Give the test runner user a name to make provenance happy. if: ${{ matrix.step == 'unit' || matrix.step == 'mypy' }} @@ -132,10 +132,10 @@ jobs: with: fetch-depth: 0 - - name: Set up Singularity + - name: Set up Singularity and environment-modules run: | wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-focal_amd64.deb - sudo apt-get install -y ./singularity-ce_3.10.4-focal_amd64.deb + sudo apt-get install -y ./singularity-ce_3.10.4-focal_amd64.deb environment-modules - name: Give the test runner user a name to make provenance happy. run: sudo usermod -c 'CI Runner' "$(whoami)" @@ -180,11 +180,11 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Set up Singularity + - name: Set up Singularity and environment-modules if: ${{ matrix.container == 'singularity' }} run: | wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-jammy_amd64.deb - sudo apt-get install -y ./singularity-ce_3.10.4-jammy_amd64.deb + sudo apt-get install -y ./singularity-ce_3.10.4-jammy_amd64.deb environment-modules - name: Singularity cache if: ${{ matrix.container == 'singularity' }} @@ -229,10 +229,10 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Set up Singularity + - name: Set up Singularity and environment-modules run: | wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-jammy_amd64.deb - sudo apt-get install -y ./singularity-ce_3.10.4-jammy_amd64.deb + sudo apt-get install -y ./singularity-ce_3.10.4-jammy_amd64.deb environment-modules - name: Set up Python uses: actions/setup-python@v5 diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml new file mode 100644 index 000000000..9c14eb4e7 --- /dev/null +++ b/.github/workflows/wheels.yml @@ -0,0 +1,130 @@ +name: Python package build and publish + +on: + release: + types: [published] + workflow_dispatch: {} + repository_dispatch: {} + pull_request: + push: + branches: + - main + +concurrency: + group: wheels-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + build_wheels: + name: ${{ matrix.image }} wheels + runs-on: ubuntu-24.04 + strategy: + matrix: + include: + - image: manylinux_2_28_x86_64 + build: "*manylinux*" + - image: musllinux_1_1_x86_64 + build: "*musllinux*" + - image: musllinux_1_2_x86_64 + build: "*musllinux*" + + steps: + - uses: actions/checkout@v4 + if: ${{ github.event_name != 'repository_dispatch' }} + with: + fetch-depth: 0 # slow, but gets all the tags + - uses: actions/checkout@v4 + if: ${{ github.event_name == 'repository_dispatch' }} + with: + fetch-depth: 0 # slow, but gets all the tags + ref: ${{ github.event.client_payload.ref }} + + # - name: Set up QEMU + # if: runner.os == 'Linux' + # uses: docker/setup-qemu-action@v2 + # with: + # platforms: all + + - name: Build wheels + uses: pypa/cibuildwheel@v2.21.3 + env: + CIBW_BUILD: ${{ matrix.build }} + CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/${{ matrix.image }} + CIBW_MUSLLINUX_X86_64_IMAGE: quay.io/pypa/${{ matrix.image }} + # configure cibuildwheel to build native 64-bit archs ('auto64'), and some + # emulated ones + # Linux arm64 wheels are built on circleci + CIBW_ARCHS_LINUX: auto64 # ppc64le s390x + + - uses: actions/upload-artifact@v4 + with: + name: artifact-${{ matrix.image }} + path: ./wheelhouse/*.whl + + build_sdist: + name: Build source distribution + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + if: ${{ github.event_name != 'repository_dispatch' }} + with: + fetch-depth: 0 # slow, but gets all the tags + - uses: actions/checkout@v4 + if: ${{ github.event_name == 'repository_dispatch' }} + with: + fetch-depth: 0 # slow, but gets all the tags + ref: ${{ github.event.client_payload.ref }} + + - name: Build sdist + run: pipx run build --sdist + + - uses: actions/upload-artifact@v4 + with: + name: artifact-source + path: dist/*.tar.gz + + build_wheels_macos: + name: Build wheels on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + # macos-13 is an intel runner, macos-14 is apple silicon + os: [macos-13, macos-14] + steps: + - uses: actions/checkout@v4 + if: ${{ github.event_name != 'repository_dispatch' }} + with: + fetch-depth: 0 # slow, but gets all the tags + - uses: actions/checkout@v4 + if: ${{ github.event_name == 'repository_dispatch' }} + with: + fetch-depth: 0 # slow, but gets all the tags + ref: ${{ github.event.client_payload.ref }} + + - name: Build wheels + uses: pypa/cibuildwheel@v2.21.3 + + - uses: actions/upload-artifact@v4 + with: + name: artifact-${{ matrix.os }}-${{ strategy.job-index }} + path: ./wheelhouse/*.whl + + # upload_pypi: + # needs: [build_wheels, build_sdist] + # runs-on: ubuntu-24.04 + # environment: deploy + # permissions: + # id-token: write + # if: (github.event_name == 'release' && github.event.action == 'published') || (github.event_name == 'repository_dispatch' && github.event.client_payload.publish_wheel == true) + # steps: + # - uses: actions/download-artifact@v4 + # with: + # # unpacks default artifact into dist/ + # pattern: artifact-* + # merge-multiple: true + # path: dist + + # - name: Publish package distributions to PyPI + # uses: pypa/gh-action-pypi-publish@release/v1 + # with: + # skip-existing: true diff --git a/MANIFEST.in b/MANIFEST.in index 187d19bea..7ee34f35e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -19,6 +19,7 @@ include tests/reloc/dir2/* include tests/checker_wf/* include tests/subgraph/* include tests/input_deps/* +recursive-include tests/test_deps_env include tests/trs/* include tests/wf/generator/* include cwltool/py.typed diff --git a/cibw-requirements.txt b/cibw-requirements.txt new file mode 100644 index 000000000..c4511439c --- /dev/null +++ b/cibw-requirements.txt @@ -0,0 +1 @@ +cibuildwheel==2.21.3 diff --git a/cwltool/software_requirements.py b/cwltool/software_requirements.py index 6ad84da4b..3d4d48f6b 100644 --- a/cwltool/software_requirements.py +++ b/cwltool/software_requirements.py @@ -50,6 +50,8 @@ class DependenciesConfiguration: def __init__(self, args: argparse.Namespace) -> None: """Initialize.""" + self.tool_dependency_dir: Optional[str] = None + self.dependency_resolvers_config_file: Optional[str] = None conf_file = getattr(args, "beta_dependency_resolvers_configuration", None) tool_dependency_dir = getattr(args, "beta_dependencies_directory", None) conda_dependencies = getattr(args, "beta_conda_dependencies", None) diff --git a/pyproject.toml b/pyproject.toml index cec213f52..a69720739 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,19 @@ build-backend = "setuptools.build_meta" [tool.setuptools_scm] write_to = "cwltool/_version.py" +[tool.cibuildwheel] +test-command = "python -m pytest -n 2 --junitxml={project}/test-results/junit_$(python -V | awk '{print $2}')_${AUDITWHEEL_PLAT}.xml -k 'not (test_bioconda or test_env_filtering or test_udocker)' --pyargs cwltool" +test-requires = "-r test-requirements.txt" +test-extras = "deps" +skip = "pp*" +# ^ skip building wheels on PyPy (any version) +build-verbosity = 1 +environment = { CWLTOOL_USE_MYPYC="1", MYPYPATH="$(pwd)/mypy-stubs" } + +# Install system library +[tool.cibuildwheel.linux] +before-all = "apk add libxml2-dev libxslt-dev nodejs || yum install -y libxml2-devel libxslt-devel nodejs environment-modules || apt-get install -y --no-install-recommends libxml2-dev libxslt-dev nodejs environment-modules" + [tool.black] line-length = 100 target-version = [ "py39" ] diff --git a/test-requirements.txt b/test-requirements.txt index e545ee65a..8b0908f2e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,7 +3,7 @@ pytest>= 6.2,< 8.4 pytest-xdist>=3.2.0 # for the worksteal scheduler psutil # enhances pytest-xdist to allow "-n logical" pytest-httpserver -pytest-retry;python_version>'3.9' +pytest-retry;python_version>='3.9' mock>=2.0.0 pytest-mock>=1.10.0 pytest-cov diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index b903c04d6..f5ac0274b 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -119,15 +119,15 @@ def test_modules(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: """Do a basic smoke test using environment modules to satisfy a SoftwareRequirement.""" wflow = get_data("tests/random_lines.cwl") job = get_data("tests/random_lines_job.json") - monkeypatch.setenv("MODULEPATH", os.path.join(os.getcwd(), "tests/test_deps_env/modulefiles")) + monkeypatch.setenv("MODULEPATH", get_data("tests/test_deps_env/modulefiles")) error_code, _, stderr = get_main_output( [ "--outdir", str(tmp_path / "out"), - "--beta-dependency-resolvers-configuration", "--beta-dependencies-directory", str(tmp_path / "deps"), - "tests/test_deps_env_modules_resolvers_conf.yml", + "--beta-dependency-resolvers-configuration", + get_data("tests/test_deps_env_modules_resolvers_conf.yml"), "--debug", wflow, job, @@ -145,7 +145,7 @@ def test_modules_environment(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Do so by by running `env` as the tool and parsing its output. """ - monkeypatch.setenv("MODULEPATH", os.path.join(os.getcwd(), "tests/test_deps_env/modulefiles")) + monkeypatch.setenv("MODULEPATH", get_data("tests/test_deps_env/modulefiles")) tool_env = get_tool_env( tmp_path, [ @@ -155,6 +155,6 @@ def test_modules_environment(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> get_data("tests/env_with_software_req.yml"), ) - assert tool_env["TEST_VAR_MODULE"] == "environment variable ends in space " + assert tool_env["TEST_VAR_MODULE"] == "environment variable ends in space ", tool_env tool_path = tool_env["PATH"].split(":") assert get_data("tests/test_deps_env/random-lines/1.0/scripts") in tool_path From 6cfef62c21330672538fd5e9b45ec888569c0a6f Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Thu, 24 Oct 2024 15:35:55 +0200 Subject: [PATCH 19/43] binary wheels: only build on main and other optimizations - cibuildwheel: adjust pytest parallel execution - cicircle: upgrade to ubuntu 24.04 - Stop building for musllinux_1_1 https://github.com/pypa/manylinux/issues/1629 > musl libc 1.1 is EOL and Alpine Linux 3.12 also (support ended 2 years ago, May 1st, 2022). --- .circleci/config.yml | 15 ++++++--------- .github/workflows/wheels.yml | 3 --- pyproject.toml | 2 +- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2127e18be..fce789feb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,8 +15,8 @@ jobs: type: string machine: - image: ubuntu-2204:current - resource_class: arm.medium # two vCPUs + image: ubuntu-2404:current + resource_class: arm.medium # 2 vCPUs environment: CIBW_ARCHS: "aarch64" @@ -84,20 +84,17 @@ workflows: - arm-wheels: name: arm-wheels-manylinux_2_28 filters: + branches: + only: main tags: only: /.*/ build: "*manylinux*" image: quay.io/pypa/manylinux_2_28_aarch64 - - arm-wheels: - name: arm-wheels-musllinux_1_1 - filters: - tags: - only: /.*/ - build: "*musllinux*" - image: quay.io/pypa/musllinux_1_1_aarch64 - arm-wheels: name: arm-wheels-musllinux_1_2 filters: + branches: + only: main tags: only: /.*/ build: "*musllinux*" diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 9c14eb4e7..fb54729d4 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -5,7 +5,6 @@ on: types: [published] workflow_dispatch: {} repository_dispatch: {} - pull_request: push: branches: - main @@ -23,8 +22,6 @@ jobs: include: - image: manylinux_2_28_x86_64 build: "*manylinux*" - - image: musllinux_1_1_x86_64 - build: "*musllinux*" - image: musllinux_1_2_x86_64 build: "*musllinux*" diff --git a/pyproject.toml b/pyproject.toml index a69720739..cec96b76b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ build-backend = "setuptools.build_meta" write_to = "cwltool/_version.py" [tool.cibuildwheel] -test-command = "python -m pytest -n 2 --junitxml={project}/test-results/junit_$(python -V | awk '{print $2}')_${AUDITWHEEL_PLAT}.xml -k 'not (test_bioconda or test_env_filtering or test_udocker)' --pyargs cwltool" +test-command = "python -m pytest --ignore cwltool/schemas -n logical --dist worksteal --junitxml={project}/test-results/junit_$(python -V | awk '{print $2}')_${AUDITWHEEL_PLAT}.xml -k 'not (test_bioconda or test_env_filtering or test_udocker)' --pyargs cwltool" test-requires = "-r test-requirements.txt" test-extras = "deps" skip = "pp*" From d810958c3cb262db8f099c08a0d6e50989edbefd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 1 Nov 2024 07:08:45 +0000 Subject: [PATCH 20/43] Update flake8-bugbear requirement from <24.9 to <24.11 Updates the requirements on [flake8-bugbear](https://github.com/PyCQA/flake8-bugbear) to permit the latest version. - [Release notes](https://github.com/PyCQA/flake8-bugbear/releases) - [Commits](https://github.com/PyCQA/flake8-bugbear/compare/16.4.1...24.10.31) --- updated-dependencies: - dependency-name: flake8-bugbear dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- lint-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lint-requirements.txt b/lint-requirements.txt index 5af76cd93..ec1e6e1fe 100644 --- a/lint-requirements.txt +++ b/lint-requirements.txt @@ -1,3 +1,3 @@ -flake8-bugbear<24.9 +flake8-bugbear<24.11 black==24.* codespell From 9cda157cb4380e9d30dec29f0452c56d0c10d064 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Sat, 2 Nov 2024 10:30:37 +0100 Subject: [PATCH 21/43] cpu_count can be None, so fallback to 1 --- cwltool/executors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cwltool/executors.py b/cwltool/executors.py index 6070462ab..03fa948e7 100644 --- a/cwltool/executors.py +++ b/cwltool/executors.py @@ -273,7 +273,7 @@ def __init__(self) -> None: self.pending_jobs_lock = threading.Lock() self.max_ram = int(psutil.virtual_memory().available / 2**20) - self.max_cores = float(psutil.cpu_count()) + self.max_cores = float(psutil.cpu_count() or 1) self.max_cuda = cuda_version_and_device_count()[1] self.allocated_ram = float(0) self.allocated_cores = float(0) @@ -429,7 +429,7 @@ def run_jobs( logger: logging.Logger, runtime_context: RuntimeContext, ) -> None: - self.taskqueue: TaskQueue = TaskQueue(threading.Lock(), psutil.cpu_count()) + self.taskqueue: TaskQueue = TaskQueue(threading.Lock(), psutil.cpu_count() or 1) try: jobiter = process.job(job_order_object, self.output_callback, runtime_context) From ee30368e4f9175c76e194c9f5fa734a5ca4b767c Mon Sep 17 00:00:00 2001 From: Dominik Brilhaus Date: Fri, 8 Nov 2024 09:46:13 +0100 Subject: [PATCH 22/43] add note on docker platform issue (#2064) Co-authored-by: Michael R. Crusoe <1330696+mr-c@users.noreply.github.com> --- README.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/README.rst b/README.rst index db40d0420..dce55fc95 100644 --- a/README.rst +++ b/README.rst @@ -189,6 +189,22 @@ and ``--tmp-outdir-prefix`` to somewhere under ``/Users``:: $ cwl-runner --tmp-outdir-prefix=/Users/username/project --tmpdir-prefix=/Users/username/project wc-tool.cwl wc-job.json + +Docker default platform on macOS with Apple Silicon +=================================================== + +If macOS users want to run CWL tools/workflows using ``cwltool`` with Docker and their software containers only have support for amd64 (64-bit x86) CPUs, but they have an Apple Silicon (aarch64/arm64) CPU, +they run into the error: + + WARNING: The requested image's platform (linux/amd64) does not match the detected host platform (linux/arm64/v8) and no specific platform was requested. + +To fix this, export the following environment variable before executing `cwltool`: + +``export DOCKER_DEFAULT_PLATFORM=linux/amd64`` + +To automatically have this variable set in the future, add the same command to ones respective shell profile (e.g. ``~/.zshrc``, ``~/.bash_profile``). + + Using uDocker ============= From 0b649350fdb7f9dc4acff3bb33fcfd062334df9f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Nov 2024 09:52:36 +0000 Subject: [PATCH 23/43] Update ruamel-yaml requirement from <0.18,>=0.16.0 to >=0.16.0,<0.19 (#2066) Updates the requirements on [ruamel-yaml]() to permit the latest version. --- updated-dependencies: - dependency-name: ruamel-yaml dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index cec96b76b..7ddf547f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ requires = [ "types-requests", "types-psutil", "importlib_resources>=1.4;python_version<'3.9'", - "ruamel.yaml>=0.16.0,<0.18", + "ruamel.yaml>=0.16.0,<0.19", "schema-salad>=8.7,<9", "cwl-utils>=0.32", "toml", From 048eb55aefd8d71d161fbc89ec0e888b8bfa0aa1 Mon Sep 17 00:00:00 2001 From: Kostas Mavrommatis Date: Mon, 11 Nov 2024 13:38:32 +0100 Subject: [PATCH 24/43] use max_cores in taskQueue instead of system cores (#2038) The class TascQueue accepts an argument `thread_count` that is used for the max size of a queue. In the `MultithreadedJobExecutor `class there is a variable defined (`max_cores`) that is getting its value from the available cores of the machine. Further down in the same class when TaskQueue is called instead of using the `max_cores` it is using `psutil.cpu_count()`. I suggest to use the self.max_cores in the call of TaskQueue in the file executor.py instead of psutil.cpu_count() Use case: when a job executor is setup as MultithreadedJobExecutor one can override the max_cores after the initialization of the object and limit the use to the specified cores. Additional enhancement would be to include an argument to allow the use to provide the number of cores available for use Co-authored-by: Michael R. Crusoe <1330696+mr-c@users.noreply.github.com> --- cwltool/executors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cwltool/executors.py b/cwltool/executors.py index 03fa948e7..e25426c9d 100644 --- a/cwltool/executors.py +++ b/cwltool/executors.py @@ -429,7 +429,7 @@ def run_jobs( logger: logging.Logger, runtime_context: RuntimeContext, ) -> None: - self.taskqueue: TaskQueue = TaskQueue(threading.Lock(), psutil.cpu_count() or 1) + self.taskqueue: TaskQueue = TaskQueue(threading.Lock(), int(math.ceil(self.max_cores))) try: jobiter = process.job(job_order_object, self.output_callback, runtime_context) From 1557c8ded38dc1af4a296e8fdd3116d7dfc5282e Mon Sep 17 00:00:00 2001 From: Sameeul Samee Date: Sun, 10 Nov 2024 06:23:56 -0500 Subject: [PATCH 25/43] Use "run" with singularity/apptainer instead of "exec", when possible Co-authored-by: Michael R. Crusoe <1330696+mr-c@users.noreply.github.com> --- cwltool/singularity.py | 20 +++++++++++++++++++- tests/test_environment.py | 3 +++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/cwltool/singularity.py b/cwltool/singularity.py index 0029d3950..40d1bc9b5 100644 --- a/cwltool/singularity.py +++ b/cwltool/singularity.py @@ -75,6 +75,14 @@ def is_apptainer_1_or_newer() -> bool: return v[0][0] >= 1 +def is_apptainer_1_1_or_newer() -> bool: + """Check if apptainer singularity distribution is version 1.1 or higher.""" + v = get_version() + if v[1] != "apptainer": + return False + return v[0][0] >= 2 or (v[0][0] >= 1 and v[0][1] >= 1) + + def is_version_2_6() -> bool: """ Check if this singularity version is exactly version 2.6. @@ -119,6 +127,12 @@ def is_version_3_9_or_newer() -> bool: return v[0][0] >= 4 or (v[0][0] == 3 and v[0][1] >= 9) +def is_version_3_10_or_newer() -> bool: + """Detect if Singularity v3.10+ is available.""" + v = get_version() + return v[0][0] >= 4 or (v[0][0] == 3 and v[0][1] >= 10) + + def _normalize_image_id(string: str) -> str: return string.replace("/", "_") + ".img" @@ -464,14 +478,18 @@ def create_runtime( ) -> tuple[list[str], Optional[str]]: """Return the Singularity runtime list of commands and options.""" any_path_okay = self.builder.get_requirement("DockerRequirement")[1] or False + runtime = [ "singularity", "--quiet", - "exec", + "run" if is_apptainer_1_1_or_newer() or is_version_3_10_or_newer() else "exec", "--contain", "--ipc", "--cleanenv", ] + if is_apptainer_1_1_or_newer() or is_version_3_10_or_newer(): + runtime.append("--no-eval") + if singularity_supports_userns(): runtime.append("--userns") else: diff --git a/tests/test_environment.py b/tests/test_environment.py index a4bfd1ac3..488477aa7 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -159,6 +159,9 @@ def BIND(v: str, env: Env) -> bool: return v.startswith(tmp_prefix) and v.endswith(":/tmp") sing_vars["SINGULARITY_BIND"] = BIND + if vminor >= 10: + sing_vars["SINGULARITY_COMMAND"] = "run" + sing_vars["SINGULARITY_NO_EVAL"] = None result.update(sing_vars) From c3c92ebb7c5d485ace93c6a31139b05f2e9d82d8 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" <1330696+mr-c@users.noreply.github.com> Date: Tue, 12 Nov 2024 14:12:02 +0100 Subject: [PATCH 26/43] conformances testing: no longer skip tests for singularity/apptainer --- conformance-test.sh | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/conformance-test.sh b/conformance-test.sh index 36ea23b17..6df14a63c 100755 --- a/conformance-test.sh +++ b/conformance-test.sh @@ -97,19 +97,7 @@ if [[ "$VERSION" = *dev* ]] then CWLTOOL_OPTIONS+=" --enable-dev" fi -if [[ "$CONTAINER" = "singularity" ]]; then - CWLTOOL_OPTIONS+=" --singularity" - # This test fails because Singularity and Docker have - # different views on how to deal with this. - exclusions+=(docker_entrypoint) - if [[ "${VERSION}" = "v1.1" ]]; then - # This fails because of a difference (in Singularity vs Docker) in - # the way filehandles are passed to processes in the container and - # wc can tell somehow. - # See issue #1440 - exclusions+=(stdin_shorcut) - fi -elif [[ "$CONTAINER" = "podman" ]]; then +if [[ "$CONTAINER" = "podman" ]]; then CWLTOOL_OPTIONS+=" --podman" fi From e4d42b85d24cd5088e14cc31d67c2dee0c6fc40a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 15 Nov 2024 07:39:02 +0000 Subject: [PATCH 27/43] Bump codecov/codecov-action from 4 to 5 Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 4 to 5. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v4...v5) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci-tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index e5d9a7e83..6608e9a22 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -81,7 +81,7 @@ jobs: - name: Upload coverage to Codecov if: ${{ matrix.step == 'unit' }} - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@v5 with: fail_ci_if_error: true env: @@ -217,7 +217,7 @@ jobs: path: | **/cwltool_conf*.xml - name: Upload coverage to Codecov - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@v5 with: fail_ci_if_error: true env: @@ -302,7 +302,7 @@ jobs: - name: Test with tox run: tox - name: Upload coverage to Codecov - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@v5 with: fail_ci_if_error: true env: From 6c86caa0571fd186d90a6600e0bb405596d4a5e0 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Wed, 27 Nov 2024 16:37:33 +0100 Subject: [PATCH 28/43] make-template: fix type error --- cwltool/main.py | 4 +- tests/CometAdapter.cwl | 228 +++++++++++++++++++++++++++++++++++++++++ tests/test_examples.py | 12 +++ 3 files changed, 242 insertions(+), 2 deletions(-) create mode 100644 tests/CometAdapter.cwl diff --git a/cwltool/main.py b/cwltool/main.py index 99928d0bd..7aedce6b1 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -330,10 +330,10 @@ def generate_input_template(tool: Process) -> CWLObjectType: """Generate an example input object for the given CWL process.""" template = ruamel.yaml.comments.CommentedMap() for inp in cast( - list[MutableMapping[str, str]], + list[CWLObjectType], realize_input_schema(tool.tool["inputs"], tool.schemaDefs), ): - name = shortname(inp["id"]) + name = shortname(cast(str, inp["id"])) value, comment = generate_example_input(inp["type"], inp.get("default", None)) template.insert(0, name, value, comment) return template diff --git a/tests/CometAdapter.cwl b/tests/CometAdapter.cwl new file mode 100644 index 000000000..ed1b24bb6 --- /dev/null +++ b/tests/CometAdapter.cwl @@ -0,0 +1,228 @@ +# Copyright (c) 2002-present, The OpenMS Team -- EKU Tuebingen, ETH Zurich, and FU Berlin +# SPDX-License-Identifier: Apache-2.0 +label: CometAdapter +doc: Annotates MS/MS spectra using Comet. +inputs: + in: + doc: Input file + type: File + out: + doc: Output file + type: string + database: + doc: FASTA file + type: File + comet_executable: + doc: The Comet executable. Provide a full or relative path, or make sure it can be found in your PATH environment. + type: File + pin_out: + doc: Output file - for Percolator input + type: string? + default_params_file: + doc: Default Comet params file. All parameters of this take precedence. A template file can be generated using 'comet.exe -p' + type: File? + precursor_mass_tolerance: + doc: "Precursor monoisotopic mass tolerance (Comet parameter: peptide_mass_tolerance). See also precursor_error_units to set the unit." + type: double? + precursor_error_units: + doc: "Unit of precursor monoisotopic mass tolerance for parameter precursor_mass_tolerance (Comet parameter: peptide_mass_units)" + type: string? + isotope_error: + doc: This parameter controls whether the peptide_mass_tolerance takes into account possible isotope errors in the precursor mass measurement. Use -8/-4/0/4/8 only for SILAC. + type: string? + fragment_mass_tolerance: + doc: "This is half the bin size, which is used to segment the MS/MS spectrum. Thus, the value should be a bit higher than for other search engines, since the bin might not be centered around the peak apex (see 'fragment_bin_offset').CAUTION: Low tolerances have heavy impact on RAM usage (since Comet uses a lot of bins in this case). Consider using use_sparse_matrix and/or spectrum_batch_size." + type: double? + fragment_error_units: + doc: Fragment monoisotopic mass error units + type: string? + fragment_bin_offset: + doc: "Offset of fragment bins. Recommended by Comet: low-res: 0.4, high-res: 0.0" + type: double? + instrument: + doc: "Comets theoretical_fragment_ions parameter: theoretical fragment ion peak representation, high-res: sum of intensities plus flanking bins, ion trap (low-res) ms/ms: sum of intensities of central M bin only" + type: string? + use_A_ions: + doc: use A ions for PSM + type: boolean? + use_B_ions: + doc: use B ions for PSM + type: boolean? + use_C_ions: + doc: use C ions for PSM + type: boolean? + use_X_ions: + doc: use X ions for PSM + type: boolean? + use_Y_ions: + doc: use Y ions for PSM + type: boolean? + use_Z_ions: + doc: use Z ions for PSM + type: boolean? + use_NL_ions: + doc: use neutral loss (NH3, H2O) ions from b/y for PSM + type: boolean? + enzyme: + doc: The enzyme used for peptide digestion. + type: string? + second_enzyme: + doc: Additional enzyme used for peptide digestion. + type: string? + num_enzyme_termini: + doc: Specify the termini where the cleavage rule has to match + type: string? + missed_cleavages: + doc: Number of possible cleavage sites missed by the enzyme. It has no effect if enzyme is unspecific cleavage. + type: long? + min_peptide_length: + doc: Minimum peptide length to consider. + type: long? + max_peptide_length: + doc: Maximum peptide length to consider. + type: long? + num_hits: + doc: Number of peptide hits (PSMs) per spectrum in output file + type: long? + precursor_charge: + doc: "Precursor charge range to search (if spectrum is not annotated with a charge or if override_charge!=keep any known): 0:[num] == search all charges, 2:6 == from +2 to +6, 3:3 == +3" + type: string? + override_charge: + doc: "_keep any known_: keep any precursor charge state (from input), _ignore known_: ignore known precursor charge state and use precursor_charge parameter, _ignore outside range_: ignore precursor charges outside precursor_charge range, _keep known search unknown_: keep any known precursor charge state. For unknown charge states, search as singly charged if there is no signal above the precursor m/z or use the precursor_charge range" + type: string? + ms_level: + doc: MS level to analyze, valid are levels 2 (default) or 3 + type: long? + activation_method: + doc: If not ALL, only searches spectra of the given method + type: string? + digest_mass_range: + doc: MH+ peptide mass range to analyze + type: string? + max_fragment_charge: + doc: Set maximum fragment charge state to analyze as long as still lower than precursor charge - 1. (Allowed max 5) + type: long? + max_precursor_charge: + doc: set maximum precursor charge state to analyze (allowed max 9) + type: long? + clip_nterm_methionine: + doc: If set to true, also considers the peptide sequence w/o N-term methionine separately and applies appropriate N-term mods to it + type: boolean? + spectrum_batch_size: + doc: max. number of spectra to search at a time; use 0 to search the entire scan range in one batch + type: long? + mass_offsets: + doc: One or more mass offsets to search (values subtracted from deconvoluted precursor mass). Has to include 0.0 if you want the default mass to be searched. + type: double[]? + minimum_peaks: + doc: Required minimum number of peaks in spectrum to search (default 10) + type: long? + minimum_intensity: + doc: Minimum intensity value to read in + type: double? + remove_precursor_peak: + doc: no = no removal, yes = remove all peaks around precursor m/z, charge_reduced = remove all charge reduced precursor peaks (for ETD/ECD). phosphate_loss = remove the HPO3 (-80) and H3PO4 (-98) precursor phosphate neutral loss peaks. See also remove_precursor_tolerance + type: string? + remove_precursor_tolerance: + doc: one-sided tolerance for precursor removal in Thompson + type: double? + clear_mz_range: + doc: for iTRAQ/TMT type data; will clear out all peaks in the specified m/z range, if not 0:0 + type: string? + fixed_modifications: + doc: Fixed modifications, specified using Unimod (www.unimod.org) terms, e.g. 'Carbamidomethyl (C)' or 'Oxidation (M)' + type: string[]? + variable_modifications: + doc: Variable modifications, specified using Unimod (www.unimod.org) terms, e.g. 'Carbamidomethyl (C)' or 'Oxidation (M)' + type: string[]? + binary_modifications: + doc: "List of modification group indices. Indices correspond to the binary modification index used by comet to group individually searched lists of variable modifications.\nNote: if set, both variable_modifications and binary_modifications need to have the same number of entries as the N-th entry corresponds to the N-th variable_modification.\n if left empty (default), all entries are internally set to 0 generating all permutations of modified and unmodified residues.\n For a detailed explanation please see the parameter description in the Comet help." + type: long[]? + max_variable_mods_in_peptide: + doc: Set a maximum number of variable modifications per peptide + type: long? + require_variable_mod: + doc: If true, requires at least one variable modification per peptide + type: boolean? + reindex: + doc: Recalculate peptide to protein association using OpenMS. Annotates target-decoy information. + type: string? + log: + doc: Name of log file (created only when specified) + type: string? + debug: + doc: Sets the debug level + type: long? + threads: + doc: Sets the number of threads allowed to be used by the TOPP tool + type: long? + no_progress: + doc: Disables progress logging to command line + type: boolean? + force: + doc: Overrides tool-specific checks + type: boolean? + test: + doc: Enables the test mode (needed for internal use only) + type: boolean? + PeptideIndexing__decoy_string: + doc: String that was appended (or prefixed - see 'decoy_string_position' flag below) to the accessions in the protein database to indicate decoy proteins. If empty (default), it's determined automatically (checking for common terms, both as prefix and suffix). + type: string? + PeptideIndexing__decoy_string_position: + doc: Is the 'decoy_string' prepended (prefix) or appended (suffix) to the protein accession? (ignored if decoy_string is empty) + type: string? + PeptideIndexing__missing_decoy_action: + doc: "Action to take if NO peptide was assigned to a decoy protein (which indicates wrong database or decoy string): 'error' (exit with error, no output), 'warn' (exit with success, warning message), 'silent' (no action is taken, not even a warning)" + type: string? + PeptideIndexing__write_protein_sequence: + doc: If set, the protein sequences are stored as well. + type: boolean? + PeptideIndexing__write_protein_description: + doc: If set, the protein description is stored as well. + type: boolean? + PeptideIndexing__keep_unreferenced_proteins: + doc: If set, protein hits which are not referenced by any peptide are kept. + type: boolean? + PeptideIndexing__unmatched_action: + doc: "If peptide sequences cannot be matched to any protein: 1) raise an error; 2) warn (unmatched PepHits will miss target/decoy annotation with downstream problems); 3) remove the hit." + type: string? + PeptideIndexing__aaa_max: + doc: Maximal number of ambiguous amino acids (AAAs) allowed when matching to a protein database with AAAs. AAAs are 'B', 'J', 'Z' and 'X'. + type: long? + PeptideIndexing__mismatches_max: + doc: Maximal number of mismatched (mm) amino acids allowed when matching to a protein database. The required runtime is exponential in the number of mm's; apply with care. MM's are allowed in addition to AAA's. + type: long? + PeptideIndexing__IL_equivalent: + doc: Treat the isobaric amino acids isoleucine ('I') and leucine ('L') as equivalent (indistinguishable). Also occurrences of 'J' will be treated as 'I' thus avoiding ambiguous matching. + type: boolean? + PeptideIndexing__allow_nterm_protein_cleavage: + doc: Allow the protein N-terminus amino acid to clip. + type: string? + PeptideIndexing__enzyme__name: + doc: "Enzyme which determines valid cleavage sites - e.g. trypsin cleaves after lysine (K) or arginine (R), but not before proline (P). Default: deduce from input" + type: string? + PeptideIndexing__enzyme__specificity: + doc: "Specificity of the enzyme. Default: deduce from input.\n 'full': both internal cleavage sites must match.\n 'semi': one of two internal cleavage sites must match.\n 'none': allow all peptide hits no matter their context (enzyme is irrelevant)." + type: string? +outputs: + out: + type: File + outputBinding: + glob: $(inputs.out) + pin_out: + type: File? + outputBinding: + glob: $(inputs.pin_out) +cwlVersion: v1.2 +class: CommandLineTool +baseCommand: + - CometAdapter +requirements: + InlineJavascriptRequirement: {} + InitialWorkDirRequirement: + listing: + - entryname: cwl_inputs.json + entry: $(JSON.stringify(inputs)) +arguments: + - -ini + - cwl_inputs.json diff --git a/tests/test_examples.py b/tests/test_examples.py index 23d17dcb2..c6ec6d06a 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -1894,3 +1894,15 @@ def test_input_named_id() -> None: ] ) assert exit_code == 0, stderr + + +def test_make_template() -> None: + """End-to-end test of --make-template, especially for mypyc mode.""" + exit_code, stdout, stderr = get_main_output( + [ + "--make-template", + "--debug", + get_data("tests/CometAdapter.cwl"), + ] + ) + assert exit_code == 0, stderr From 7fa57ddbf627b0a90efcde0151b92968620c9f9f Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Mon, 9 Dec 2024 12:12:20 +0100 Subject: [PATCH 29/43] --no-warnings includes schema-salad; --validate should complain to stdout --- cwltool/loghandler.py | 22 +++++++----- cwltool/main.py | 18 +++++----- tests/test_examples.py | 13 ++++--- tests/test_validate.py | 80 +++++++++++++++++++++++++++++++++++++++--- tests/util.py | 5 +-- 5 files changed, 107 insertions(+), 31 deletions(-) diff --git a/cwltool/loghandler.py b/cwltool/loghandler.py index 76daa8be9..c76830816 100644 --- a/cwltool/loghandler.py +++ b/cwltool/loghandler.py @@ -11,7 +11,7 @@ def configure_logging( - stderr_handler: logging.Handler, + err_handler: logging.Handler, no_warnings: bool, quiet: bool, debug: bool, @@ -21,25 +21,29 @@ def configure_logging( ) -> None: """Configure logging.""" rdflib_logger = logging.getLogger("rdflib.term") - rdflib_logger.addHandler(stderr_handler) + rdflib_logger.addHandler(err_handler) rdflib_logger.setLevel(logging.ERROR) deps_logger = logging.getLogger("galaxy.tool_util.deps") - deps_logger.addHandler(stderr_handler) + deps_logger.addHandler(err_handler) ss_logger = logging.getLogger("salad") - ss_logger.addHandler(stderr_handler) if no_warnings: - stderr_handler.setLevel(logging.ERROR) - if quiet: + err_handler.setLevel(logging.ERROR) + ss_logger.setLevel(logging.ERROR) + elif quiet: # Silence STDERR, not an eventual provenance log file - stderr_handler.setLevel(logging.WARN) + err_handler.setLevel(logging.WARN) + ss_logger.setLevel(logging.WARN) + else: + err_handler.setLevel(logging.INFO) + ss_logger.setLevel(logging.INFO) if debug: # Increase to debug for both stderr and provenance log file base_logger.setLevel(logging.DEBUG) - stderr_handler.setLevel(logging.DEBUG) + err_handler.setLevel(logging.DEBUG) rdflib_logger.setLevel(logging.DEBUG) deps_logger.setLevel(logging.DEBUG) fmtclass = coloredlogs.ColoredFormatter if enable_color else logging.Formatter formatter = fmtclass("%(levelname)s %(message)s") if timestamps: formatter = fmtclass("[%(asctime)s] %(levelname)s %(message)s", "%Y-%m-%d %H:%M:%S") - stderr_handler.setFormatter(formatter) + err_handler.setFormatter(formatter) diff --git a/cwltool/main.py b/cwltool/main.py index 7aedce6b1..17ccb11ce 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -967,12 +967,6 @@ def main( stdout = cast(IO[str], stdout) _logger.removeHandler(defaultStreamHandler) - stderr_handler = logger_handler - if stderr_handler is not None: - _logger.addHandler(stderr_handler) - else: - coloredlogs.install(logger=_logger, stream=stderr) - stderr_handler = _logger.handlers[-1] workflowobj = None prov_log_handler: Optional[logging.StreamHandler[ProvOut]] = None global docker_exe @@ -997,6 +991,13 @@ def main( if not args.cidfile_dir: args.cidfile_dir = os.getcwd() del args.record_container_id + if logger_handler is not None: + err_handler = logger_handler + _logger.addHandler(err_handler) + else: + coloredlogs.install(logger=_logger, stream=stdout if args.validate else stderr) + err_handler = _logger.handlers[-1] + logging.getLogger("salad").handlers = _logger.handlers if runtimeContext is None: runtimeContext = RuntimeContext(vars(args)) @@ -1015,7 +1016,7 @@ def main( setattr(args, key, val) configure_logging( - stderr_handler, + err_handler, args.no_warnings, args.quiet, runtimeContext.debug, @@ -1413,8 +1414,7 @@ def loc_to_path(obj: CWLObjectType) -> None: # public API for logging.StreamHandler prov_log_handler.close() close_ro(research_obj, args.provenance) - - _logger.removeHandler(stderr_handler) + _logger.removeHandler(err_handler) _logger.addHandler(defaultStreamHandler) diff --git a/tests/test_examples.py b/tests/test_examples.py index c6ec6d06a..f413976fd 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -1820,9 +1820,9 @@ def test_validate_optional_src_with_mandatory_sink() -> None: ["--validate", get_data("tests/wf/optional_src_mandatory_sink.cwl")] ) assert exit_code == 0 - stderr = re.sub(r"\s\s+", " ", stderr) - assert 'Source \'opt_file\' of type ["null", "File"] may be incompatible' in stderr - assert "with sink 'r' of type \"File\"" in stderr + stdout = re.sub(r"\s\s+", " ", stdout) + assert 'Source \'opt_file\' of type ["null", "File"] may be incompatible' in stdout + assert "with sink 'r' of type \"File\"" in stdout def test_res_req_expr_float_1_0() -> None: @@ -1875,12 +1875,11 @@ def test_invalid_nested_array() -> None: ] ) assert exit_code == 1, stderr - stderr = re.sub(r"\n\s+", " ", stderr) - stderr = re.sub(r"\s\s+", " ", stderr) - assert "Tool definition failed validation:" in stderr + stdout = re.sub(r"\s\s+", " ", stdout) + assert "Tool definition failed validation:" in stdout assert ( "tests/nested-array.cwl:6:5: Field 'type' references unknown identifier 'string[][]'" - ) in stderr + ) in stdout def test_input_named_id() -> None: diff --git a/tests/test_validate.py b/tests/test_validate.py index 171a6b6c1..f2d89e473 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -1,5 +1,7 @@ """Tests --validation.""" +import io +import logging import re from .util import get_data, get_main_output @@ -43,13 +45,83 @@ def test_validate_with_invalid_input_object() -> None: ] ) assert exit_code == 1 - stderr = re.sub(r"\s\s+", " ", stderr) - assert "Invalid job input record" in stderr + stdout = re.sub(r"\s\s+", " ", stdout) + assert "Invalid job input record" in stdout assert ( "tests/wf/1st-workflow_bad_inputs.yml:2:1: * the 'ex' field is not " - "valid because the value is not string" in stderr + "valid because the value is not string" in stdout ) assert ( "tests/wf/1st-workflow_bad_inputs.yml:1:1: * the 'inp' field is not " - "valid because is not a dict. Expected a File object." in stderr + "valid because is not a dict. Expected a File object." in stdout + ) + + +def test_validate_quiet() -> None: + """Ensure that --validate --quiet prints the correct amount of information.""" + exit_code, stdout, stderr = get_main_output( + [ + "--validate", + "--quiet", + get_data("tests/CometAdapter.cwl"), + ] + ) + assert exit_code == 0 + stdout = re.sub(r"\s\s+", " ", stdout) + assert "INFO" not in stdout + assert "INFO" not in stderr + assert "tests/CometAdapter.cwl:9:3: object id" in stdout + assert "tests/CometAdapter.cwl#out' previously defined" in stdout + + +def test_validate_no_warnings() -> None: + """Ensure that --validate --no-warnings doesn't print any warnings.""" + exit_code, stdout, stderr = get_main_output( + [ + "--validate", + "--no-warnings", + get_data("tests/CometAdapter.cwl"), + ] ) + assert exit_code == 0 + stdout = re.sub(r"\s\s+", " ", stdout) + stderr = re.sub(r"\s\s+", " ", stderr) + assert "INFO" not in stdout + assert "INFO" not in stderr + assert "WARNING" not in stdout + assert "WARNING" not in stderr + assert "tests/CometAdapter.cwl:9:3: object id" not in stdout + assert "tests/CometAdapter.cwl:9:3: object id" not in stderr + assert "tests/CometAdapter.cwl#out' previously defined" not in stdout + assert "tests/CometAdapter.cwl#out' previously defined" not in stderr + + +def test_validate_custom_logger() -> None: + """Custom log handling test.""" + custom_log = io.StringIO() + handler = logging.StreamHandler(custom_log) + handler.setLevel(logging.DEBUG) + exit_code, stdout, stderr = get_main_output( + [ + "--validate", + get_data("tests/CometAdapter.cwl"), + ], + logger_handler=handler, + ) + custom_log_text = custom_log.getvalue() + assert exit_code == 0 + custom_log_text = re.sub(r"\s\s+", " ", custom_log_text) + stdout = re.sub(r"\s\s+", " ", stdout) + stderr = re.sub(r"\s\s+", " ", stderr) + assert "INFO" not in stdout + assert "INFO" not in stderr + assert "INFO" in custom_log_text + assert "WARNING" not in stdout + assert "WARNING" not in stderr + assert "WARNING" in custom_log_text + assert "tests/CometAdapter.cwl:9:3: object id" not in stdout + assert "tests/CometAdapter.cwl:9:3: object id" not in stderr + assert "tests/CometAdapter.cwl:9:3: object id" in custom_log_text + assert "tests/CometAdapter.cwl#out' previously defined" not in stdout + assert "tests/CometAdapter.cwl#out' previously defined" not in stderr + assert "tests/CometAdapter.cwl#out' previously defined" in custom_log_text diff --git a/tests/util.py b/tests/util.py index 44d2f108c..8dd0bf74e 100644 --- a/tests/util.py +++ b/tests/util.py @@ -11,7 +11,7 @@ from collections.abc import Generator, Mapping from contextlib import ExitStack from pathlib import Path -from typing import Optional, Union +from typing import Any, Optional, Union import pytest @@ -88,6 +88,7 @@ def get_main_output( replacement_env: Optional[Mapping[str, str]] = None, extra_env: Optional[Mapping[str, str]] = None, monkeypatch: Optional[pytest.MonkeyPatch] = None, + **extra_kwargs: Any, ) -> tuple[Optional[int], str, str]: """Run cwltool main. @@ -113,7 +114,7 @@ def get_main_output( monkeypatch.setenv(k, v) try: - rc = main(argsl=args, stdout=stdout, stderr=stderr) + rc = main(argsl=args, stdout=stdout, stderr=stderr, **extra_kwargs) except SystemExit as e: if isinstance(e.code, int): rc = e.code From 604fd1242f4923c64d4cb396474ce9be0c134f8a Mon Sep 17 00:00:00 2001 From: Sameeul Bashir Samee Date: Wed, 11 Dec 2024 09:58:33 -0500 Subject: [PATCH 30/43] Append "_latest" to image id if no tag is present (#2085) --- cwltool/singularity.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cwltool/singularity.py b/cwltool/singularity.py index 40d1bc9b5..d0e46fb27 100644 --- a/cwltool/singularity.py +++ b/cwltool/singularity.py @@ -134,10 +134,14 @@ def is_version_3_10_or_newer() -> bool: def _normalize_image_id(string: str) -> str: + if ":" not in string: + string += "_latest" return string.replace("/", "_") + ".img" def _normalize_sif_id(string: str) -> str: + if ":" not in string: + string += "_latest" return string.replace("/", "_") + ".sif" From d3c7bd5d6c409e857b98f9034a55952ca95afdb3 Mon Sep 17 00:00:00 2001 From: Iacopo Colonnelli Date: Thu, 12 Dec 2024 13:19:58 +0100 Subject: [PATCH 31/43] Fix `cwltool:Loop` extension definition (#2081) This commit fixes one small error in the `cwltool:Loop` definition that were breaking the Schema SALAD codegen procedure. --- cwltool/extensions-v1.2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cwltool/extensions-v1.2.yml b/cwltool/extensions-v1.2.yml index c39b15d07..ae371c671 100644 --- a/cwltool/extensions-v1.2.yml +++ b/cwltool/extensions-v1.2.yml @@ -236,7 +236,7 @@ $graph: name: LoopOutputModes symbols: [ last, all ] default: last - doc: + doc: | - Specify the desired method of dealing with loop outputs - Default. Propagates only the last computed element to the subsequent steps when the loop terminates. - Propagates a single array with all output values to the subsequent steps when the loop terminates. From 37d85390827b17c13bc90b8e4e707cad359bfa03 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2024 09:41:09 +0100 Subject: [PATCH 32/43] Update flake8-bugbear requirement from <24.11 to <24.13 (#2086) Updates the requirements on [flake8-bugbear](https://github.com/PyCQA/flake8-bugbear) to permit the latest version. - [Release notes](https://github.com/PyCQA/flake8-bugbear/releases) - [Commits](https://github.com/PyCQA/flake8-bugbear/compare/16.4.1...24.12.12) --- updated-dependencies: - dependency-name: flake8-bugbear dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- lint-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lint-requirements.txt b/lint-requirements.txt index ec1e6e1fe..abb223c85 100644 --- a/lint-requirements.txt +++ b/lint-requirements.txt @@ -1,3 +1,3 @@ -flake8-bugbear<24.11 +flake8-bugbear<24.13 black==24.* codespell From 56e159f1fff80c4f89b0279fffb5a38f223008df Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2024 10:45:18 +0100 Subject: [PATCH 33/43] Bump cibuildwheel from 2.21.3 to 2.22.0 (#2077) Bumps [cibuildwheel](https://github.com/pypa/cibuildwheel) from 2.21.3 to 2.22.0. - [Release notes](https://github.com/pypa/cibuildwheel/releases) - [Changelog](https://github.com/pypa/cibuildwheel/blob/main/docs/changelog.md) - [Commits](https://github.com/pypa/cibuildwheel/compare/v2.21.3...v2.22) --- updated-dependencies: - dependency-name: cibuildwheel dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- cibw-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibw-requirements.txt b/cibw-requirements.txt index c4511439c..833aca23d 100644 --- a/cibw-requirements.txt +++ b/cibw-requirements.txt @@ -1 +1 @@ -cibuildwheel==2.21.3 +cibuildwheel==2.22.0 From cc6772e01d523f7103a96ae99cc980f32d0ffcb9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2024 10:45:53 +0100 Subject: [PATCH 34/43] Bump sphinx-rtd-theme from 3.0.1 to 3.0.2 (#2069) Bumps [sphinx-rtd-theme](https://github.com/readthedocs/sphinx_rtd_theme) from 3.0.1 to 3.0.2. - [Changelog](https://github.com/readthedocs/sphinx_rtd_theme/blob/master/docs/changelog.rst) - [Commits](https://github.com/readthedocs/sphinx_rtd_theme/compare/3.0.1...3.0.2) --- updated-dependencies: - dependency-name: sphinx-rtd-theme dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index d614584fc..fd3033b91 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,5 @@ sphinx >= 2.2 -sphinx-rtd-theme==3.0.1 +sphinx-rtd-theme==3.0.2 sphinx-autoapi sphinx-autodoc-typehints sphinxcontrib-autoprogram From 6b8f06a9f6f6a570142c7aedc767fea2efa2a0cc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2024 10:47:09 +0100 Subject: [PATCH 35/43] Bump pypa/cibuildwheel from 2.21.3 to 2.22.0 (#2076) Bumps [pypa/cibuildwheel](https://github.com/pypa/cibuildwheel) from 2.21.3 to 2.22.0. - [Release notes](https://github.com/pypa/cibuildwheel/releases) - [Changelog](https://github.com/pypa/cibuildwheel/blob/main/docs/changelog.md) - [Commits](https://github.com/pypa/cibuildwheel/compare/v2.21.3...v2.22.0) --- updated-dependencies: - dependency-name: pypa/cibuildwheel dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/wheels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index fb54729d4..ed10cd17d 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -43,7 +43,7 @@ jobs: # platforms: all - name: Build wheels - uses: pypa/cibuildwheel@v2.21.3 + uses: pypa/cibuildwheel@v2.22.0 env: CIBW_BUILD: ${{ matrix.build }} CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/${{ matrix.image }} @@ -99,7 +99,7 @@ jobs: ref: ${{ github.event.client_payload.ref }} - name: Build wheels - uses: pypa/cibuildwheel@v2.21.3 + uses: pypa/cibuildwheel@v2.22.0 - uses: actions/upload-artifact@v4 with: From f1d192dd2b28902fd0098e133c2ef241557d27a8 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Sat, 30 Nov 2024 13:07:55 +0100 Subject: [PATCH 36/43] argpase: colorize the --help output with rich-argparse --- .github/workflows/ci-tests.yml | 2 +- cwltool/argparser.py | 352 ++++++++++++++++++--------------- cwltool/main.py | 7 +- pyproject.toml | 1 + requirements.txt | 1 + setup.py | 1 + tests/test_environment.py | 12 +- tests/test_misc_cli.py | 10 +- tests/test_singularity.py | 3 +- 9 files changed, 215 insertions(+), 174 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 6608e9a22..1f01160c8 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -157,7 +157,7 @@ jobs: chmod a-w . - name: run tests - run: APPTAINER_TMPDIR=${RUNNER_TEMP} make test + run: APPTAINER_TMPDIR=${RUNNER_TEMP} make test PYTEST_EXTRA=-vvv conformance_tests: diff --git a/cwltool/argparser.py b/cwltool/argparser.py index 7b3125d94..3622b627f 100644 --- a/cwltool/argparser.py +++ b/cwltool/argparser.py @@ -6,6 +6,9 @@ from collections.abc import MutableMapping, MutableSequence, Sequence from typing import Any, Callable, Optional, Union, cast +import rich.markup +from rich_argparse import HelpPreviewAction, RichHelpFormatter + from .loghandler import _logger from .process import Process, shortname from .resolver import ga4gh_tool_registries @@ -14,9 +17,11 @@ def arg_parser() -> argparse.ArgumentParser: + RichHelpFormatter.group_name_formatter = str parser = argparse.ArgumentParser( + formatter_class=RichHelpFormatter, description="Reference executor for Common Workflow Language standards. " - "Not for production use." + "Not for production use.", ) parser.add_argument("--basedir", type=str) parser.add_argument( @@ -26,23 +31,15 @@ def arg_parser() -> argparse.ArgumentParser: help="Output directory. The default is the current directory.", ) - parser.add_argument( - "--log-dir", - type=str, - default="", - help="Log your tools stdout/stderr to this location outside of container " - "This will only log stdout/stderr if you specify stdout/stderr in their " - "respective fields or capture it as an output", - ) - parser.add_argument( "--parallel", action="store_true", default=False, help="Run jobs in parallel. ", ) - envgroup = parser.add_mutually_exclusive_group() - envgroup.add_argument( + envgroup = parser.add_argument_group(title="Control environment variables") + env_exclusive = envgroup.add_mutually_exclusive_group() + env_exclusive.add_argument( "--preserve-environment", type=str, action="append", @@ -53,7 +50,7 @@ def arg_parser() -> argparse.ArgumentParser: default=[], dest="preserve_environment", ) - envgroup.add_argument( + env_exclusive.add_argument( "--preserve-entire-environment", action="store_true", help="Preserve all environment variables when running CommandLineTools " @@ -62,54 +59,10 @@ def arg_parser() -> argparse.ArgumentParser: dest="preserve_entire_environment", ) - containergroup = parser.add_mutually_exclusive_group() - containergroup.add_argument( - "--rm-container", - action="store_true", - default=True, - help="Delete Docker container used by jobs after they exit (default)", - dest="rm_container", - ) - - containergroup.add_argument( - "--leave-container", - action="store_false", - default=True, - help="Do not delete Docker container used by jobs after they exit", - dest="rm_container", - ) - - cidgroup = parser.add_argument_group( - "Options for recording the Docker container identifier into a file." - ) - cidgroup.add_argument( - # Disabled as containerid is now saved by default - "--record-container-id", - action="store_true", - default=False, - help=argparse.SUPPRESS, - dest="record_container_id", - ) - - cidgroup.add_argument( - "--cidfile-dir", - type=str, - help="Store the Docker container ID into a file in the specified directory.", - default=None, - dest="cidfile_dir", - ) - - cidgroup.add_argument( - "--cidfile-prefix", - type=str, - help="Specify a prefix to the container ID filename. " - "Final file name will be followed by a timestamp. " - "The default is no prefix.", - default=None, - dest="cidfile_prefix", + files_group = parser.add_argument_group( + title="Manage intermediate, temporary, or final output files" ) - - parser.add_argument( + files_group.add_argument( "--tmpdir-prefix", type=str, help="Path prefix for temporary directories. If --tmpdir-prefix is not " @@ -119,7 +72,7 @@ def arg_parser() -> argparse.ArgumentParser: default=DEFAULT_TMP_PREFIX, ) - intgroup = parser.add_mutually_exclusive_group() + intgroup = files_group.add_mutually_exclusive_group() intgroup.add_argument( "--tmp-outdir-prefix", type=str, @@ -137,7 +90,7 @@ def arg_parser() -> argparse.ArgumentParser: "troubleshooting of CWL documents.", ) - tmpgroup = parser.add_mutually_exclusive_group() + tmpgroup = files_group.add_mutually_exclusive_group() tmpgroup.add_argument( "--rm-tmpdir", action="store_true", @@ -154,7 +107,7 @@ def arg_parser() -> argparse.ArgumentParser: dest="rm_tmpdir", ) - outgroup = parser.add_mutually_exclusive_group() + outgroup = files_group.add_mutually_exclusive_group() outgroup.add_argument( "--move-outputs", action="store_const", @@ -184,30 +137,6 @@ def arg_parser() -> argparse.ArgumentParser: dest="move_outputs", ) - pullgroup = parser.add_mutually_exclusive_group() - pullgroup.add_argument( - "--enable-pull", - default=True, - action="store_true", - help="Try to pull Docker images", - dest="pull_image", - ) - - pullgroup.add_argument( - "--disable-pull", - default=True, - action="store_false", - help="Do not try to pull Docker images", - dest="pull_image", - ) - - parser.add_argument( - "--rdf-serializer", - help="Output RDF serialization format used by --print-rdf (one of " - "turtle (default), n3, nt, xml)", - default="turtle", - ) - parser.add_argument( "--eval-timeout", help="Time to wait for a Javascript expression to evaluate before giving " @@ -216,9 +145,7 @@ def arg_parser() -> argparse.ArgumentParser: default=60, ) - provgroup = parser.add_argument_group( - "Options for recording provenance information of the execution" - ) + provgroup = parser.add_argument_group("Recording provenance information of the execution") provgroup.add_argument( "--provenance", help="Save provenance to specified folder as a " @@ -276,7 +203,8 @@ def arg_parser() -> argparse.ArgumentParser: type=str, ) - printgroup = parser.add_mutually_exclusive_group() + non_exec_group = parser.add_argument_group(title="Non-execution options") + printgroup = non_exec_group.add_mutually_exclusive_group() printgroup.add_argument( "--print-rdf", action="store_true", @@ -324,6 +252,15 @@ def arg_parser() -> argparse.ArgumentParser: printgroup.add_argument( "--make-template", action="store_true", help="Generate a template input object" ) + non_exec_group.add_argument( + "--rdf-serializer", + help="Output RDF serialization format used by --print-rdf (one of " + "turtle (default), n3, nt, xml)", + default="turtle", + ) + non_exec_group.add_argument( + "--tool-help", action="store_true", help="Print command line help for tool" + ) strictgroup = parser.add_mutually_exclusive_group() strictgroup.add_argument( @@ -365,11 +302,27 @@ def arg_parser() -> argparse.ArgumentParser: dest="doc_cache", ) - volumegroup = parser.add_mutually_exclusive_group() - volumegroup.add_argument("--verbose", action="store_true", help="Default logging") - volumegroup.add_argument("--no-warnings", action="store_true", help="Only print errors.") - volumegroup.add_argument("--quiet", action="store_true", help="Only print warnings and errors.") - volumegroup.add_argument("--debug", action="store_true", help="Print even more logging") + volumegroup = parser.add_argument_group(title="Configure logging") + volume_exclusive = volumegroup.add_mutually_exclusive_group() + volume_exclusive.add_argument("--verbose", action="store_true", help="Default logging") + volume_exclusive.add_argument("--no-warnings", action="store_true", help="Only print errors.") + volume_exclusive.add_argument( + "--quiet", action="store_true", help="Only print warnings and errors." + ) + volume_exclusive.add_argument("--debug", action="store_true", help="Print even more logging") + volumegroup.add_argument( + "--log-dir", + type=str, + default="", + help="Log your tools stdout/stderr to this location outside of container " + "This will only log stdout/stderr if you specify stdout/stderr in their " + "respective fields or capture it as an output", + ) + volumegroup.add_argument( + "--timestamps", + action="store_true", + help="Add timestamps to the errors, warnings, and notifications.", + ) parser.add_argument( "--write-summary", @@ -380,30 +333,6 @@ def arg_parser() -> argparse.ArgumentParser: dest="write_summary", ) - parser.add_argument( - "--strict-memory-limit", - action="store_true", - help="When running with " - "software containers and the Docker engine, pass either the " - "calculated memory allocation from ResourceRequirements or the " - "default of 1 gigabyte to Docker's --memory option.", - ) - - parser.add_argument( - "--strict-cpu-limit", - action="store_true", - help="When running with " - "software containers and the Docker engine, pass either the " - "calculated cpu allocation from ResourceRequirements or the " - "default of 1 core to Docker's --cpu option. " - "Requires docker version >= v1.13.", - ) - - parser.add_argument( - "--timestamps", - action="store_true", - help="Add timestamps to the errors, warnings, and notifications.", - ) parser.add_argument( "--js-console", action="store_true", help="Enable javascript console output" ) @@ -418,7 +347,105 @@ def arg_parser() -> argparse.ArgumentParser: help="File of options to pass to jshint. " 'This includes the added option "includewarnings". ', ) - dockergroup = parser.add_mutually_exclusive_group() + container_group = parser.add_argument_group( + title="Software container engine selection and configuration" + ) + pullgroup = container_group.add_mutually_exclusive_group() + pullgroup.add_argument( + "--enable-pull", + default=True, + action="store_true", + help="Try to pull Docker images", + dest="pull_image", + ) + + pullgroup.add_argument( + "--disable-pull", + default=True, + action="store_false", + help="Do not try to pull Docker images", + dest="pull_image", + ) + container_group.add_argument( + "--force-docker-pull", + action="store_true", + default=False, + help="Pull latest software container image even if it is locally present", + dest="force_docker_pull", + ) + container_group.add_argument( + "--no-read-only", + action="store_true", + default=False, + help="Do not set root directory in the container as read-only", + dest="no_read_only", + ) + + container_group.add_argument( + "--default-container", + help="Specify a default software container to use for any " + "CommandLineTool without a DockerRequirement.", + ) + container_group.add_argument( + "--no-match-user", + action="store_true", + help="Disable passing the current uid to `docker run --user`", + ) + container_group.add_argument( + "--custom-net", + type=str, + help="Passed to `docker run` as the `--net` parameter when " + "NetworkAccess is true, which is its default setting.", + ) + + container_cleanup = container_group.add_mutually_exclusive_group() + container_cleanup.add_argument( + "--rm-container", + action="store_true", + default=True, + help="Delete Docker container used by jobs after they exit (default)", + dest="rm_container", + ) + + container_cleanup.add_argument( + "--leave-container", + action="store_false", + default=True, + help="Do not delete Docker container used by jobs after they exit", + dest="rm_container", + ) + + cidgroup = container_group.add_argument_group( + "Recording the Docker container identifier into a file" + ) + cidgroup.add_argument( + # Disabled as containerid is now saved by default + "--record-container-id", + action="store_true", + default=False, + help=argparse.SUPPRESS, + dest="record_container_id", + ) + + cidgroup.add_argument( + "--cidfile-dir", + type=str, + help="Store the Docker container ID into a file in the specified directory.", + default=None, + dest="cidfile_dir", + ) + + cidgroup.add_argument( + "--cidfile-prefix", + type=str, + help="Specify a prefix to the container ID filename. " + "Final file name will be followed by a timestamp. " + "The default is no prefix.", + default=None, + dest="cidfile_prefix", + ) + + dockergroup = container_group.add_mutually_exclusive_group() dockergroup.add_argument( "--user-space-docker-cmd", metavar="CMD", @@ -458,6 +485,24 @@ def arg_parser() -> argparse.ArgumentParser: "is specified under `hints`.", dest="use_container", ) + container_group.add_argument( + "--strict-memory-limit", + action="store_true", + help="When running with " + "software containers and the Docker engine, pass either the " + "calculated memory allocation from ResourceRequirements or the " + "default of 1 gigabyte to Docker's --memory option.", + ) + + container_group.add_argument( + "--strict-cpu-limit", + action="store_true", + help="When running with " + "software containers and the Docker engine, pass either the " + "calculated cpu allocation from ResourceRequirements or the " + "default of 1 core to Docker's --cpu option. " + "Requires docker version >= v1.13.", + ) dependency_resolvers_configuration_help = argparse.SUPPRESS dependencies_directory_help = argparse.SUPPRESS @@ -467,7 +512,7 @@ def arg_parser() -> argparse.ArgumentParser: if SOFTWARE_REQUIREMENTS_ENABLED: dependency_resolvers_configuration_help = ( "Dependency resolver " - "configuration file describing how to adapt 'SoftwareRequirement' " + "configuration file describing how to adapt `SoftwareRequirement` " "packages to current system." ) dependencies_directory_help = ( @@ -476,7 +521,7 @@ def arg_parser() -> argparse.ArgumentParser: use_biocontainers_help = ( "Use biocontainers for tools without an " "explicitly annotated Docker container." ) - conda_dependencies = "Short cut to use Conda to resolve 'SoftwareRequirement' packages." + conda_dependencies = "Short cut to use Conda to resolve `SoftwareRequirement` packages." parser.add_argument( "--beta-dependency-resolvers-configuration", @@ -499,8 +544,6 @@ def arg_parser() -> argparse.ArgumentParser: action="store_true", ) - parser.add_argument("--tool-help", action="store_true", help="Print command line help for tool") - parser.add_argument( "--relative-deps", choices=["primary", "cwd"], @@ -519,7 +562,7 @@ def arg_parser() -> argparse.ArgumentParser: parser.add_argument( "--enable-ext", action="store_true", - help="Enable loading and running 'cwltool:' extensions to the CWL standards.", + help="Enable loading and running `cwltool:` extensions to the CWL standards.", default=False, ) @@ -537,22 +580,6 @@ def arg_parser() -> argparse.ArgumentParser: help="Disable colored logging (default false)", ) - parser.add_argument( - "--default-container", - help="Specify a default software container to use for any " - "CommandLineTool without a DockerRequirement.", - ) - parser.add_argument( - "--no-match-user", - action="store_true", - help="Disable passing the current uid to `docker run --user`", - ) - parser.add_argument( - "--custom-net", - type=str, - help="Passed to `docker run` as the '--net' parameter when " - "NetworkAccess is true, which is its default setting.", - ) parser.add_argument( "--disable-validate", dest="do_validate", @@ -595,9 +622,9 @@ def arg_parser() -> argparse.ArgumentParser: parser.add_argument( "--on-error", - help="Desired workflow behavior when a step fails. One of 'stop' (do " - "not submit any more steps) or 'continue' (may submit other steps that " - "are not downstream from the error). Default is 'stop'.", + help="Desired workflow behavior when a step fails. One of `stop` (do " + "not submit any more steps) or `continue` (may submit other steps that " + "are not downstream from the error). Default is `stop`.", default="stop", choices=("stop", "continue"), ) @@ -625,21 +652,6 @@ def arg_parser() -> argparse.ArgumentParser: dest="relax_path_checks", ) - parser.add_argument( - "--force-docker-pull", - action="store_true", - default=False, - help="Pull latest software container image even if it is locally present", - dest="force_docker_pull", - ) - parser.add_argument( - "--no-read-only", - action="store_true", - default=False, - help="Do not set root directory in the container as read-only", - dest="no_read_only", - ) - parser.add_argument( "--overrides", type=str, @@ -647,7 +659,8 @@ def arg_parser() -> argparse.ArgumentParser: help="Read process requirement overrides from file.", ) - subgroup = parser.add_mutually_exclusive_group() + target_group = parser.add_argument_group(title="Target selection (optional)") + subgroup = target_group.add_mutually_exclusive_group() subgroup.add_argument( "--target", "-t", @@ -668,8 +681,8 @@ def arg_parser() -> argparse.ArgumentParser: default=None, help="Only executes the underlying Process (CommandLineTool, " "ExpressionTool, or sub-Workflow) for the given step in a workflow. " - "This will not include any step-level processing: 'scatter', 'when'; " - "and there will be no processing of step-level 'default', or 'valueFrom' " + "This will not include any step-level processing: `scatter`, `when`; " + "and there will be no processing of step-level `default`, or `valueFrom` " "input modifiers. However, requirements/hints from the step or parent " "workflow(s) will be inherited as usual." "The input object must match that Process's inputs.", @@ -703,7 +716,11 @@ def arg_parser() -> argparse.ArgumentParser: "formatted description of the required input values for the given " "`cwl_document`.", ) - + parser.add_argument( + "--generate-help-preview", + action=HelpPreviewAction, + path="help-preview.svg", # (optional) or "help-preview.html" or "help-preview.txt" + ) return parser @@ -855,6 +872,7 @@ def add_argument( urljoin: Callable[[str, str], str] = urllib.parse.urljoin, base_uri: str = "", ) -> None: + description = rich.markup.escape(description) if len(name) == 1: flag = "-" else: @@ -980,4 +998,10 @@ def generate_parser( base_uri, ) + toolparser.add_argument( + "--generate-help-preview", + action=HelpPreviewAction, + path="help-preview.svg", # (optional) or "help-preview.html" or "help-preview.txt" + ) + return toolparser diff --git a/cwltool/main.py b/cwltool/main.py index 17ccb11ce..b7ba40d40 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -22,6 +22,7 @@ import coloredlogs import requests import ruamel.yaml +from rich_argparse import RichHelpFormatter from ruamel.yaml.comments import CommentedMap, CommentedSeq from ruamel.yaml.main import YAML from schema_salad.exceptions import ValidationException @@ -413,7 +414,10 @@ def init_job_order( namemap: dict[str, str] = {} records: list[str] = [] toolparser = generate_parser( - argparse.ArgumentParser(prog=args.workflow), + argparse.ArgumentParser( + prog=args.workflow, + formatter_class=RichHelpFormatter, + ), process, namemap, records, @@ -976,6 +980,7 @@ def main( user_agent += f" {progname}" # append the real program name as well append_word_to_default_user_agent(user_agent) + err_handler: logging.Handler = defaultStreamHandler try: if args is None: if argsl is None: diff --git a/pyproject.toml b/pyproject.toml index 7ddf547f2..248c0e69d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ requires = [ "cwl-utils>=0.32", "toml", "argcomplete>=1.12.0", + "rich-argparse" ] build-backend = "setuptools.build_meta" diff --git a/requirements.txt b/requirements.txt index 3ac631838..3cbcf0027 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,3 +12,4 @@ argcomplete>=1.12.0 pyparsing!=3.0.2 # breaks --print-dot (pydot) https://github.com/pyparsing/pyparsing/issues/319 cwl-utils>=0.32 spython>=0.3.0 +rich-argparse diff --git a/setup.py b/setup.py index 9bbee10f1..d3fef7b26 100644 --- a/setup.py +++ b/setup.py @@ -135,6 +135,7 @@ "pyparsing != 3.0.2", # breaks --print-dot (pydot) https://github.com/pyparsing/pyparsing/issues/319 "cwl-utils >= 0.32", "spython >= 0.3.0", + "rich-argparse", ], extras_require={ "deps": [ diff --git a/tests/test_environment.py b/tests/test_environment.py index 488477aa7..4e9c602f1 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -4,7 +4,7 @@ from abc import ABC, abstractmethod from collections.abc import Mapping from pathlib import Path -from typing import Any, Callable, Union +from typing import Callable, Union import pytest @@ -198,7 +198,7 @@ def BIND(v: str, env: Env) -> bool: @CRT_PARAMS -def test_basic(crt_params: CheckHolder, tmp_path: Path, monkeypatch: Any) -> None: +def test_basic(crt_params: CheckHolder, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """Test that basic env vars (only) show up.""" tmp_prefix = str(tmp_path / "canary") extra_env = { @@ -218,7 +218,9 @@ def test_basic(crt_params: CheckHolder, tmp_path: Path, monkeypatch: Any) -> Non @CRT_PARAMS -def test_preserve_single(crt_params: CheckHolder, tmp_path: Path, monkeypatch: Any) -> None: +def test_preserve_single( + crt_params: CheckHolder, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: """Test that preserving a single env var works.""" tmp_prefix = str(tmp_path / "canary") extra_env = { @@ -242,7 +244,9 @@ def test_preserve_single(crt_params: CheckHolder, tmp_path: Path, monkeypatch: A @CRT_PARAMS -def test_preserve_all(crt_params: CheckHolder, tmp_path: Path, monkeypatch: Any) -> None: +def test_preserve_all( + crt_params: CheckHolder, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: """Test that preserving all works.""" tmp_prefix = str(tmp_path / "canary") extra_env = { diff --git a/tests/test_misc_cli.py b/tests/test_misc_cli.py index 307153e16..be314cad4 100644 --- a/tests/test_misc_cli.py +++ b/tests/test_misc_cli.py @@ -1,5 +1,7 @@ """Tests for various command line options.""" +import pytest + from cwltool.utils import versionstring from .util import get_data, get_main_output, needs_docker @@ -26,9 +28,13 @@ def test_empty_cmdling() -> None: assert "CWL document required, no input file was provided" in stderr -def test_tool_help() -> None: +def test_tool_help(monkeypatch: pytest.MonkeyPatch) -> None: """Test --tool-help.""" - return_code, stdout, stderr = get_main_output(["--tool-help", get_data("tests/echo.cwl")]) + return_code, stdout, stderr = get_main_output( + ["--tool-help", get_data("tests/echo.cwl")], + extra_env={"NO_COLOR": "1"}, + monkeypatch=monkeypatch, + ) assert return_code == 0 assert "job_order Job input json file" in stdout diff --git a/tests/test_singularity.py b/tests/test_singularity.py index 0512f2e28..1139dfbc7 100644 --- a/tests/test_singularity.py +++ b/tests/test_singularity.py @@ -2,7 +2,6 @@ import shutil from pathlib import Path -from typing import Any import pytest @@ -19,7 +18,7 @@ @needs_singularity_2_6 -def test_singularity_pullfolder(tmp_path: Path, monkeypatch: Any) -> None: +def test_singularity_pullfolder(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """Test singularity respects SINGULARITY_PULLFOLDER.""" workdir = tmp_path / "working_dir_new" workdir.mkdir() From 527884b7ae7a815798903887f8f58b89bd0bc2a9 Mon Sep 17 00:00:00 2001 From: Francis Charette-Migneault Date: Mon, 16 Dec 2024 17:34:50 -0500 Subject: [PATCH 37/43] Proposal: Improved `ProvenanceProfile` definition (#2082) Co-authored-by: Michael R. Crusoe <1330696+mr-c@users.noreply.github.com> --- .gitignore | 1 + cwltool/context.py | 2 + cwltool/cwlprov/provenance_profile.py | 45 +------- cwltool/cwlprov/ro.py | 97 ++++++++++++++++-- cwltool/executors.py | 11 +- cwltool/main.py | 5 + cwltool/workflow.py | 3 +- mypy-stubs/rdflib/graph.pyi | 4 +- tests/test_provenance.py | 142 +++++++++++++++++++++----- 9 files changed, 225 insertions(+), 85 deletions(-) diff --git a/.gitignore b/.gitignore index fbe4b24fc..b4cab0e66 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ eggs/ *.egg .tox/ .pytest_cache +*.so # Editor Temps .*.sw? diff --git a/cwltool/context.py b/cwltool/context.py index 237a90968..bb281fd88 100644 --- a/cwltool/context.py +++ b/cwltool/context.py @@ -183,6 +183,8 @@ def __init__(self, kwargs: Optional[dict[str, Any]] = None) -> None: self.orcid: str = "" self.cwl_full_name: str = "" self.process_run_id: Optional[str] = None + self.prov_host: bool = False + self.prov_user: bool = False self.prov_obj: Optional[ProvenanceProfile] = None self.mpi_config: MpiConfig = MpiConfig() self.default_stdout: Optional[Union[IO[bytes], TextIO]] = None diff --git a/cwltool/cwlprov/provenance_profile.py b/cwltool/cwlprov/provenance_profile.py index d4dfd6cb4..e8538e51b 100644 --- a/cwltool/cwlprov/provenance_profile.py +++ b/cwltool/cwlprov/provenance_profile.py @@ -6,7 +6,6 @@ from collections.abc import MutableMapping, MutableSequence, Sequence from io import BytesIO from pathlib import PurePath, PurePosixPath -from socket import getfqdn from typing import TYPE_CHECKING, Any, Optional, Union, cast from prov.identifier import Identifier, QualifiedName @@ -24,12 +23,10 @@ ACCOUNT_UUID, CWLPROV, ENCODING, - FOAF, METADATA, ORE, PROVENANCE, RO, - SCHEMA, SHA1, SHA256, TEXT_PLAIN, @@ -108,25 +105,6 @@ def __str__(self) -> str: def generate_prov_doc(self) -> tuple[str, ProvDocument]: """Add basic namespaces.""" - - def host_provenance(document: ProvDocument) -> None: - """Record host provenance.""" - document.add_namespace(CWLPROV) - document.add_namespace(UUID) - document.add_namespace(FOAF) - - hostname = getfqdn() - # won't have a foaf:accountServiceHomepage for unix hosts, but - # we can at least provide hostname - document.agent( - ACCOUNT_UUID, - { - PROV_TYPE: FOAF["OnlineAccount"], - "prov:location": hostname, - CWLPROV["hostname"]: hostname, - }, - ) - self.cwltool_version = f"cwltool {versionstring().split()[-1]}" self.document.add_namespace("wfprov", "http://purl.org/wf4ever/wfprov#") # document.add_namespace('prov', 'http://www.w3.org/ns/prov#') @@ -165,25 +143,10 @@ def host_provenance(document: ProvDocument) -> None: # .. but we always know cwltool was launched (directly or indirectly) # by a user account, as cwltool is a command line tool account = self.document.agent(ACCOUNT_UUID) - if self.orcid or self.full_name: - person: dict[Union[str, Identifier], Any] = { - PROV_TYPE: PROV["Person"], - "prov:type": SCHEMA["Person"], - } - if self.full_name: - person["prov:label"] = self.full_name - person["foaf:name"] = self.full_name - person["schema:name"] = self.full_name - else: - # TODO: Look up name from ORCID API? - pass - agent = self.document.agent(self.orcid or uuid.uuid4().urn, person) - self.document.actedOnBehalfOf(account, agent) - else: - if self.host_provenance: - host_provenance(self.document) - if self.user_provenance: - self.research_object.user_provenance(self.document) + if self.host_provenance: + self.research_object.host_provenance(self.document) + if self.user_provenance or self.orcid or self.full_name: + self.research_object.user_provenance(self.document) # The execution of cwltool wfengine = self.document.agent( self.engine_uuid, diff --git a/cwltool/cwlprov/ro.py b/cwltool/cwlprov/ro.py index ac60afc92..f58919a6b 100644 --- a/cwltool/cwlprov/ro.py +++ b/cwltool/cwlprov/ro.py @@ -9,10 +9,11 @@ import uuid from collections.abc import MutableMapping, MutableSequence from pathlib import Path, PurePosixPath -from typing import IO, Any, Optional, Union, cast +from socket import getfqdn +from typing import IO, TYPE_CHECKING, Any, Optional, Union, cast import prov.model as provM -from prov.model import PROV, ProvDocument +from prov.model import ProvDocument from ..loghandler import _logger from ..stdfsaccess import StdFsAccess @@ -27,6 +28,7 @@ from . import Aggregate, Annotation, AuthoredBy, _valid_orcid, _whoami, checksum_copy from .provenance_constants import ( ACCOUNT_UUID, + CWLPROV, CWLPROV_VERSION, DATA, ENCODING, @@ -35,6 +37,7 @@ METADATA, ORCID, PROVENANCE, + SCHEMA, SHA1, SHA256, SHA512, @@ -46,6 +49,9 @@ Hasher, ) +if TYPE_CHECKING: + from .provenance_profile import ProvenanceProfile # pylint: disable=unused-import + class ResearchObject: """CWLProv Research Object.""" @@ -82,6 +88,34 @@ def __init__( self._initialize() _logger.debug("[provenance] Temporary research object: %s", self.folder) + def initialize_provenance( + self, + full_name: str, + host_provenance: bool, + user_provenance: bool, + orcid: str, + fsaccess: StdFsAccess, + run_uuid: Optional[uuid.UUID] = None, + ) -> "ProvenanceProfile": + """ + Provide a provenance profile initialization hook function. + + Allows overriding the default strategy to define the + provenance profile concepts and associations to extend + details as needed. + """ + from .provenance_profile import ProvenanceProfile + + return ProvenanceProfile( + research_object=self, + full_name=full_name, + host_provenance=host_provenance, + user_provenance=user_provenance, + orcid=orcid, + fsaccess=fsaccess, + run_uuid=run_uuid, + ) + def self_check(self) -> None: """Raise ValueError if this RO is closed.""" if self.closed: @@ -117,10 +151,22 @@ def _initialize_bagit(self) -> None: bag_it_file.write("BagIt-Version: 0.97\n") bag_it_file.write(f"Tag-File-Character-Encoding: {ENCODING}\n") + def resolve_user(self) -> tuple[str, str]: + """ + Provide a user provenance hook function. + + Allows overriding the default strategy to retrieve user provenance + in case the calling code can provide a better resolution. + The function must return a tuple of the (username, fullname) + that identifies the user. This user will be applied on top + to any provided ORCID or fullname by agent association. + """ + return _whoami() + def user_provenance(self, document: ProvDocument) -> None: """Add the user provenance.""" self.self_check() - (username, fullname) = _whoami() + (username, fullname) = self.resolve_user() if not self.full_name: self.full_name = fullname @@ -132,19 +178,21 @@ def user_provenance(self, document: ProvDocument) -> None: ACCOUNT_UUID, { provM.PROV_TYPE: FOAF["OnlineAccount"], - "prov:label": username, + provM.PROV_LABEL: username, FOAF["accountName"]: username, }, ) user = document.agent( self.orcid or USER_UUID, - { - provM.PROV_TYPE: PROV["Person"], - "prov:label": self.full_name, - FOAF["name"]: self.full_name, - FOAF["account"]: account, - }, + [ + (provM.PROV_TYPE, SCHEMA["Person"]), + (provM.PROV_TYPE, provM.PROV["Person"]), + (provM.PROV_LABEL, self.full_name), + (FOAF["name"], self.full_name), + (FOAF["account"], account), + (SCHEMA["name"], self.full_name), + ], ) # cwltool may be started on the shell (directly by user), # by shell script (indirectly by user) @@ -156,6 +204,35 @@ def user_provenance(self, document: ProvDocument) -> None: # get their name wrong!) document.actedOnBehalfOf(account, user) + def resolve_host(self) -> tuple[str, str]: + """ + Provide a host provenance hook function. + + Allows overriding the default strategy to retrieve host provenance + in case the calling code can provide a better resolution. + The function must return a tuple of the (fqdn, uri) that identifies the host. + """ + fqdn = getfqdn() + return fqdn, fqdn # allow for (fqdn, uri) to be distinct, but the same by default + + def host_provenance(self, document: ProvDocument) -> None: + """Record host provenance.""" + document.add_namespace(CWLPROV) + document.add_namespace(UUID) + document.add_namespace(FOAF) + + hostname, uri = self.resolve_host() + # won't have a foaf:accountServiceHomepage for unix hosts, but + # we can at least provide hostname + document.agent( + ACCOUNT_UUID, + { + provM.PROV_TYPE: FOAF["OnlineAccount"], + provM.PROV_LOCATION: uri, + CWLPROV["hostname"]: hostname, + }, + ) + def add_tagfile(self, path: str, timestamp: Optional[datetime.datetime] = None) -> None: """Add tag files to our research object.""" self.self_check() diff --git a/cwltool/executors.py b/cwltool/executors.py index e25426c9d..33198d854 100644 --- a/cwltool/executors.py +++ b/cwltool/executors.py @@ -19,7 +19,6 @@ from .command_line_tool import CallbackJob, ExpressionJob from .context import RuntimeContext, getdefault from .cuda import cuda_version_and_device_count -from .cwlprov.provenance_profile import ProvenanceProfile from .errors import WorkflowException from .job import JobBase from .loghandler import _logger @@ -194,11 +193,13 @@ def run_jobs( # define provenance profile for single commandline tool if not isinstance(process, Workflow) and runtime_context.research_obj is not None: - process.provenance_object = ProvenanceProfile( - runtime_context.research_obj, + process.provenance_object = runtime_context.research_obj.initialize_provenance( full_name=runtime_context.cwl_full_name, - host_provenance=False, - user_provenance=False, + # following are only set from main when directly command line tool + # when nested in a workflow, they should be disabled since they would + # already have been provided/initialized by the parent workflow prov-obj + host_provenance=runtime_context.prov_host, + user_provenance=runtime_context.prov_user, orcid=runtime_context.orcid, # single tool execution, so RO UUID = wf UUID = tool UUID run_uuid=runtime_context.research_obj.ro_uuid, diff --git a/cwltool/main.py b/cwltool/main.py index b7ba40d40..a137d8a4f 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -1065,6 +1065,11 @@ def main( loadingContext = setup_loadingContext(loadingContext, runtimeContext, args) + if loadingContext.research_obj: + # early forward parameters required for a single command line tool + runtimeContext.prov_host = loadingContext.host_provenance + runtimeContext.prov_user = loadingContext.user_provenance + uri, tool_file_uri = resolve_tool_uri( args.workflow, resolver=loadingContext.resolver, diff --git a/cwltool/workflow.py b/cwltool/workflow.py index 3bf32251f..899ac4643 100644 --- a/cwltool/workflow.py +++ b/cwltool/workflow.py @@ -72,8 +72,7 @@ def __init__( if is_main: run_uuid = loadingContext.research_obj.ro_uuid - self.provenance_object = ProvenanceProfile( - loadingContext.research_obj, + self.provenance_object = loadingContext.research_obj.initialize_provenance( full_name=loadingContext.cwl_full_name, host_provenance=loadingContext.host_provenance, user_provenance=loadingContext.user_provenance, diff --git a/mypy-stubs/rdflib/graph.pyi b/mypy-stubs/rdflib/graph.pyi index d3e6f2f54..9764972b2 100644 --- a/mypy-stubs/rdflib/graph.pyi +++ b/mypy-stubs/rdflib/graph.pyi @@ -16,7 +16,7 @@ from rdflib import query from rdflib.collection import Collection from rdflib.paths import Path from rdflib.resource import Resource -from rdflib.term import BNode, Identifier, Node +from rdflib.term import BNode, Identifier, Literal, Node class Graph(Node): base: Any = ... @@ -66,7 +66,7 @@ class Graph(Node): ) -> Iterable[Node]: ... def objects( self, subject: Optional[Any] = ..., predicate: Optional[Any] = ... - ) -> Iterable[Identifier]: ... + ) -> Iterable[Union[Identifier, Literal]]: ... def subject_predicates(self, object: Optional[Any] = ...) -> None: ... def subject_objects(self, predicate: Optional[Any] = ...) -> None: ... def predicate_objects(self, subject: Optional[Any] = ...) -> None: ... diff --git a/tests/test_provenance.py b/tests/test_provenance.py index e8d8416be..d7a2a698b 100644 --- a/tests/test_provenance.py +++ b/tests/test_provenance.py @@ -32,12 +32,23 @@ SCHEMA = Namespace("http://schema.org/") CWLPROV = Namespace("https://w3id.org/cwl/prov#") OA = Namespace("http://www.w3.org/ns/oa#") +FOAF = Namespace("http://xmlns.com/foaf/0.1/") -def cwltool(tmp_path: Path, *args: Any) -> Path: +TEST_ORCID = "https://orcid.org/0000-0003-4862-3349" + + +def cwltool(tmp_path: Path, *args: Any, with_orcid: bool = False) -> Path: prov_folder = tmp_path / "provenance" prov_folder.mkdir() - new_args = ["--provenance", str(prov_folder)] + new_args = [ + "--enable-user-provenance", + "--enable-host-provenance", + "--provenance", + str(prov_folder), + ] + if with_orcid: + new_args.extend(["--orcid", TEST_ORCID]) new_args.extend(args) # Run within a temporary directory to not pollute git checkout tmp_dir = tmp_path / "cwltool-run" @@ -49,61 +60,81 @@ def cwltool(tmp_path: Path, *args: Any) -> Path: @needs_docker -def test_hello_workflow(tmp_path: Path) -> None: +@pytest.mark.parametrize("with_orcid", [True, False]) +def test_hello_workflow(tmp_path: Path, with_orcid: bool) -> None: check_provenance( cwltool( tmp_path, get_data("tests/wf/hello-workflow.cwl"), "--usermessage", "Hello workflow", - ) + with_orcid=with_orcid, + ), + with_orcid=with_orcid, ) @needs_docker -def test_hello_single_tool(tmp_path: Path) -> None: +@pytest.mark.parametrize("with_orcid", [True, False]) +def test_hello_single_tool(tmp_path: Path, with_orcid: bool) -> None: check_provenance( cwltool( tmp_path, get_data("tests/wf/hello_single_tool.cwl"), "--message", "Hello tool", + with_orcid=with_orcid, ), single_tool=True, + with_orcid=with_orcid, ) @needs_docker -def test_revsort_workflow(tmp_path: Path) -> None: +@pytest.mark.parametrize("with_orcid", [True, False]) +def test_revsort_workflow(tmp_path: Path, with_orcid: bool) -> None: folder = cwltool( tmp_path, get_data("tests/wf/revsort.cwl"), get_data("tests/wf/revsort-job.json"), + with_orcid=with_orcid, ) check_output_object(folder) - check_provenance(folder) + check_provenance(folder, with_orcid=with_orcid) @needs_docker -def test_revsort_workflow_shortcut(tmp_path: Path) -> None: +@pytest.mark.parametrize("with_orcid", [True, False]) +def test_revsort_workflow_shortcut(tmp_path: Path, with_orcid: bool) -> None: """Confirm that using 'cwl:tool' shortcut still snapshots the CWL files.""" folder = cwltool( tmp_path, get_data("tests/wf/revsort-job-shortcut.json"), + with_orcid=with_orcid, ) check_output_object(folder) - check_provenance(folder) + check_provenance(folder, with_orcid=with_orcid) assert not (folder / "snapshot" / "revsort-job-shortcut.json").exists() assert len(list((folder / "snapshot").iterdir())) == 4 @needs_docker -def test_nested_workflow(tmp_path: Path) -> None: - check_provenance(cwltool(tmp_path, get_data("tests/wf/nested.cwl")), nested=True) +@pytest.mark.parametrize("with_orcid", [True, False]) +def test_nested_workflow(tmp_path: Path, with_orcid: bool) -> None: + check_provenance( + cwltool( + tmp_path, + get_data("tests/wf/nested.cwl"), + with_orcid=with_orcid, + ), + nested=True, + with_orcid=with_orcid, + ) @needs_docker -def test_secondary_files_implicit(tmp_path: Path) -> None: +@pytest.mark.parametrize("with_orcid", [True, False]) +def test_secondary_files_implicit(tmp_path: Path, with_orcid: bool) -> None: file1 = tmp_path / "foo1.txt" file1idx = tmp_path / "foo1.txt.idx" @@ -113,13 +144,20 @@ def test_secondary_files_implicit(tmp_path: Path) -> None: f.write("bar") # secondary will be picked up by .idx - folder = cwltool(tmp_path, get_data("tests/wf/sec-wf.cwl"), "--file1", str(file1)) - check_provenance(folder, secondary_files=True) + folder = cwltool( + tmp_path, + get_data("tests/wf/sec-wf.cwl"), + "--file1", + str(file1), + with_orcid=with_orcid, + ) + check_provenance(folder, secondary_files=True, with_orcid=with_orcid) check_secondary_files(folder) @needs_docker -def test_secondary_files_explicit(tmp_path: Path) -> None: +@pytest.mark.parametrize("with_orcid", [True, False]) +def test_secondary_files_explicit(tmp_path: Path, with_orcid: bool) -> None: # Deliberately do NOT have common basename or extension file1dir = tmp_path / "foo" file1dir.mkdir() @@ -154,22 +192,33 @@ def test_secondary_files_explicit(tmp_path: Path) -> None: j = json.dumps(job, ensure_ascii=True) fp.write(j.encode("ascii")) - folder = cwltool(tmp_path, get_data("tests/wf/sec-wf.cwl"), str(jobJson)) - check_provenance(folder, secondary_files=True) + folder = cwltool( + tmp_path, + get_data("tests/wf/sec-wf.cwl"), + str(jobJson), + with_orcid=with_orcid, + ) + check_provenance(folder, secondary_files=True, with_orcid=with_orcid) check_secondary_files(folder) @needs_docker -def test_secondary_files_output(tmp_path: Path) -> None: +@pytest.mark.parametrize("with_orcid", [True, False]) +def test_secondary_files_output(tmp_path: Path, with_orcid: bool) -> None: # secondary will be picked up by .idx - folder = cwltool(tmp_path, get_data("tests/wf/sec-wf-out.cwl")) - check_provenance(folder, secondary_files=True) + folder = cwltool( + tmp_path, + get_data("tests/wf/sec-wf-out.cwl"), + with_orcid=with_orcid, + ) + check_provenance(folder, secondary_files=True, with_orcid=with_orcid) # Skipped, not the same secondary files as above # self.check_secondary_files() @needs_docker -def test_directory_workflow(tmp_path: Path) -> None: +@pytest.mark.parametrize("with_orcid", [True, False]) +def test_directory_workflow(tmp_path: Path, with_orcid: bool) -> None: dir2 = tmp_path / "dir2" dir2.mkdir() sha1 = { @@ -185,8 +234,14 @@ def test_directory_workflow(tmp_path: Path) -> None: with open(dir2 / x, "w", encoding="ascii") as f: f.write(x) - folder = cwltool(tmp_path, get_data("tests/wf/directory.cwl"), "--dir", str(dir2)) - check_provenance(folder, directory=True) + folder = cwltool( + tmp_path, + get_data("tests/wf/directory.cwl"), + "--dir", + str(dir2), + with_orcid=with_orcid, + ) + check_provenance(folder, directory=True, with_orcid=with_orcid) # Output should include ls stdout of filenames a b c on each line file_list = ( @@ -209,10 +264,12 @@ def test_directory_workflow(tmp_path: Path) -> None: @needs_docker -def test_no_data_files(tmp_path: Path) -> None: +@pytest.mark.parametrize("with_orcid", [True, False]) +def test_no_data_files(tmp_path: Path, with_orcid: bool) -> None: folder = cwltool( tmp_path, get_data("tests/wf/conditional_step_no_inputs.cwl"), + with_orcid=with_orcid, ) check_bagit(folder) @@ -263,6 +320,7 @@ def check_provenance( single_tool: bool = False, directory: bool = False, secondary_files: bool = False, + with_orcid: bool = False, ) -> None: check_folders(base_path) check_bagit(base_path) @@ -273,6 +331,7 @@ def check_provenance( single_tool=single_tool, directory=directory, secondary_files=secondary_files, + with_orcid=with_orcid, ) @@ -463,6 +522,7 @@ def check_prov( single_tool: bool = False, directory: bool = False, secondary_files: bool = False, + with_orcid: bool = False, ) -> None: prov_file = base_path / "metadata" / "provenance" / "primary.cwlprov.nt" assert prov_file.is_file(), f"Can't find {prov_file}" @@ -485,7 +545,6 @@ def check_prov( # the has_provenance annotations in manifest.json instead # run should have been started by a wf engine - engines = set(g.subjects(RDF.type, WFPROV.WorkflowEngine)) assert engines, "Could not find WorkflowEngine" assert len(engines) == 1, "Found too many WorkflowEngines: %s" % engines @@ -502,6 +561,39 @@ def check_prov( PROV.SoftwareAgent, ) in g, "Engine not declared as SoftwareAgent" + # run should be associated to the user + accounts = set(g.subjects(RDF.type, FOAF.OnlineAccount)) + assert len(accounts) == 1 + account = accounts.pop() + people = set(g.subjects(RDF.type, SCHEMA.Person)) + assert len(people) == 1, "Can't find associated person in workflow run" + person = people.pop() + if with_orcid: + assert person == URIRef(TEST_ORCID) + else: + account_names = set(g.objects(account, FOAF.accountName)) + assert len(account_names) == 1 + account_name = cast(Literal, account_names.pop()) + machine_user = provenance._whoami()[0] + assert account_name.value == machine_user + + # find the random UUID assigned to cwltool + tool_agents = set(g.subjects(RDF.type, PROV.SoftwareAgent)) + n_all_agents = 2 + len(tool_agents) + agents = set(g.subjects(RDF.type, PROV.Agent)) + assert ( + len(agents) == n_all_agents + ), "There should be 1 agent per tool (engine), 1 user agent, and 1 cwltool agent" + agents.remove(person) + agents.remove(engine) # the main tool + remain_agents = agents - tool_agents + assert len(remain_agents) == 1 + assert ( + account, + PROV.actedOnBehalfOf, + person, + ) in g, "Association of cwltool agent acting for user is missing" + if single_tool: activities = set(g.subjects(RDF.type, PROV.Activity)) assert len(activities) == 1, "Too many activities: %s" % activities From c6782ff11d345dcc86a79e1d582fbbaa8a61e8c1 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Tue, 17 Dec 2024 17:14:26 +0100 Subject: [PATCH 38/43] singularity: improve testing on version 4.x+ --- .github/workflows/ci-tests.yml | 16 ++++++++-------- tests/test_environment.py | 33 ++++++++++++++++----------------- 2 files changed, 24 insertions(+), 25 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 1f01160c8..2ebb14c5f 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -47,8 +47,8 @@ jobs: - name: Set up Singularity and environment-modules if: ${{ matrix.step == 'unit' || matrix.step == 'mypy' }} run: | - wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-focal_amd64.deb - sudo apt-get install -y ./singularity-ce_3.10.4-focal_amd64.deb environment-modules + wget --no-verbose https://github.com/sylabs/singularity/releases/download/v4.2.1/singularity-ce_4.2.1-focal_amd64.deb + sudo apt-get install -y ./singularity-ce_4.2.1-focal_amd64.deb environment-modules - name: Give the test runner user a name to make provenance happy. if: ${{ matrix.step == 'unit' || matrix.step == 'mypy' }} @@ -134,8 +134,8 @@ jobs: - name: Set up Singularity and environment-modules run: | - wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-focal_amd64.deb - sudo apt-get install -y ./singularity-ce_3.10.4-focal_amd64.deb environment-modules + wget --no-verbose https://github.com/sylabs/singularity/releases/download/v4.2.1/singularity-ce_4.2.1-focal_amd64.deb + sudo apt-get install -y ./singularity-ce_4.2.1-focal_amd64.deb environment-modules - name: Give the test runner user a name to make provenance happy. run: sudo usermod -c 'CI Runner' "$(whoami)" @@ -183,8 +183,8 @@ jobs: - name: Set up Singularity and environment-modules if: ${{ matrix.container == 'singularity' }} run: | - wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-jammy_amd64.deb - sudo apt-get install -y ./singularity-ce_3.10.4-jammy_amd64.deb environment-modules + wget --no-verbose https://github.com/sylabs/singularity/releases/download/v4.2.1/singularity-ce_4.2.1-focal_amd64.deb + sudo apt-get install -y ./singularity-ce_4.2.1-focal_amd64.deb environment-modules - name: Singularity cache if: ${{ matrix.container == 'singularity' }} @@ -231,8 +231,8 @@ jobs: - name: Set up Singularity and environment-modules run: | - wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-jammy_amd64.deb - sudo apt-get install -y ./singularity-ce_3.10.4-jammy_amd64.deb environment-modules + wget --no-verbose https://github.com/sylabs/singularity/releases/download/v4.2.1/singularity-ce_4.2.1-focal_amd64.deb + sudo apt-get install -y ./singularity-ce_4.2.1-focal_amd64.deb environment-modules - name: Set up Python uses: actions/setup-python@v5 diff --git a/tests/test_environment.py b/tests/test_environment.py index 4e9c602f1..fa29fb924 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -7,6 +7,7 @@ from typing import Callable, Union import pytest +from packaging.version import Version from cwltool.singularity import get_version @@ -133,33 +134,31 @@ def PWD(v: str, env: Env) -> bool: } # Singularity variables appear to be in flux somewhat. - version = get_version()[0] - vmajor = version[0] - assert vmajor == 3, "Tests only work for Singularity 3" - vminor = version[1] + version = Version(".".join(map(str, get_version()[0]))) + assert version >= Version("3"), "Tests only work for Singularity 3+" sing_vars: EnvChecks = { "SINGULARITY_CONTAINER": None, "SINGULARITY_NAME": None, } - if vminor < 5: + if version < Version("3.5"): sing_vars["SINGULARITY_APPNAME"] = None - if vminor >= 5: + if (version >= Version("3.5")) and (version < Version("3.6")): + sing_vars["SINGULARITY_INIT"] = "1" + if version >= Version("3.5"): sing_vars["PROMPT_COMMAND"] = None sing_vars["SINGULARITY_ENVIRONMENT"] = None - if vminor == 5: - sing_vars["SINGULARITY_INIT"] = "1" - elif vminor > 5: + if version >= Version("3.6"): sing_vars["SINGULARITY_COMMAND"] = "exec" - if vminor >= 7: - if vminor > 9: - sing_vars["SINGULARITY_BIND"] = "" - else: + if version >= Version("3.7"): + if version > Version("3.9"): + sing_vars["SINGULARITY_BIND"] = "" + else: - def BIND(v: str, env: Env) -> bool: - return v.startswith(tmp_prefix) and v.endswith(":/tmp") + def BIND(v: str, env: Env) -> bool: + return v.startswith(tmp_prefix) and v.endswith(":/tmp") - sing_vars["SINGULARITY_BIND"] = BIND - if vminor >= 10: + sing_vars["SINGULARITY_BIND"] = BIND + if version >= Version("3.10"): sing_vars["SINGULARITY_COMMAND"] = "run" sing_vars["SINGULARITY_NO_EVAL"] = None From ae5fae45dcf3af00a3303f63a237bbc88cf0f67e Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" <1330696+mr-c@users.noreply.github.com> Date: Fri, 20 Dec 2024 08:48:37 -0800 Subject: [PATCH 39/43] README: Python supported versions list was outdated --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index dce55fc95..7c6ffe22d 100644 --- a/README.rst +++ b/README.rst @@ -52,7 +52,7 @@ and provide comprehensive validation of CWL files as well as provide other tools related to working with CWL. ``cwltool`` is written and tested for -`Python `_ ``3.x {x = 6, 8, 9, 10, 11}`` +`Python `_ ``3.x {x = 9, 10, 11, 12, 13}`` The reference implementation consists of two packages. The ``cwltool`` package is the primary Python module containing the reference implementation in the From 6a61bed8973ecc62dc95a5544dda428d6e9e963b Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" <1330696+mr-c@users.noreply.github.com> Date: Fri, 20 Dec 2024 09:20:13 -0800 Subject: [PATCH 40/43] docs: render the CLI groups too --- docs/cli.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cli.rst b/docs/cli.rst index d569f5586..1c6021bf4 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -3,4 +3,4 @@ cwltool Command Line Options .. autoprogram:: cwltool.argparser:arg_parser() :prog: cwltool - + :groups: From f453cdce5956fe6581f5ccdcb8aacb8c4f29f6d4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Dec 2024 07:13:14 +0000 Subject: [PATCH 41/43] Bump mypy from 1.13.0 to 1.14.0 Bumps [mypy](https://github.com/python/mypy) from 1.13.0 to 1.14.0. - [Changelog](https://github.com/python/mypy/blob/master/CHANGELOG.md) - [Commits](https://github.com/python/mypy/compare/v1.13.0...v1.14.0) --- updated-dependencies: - dependency-name: mypy dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- mypy-requirements.txt | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index 5f18fa03a..f5944d2e8 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -1,4 +1,4 @@ -mypy==1.13.0 # also update pyproject.toml +mypy==1.14.0 # also update pyproject.toml ruamel.yaml>=0.16.0,<0.19 cwl-utils>=0.32 cwltest diff --git a/pyproject.toml b/pyproject.toml index 248c0e69d..deb1adc27 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ requires = [ "setuptools>=45", "setuptools_scm[toml]>=8.0.4,<9", - "mypy==1.13.0", # also update mypy-requirements.txt + "mypy==1.14.0", # also update mypy-requirements.txt "types-requests", "types-psutil", "importlib_resources>=1.4;python_version<'3.9'", From baa668bc96ade54607465d21bc6cfa15c9bff13c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 31 Dec 2024 07:48:50 +0000 Subject: [PATCH 42/43] Bump mypy from 1.14.0 to 1.14.1 Bumps [mypy](https://github.com/python/mypy) from 1.14.0 to 1.14.1. - [Changelog](https://github.com/python/mypy/blob/master/CHANGELOG.md) - [Commits](https://github.com/python/mypy/compare/v1.14.0...v1.14.1) --- updated-dependencies: - dependency-name: mypy dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- mypy-requirements.txt | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index f5944d2e8..760428998 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -1,4 +1,4 @@ -mypy==1.14.0 # also update pyproject.toml +mypy==1.14.1 # also update pyproject.toml ruamel.yaml>=0.16.0,<0.19 cwl-utils>=0.32 cwltest diff --git a/pyproject.toml b/pyproject.toml index deb1adc27..b243171fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ requires = [ "setuptools>=45", "setuptools_scm[toml]>=8.0.4,<9", - "mypy==1.14.0", # also update mypy-requirements.txt + "mypy==1.14.1", # also update mypy-requirements.txt "types-requests", "types-psutil", "importlib_resources>=1.4;python_version<'3.9'", From a67c898958f6affc8cb9de05fe87c9228a4fc63e Mon Sep 17 00:00:00 2001 From: stxue1 Date: Thu, 9 Jan 2025 10:37:05 -0800 Subject: [PATCH 43/43] Change caching pathmapper to respect the pathmapper factory --- cwltool/command_line_tool.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cwltool/command_line_tool.py b/cwltool/command_line_tool.py index e201fb12b..775360016 100644 --- a/cwltool/command_line_tool.py +++ b/cwltool/command_line_tool.py @@ -807,10 +807,10 @@ def job( cachecontext.tmpdir = "/tmp" # nosec cachecontext.stagedir = "/stage" cachebuilder = self._init_job(job_order, cachecontext) - cachebuilder.pathmapper = PathMapper( + cachebuilder.pathmapper = self.make_path_mapper( cachebuilder.files, - runtimeContext.basedir, cachebuilder.stagedir, + runtimeContext, separateDirs=False, ) _check_adjust = partial(check_adjust, self.path_check_mode.value, cachebuilder)