diff --git a/docs/devguide/changelog.rst b/docs/devguide/changelog.rst index 21b6b0e001..a32e327d6e 100644 --- a/docs/devguide/changelog.rst +++ b/docs/devguide/changelog.rst @@ -289,7 +289,7 @@ New Functionality .. code-block:: python @bash_app - def cat(inputs=[], outputs=[]): + def cat(inputs=(), outputs=()): return 'cat {} > {}'.format(inputs[0], outputs[0]) concat = cat(inputs=['hello-0.txt'], @@ -302,7 +302,7 @@ New Functionality from parsl import File @bash_app - def cat(inputs=[], outputs=[]): + def cat(inputs=(), outputs=()): return 'cat {} > {}'.format(inputs[0].filepath, outputs[0].filepath) concat = cat(inputs=[File('hello-0.txt')], @@ -316,7 +316,7 @@ New Functionality from parsl import File @bash_app - def cat(inputs=[], outputs=[]): + def cat(inputs=(), outputs=()): return 'cat {} > {}'.format(inputs[0].filepath, outputs[0].filepath) @@ -397,16 +397,16 @@ New Functionality # The following example worked until v0.8.0 @bash_app - def cat(inputs=[], outputs=[]): + def cat(inputs=(), outputs=()): return 'cat {inputs[0]} > {outputs[0]}' # <-- Relies on Parsl auto formatting the string # Following are two mechanisms that will work going forward from v0.9.0 @bash_app - def cat(inputs=[], outputs=[]): + def cat(inputs=(), outputs=()): return 'cat {} > {}'.format(inputs[0], outputs[0]) # <-- Use str.format method @bash_app - def cat(inputs=[], outputs=[]): + def cat(inputs=(), outputs=()): return f'cat {inputs[0]} > {outputs[0]}' # <-- OR use f-strings introduced in Python3.6 @@ -510,12 +510,12 @@ New Functionality # Old style: " ".join(inputs) is legal since inputs will behave like a list of strings @bash_app - def concat(inputs=[], outputs=[], stdout="stdout.txt", stderr='stderr.txt'): + def concat(inputs=(), outputs=(), stdout="stdout.txt", stderr='stderr.txt'): return "cat {0} > {1}".format(" ".join(inputs), outputs[0]) # New style: @bash_app - def concat(inputs=[], outputs=[], stdout="stdout.txt", stderr='stderr.txt'): + def concat(inputs=(), outputs=(), stdout="stdout.txt", stderr='stderr.txt'): return "cat {0} > {1}".format(" ".join(list(map(str,inputs))), outputs[0]) * Cleaner user app file log management. diff --git a/docs/faq.rst b/docs/faq.rst index e6fc652a52..ac71299224 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -44,7 +44,7 @@ How can I make an App dependent on multiple inputs? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ You can pass any number of futures in to a single App either as positional arguments -or as a list of futures via the special keyword ``inputs=[]``. +or as a list of futures via the special keyword ``inputs=()``. The App will wait for all inputs to be satisfied before execution. diff --git a/docs/teaching_scripts/README.md b/docs/teaching_scripts/README.md new file mode 100644 index 0000000000..67f80899af --- /dev/null +++ b/docs/teaching_scripts/README.md @@ -0,0 +1,3 @@ +# Example Scripts + +Scripts which illustrate example from the documentation that do not run well as part of the pytest diff --git a/docs/teaching_scripts/test_apps.py b/docs/teaching_scripts/test_apps.py new file mode 100644 index 0000000000..d139ea140a --- /dev/null +++ b/docs/teaching_scripts/test_apps.py @@ -0,0 +1,83 @@ +"""Tests documentation related to building apps. Must reside outside the Parsl library to be effective""" +from typing import List, Union + +import numpy as np + +from parsl import python_app, HighThroughputExecutor, Config +import parsl + +parsl.load(Config(executors=[HighThroughputExecutor(label='htex_spawn', max_workers=1, start_method='spawn', address='127.0.0.1')])) + + +# Part 1: Explain imports +# BAD: Assumes library has been imported +@python_app(executors=['htex_spawn']) +def bad_imports(x: Union[List[float], np.ndarray], m: float, b: float): + return np.multiply(x, m) + b + + +# GOOD: Imports libraries itself +@python_app(executors=['htex_spawn']) +def good_imports(x: Union[List[float], 'np.ndarray'], m: float, b: float): + import numpy as np + return np.multiply(x, m) + b + + +future = bad_imports([1.], 1, 0) + +try: + future.result() + raise ValueError() +except NameError as e: + print('Failed, as expected. Error:', e) + +future = good_imports([1.], 1, 0) +print(f'Passed, as expected: {future.result()}') + +# Part 2: Test other types of globals +# BAD: Uses global variables +global_var = {'a': 0} + + +@python_app +def bad_global(string: str, character: str = 'a'): + global_var[character] += string.count(character) # `global_var` will not be accessible + + +# GOOD +@python_app +def good_global(string: str, character: str = 'a'): + return {character: string.count(character)} + + +try: + bad_global('parsl').result() +except NameError as e: + print(f'Failed, as expected: {e}') + +for ch, co in good_global('parsl', 'a').result().items(): + global_var[ch] += co + + +# Part 3: Mutable args + +# BAD: Assumes changes to inputs will be communicated +@python_app +def append_to_list(input_list: list, new_val): + input_list.append(new_val) + + +mutable_arg = [] +append_to_list(mutable_arg, 1).result() +assert mutable_arg == [], 'The list _was_changed' + + +# GOOD: Changes to inputs are returned +@python_app +def append_to_list(input_list: list, new_val) -> list: + input_list.append(new_val) + return input_list + + +mutable_arg = append_to_list(mutable_arg, 1).result() +assert mutable_arg == [1] diff --git a/docs/userguide/apps.rst b/docs/userguide/apps.rst index 66a415ec0b..1ef105b4fe 100644 --- a/docs/userguide/apps.rst +++ b/docs/userguide/apps.rst @@ -3,112 +3,127 @@ Apps ==== -An **app** is a Parsl construct for representing a fragment of Python code -or external Bash shell code that can be asynchronously executed. +An **App** defines a computation that will be executed asynchronously by Parsl. +Apps are Python functions marked with a decorator which +designates that the function will run asynchronously and cause it to return +a :class:`~concurrent.futures.Future` instead of the result. -A Parsl app is defined by annotating a Python function with a decorator: -the ``@python_app`` decorator for a **Python app**, the ``@bash_app`` decorator for a **Bash app**, -and the ``@join_app`` decorator for a **Join app**. +Apps can be one of three types of functions, each with their own type of decorator -Python apps encapsulate pure Python code, while Bash apps wrap calls to external applications and scripts, -and Join apps allow composition of other apps to form sub-workflows. +- ``@python_app``: Most Python functions +- ``@bash_app``: A Python function which returns a command line program to execute +- ``@join_app``: A function which launches one or more new Apps -Python and Bash apps are documented below. Join apps are documented in a later -section (see :ref:`label-joinapp`) +The intricacies of Python and Bash apps are documented below. Join apps are documented in a later +section (see :ref:`label-joinapp`). Python Apps ----------- -The following code snippet shows a Python function ``double(x: int)``, which returns double the input -value. -The ``@python_app`` decorator defines the function as a Parsl Python app. - .. code-block:: python - @python_app - def double(x): - return x * 2 + @python_app + def hello_world(name: str) -> str: + return f'Hello, {name}!' - double(42) + print(hello_world('user').result()) -As a Parsl Python app is executed asynchronously, and potentially remotely, the function -cannot assume access to shared program state. For example, it must explicitly import any -required modules and cannot refer to variables used outside the function. -Thus while the following code fragment is valid Python, it is not valid Parsl, -as the ``bad_double()`` function requires the `random` module and refers to the external -variable ``factor``. -.. code-block:: python +Python Apps run Python functions. The code inside a function marked by ``@python_app`` is what will +be executed either locally or on a remote system. + +Most functions can run without modification. +Limitations on the content of the functions and their inputs/outputs are described below. + +Rules for Function Contents +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. _function-rules: + +Parsl apps have access to less information from the script that defined them +than functions run via Python's native multiprocessing libraries. +The reason is that functions are executed on workers that +lack access to the global variables in the script that defined them. +Practically, this means + +1. *Functions may need to re-import libraries.* + Place the import statements that define functions or classes inside the function. + Type annotations should not use libraries defined in the function. + + + .. code-block:: python + + import numpy as np + + # BAD: Assumes library has been imported + @python_app + def linear_model(x: list[float] | np.ndarray, m: float, b: float): + return np.multiply(x, m) + b - import random - factor = 5 + # GOOD: Function imports libraries on remote worker + @python_app + def linear_model(x: list[float] | 'np.ndarray', m: float, b: float): + import numpy as np + return np.multiply(x, m) + b - @python_app - def bad_double(x): - return x * random.random() * factor - print(bad_double(42)) - -The following alternative formulation is valid Parsl. +2. *Global variables are inaccessible*. + Functions should not use variables defined outside the function. + Likewise, do not assume that variables created inside the function are visible elsewhere. + .. code-block:: python - factor = 5 + # BAD: Uses global variables + global_var = {'a': 0} + + @python_app + def counter_func(string: str, character: str = 'a'): + global_var[character] += string.count(character) # `global_var` will not be accessible + - @python_app - def good_double(x, f): - import random - return x * random.random() * f + # GOOD + @python_app + def counter_func(string: str, character: str = 'a'): + return {character: string.count(character)} - print(good_double(42, factor)) + for ch, co in good_global('parsl', 'a').result().items(): + global_var[ch] += co -Python apps may be passed any Python input argument, including primitive types, -files, and other complex types that can be serialized (e.g., numpy array, -scikit-learn model). They may also be passed a Parsl ``Future`` (see :ref:`label-futures`) -returned by another Parsl app. -In this case, Parsl will establish a dependency between the two apps and will not -execute the dependent app until all dependent futures are resolved. -Further detail is provided in :ref:`label-futures`. -A Python app may also act upon files. In order to make Parsl aware of these files, -they must be specified by using the ``inputs`` and/or ``outputs`` keyword arguments, -as in following code snippet, which copies the contents of one file (``in.txt``) to another (``out.txt``). + +3. *Outputs are only available through return statements*. + Parsl does not support generator functions (i.e., those which use ``yield`` statements) and + any changes to input arguments will not be communicated. .. code-block:: python - @python_app - def echo(inputs=[], outputs=[]): - with open(inputs[0], 'r') as in_file, open(outputs[0], 'w') as out_file: - out_file.write(in_file.readline()) + # BAD: Assumes changes to inputs will be communicated + @python_app + def append_to_list(input_list: list, new_val): + input_list.append(new_val) - echo(inputs=[in.txt], outputs=[out.txt]) -Special Keyword Arguments -^^^^^^^^^^^^^^^^^^^^^^^^^^ + # GOOD: Changes to inputs are returned + @python_app + def append_to_list(input_list: list, new_val) -> list: + input_list.append(new_val) + return input_list -Any Parsl app (a Python function decorated with the ``@python_app`` or ``@bash_app`` decorator) can use the following special reserved keyword arguments. -1. inputs: (list) This keyword argument defines a list of input :ref:`label-futures` or files. - Parsl will wait for the results of any listed :ref:`label-futures` to be resolved before executing the app. - The ``inputs`` argument is useful both for passing files as arguments - and when one wishes to pass in an arbitrary number of futures at call time. -2. outputs: (list) This keyword argument defines a list of files that - will be produced by the app. For each file thus listed, Parsl will create a future, - track the file, and ensure that it is correctly created. The future - can then be passed to other apps as an input argument. -3. walltime: (int) This keyword argument places a limit on the app's - runtime in seconds. If the walltime is exceed, Parsl will raise an `parsl.app.errors.AppTimeout` exception. +Functions from Modules +++++++++++++++++++++++ -Serializing Functions from Libraries -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The above rules assume that the user is running the example code from a standalone script or Jupyter Notebook. +Functions that are defined in an installed Python module do not need to abide by these guidelines, +as they are sent to workers differently than functions defined locally within a script. -Parsl can create Apps directly from functions defined in Python modules. -Supply the function as an argument to ``python_app`` rather than creating a new function which is decorated. +Directly convert a function from a library to a Python App by passing it as an argument to ``python_app``: .. code-block:: python from module import function - function_app = python_app(function, executors='all') + function_app = python_app(function) ``function_app`` will act as Parsl App function of ``function``. @@ -124,21 +139,140 @@ to include attributes from the original function (e.g., its name). my_max = update_wrapper(my_max, max) # Copy over the names my_max_app = python_app(my_max) +The above example is equivalent to creating a new function (as below) + +.. code-block:: python + + @python_app + def my_max_app(*args, **kwargs): + import numpy as np + return np.max(*args, keepdims=True, axis=0, **kwargs) + +Inputs and Outputs +^^^^^^^^^^^^^^^^^^ + +Python apps may be passed any Python type as an input and return any Python type, with a few exceptions. +There are several classes of allowed types, each with different rules. + +- *Python Objects*: Any Python object that can be saved with + `pickle `_ or `dill `_ + can be used as an import or output. + All primitive types (e.g., floats, strings) are valid as are many complex types (e.g., numpy arrays). +- *Files*: Pass files as inputs as a :py:class:`~parsl.data_provider.files.File` object. + Parsl can transfer them to a remote system and update the ``File`` object with a new path. + Access the new path with ``File.filepath`` attribute. + + .. code-block:: python + + @python_app + def read_first_line(x: File): + with open(x.filepath, 'r') as fp: + return fp.readline() + + Files can also be outputs of a function, but only through the ``outputs`` kwargs (described below). +- *Parsl Futures*. Functions can receive results from other Apps as Parsl ``Future`` objects. + Parsl will establish a dependency on the App(s) which created the Future(s) + and start executing as soon as the preceding ones complete. + + .. code-block:: python + + @python_app + def capitalize(x: str): + return x.upper() + + input_file = File('text.txt') + first_line_future = read_first_line(input_file) + capital_future = capitalize(first_line_future) + print(capital_future.result()) + + See the section on `Futures `_ for more details. + + +Learn more about the types of data allowed in `the data section `_. + +.. note:: + + Any changes to mutable input arguments will be ignored. + +Special Keyword Arguments ++++++++++++++++++++++++++ + +Some keyword arguments to the Python function are treated differently by Parsl -Apps created using ``python_app`` as a function will work just like those which use it as a decorator. +1. inputs: (list) This keyword argument defines a list of input :ref:`label-futures` or files. + Parsl will wait for the results of any listed :ref:`label-futures` to be resolved before executing the app. + The ``inputs`` argument is useful both for passing files as arguments + and when one wishes to pass in an arbitrary number of futures at call time. + +.. code-block:: python + + @python_app() + def map_app(x): + return x * 2 + + @python_app() + def reduce_app(inputs = ()): + return sum(inputs) + + map_futures = [map_app(x) for x in range(3)] + reduce_future = reduce_app(inputs=map_futures) + + print(reduce_future.result()) # 0 + 1 * 2 + 2 * 2 = 6 + +2. outputs: (list) This keyword argument defines a list of files that + will be produced by the app. For each file thus listed, Parsl will create a future, + track the file, and ensure that it is correctly created. The future + can then be passed to other apps as an input argument. + +.. code-block:: python + + @python_app() + def write_app(message, outputs=()): + """Write a single message to every file in outputs""" + for path in outputs: + with open(path, 'w') as fp: + print(message, file=fp) + + to_write = [ + File(Path(tmpdir) / 'output-0.txt'), + File(Path(tmpdir) / 'output-1.txt') + ] + write_app('Hello!', outputs=to_write).result() + for path in to_write: + with open(path) as fp: + assert fp.read() == 'Hello!\n' -Returns -^^^^^^^ +3. walltime: (int) This keyword argument places a limit on the app's + runtime in seconds. If the walltime is exceed, Parsl will raise an `parsl.app.errors.AppTimeout` exception. + +Outputs ++++++++ A Python app returns an AppFuture (see :ref:`label-futures`) as a proxy for the results that will be returned by the app once it is executed. This future can be inspected to obtain task status; and it can be used to wait for the result, and when complete, present the output Python object(s) returned by the app. In case of an error or app failure, the future holds the exception raised by the app. +Options for Python Apps +^^^^^^^^^^^^^^^^^^^^^^^ + +The :meth:`~parsl.app.app.python_app` decorator has a few options which controls how Parsl executes all tasks +run with that application. +For example, you can ensure that Parsl caches the results of the function and executes tasks on specific sites. + +.. code-block:: python + + @python_app(cache=True, executors=['gpu']) + def expensive_gpu_function(): + # ... + return + +See the Parsl documentation for full details. + Limitations ^^^^^^^^^^^ -There are some limitations on the Python functions that can be converted to apps: +To summarize, any Python function can be made a Python App with a few restrictions 1. Functions should act only on defined input arguments. That is, they should not use script-level or global variables. 2. Functions must explicitly import any required modules if they are defined in script which starts Parsl. @@ -149,82 +283,65 @@ There are some limitations on the Python functions that can be converted to apps Bash Apps --------- -A Parsl Bash app is used to execute an external application, script, or code written in another language. -It is defined by a ``@bash_app`` decorator and the Python code that forms the body of the -function must return a fragment of Bash shell code to be executed by Parsl. -The Bash shell code executed by a Bash app can be arbitrarily long. - -The following code snippet presents an example of a Bash app ``echo_hello``, -which returns the bash command ``'echo "Hello World!"'`` as a string. -This string will be executed by Parsl as a Bash command. - .. code-block:: python @bash_app - def echo_hello(stderr='std.err', stdout='std.out'): - return 'echo "Hello World!"' + def echo( + name: str, + stdout=parsl.AUTO_LOGNAME # Requests Parsl to return the stdout + ): + return f'echo "Hello, {name}!"' - # echo_hello() when called will execute the shell command and - # create a std.out file with the contents "Hello World!" - echo_hello() + future = echo('user') + future.result() # block until task has completed + with open(future.stdout, 'r') as f: + print(f.read()) -Unlike a Python app, a Bash app cannot return Python objects. -Instead, Bash apps communicate with other apps via files. -A decorated ``@bash_app`` exposes the ``inputs`` and ``outputs`` keyword arguments -described above for tracking input and output files. -It also includes, as described below, keyword arguments for capturing the STDOUT and STDERR streams and recording -them in files that are managed by Parsl. -Special Keywords -^^^^^^^^^^^^^^^^ +A Parsl Bash app executes an external application by making a command-line execution. +Parsl will execute the string returned by the function as a command-line script on a remote worker. -In addition to the ``inputs``, ``outputs``, and ``walltime`` keyword arguments -described above, a Bash app can accept the following keywords: +Rules for Function Contents +^^^^^^^^^^^^^^^^^^^^^^^^^^^ -1. stdout: (string, tuple or ``parsl.AUTO_LOGNAME``) The path to a file to which standard output should be redirected. If set to ``parsl.AUTO_LOGNAME``, the log will be automatically named according to task id and saved under ``task_logs`` in the run directory. If set to a tuple ``(filename, mode)``, standard output will be redirected to the named file, opened with the specified mode as used by the Python `open `_ function. -2. stderr: (string or ``parsl.AUTO_LOGNAME``) Like stdout, but for the standard error stream. -3. label: (string) If the app is invoked with ``stdout=parsl.AUTO_LOGNAME`` or ``stderr=parsl.AUTO_LOGNAME``, this arugment will be appended to the log name. +Bash Apps follow the same rules :ref:`as Python Apps `. +For example, imports may need to be inside functions and global variables will be inaccessible. -A Bash app can construct the Bash command string to be executed from arguments passed -to the decorated function. +Inputs and Outputs +^^^^^^^^^^^^^^^^^^ -.. code-block:: python +Bash Apps can use the same kinds of inputs as Python Apps, but only communicate results with Files. - @bash_app - def echo(arg, inputs=[], stderr=parsl.AUTO_LOGNAME, stdout=parsl.AUTO_LOGNAME): - return 'echo {} {} {}'.format(arg, inputs[0], inputs[1]) - - future = echo('Hello', inputs=['World', '!']) - future.result() # block until task has completed +The Bash Apps, unlike Python Apps, can also return the content printed to the Standard Output and Error. - with open(future.stdout, 'r') as f: - print(f.read()) # prints "Hello World !" +Special Keywords Arguments +++++++++++++++++++++++++++ +In addition to the ``inputs``, ``outputs``, and ``walltime`` keyword arguments +described above, a Bash app can accept the following keywords: -Returns -^^^^^^^ +1. stdout: (string, tuple or ``parsl.AUTO_LOGNAME``) The path to a file to which standard output should be redirected. If set to ``parsl.AUTO_LOGNAME``, the log will be automatically named according to task id and saved under ``task_logs`` in the run directory. If set to a tuple ``(filename, mode)``, standard output will be redirected to the named file, opened with the specified mode as used by the Python `open `_ function. +2. stderr: (string or ``parsl.AUTO_LOGNAME``) Like stdout, but for the standard error stream. +3. label: (string) If the app is invoked with ``stdout=parsl.AUTO_LOGNAME`` or ``stderr=parsl.AUTO_LOGNAME``, this argument will be appended to the log name. -A Bash app, like a Python app, returns an AppFuture, which can be used to obtain -task status, determine when the app has completed (e.g., via ``future.result()`` as in the preceding code fragment), and access exceptions. -As a Bash app can only return results via files specified via ``outputs``, ``stderr``, or ``stdout``; the value returned by the AppFuture has no meaning. +Outputs ++++++++ If the Bash app exits with Unix exit code 0, then the AppFuture will complete. If the Bash app exits with any other code, Parsl will treat this as a failure, and the AppFuture will instead contain an `BashExitFailure` exception. The Unix exit code can be accessed through the ``exitcode`` attribute of that `BashExitFailure`. -Limitations -^^^^^^^^^^^ - -The following limitation applies to Bash apps: -1. Environment variables are not supported. +Execution Options +^^^^^^^^^^^^^^^^^ +Bash Apps have the same execution options (e.g., pinning to specific sites) as the Python Apps. MPI Apps --------- +^^^^^^^^ -Applications which employ MPI to span multiple nodes are a special case of Bash or Python apps, +Applications which employ MPI to span multiple nodes are a special case of Bash apps, and require special modification of Parsl's `execution environment `_ to function. Support for MPI applications is described `in a later section `_. diff --git a/docs/userguide/configuring.rst b/docs/userguide/configuring.rst index 91685e275c..12ec881a46 100644 --- a/docs/userguide/configuring.rst +++ b/docs/userguide/configuring.rst @@ -48,13 +48,46 @@ used by the infiniband interface with ``address_by_interface('ib0')`` .. contents:: Configuration How-To and Examples: -.. note:: - All configuration examples below must be customized for the user's - allocation, Python environment, file system, etc. + +Creating and Using Config Objects +--------------------------------- + +:class:`~parsl.config.Config` objects are loaded to define the "Data Flow Kernel" (DFK) that will manage tasks. +All Parsl applications start by creating or importing a configuration then calling the load function. + +.. code-block:: python + + from parsl.configs.htex_local import config + import parsl + + parsl.load(config) + +The ``load`` statement can happen after Apps are defined but must occur before tasks are started. + +The :class:`~parsl.config.Config` object may not be used again after loaded. +Consider a configuration function if the application will shut down and re-launch the DFK. + +.. code-block:: python + + from parsl.config import Config + import parsl + + def make_config() -> Config: + return Config(...) + + parsl.load(make_config()) + parsl.clear() # Stops Parsl + parsl.load(make_config()) # Re-launches with a fresh configuration + How to Configure ---------------- +.. note:: + All configuration examples below must be customized for the user's + allocation, Python environment, file system, etc. + + The configuration specifies what, and how, resources are to be used for executing the Parsl program and its apps. It is important to carefully consider the needs of the Parsl program and its apps, diff --git a/docs/userguide/data.rst b/docs/userguide/data.rst index 4b04d1e72b..9350a6d96f 100644 --- a/docs/userguide/data.rst +++ b/docs/userguide/data.rst @@ -97,7 +97,7 @@ irrespective of where that app executes. .. code-block:: python @python_app - def print_file(inputs=[]): + def print_file(inputs=()): with open(inputs[0].filepath, 'r') as inp: content = inp.read() return(content) @@ -107,7 +107,7 @@ irrespective of where that app executes. # call the print_file app with the Parsl file r = print_file(inputs=[f]) - r.result() + r.result() As described below, the method by which this files are transferred depends on the scheme and the staging providers specified in the Parsl @@ -219,7 +219,7 @@ The following example illustrates how the remote file is implicitly downloaded f .. code-block:: python @python_app - def convert(inputs=[], outputs=[]): + def convert(inputs=(), outputs=()): with open(inputs[0].filepath, 'r') as inp: content = inp.read() with open(outputs[0].filepath, 'w') as out: diff --git a/docs/userguide/execution.rst b/docs/userguide/execution.rst index b51eceb278..0601b38802 100644 --- a/docs/userguide/execution.rst +++ b/docs/userguide/execution.rst @@ -334,11 +334,11 @@ The following code snippet shows how apps can specify suitable executors in the # A mock molecular dynamics simulation app @bash_app(executors=["Theta.Phi"]) - def MD_Sim(arg, outputs=[]): + def MD_Sim(arg, outputs=()): return "MD_simulate {} -o {}".format(arg, outputs[0]) # Visualize results from the mock MD simulation app @bash_app(executors=["Cooley.GPU"]) - def visualize(inputs=[], outputs=[]): + def visualize(inputs=(), outputs=()): bash_array = " ".join(inputs) return "viz {} -o {}".format(bash_array, outputs[0]) diff --git a/docs/userguide/futures.rst b/docs/userguide/futures.rst index f1655da88f..13d22a211b 100644 --- a/docs/userguide/futures.rst +++ b/docs/userguide/futures.rst @@ -151,7 +151,7 @@ be created (``hello.outputs[0].result()``). # This app echoes the input string to the first file specified in the # outputs list @bash_app - def echo(message, outputs=[]): + def echo(message, outputs=()): return 'echo {} &> {}'.format(message, outputs[0]) # Call echo specifying the output file diff --git a/docs/userguide/workflow.rst b/docs/userguide/workflow.rst index b2f267b941..d197e7a3f3 100644 --- a/docs/userguide/workflow.rst +++ b/docs/userguide/workflow.rst @@ -59,11 +59,11 @@ Sequential workflows can be created by passing an AppFuture from one task to ano def generate(limit): from random import randint """Generate a random integer and return it""" - return randint(1,limit) + return randint(1, limit) # Write a message to a file @bash_app - def save(message, outputs=[]): + def save(message, outputs=()): return 'echo {} &> {}'.format(message, outputs[0]) message = generate(10) @@ -119,8 +119,8 @@ A common approach to executing Parsl apps in parallel is via loops. The followin @python_app def generate(limit): - from random import randint """Generate a random integer and return it""" + from random import randint return randint(1, limit) rand_nums = [] @@ -138,6 +138,13 @@ The :class:`~parsl.concurrent.ParslPoolExecutor` simplifies this pattern using t from parsl.concurrent import ParslPoolExecutor from parsl.configs.htex_local import config + # NOTE: Functions used by the ParslPoolExecutor do _not_ use decorators + def generate(limit): + """Generate a random integer and return it""" + from random import randint + return randint(1, limit) + + with ParslPoolExecutor(config) as exec: outputs = pool.map(generate, range(1, 5)) @@ -152,15 +159,15 @@ In other cases, it can be convenient to pass data in files, as in the following parsl.load() @bash_app - def generate(outputs=[]): + def generate(outputs=()): return 'echo $(( RANDOM % (10 - 5 + 1 ) + 5 )) &> {}'.format(outputs[0]) @bash_app - def concat(inputs=[], outputs=[], stdout='stdout.txt', stderr='stderr.txt'): + def concat(inputs=(), outputs=(), stdout='stdout.txt', stderr='stderr.txt'): return 'cat {0} >> {1}'.format(' '.join(inputs), outputs[0]) @python_app - def total(inputs=[]): + def total(inputs=()): total = 0 with open(inputs[0].filepath, 'r') as f: for l in f: @@ -202,7 +209,7 @@ the sum of those results. # Reduce function that returns the sum of a list @python_app - def app_sum(inputs=[]): + def app_sum(inputs=()): return sum(inputs) # Create a list of integers diff --git a/parsl/tests/test_docs/test_kwargs.py b/parsl/tests/test_docs/test_kwargs.py new file mode 100644 index 0000000000..4b811c53f3 --- /dev/null +++ b/parsl/tests/test_docs/test_kwargs.py @@ -0,0 +1,37 @@ +"""Functions used to explain kwargs""" +from pathlib import Path + +from parsl import python_app, File + + +def test_inputs(): + @python_app() + def map_app(x): + return x * 2 + + @python_app() + def reduce_app(inputs=()): + return sum(inputs) + + map_futures = [map_app(x) for x in range(3)] + reduce_future = reduce_app(inputs=map_futures) + + assert reduce_future.result() == 6 + + +def test_outputs(tmpdir): + @python_app() + def write_app(message, outputs=()): + """Write a single message to every file in outputs""" + for path in outputs: + with open(path, 'w') as fp: + print(message, file=fp) + + to_write = [ + File(Path(tmpdir) / 'output-0.txt'), + File(Path(tmpdir) / 'output-1.txt') + ] + write_app('Hello!', outputs=to_write).result() + for path in to_write: + with open(path) as fp: + assert fp.read() == 'Hello!\n'