Add --extra-args option

.github/workflows/bench-example.yml

@@ -27,13 +27,16 @@ jobs:
         python -m pip install ".[dev]"
     - name: Basic benchmark test
       run: |
-         spinner -c docs/examples/sleep_benchmark.yaml -r T -e F
+         spinner -c docs/examples/sleep_benchmark.yaml -r
     - name: Timeout test
       run: |
-         spinner -c docs/examples/timeout_test.yaml -r T -e F
+         spinner -c docs/examples/timeout_test.yaml -r
+    - name: Extra Args test
+      run: |
+         spinner -c docs/examples/extra_args.yaml -r --extra-args "extra_time=4"
     - name: Capture output test
       run: |
-         spinner -c docs/examples/capture_output.yaml -r T -e T
+         spinner -c docs/examples/capture_output.yaml -r -e
     - name: upload produced graphs
       uses: actions/upload-artifact@v4
       with:

README.md

@@ -25,9 +25,11 @@ spinner --help
 Usage: spinner [OPTIONS]
 
 Options:
-  -c, --config TEXT     Benchmark configuration file
-  -r, --run BOOLEAN     Run all benchmarks
-  -e, --export BOOLEAN  Export results to report.html
-  -h, --hosts TEXT      Hosts list
-  --help                Show this message and exit.
+  -c, --config TEXT       Benchmark configuration file
+  -r, --run               Run all benchmarks
+  -e, --export            Export results to report.html
+  -o, --output TEXT       Output File (.pkl)
+  -ea, --extra-args TEXT  Extra arguments as key=value pairs, separated by
+                          semicolons.
+  --help                  Show this message and exit.
 ```

docs/examples/extra_args.yaml

@@ -0,0 +1,17 @@
+metadata:
+  description: Sleep check using --extra-args flag
+  version: "1.0"
+  runs: 1
+  timeout: 10
+  retry: True
+  retry_limit: 1
+
+  sleep_test:
+    command:
+      template: >
+        sleep {{sleep_ammount + (extra_time | int)}}
+
+sleep_test:
+  sleep_ammount:
+    - 1
+    - 2

docs/slurm.MD

@@ -15,7 +15,7 @@ set -e
 # Load required modules
 module purge
 module load cmake/3.27.9
-module load mpich/4.0.2-cuda-12.4.0-ucx
+module load mpich/4.0.2
 module load singularity/3.7.1
 
 # Load Spinner env
@@ -24,6 +24,6 @@ source spinner/.venv/bin/activate
 # Get MPI host lsit from Slurm and format it for MPI
 host_list=$(scontrol show hostname $(echo "$SLURM_JOB_NODELIST" | head -n 4 | tr '\n' ',' | sed 's/,$//'))
 
-spinner  -c bench_settings.yaml -r T -e T --hosts $host_list
+spinner -c bench_settings.yaml -r -e -o mpi_bench.pkl --extra-args "hosts=$host_list"
 
 ```

spinner/cli/main.py

@@ -6,6 +6,20 @@
 from runner.utilities import run_benchmarks
 
 
+def parse_extra_args(extra_args):
+    args_dict = {}
+    if extra_args:
+        # Split by ';' first to handle key-value pairs
+        pairs = extra_args.split(";")
+        for pair in pairs:
+            if "=" in pair:
+                # Split by '=' to separate key and value
+                # The '1' ensures only the first '=' is split
+                key, value = pair.split("=", 1)
+                args_dict[key.strip()] = value.strip()
+    return args_dict
+
+
 @click.command()
 @click.option(
     "--config",
@@ -14,18 +28,40 @@
     type=str,
     help="Benchmark configuration file",
 )
-@click.option("--run", "-r", default=False, type=bool, help="Run all benchmarks")
 @click.option(
-    "--export", "-e", default=True, type=bool, help="Export results to report.html"
+    "--run",
+    "-r",
+    default=False,
+    is_flag=True,
+    show_default=True,
+    help="Run all benchmarks",
+)
+@click.option(
+    "--export",
+    "-e",
+    default=False,
+    is_flag=True,
+    show_default=True,
+    help="Export results to report.html",
+)
+@click.option(
+    "--output", "-o", default="bench_metadata.pkl", type=str, help="Output File (.pkl)"
+)
+@click.option(
+    "--extra-args",
+    "-ea",
+    default="",
+    type=str,
+    help="Extra arguments as key=value pairs, separated by semicolons.",
 )
-@click.option("--hosts", "-h", default=None, type=str, help="Hosts list")
-def cli(run, export, config, hosts):
-    main(run, export, config, hosts)
+def cli(run, export, config, output, extra_args):
+    extra_args_d = parse_extra_args(extra_args)
+    main(run, export, config, output, extra_args_d)
 
 
-def main(run, export, config, hosts):
+def main(run, export, config, output, extra_args):
     if run:
-        run_benchmarks(config, hosts)
+        run_benchmarks(config, output, extra_args)
 
     if export:
         reporter_path = importlib.resources.files("exporter") / "reporter.ipynb"

spinner/runner/utilities.py

@@ -10,15 +10,14 @@
 from runner.instance_runner import InstanceRunner
 
 
-def run_benchmarks(config, hosts):
+def run_benchmarks(config, output, extra_args):
     """
     Generate execution matrix from input configuration and run all benchmarks.
     """
     bench_config = yaml.safe_load(open(config))
     bench_metadata = bench_config["metadata"]
     bench_metadata["start_timestamp"] = str(pd.Timestamp.now())
-
-    bench_metadata["hosts"] = hosts
+    bench_metadata.update(extra_args)
     bench_metadata["runner_hostname"] = str(os.uname()[1])
     bench_metadata["start_env"] = str(os.environ.copy())
 
@@ -96,5 +95,5 @@ def _update_progress(
     bench_metadata["end_env"] = str(os.environ.copy())
 
     rprint(execution_df)
-    with open("bench_metadata.pkl", "wb") as f:
+    with open(output, "wb") as f:
         pickle.dump({"metadata": bench_metadata, "dataframe": execution_df}, f)