flux-framework · hariharan-devarajan · Aug 10, 2024 · Aug 13, 2024 · Aug 13, 2024 · Jul 29, 2024
diff --git a/.github/workflows/docker-builds.yaml b/.github/workflows/docker-builds.yaml
@@ -10,9 +10,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        test: [["2024-RADIUSS-AWS/JupyterNotebook", "docker/Dockerfile.hub", "ghcr.io/flux-framework/flux-jupyter-hub:radiuss-2024"],
-               ["2024-RADIUSS-AWS/JupyterNotebook", "docker/Dockerfile.init", "ghcr.io/flux-framework/flux-jupyter-init:radiuss-2024"],
-               ["2024-RADIUSS-AWS/JupyterNotebook", "docker/Dockerfile.spawn", "ghcr.io/flux-framework/flux-jupyter-spawn:radiuss-2024"]]
+        test: [["2024-HPCIC-AWS/JupyterNotebook", "docker/Dockerfile.hub", "ghcr.io/flux-framework/flux-jupyter-hub:hpcic-2024"],
+               ["2024-HPCIC-AWS/JupyterNotebook", "docker/Dockerfile.init", "ghcr.io/flux-framework/flux-jupyter-init:hpcic-2024"],
+               ["2024-HPCIC-AWS/JupyterNotebook", "docker/Dockerfile.spawn", "ghcr.io/flux-framework/flux-jupyter-spawn:hpcic-2024"]]
 
 # Tutorials are over - these builds are disabled
 # ["2023-RADIUSS-AWS/JupyterNotebook", "docker/Dockerfile.hub", "ghcr.io/flux-framework/flux-jupyter-hub:2023"],

diff --git a/2024-RADIUSS-AWS/JupyterNotebook/README.md → 2024-HPCIC-AWS/JupyterNotebook/README.md b/2024-RADIUSS-AWS/JupyterNotebook/README.md → 2024-HPCIC-AWS/JupyterNotebook/README.md
@@ -22,9 +22,9 @@ Let's build a set of images - one spawner and one hub, and an init. You can cust
 Remember that if you just want to test locally, you can jump to the [local usage](#local-usage) section.
 
 ```bash
-docker build -t ghcr.io/flux-framework/flux-jupyter-hub:radiuss-2024 -f docker/Dockerfile.hub .
-docker build -t ghcr.io/flux-framework/flux-jupyter-spawn:radiuss-2024 -f docker/Dockerfile.spawn .
-docker build -t ghcr.io/flux-framework/flux-jupyter-init:radiuss-2024 -f docker/Dockerfile.init .
+docker build -t ghcr.io/flux-framework/flux-jupyter-hub:hpcic-2024 -f docker/Dockerfile.hub .
+docker build -t ghcr.io/flux-framework/flux-jupyter-spawn:hpcic-2024 -f docker/Dockerfile.spawn .
+docker build -t ghcr.io/flux-framework/flux-jupyter-init:hpcic-2024 -f docker/Dockerfile.init .
 ```
 
 Note that these are available under the flux-framework organization GitHub packages, so you shouldn't need
@@ -79,8 +79,8 @@ you should install.
 # Create an EKS cluster with autoscaling with default storage
 eksctl create cluster --config-file aws/eksctl-config.yaml
 
-# Create an EKS cluster with io1 node storage but no autoscaling, used for the RADIUSS 2023 tutorial
-eksctl create cluster --config-file aws/eksctl-radiuss-tutorial-2023.yaml
+# Create an EKS cluster with io1 node storage but no autoscaling, used for the HPCIC 2024 tutorial
+eksctl create cluster --config-file aws/eksctl-hpcic-tutorial-2023.yaml
 ```
 
 You can find vanilla (manual) instructions [here](https://z2jh.jupyter.org/en/stable/kubernetes/amazon/step-zero-aws-eks.html) if you

diff --git a/.../aws/cluster-autoscaler-autodiscover.yaml → .../aws/cluster-autoscaler-autodiscover.yaml b/.../aws/cluster-autoscaler-autodiscover.yaml → .../aws/cluster-autoscaler-autodiscover.yaml
diff --git a/...S/JupyterNotebook/aws/config-aws-ssl.yaml → ...S/JupyterNotebook/aws/config-aws-ssl.yaml b/...S/JupyterNotebook/aws/config-aws-ssl.yaml → ...S/JupyterNotebook/aws/config-aws-ssl.yaml
@@ -23,7 +23,7 @@ hub:
   # This is the image I built based off of jupyterhub/k8s-hub, 3.0.2 at time of writing this
   image: 
     name: ghcr.io/flux-framework/flux-jupyter-hub
-    tag: "radiuss-2024"
+    tag: "hpcic-2024"
     pullPolicy: Always
 
 # # https://z2jh.jupyter.org/en/latest/administrator/optimization.html#scaling-up-in-time-user-placeholders
@@ -46,7 +46,7 @@ proxy:
 singleuser:
   image:
     name: ghcr.io/flux-framework/flux-jupyter-spawn
-    tag: "radiuss-2024"
+    tag: "hpcic-2024"
     pullPolicy: Always
   cpu:
     limit: 2
@@ -59,7 +59,7 @@ singleuser:
   # This runs as the root user, who clones and changes ownership to uid 1000
   initContainers:
     - name: init-myservice
-      image: ghcr.io/flux-framework/flux-jupyter-init:radiuss-2024
+      image: ghcr.io/flux-framework/flux-jupyter-init:hpcic-2024
       command: ["/entrypoint.sh"]
       volumeMounts:
         - name: flux-tutorial

diff --git a/...S-AWS/JupyterNotebook/aws/config-aws.yaml → ...C-AWS/JupyterNotebook/aws/config-aws.yaml b/...S-AWS/JupyterNotebook/aws/config-aws.yaml → ...C-AWS/JupyterNotebook/aws/config-aws.yaml
@@ -17,7 +17,7 @@ hub:
   # This is the image I built based off of jupyterhub/k8s-hub, 3.0.2 at time of writing this
   image: 
     name: ghcr.io/flux-framework/flux-jupyter-hub
-    tag: "radiuss-2024"
+    tag: "hpcic-2024"
     pullPolicy: Always
 
 # https://z2jh.jupyter.org/en/latest/administrator/optimization.html#scaling-up-in-time-user-placeholders
@@ -32,7 +32,7 @@ scheduling:
 singleuser:
   image:
     name: ghcr.io/flux-framework/flux-jupyter-spawn
-    tag: "radiuss-2024"
+    tag: "hpcic-2024"
     pullPolicy: Always
   cpu:
     limit: 1
@@ -43,7 +43,7 @@ singleuser:
   # This runs as the root user, who clones and changes ownership to uid 1000
   initContainers:
     - name: init-myservice
-      image: ghcr.io/flux-framework/flux-jupyter-init:radiuss-2024
+      image: ghcr.io/flux-framework/flux-jupyter-init:hpcic-2024
       command: ["/entrypoint.sh"]
       volumeMounts:
         - name: flux-tutorial

diff --git a/...WS/JupyterNotebook/aws/eksctl-config.yaml → ...WS/JupyterNotebook/aws/eksctl-config.yaml b/...WS/JupyterNotebook/aws/eksctl-config.yaml → ...WS/JupyterNotebook/aws/eksctl-config.yaml
diff --git a/...ook/aws/eksctl-radiuss-tutorial-2024.yaml → ...ebook/aws/eksctl-hpcic-tutorial-2024.yaml b/...ook/aws/eksctl-radiuss-tutorial-2024.yaml → ...ebook/aws/eksctl-hpcic-tutorial-2024.yaml
diff --git a/...AWS/JupyterNotebook/aws/storageclass.yaml → ...AWS/JupyterNotebook/aws/storageclass.yaml b/...AWS/JupyterNotebook/aws/storageclass.yaml → ...AWS/JupyterNotebook/aws/storageclass.yaml
diff --git a/...AWS/JupyterNotebook/docker/Dockerfile.hub → ...AWS/JupyterNotebook/docker/Dockerfile.hub b/...AWS/JupyterNotebook/docker/Dockerfile.hub → ...AWS/JupyterNotebook/docker/Dockerfile.hub
diff --git a/...WS/JupyterNotebook/docker/Dockerfile.init → ...WS/JupyterNotebook/docker/Dockerfile.init b/...WS/JupyterNotebook/docker/Dockerfile.init → ...WS/JupyterNotebook/docker/Dockerfile.init
diff --git a/...S/JupyterNotebook/docker/Dockerfile.spawn → ...S/JupyterNotebook/docker/Dockerfile.spawn b/...S/JupyterNotebook/docker/Dockerfile.spawn → ...S/JupyterNotebook/docker/Dockerfile.spawn
@@ -48,7 +48,7 @@ RUN python3 -m pip install -r requirements.txt && \
     python3 -m IPython kernel install
 
 # This is code to install DYAD
-# This was added to the RADIUSS 2023 tutorials on AWS
+# This was added to the RADIUSS 2023 tutorials on AWS (changed to HPCIC in 2024)
 RUN git clone https://github.com/openucx/ucx.git \
     && cd ucx \
     && git checkout v1.13.1 \
@@ -91,7 +91,7 @@ RUN apt-get update && apt-get install -y nodejs && apt-get clean && rm -rf /var/
 RUN wget https://nodejs.org/dist/v20.15.0/node-v20.15.0-linux-x64.tar.xz && \
     apt-get update && apt-get install -y xz-utils && rm -rf /var/lib/apt/lists/* && \
     xz -d -v node-v20.15.0-linux-x64.tar.xz && \
-    tar -C /usr/local --strip-components=1 -xvf node-v20.15.0-linux-x64.tar 
+    tar -C /usr/local --strip-components=1 -xvf node-v20.15.0-linux-x64.tar
 
 # This customizes the launcher UI
 # https://jupyter-app-launcher.readthedocs.io/en/latest/usage.html
@@ -113,6 +113,11 @@ COPY ./docker/flux-icon.png $HOME/flux-icon.png
 # note that previous examples are added via git volume in config.yaml
 ENV SHELL=/usr/bin/bash
 ENV FLUX_URI_RESOLVE_LOCAL=t
+# Prepend /usr/lib to LD_LIBRARY_PATH because Ubuntu Jammy comes with
+# UCX 1.12. Without this, DYAD will build (correctly) with UCX 1.13.1, but
+# it will try to run with the system install of UCX 1.12, which can cause
+# either a crash or a hang.
+ENV LD_LIBRARY_PATH="/usr/lib:$LD_LIBRARY_PATH"
 
 EXPOSE 8888
 ENTRYPOINT ["tini", "--"]
@@ -132,7 +137,7 @@ RUN mkdir -p $HOME/.local/share && \
 #    flux start flux account add-user --username=jovyan --bank=default && \
 #    flux start flux jobtap load mf_priority.so && \
 #    flux start flux account-update-db
-    
+
 USER ${NB_USER}
 
 CMD ["flux", "start", "--test-size=4", "jupyter", "lab"]
diff --git a/...-AWS/JupyterNotebook/docker/entrypoint.sh → ...-AWS/JupyterNotebook/docker/entrypoint.sh b/...-AWS/JupyterNotebook/docker/entrypoint.sh → ...-AWS/JupyterNotebook/docker/entrypoint.sh
diff --git a/...-AWS/JupyterNotebook/docker/flux-icon.png → ...-AWS/JupyterNotebook/docker/flux-icon.png b/...-AWS/JupyterNotebook/docker/flux-icon.png → ...-AWS/JupyterNotebook/docker/flux-icon.png
diff --git a/...JupyterNotebook/docker/init-entrypoint.sh → ...JupyterNotebook/docker/init-entrypoint.sh b/...JupyterNotebook/docker/init-entrypoint.sh → ...JupyterNotebook/docker/init-entrypoint.sh
@@ -7,5 +7,4 @@
 
 # We need to clone to the user home, and then change permissions to uid 1000
 # That uid is shared by jovyan here and the spawn container
-# git clone https://github.com/rse-ops/flux-radiuss-tutorial-2023 /home/jovyan/flux-tutorial
 chown -R 1000 /home/jovyan
diff --git a/...yterNotebook/docker/jupyter-launcher.yaml → ...yterNotebook/docker/jupyter-launcher.yaml b/...yterNotebook/docker/jupyter-launcher.yaml → ...yterNotebook/docker/jupyter-launcher.yaml
diff --git a/...USS-AWS/JupyterNotebook/docker/login.html → ...CIC-AWS/JupyterNotebook/docker/login.html b/...USS-AWS/JupyterNotebook/docker/login.html → ...CIC-AWS/JupyterNotebook/docker/login.html
diff --git a/...DIUSS-AWS/JupyterNotebook/docker/start.sh → ...HPCIC-AWS/JupyterNotebook/docker/start.sh b/...DIUSS-AWS/JupyterNotebook/docker/start.sh → ...HPCIC-AWS/JupyterNotebook/docker/start.sh
diff --git a/...S-AWS/JupyterNotebook/flux-tree/flux-tree → ...C-AWS/JupyterNotebook/flux-tree/flux-tree b/...S-AWS/JupyterNotebook/flux-tree/flux-tree → ...C-AWS/JupyterNotebook/flux-tree/flux-tree
diff --git a/...terNotebook/flux-tree/flux-tree-helper.py → ...terNotebook/flux-tree/flux-tree-helper.py b/...terNotebook/flux-tree/flux-tree-helper.py → ...terNotebook/flux-tree/flux-tree-helper.py
diff --git a/...DIUSS-AWS/JupyterNotebook/gcp/config.yaml → ...HPCIC-AWS/JupyterNotebook/gcp/config.yaml b/...DIUSS-AWS/JupyterNotebook/gcp/config.yaml → ...HPCIC-AWS/JupyterNotebook/gcp/config.yaml
@@ -17,7 +17,7 @@ hub:
   # This is the image I built based off of jupyterhub/k8s-hub, 3.0.2 at time of writing this
   image: 
     name: ghcr.io/flux-framework/flux-jupyter-hub
-    tag: "radiuss-2024"
+    tag: "hpcic-2024"
     pullPolicy: Always
 
 # https://z2jh.jupyter.org/en/latest/administrator/optimization.html#scaling-up-in-time-user-placeholders
@@ -32,22 +32,14 @@ scheduling:
 singleuser:
   image:
     name: ghcr.io/flux-framework/flux-jupyter-spawn
-    tag: "radiuss-2024"
+    tag: "hpcic-2024"
     pullPolicy: Always
   cpu:
     limit: 1
   memory:
     limit: '4G'
   cmd: /entrypoint.sh
 
-#  initContainers:
-#    - name: init-myservice
-#      image: alpine/git
-#      command: ["git", "clone", "https://github.com/rse-ops/flux-radiuss-tutorial-2023", "/home/jovyan/flux-tutorial"]
-#      volumeMounts:
-#        - name: flux-tutorial
-#          mountPath: /home/jovyan
-
   # This is how we get the tutorial files added
   storage:
     type: none

diff --git a/...IUSS-AWS/JupyterNotebook/requirements.txt → ...PCIC-AWS/JupyterNotebook/requirements.txt b/...IUSS-AWS/JupyterNotebook/requirements.txt → ...PCIC-AWS/JupyterNotebook/requirements.txt
diff --git a/...AWS/JupyterNotebook/requirements_venv.txt → ...AWS/JupyterNotebook/requirements_venv.txt b/...AWS/JupyterNotebook/requirements_venv.txt → ...AWS/JupyterNotebook/requirements_venv.txt
diff --git a/...S-AWS/JupyterNotebook/tutorial/.gitignore → ...C-AWS/JupyterNotebook/tutorial/.gitignore b/...S-AWS/JupyterNotebook/tutorial/.gitignore → ...C-AWS/JupyterNotebook/tutorial/.gitignore
diff --git a/...rNotebook/tutorial/01_flux_tutorial.ipynb → ...rNotebook/tutorial/01_flux_tutorial.ipynb b/...rNotebook/tutorial/01_flux_tutorial.ipynb → ...rNotebook/tutorial/01_flux_tutorial.ipynb
@@ -86,7 +86,7 @@
     "\n",
     "# Getting started with Flux\n",
     "\n",
-    "The code and examples that this tutorial is based on can be found at [flux-framework/Tutorials](https://github.com/flux-framework/Tutorials/tree/master/2024-RADIUSS-AWS). You can also find python examples in the `flux-workflow-examples` directory from the sidebar navigation in this JupyterLab instance. "
+    "The code and examples that this tutorial is based on can be found at [flux-framework/Tutorials](https://github.com/flux-framework/Tutorials/tree/master/2024-HPCIC-AWS). You can also find python examples in the `flux-workflow-examples` directory from the sidebar navigation in this JupyterLab instance. "
    ]
   },
   {

diff --git a/...Notebook/tutorial/02_flux_framework.ipynb → ...Notebook/tutorial/02_flux_framework.ipynb b/...Notebook/tutorial/02_flux_framework.ipynb → ...Notebook/tutorial/02_flux_framework.ipynb
diff --git a/...torial/03_flux_tutorial_conclusions.ipynb → ...torial/03_flux_tutorial_conclusions.ipynb b/...torial/03_flux_tutorial_conclusions.ipynb → ...torial/03_flux_tutorial_conclusions.ipynb
@@ -27,7 +27,7 @@
     "\n",
     "> But what do I do now?\n",
     "\n",
-    "Feel free to experiment more with Flux here, or (for more freedom) in the terminal. You can try more of the examples in the `flux-workflow-examples` directory in the window to the left. If you're using a shared system like the one on the RADIUSS AWS tutorial please be mindful of other users and don't run compute intensive workloads. If you're running the tutorial in a job on an HPC cluster... compute away! ⚾️\n",
+    "Feel free to experiment more with Flux here, or (for more freedom) in the terminal. You can try more of the examples in the `flux-workflow-examples` directory in the window to the left. If you're using a shared system like the one on the HPCIC AWS tutorial please be mindful of other users and don't run compute intensive workloads. If you're running the tutorial in a job on an HPC cluster... compute away! ⚾️\n",
     "\n",
     "> Where can I learn to set this up on my own?\n",
     "\n",

diff --git a/...WS/JupyterNotebook/tutorial/Flux-logo.svg → ...WS/JupyterNotebook/tutorial/Flux-logo.svg b/...WS/JupyterNotebook/tutorial/Flux-logo.svg → ...WS/JupyterNotebook/tutorial/Flux-logo.svg
diff --git a/...WS/JupyterNotebook/tutorial/flux-icon.png → ...WS/JupyterNotebook/tutorial/flux-icon.png b/...WS/JupyterNotebook/tutorial/flux-icon.png → ...WS/JupyterNotebook/tutorial/flux-icon.png
diff --git a/...kflow-examples/.github/workflows/main.yml → ...kflow-examples/.github/workflows/main.yml b/...kflow-examples/.github/workflows/main.yml → ...kflow-examples/.github/workflows/main.yml
diff --git a/...orial/flux-workflow-examples/.mergify.yml → ...orial/flux-workflow-examples/.mergify.yml b/...orial/flux-workflow-examples/.mergify.yml → ...orial/flux-workflow-examples/.mergify.yml
diff --git a/.../tutorial/flux-workflow-examples/Makefile → .../tutorial/flux-workflow-examples/Makefile b/.../tutorial/flux-workflow-examples/Makefile → .../tutorial/flux-workflow-examples/Makefile
diff --git a/...tutorial/flux-workflow-examples/README.md → ...tutorial/flux-workflow-examples/README.md b/...tutorial/flux-workflow-examples/README.md → ...tutorial/flux-workflow-examples/README.md
diff --git a/...-examples/async-bulk-job-submit/README.md → ...-examples/async-bulk-job-submit/README.md b/...-examples/async-bulk-job-submit/README.md → ...-examples/async-bulk-job-submit/README.md
diff --git a/...mples/async-bulk-job-submit/bulksubmit.py → ...mples/async-bulk-job-submit/bulksubmit.py b/...mples/async-bulk-job-submit/bulksubmit.py → ...mples/async-bulk-job-submit/bulksubmit.py
diff --git a/...nc-bulk-job-submit/bulksubmit_executor.py → ...nc-bulk-job-submit/bulksubmit_executor.py b/...nc-bulk-job-submit/bulksubmit_executor.py → ...nc-bulk-job-submit/bulksubmit_executor.py
diff --git a/...al/flux-workflow-examples/bulksubmit/0.sh → ...al/flux-workflow-examples/bulksubmit/0.sh b/...al/flux-workflow-examples/bulksubmit/0.sh → ...al/flux-workflow-examples/bulksubmit/0.sh
diff --git a/...al/flux-workflow-examples/bulksubmit/1.sh → ...al/flux-workflow-examples/bulksubmit/1.sh b/...al/flux-workflow-examples/bulksubmit/1.sh → ...al/flux-workflow-examples/bulksubmit/1.sh
diff --git a/...al/flux-workflow-examples/bulksubmit/2.sh → ...al/flux-workflow-examples/bulksubmit/2.sh b/...al/flux-workflow-examples/bulksubmit/2.sh → ...al/flux-workflow-examples/bulksubmit/2.sh
diff --git a/...al/flux-workflow-examples/bulksubmit/3.sh → ...al/flux-workflow-examples/bulksubmit/3.sh b/...al/flux-workflow-examples/bulksubmit/3.sh → ...al/flux-workflow-examples/bulksubmit/3.sh
diff --git a/...al/flux-workflow-examples/bulksubmit/4.sh → ...al/flux-workflow-examples/bulksubmit/4.sh b/...al/flux-workflow-examples/bulksubmit/4.sh → ...al/flux-workflow-examples/bulksubmit/4.sh
diff --git a/...al/flux-workflow-examples/bulksubmit/5.sh → ...al/flux-workflow-examples/bulksubmit/5.sh b/...al/flux-workflow-examples/bulksubmit/5.sh → ...al/flux-workflow-examples/bulksubmit/5.sh
diff --git a/...al/flux-workflow-examples/bulksubmit/6.sh → ...al/flux-workflow-examples/bulksubmit/6.sh b/...al/flux-workflow-examples/bulksubmit/6.sh → ...al/flux-workflow-examples/bulksubmit/6.sh
diff --git a/...x-workflow-examples/comms-module/Makefile → ...x-workflow-examples/comms-module/Makefile b/...x-workflow-examples/comms-module/Makefile → ...x-workflow-examples/comms-module/Makefile
diff --git a/...-workflow-examples/comms-module/README.md → ...-workflow-examples/comms-module/README.md b/...-workflow-examples/comms-module/README.md → ...-workflow-examples/comms-module/README.md
diff --git a/...flux-workflow-examples/comms-module/app.c → ...flux-workflow-examples/comms-module/app.c b/...flux-workflow-examples/comms-module/app.c → ...flux-workflow-examples/comms-module/app.c
diff --git a/...orkflow-examples/comms-module/compute.lua → ...orkflow-examples/comms-module/compute.lua b/...orkflow-examples/comms-module/compute.lua → ...orkflow-examples/comms-module/compute.lua
diff --git a/...w-examples/comms-module/io-forwarding.lua → ...w-examples/comms-module/io-forwarding.lua b/...w-examples/comms-module/io-forwarding.lua → ...w-examples/comms-module/io-forwarding.lua
diff --git a/...k/tutorial/flux-workflow-examples/conf.py → ...k/tutorial/flux-workflow-examples/conf.py b/...k/tutorial/flux-workflow-examples/conf.py → ...k/tutorial/flux-workflow-examples/conf.py
diff --git a/...x-workflow-examples/data-conduit/Makefile → ...x-workflow-examples/data-conduit/Makefile b/...x-workflow-examples/data-conduit/Makefile → ...x-workflow-examples/data-conduit/Makefile
diff --git a/...-workflow-examples/data-conduit/README.md → ...-workflow-examples/data-conduit/README.md b/...-workflow-examples/data-conduit/README.md → ...-workflow-examples/data-conduit/README.md
diff --git a/...workflow-examples/data-conduit/compute.py → ...workflow-examples/data-conduit/compute.py b/...workflow-examples/data-conduit/compute.py → ...workflow-examples/data-conduit/compute.py
diff --git a/...-workflow-examples/data-conduit/conduit.c → ...-workflow-examples/data-conduit/conduit.c b/...-workflow-examples/data-conduit/conduit.c → ...-workflow-examples/data-conduit/conduit.c
diff --git a/...rkflow-examples/data-conduit/datastore.py → ...rkflow-examples/data-conduit/datastore.py b/...rkflow-examples/data-conduit/datastore.py → ...rkflow-examples/data-conduit/datastore.py
diff --git a/...examples/hierarchical-launching/README.md → ...examples/hierarchical-launching/README.md b/...examples/hierarchical-launching/README.md → ...examples/hierarchical-launching/README.md
diff --git a/...amples/hierarchical-launching/ensemble.sh → ...amples/hierarchical-launching/ensemble.sh b/...amples/hierarchical-launching/ensemble.sh → ...amples/hierarchical-launching/ensemble.sh
diff --git a/...examples/hierarchical-launching/parent.sh → ...examples/hierarchical-launching/parent.sh b/...examples/hierarchical-launching/parent.sh → ...examples/hierarchical-launching/parent.sh
diff --git a/...tutorial/flux-workflow-examples/index.rst → ...tutorial/flux-workflow-examples/index.rst b/...tutorial/flux-workflow-examples/index.rst → ...tutorial/flux-workflow-examples/index.rst
diff --git a/...ux-workflow-examples/job-cancel/README.md → ...ux-workflow-examples/job-cancel/README.md b/...ux-workflow-examples/job-cancel/README.md → ...ux-workflow-examples/job-cancel/README.md
diff --git a/...workflow-examples/job-cancel/submitter.py → ...workflow-examples/job-cancel/submitter.py b/...workflow-examples/job-cancel/submitter.py → ...workflow-examples/job-cancel/submitter.py
diff --git a/...-workflow-examples/job-ensemble/README.md → ...-workflow-examples/job-ensemble/README.md b/...-workflow-examples/job-ensemble/README.md → ...-workflow-examples/job-ensemble/README.md
diff --git a/...orkflow-examples/job-ensemble/compute.lua → ...orkflow-examples/job-ensemble/compute.lua b/...orkflow-examples/job-ensemble/compute.lua → ...orkflow-examples/job-ensemble/compute.lua
diff --git a/...orkflow-examples/job-ensemble/ensemble.sh → ...orkflow-examples/job-ensemble/ensemble.sh b/...orkflow-examples/job-ensemble/ensemble.sh → ...orkflow-examples/job-ensemble/ensemble.sh
diff --git a/...w-examples/job-ensemble/io-forwarding.lua → ...w-examples/job-ensemble/io-forwarding.lua b/...w-examples/job-ensemble/io-forwarding.lua → ...w-examples/job-ensemble/io-forwarding.lua
diff --git a/...examples/job-ensemble/kvs-watch-until.lua → ...examples/job-ensemble/kvs-watch-until.lua b/...examples/job-ensemble/kvs-watch-until.lua → ...examples/job-ensemble/kvs-watch-until.lua
diff --git a/...low-examples/job-status-control/README.md → ...low-examples/job-status-control/README.md b/...low-examples/job-status-control/README.md → ...low-examples/job-status-control/README.md
diff --git a/...examples/job-status-control/bookkeeper.py → ...examples/job-status-control/bookkeeper.py b/...examples/job-status-control/bookkeeper.py → ...examples/job-status-control/bookkeeper.py
diff --git a/...ow-examples/job-status-control/compute.py → ...ow-examples/job-status-control/compute.py b/...ow-examples/job-status-control/compute.py → ...ow-examples/job-status-control/compute.py
diff --git a/...mples/job-status-control/io-forwarding.py → ...mples/job-status-control/io-forwarding.py b/...mples/job-status-control/io-forwarding.py → ...mples/job-status-control/io-forwarding.py
diff --git a/...orkflow-examples/job-submit-api/README.md → ...orkflow-examples/job-submit-api/README.md b/...orkflow-examples/job-submit-api/README.md → ...orkflow-examples/job-submit-api/README.md
diff --git a/...rkflow-examples/job-submit-api/compute.py → ...rkflow-examples/job-submit-api/compute.py b/...rkflow-examples/job-submit-api/compute.py → ...rkflow-examples/job-submit-api/compute.py
diff --git a/...-examples/job-submit-api/io-forwarding.py → ...-examples/job-submit-api/io-forwarding.py b/...-examples/job-submit-api/io-forwarding.py → ...-examples/job-submit-api/io-forwarding.py
diff --git a/...flow-examples/job-submit-api/submitter.py → ...flow-examples/job-submit-api/submitter.py b/...flow-examples/job-submit-api/submitter.py → ...flow-examples/job-submit-api/submitter.py
diff --git a/...low-examples/job-submit-api/submitter2.py → ...low-examples/job-submit-api/submitter2.py b/...low-examples/job-submit-api/submitter2.py → ...low-examples/job-submit-api/submitter2.py
diff --git a/...orkflow-examples/job-submit-cli/README.md → ...orkflow-examples/job-submit-cli/README.md b/...orkflow-examples/job-submit-cli/README.md → ...orkflow-examples/job-submit-cli/README.md
diff --git a/...kflow-examples/job-submit-cli/compute.lua → ...kflow-examples/job-submit-cli/compute.lua b/...kflow-examples/job-submit-cli/compute.lua → ...kflow-examples/job-submit-cli/compute.lua
diff --git a/...rkflow-examples/job-submit-cli/compute.py → ...rkflow-examples/job-submit-cli/compute.py b/...rkflow-examples/job-submit-cli/compute.py → ...rkflow-examples/job-submit-cli/compute.py
diff --git a/...examples/job-submit-cli/io-forwarding.lua → ...examples/job-submit-cli/io-forwarding.lua b/...examples/job-submit-cli/io-forwarding.lua → ...examples/job-submit-cli/io-forwarding.lua
diff --git a/...-examples/job-submit-cli/io-forwarding.py → ...-examples/job-submit-cli/io-forwarding.py b/...-examples/job-submit-cli/io-forwarding.py → ...-examples/job-submit-cli/io-forwarding.py
diff --git a/...rkflow-examples/job-submit-wait/README.md → ...rkflow-examples/job-submit-wait/README.md b/...rkflow-examples/job-submit-wait/README.md → ...rkflow-examples/job-submit-wait/README.md
diff --git a/...kflow-examples/job-submit-wait/compute.py → ...kflow-examples/job-submit-wait/compute.py b/...kflow-examples/job-submit-wait/compute.py → ...kflow-examples/job-submit-wait/compute.py
diff --git a/...b-submit-wait/submitter_sliding_window.py → ...b-submit-wait/submitter_sliding_window.py b/...b-submit-wait/submitter_sliding_window.py → ...b-submit-wait/submitter_sliding_window.py
diff --git a/...les/job-submit-wait/submitter_wait_any.py → ...les/job-submit-wait/submitter_wait_any.py b/...les/job-submit-wait/submitter_wait_any.py → ...les/job-submit-wait/submitter_wait_any.py
diff --git a/...ob-submit-wait/submitter_wait_in_order.py → ...ob-submit-wait/submitter_wait_in_order.py b/...ob-submit-wait/submitter_wait_in_order.py → ...ob-submit-wait/submitter_wait_in_order.py
diff --git a/...-workflow-examples/job-watch/job-watch.sh → ...-workflow-examples/job-watch/job-watch.sh b/...-workflow-examples/job-watch/job-watch.sh → ...-workflow-examples/job-watch/job-watch.sh
diff --git a/...ow-examples/kvs-python-bindings/README.md → ...ow-examples/kvs-python-bindings/README.md b/...ow-examples/kvs-python-bindings/README.md → ...ow-examples/kvs-python-bindings/README.md
diff --git a/...les/kvs-python-bindings/kvsput-usrdata.py → ...les/kvs-python-bindings/kvsput-usrdata.py b/...les/kvs-python-bindings/kvsput-usrdata.py → ...les/kvs-python-bindings/kvsput-usrdata.py
diff --git a/...l/flux-workflow-examples/requirements.txt → ...l/flux-workflow-examples/requirements.txt b/...l/flux-workflow-examples/requirements.txt → ...l/flux-workflow-examples/requirements.txt
diff --git a/...low-examples/synchronize-events/README.md → ...low-examples/synchronize-events/README.md b/...low-examples/synchronize-events/README.md → ...low-examples/synchronize-events/README.md
diff --git a/...w-examples/synchronize-events/compute.lua → ...w-examples/synchronize-events/compute.lua b/...w-examples/synchronize-events/compute.lua → ...w-examples/synchronize-events/compute.lua
diff --git a/...ples/synchronize-events/io-forwarding.lua → ...ples/synchronize-events/io-forwarding.lua b/...ples/synchronize-events/io-forwarding.lua → ...ples/synchronize-events/io-forwarding.lua
diff --git a/...S/JupyterNotebook/tutorial/hello-batch.sh → ...S/JupyterNotebook/tutorial/hello-batch.sh b/...S/JupyterNotebook/tutorial/hello-batch.sh → ...S/JupyterNotebook/tutorial/hello-batch.sh
diff --git a/...rNotebook/tutorial/img/dl-training-io.png → ...rNotebook/tutorial/img/dl-training-io.png b/...rNotebook/tutorial/img/dl-training-io.png → ...rNotebook/tutorial/img/dl-training-io.png
diff --git a/...pyterNotebook/tutorial/img/flux-batch.jpg → ...pyterNotebook/tutorial/img/flux-batch.jpg b/...pyterNotebook/tutorial/img/flux-batch.jpg → ...pyterNotebook/tutorial/img/flux-batch.jpg
diff --git a/...ebook/tutorial/img/flux-broker-design.png → ...ebook/tutorial/img/flux-broker-design.png b/...ebook/tutorial/img/flux-broker-design.png → ...ebook/tutorial/img/flux-broker-design.png
diff --git a/...k/tutorial/img/flux-instance-pre-tbon.png → ...k/tutorial/img/flux-instance-pre-tbon.png b/...k/tutorial/img/flux-instance-pre-tbon.png → ...k/tutorial/img/flux-instance-pre-tbon.png
diff --git a/...ook/tutorial/img/flux-instance-w-tbon.png → ...ook/tutorial/img/flux-instance-w-tbon.png b/...ook/tutorial/img/flux-instance-w-tbon.png → ...ook/tutorial/img/flux-instance-w-tbon.png
diff --git a/...upyterNotebook/tutorial/img/flux-tree.png → ...upyterNotebook/tutorial/img/flux-tree.png b/...upyterNotebook/tutorial/img/flux-tree.png → ...upyterNotebook/tutorial/img/flux-tree.png
diff --git a/...Notebook/tutorial/img/instance-submit.png → ...Notebook/tutorial/img/instance-submit.png b/...Notebook/tutorial/img/instance-submit.png → ...Notebook/tutorial/img/instance-submit.png
diff --git a/...erNotebook/tutorial/img/scaled-submit.png → ...erNotebook/tutorial/img/scaled-submit.png b/...erNotebook/tutorial/img/scaled-submit.png → ...erNotebook/tutorial/img/scaled-submit.png
diff --git a/...erNotebook/tutorial/img/single-submit.png → ...erNotebook/tutorial/img/single-submit.png b/...erNotebook/tutorial/img/single-submit.png → ...erNotebook/tutorial/img/single-submit.png
diff --git a/...S/JupyterNotebook/tutorial/sleep_batch.sh → ...S/JupyterNotebook/tutorial/sleep_batch.sh b/...S/JupyterNotebook/tutorial/sleep_batch.sh → ...S/JupyterNotebook/tutorial/sleep_batch.sh
diff --git a/...-AWS/JupyterNotebook/tutorial/sub_job1.sh → ...-AWS/JupyterNotebook/tutorial/sub_job1.sh b/...-AWS/JupyterNotebook/tutorial/sub_job1.sh → ...-AWS/JupyterNotebook/tutorial/sub_job1.sh
diff --git a/...-AWS/JupyterNotebook/tutorial/sub_job2.sh → ...-AWS/JupyterNotebook/tutorial/sub_job2.sh b/...-AWS/JupyterNotebook/tutorial/sub_job2.sh → ...-AWS/JupyterNotebook/tutorial/sub_job2.sh
diff --git a/...ns/configs/workload/dyad_unet3d_demo.yaml → ...ns/configs/workload/dyad_unet3d_demo.yaml b/...ns/configs/workload/dyad_unet3d_demo.yaml → ...ns/configs/workload/dyad_unet3d_demo.yaml
@@ -20,7 +20,7 @@ reader:
   read_threads: 1
   file_shuffle: seed
   sample_shuffle: seed
-  multiprocessing_context: spawn
+  multiprocessing_context: fork
   data_loader_classname: dyad_torch_data_loader.DyadTorchDataLoader
   data_loader_sampler: index
 
@@ -32,4 +32,4 @@ checkpoint:
   checkpoint_folder: checkpoints/unet3d
   checkpoint_after_epoch: 5
   epochs_between_checkpoints: 2
-  model_size: 499153191
+  model_size: 499153191
diff --git a/...dlio_extensions/dyad_torch_data_loader.py → ...dlio_extensions/dyad_torch_data_loader.py b/...dlio_extensions/dyad_torch_data_loader.py → ...dlio_extensions/dyad_torch_data_loader.py
@@ -48,11 +48,11 @@ def __init__(self, format_type, dataset_type, epoch, num_samples, num_workers, b
         self.reader = None
         self.num_images_read = 0
         self.batch_size = batch_size
+        self.broker_per_node = 1
         args = ConfigArguments.get_instance()
         self.serial_args = pickle.dumps(args)
         if num_workers == 0:
             self.worker_init(-1)
-        self.broker_per_node = 1
 
     def worker_init(self, worker_id):
         # Configure PyTorch components
@@ -138,12 +138,10 @@ def read(self):
             prefetch_factor = math.ceil(self._args.prefetch_size / self._args.read_threads)
         else:
             prefetch_factor = self._args.prefetch_size
-        if prefetch_factor > 0:
-            if self._args.my_rank == 0:
-        else:
+        if prefetch_factor <= 0:
             prefetch_factor = 2
             if self._args.my_rank == 0:
-        logging.debug(f"{utcnow()} Setup dataloader with {self._args.read_threads} workers {torch.__version__}")
+                logging.debug(f"{utcnow()} Setup dataloader with {self._args.read_threads} workers {torch.__version__}")
         if self._args.read_threads==0:
             kwargs={}
         else:
-Original file line number
+Diff line change
@@ Expand Up / @@ -86,7 +86,7 @@ @@
         "\n",
         "# Getting started with Flux\n",
         "\n",
-        "The code and examples that this tutorial is based on can be found at [flux-framework/Tutorials](https://github.com/flux-framework/Tutorials/tree/master/2024-RADIUSS-AWS). You can also find python examples in the `flux-workflow-examples` directory from the sidebar navigation in this JupyterLab instance. "
+        "The code and examples that this tutorial is based on can be found at [flux-framework/Tutorials](https://github.com/flux-framework/Tutorials/tree/master/2024-HPCIC-AWS). You can also find python examples in the `flux-workflow-examples` directory from the sidebar navigation in this JupyterLab instance. "
        ]
       },
       {
@@ Expand Down @@