Merge separate-postprocessing-packages into #166

.github/workflows/debug_checks.yml

@@ -18,27 +18,23 @@ jobs:
       - uses: mpi4py/setup-mpi@v1
         with:
           mpi: 'openmpi'
-      - uses: actions/setup-python@v4
-        with:
-          python-version: '3.11'
       - uses: julia-actions/setup-julia@v1
         with:
-          version: '1.8'
+          version: '1.10'
           arch: x64
       - uses: julia-actions/julia-buildpkg@v1
-        env:
-          # Use the system Python for PyCall - avoids library linking error on macOS
-          PYTHON: "${{ env.pythonLocation }}/bin/python"
+        with:
+          project: 'moment_kinetics/'
       - name: Debug test
         run: |
+          cd moment_kinetics
+
           # Hard code the debug level so that we can run without using the
           # `--compiled-modules=no` flag, which breaks Symbolics.jl at the
           # moment.
           sed -i -e "s/_debug_level = get_options.*/_debug_level = 2/" src/debugging.jl
 
-          pip3 install --user matplotlib
-          julia --project -e 'using MPIPreferences; MPIPreferences.use_system_binary()'
-          julia --project -e 'using Pkg; Pkg.build("MPI"; verbose=true)'
+          julia --project -e 'using MPIPreferences; MPIPreferences.use_system_binary(); using Pkg; Pkg.precompile()'
 
           # Need to use openmpi so that the following arguments work:
           # * `--mca rmaps_base_oversubscribe 1` allows oversubscription (more processes

.github/workflows/documentation.yml

@@ -14,11 +14,10 @@ jobs:
       - uses: actions/checkout@v4
       - uses: julia-actions/setup-julia@latest
         with:
-          version: '1.8'
+          version: '1.10'
       - name: Install dependencies
         run: |
           pip3 install --user matplotlib
-          julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()'
       - name: Build and deploy
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Authenticate with GitHub Actions token

.github/workflows/examples.yml

@@ -14,19 +14,13 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
-        with:
-          python-version: '3.11'
       - uses: julia-actions/setup-julia@v1
         with:
-          version: '1.8'
+          version: '1.10'
           arch: x64
-      - uses: julia-actions/julia-buildpkg@v1
-        env:
-          # Use the system Python for PyCall - avoids library linking error on macOS
-          PYTHON: "${{ env.pythonLocation }}/bin/python"
       - name: Test examples
         run: |
-          pip3 install --user matplotlib
+          touch Project.toml
+          julia -O3 --project -e 'import Pkg; Pkg.develop(path="moment_kinetics/"); Pkg.add("NCDatasets"); Pkg.precompile()'
           # Reduce nstep for each example to 10 to avoid the CI job taking too long
-          julia --project -O3 -e 'using moment_kinetics; for (root, dirs, files) in walkdir("examples") for file in files if endswith(file, ".toml") filename = joinpath(root, file); println(filename); input = moment_kinetics.moment_kinetics_input.read_input_file(filename); input["nstep"] = 10; run_moment_kinetics(input) end end end'
+          julia -O3 --project -e 'using moment_kinetics; for (root, dirs, files) in walkdir("examples") for file in files if endswith(file, ".toml") filename = joinpath(root, file); println(filename); input = moment_kinetics.moment_kinetics_input.read_input_file(filename); input["nstep"] = 10; run_moment_kinetics(input) end end end'

.github/workflows/longtest.yml

@@ -19,22 +19,16 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
-        with:
-          python-version: '3.11'
       - uses: julia-actions/setup-julia@v1
         with:
-          version: '1.8'
+          version: '1.10'
           arch: x64
       - uses: julia-actions/julia-buildpkg@v1
-        env:
-          # Use the system Python for PyCall - avoids library linking error on macOS
-          PYTHON: "${{ env.pythonLocation }}/bin/python"
-        # The following is copied and simplified from
-        # https://github.com/julia-actions/julia-runtest/blob/master/action.yml
-        # in order to pass customised arguments to `Pkg.test()`
-        #
+        with:
+          project: 'moment_kinetics/'
+      # The following is copied and simplified from
+      # https://github.com/julia-actions/julia-runtest/blob/master/action.yml
+      # in order to pass customised arguments to `Pkg.test()`
       - run: |
-          pip3 install --user matplotlib
-          julia --check-bounds=yes --color=yes --depwarn=yes --project=@. -e 'import Pkg; Pkg.test(; test_args=["--long"])'
+          julia --check-bounds=yes --color=yes --depwarn=yes --project=moment_kinetics/ -e 'import Pkg; Pkg.test(; test_args=["--long", "--force-optional-dependencies"])'
         shell: bash

.github/workflows/parallel_test.yml

@@ -4,44 +4,59 @@ name: Run tests in parallel
 on: [push, pull_request, workflow_dispatch]
 
 jobs:
-  test:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest, macOS-latest]
-        include:
-          - julia_version: '1.8'
-      fail-fast: false
+  test-ubuntu:
+    runs-on: ubuntu-latest
     timeout-minutes: 120
 
     steps:
       - uses: actions/checkout@v4
       - uses: mpi4py/setup-mpi@v1
         with:
           mpi: 'openmpi'
-      - uses: actions/setup-python@v4
+      - uses: julia-actions/setup-julia@v1
         with:
-          python-version: '3.11'
+          version: '1.10'
+          arch: x64
+      - run: |
+          touch Project.toml
+          julia --project -O3 --check-bounds=no -e 'import Pkg; Pkg.add(["MPI", "MPIPreferences", "NCDatasets", "Random", "SpecialFunctions", "Test"]); Pkg.develop(path="moment_kinetics/"); using MPIPreferences; MPIPreferences.use_system_binary()'
+          julia --project -O3 --check-bounds=no -e 'import Pkg; Pkg.precompile()'
+          # Need to use openmpi so that the following arguments work:
+          # * `--mca rmaps_base_oversubscribe 1` allows oversubscription (more processes
+          #   than physical cores).
+          # * `--mca mpi_yield_when_idle 1` changes a setting to prevent excessively
+          #   terrible performance when oversubscribing.
+          mpiexec -np 3 --mca rmaps_base_oversubscribe 1 julia --project -O3 --check-bounds=no moment_kinetics/test/runtests.jl --debug 1 --force-optional-dependencies
+          mpiexec -np 4 --mca rmaps_base_oversubscribe 1 julia --project -O3 --check-bounds=no moment_kinetics/test/runtests.jl --debug 1 --force-optional-dependencies
+          mpiexec -np 2 --mca rmaps_base_oversubscribe 1 julia --project -O3 --check-bounds=no moment_kinetics/test/runtests.jl --debug 1 --long --force-optional-dependencies
+          # Note: MPI.jl's default implementation is mpich, which has a similar option
+          # `--with-device=ch3:sock`, but that needs to be set when compiling mpich.
+        shell: bash
+
+  # macOS is slow at the moment, so only run one set of parallel tests
+  test-macOS:
+    runs-on: macOS-latest
+    timeout-minutes: 120
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: mpi4py/setup-mpi@v1
+        with:
+          mpi: 'openmpi'
       - uses: julia-actions/setup-julia@v1
         with:
-          version: ${{ matrix.julia_version }}
+          version: '1.10'
           arch: x64
-      - uses: julia-actions/julia-buildpkg@v1
-        env:
-          # Use the system Python for PyCall - avoids library linking error on macOS
-          PYTHON: "${{ env.pythonLocation }}/bin/python"
       - run: |
-          pip3 install --user matplotlib
-          julia --project -e 'using MPIPreferences; MPIPreferences.use_system_binary()'
-          julia --project -e 'using Pkg; Pkg.build("MPI"; verbose=true)'
+          touch Project.toml
+          julia --project -O3 --check-bounds=no -e 'import Pkg; Pkg.add(["MPI", "MPIPreferences", "NCDatasets", "Random", "SpecialFunctions", "Test"]); Pkg.develop(path="moment_kinetics/"); using MPIPreferences; MPIPreferences.use_system_binary()'
+          julia --project -O3 --check-bounds=no -e 'import Pkg; Pkg.precompile()'
           # Need to use openmpi so that the following arguments work:
           # * `--mca rmaps_base_oversubscribe 1` allows oversubscription (more processes
           #   than physical cores).
           # * `--mca mpi_yield_when_idle 1` changes a setting to prevent excessively
           #   terrible performance when oversubscribing.
-          mpiexec -np 3 --mca rmaps_base_oversubscribe 1 julia --project=@. -O3 --check-bounds=no test/runtests.jl --debug 1
-          mpiexec -np 4 --mca rmaps_base_oversubscribe 1 julia --project=@. -O3 --check-bounds=no test/runtests.jl --debug 1
-          mpiexec -np 2 --mca rmaps_base_oversubscribe 1 julia --project=@. -O3 --check-bounds=no test/runtests.jl --debug 1 --long
+          mpiexec -np 4 --mca rmaps_base_oversubscribe 1 julia --project -O3 --check-bounds=no moment_kinetics/test/runtests.jl --debug 1 --force-optional-dependencies
           # Note: MPI.jl's default implementation is mpich, which has a similar option
           # `--with-device=ch3:sock`, but that needs to be set when compiling mpich.
         shell: bash

.github/workflows/test.yml

@@ -14,17 +14,16 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
-        with:
-          python-version: '3.11'
       - uses: julia-actions/setup-julia@v1
         with:
-          version: '1.8'
+          version: '1.10'
           arch: x64
       - uses: julia-actions/julia-buildpkg@v1
-        env:
-          # Use the system Python for PyCall - avoids library linking error on macOS
-          PYTHON: "${{ env.pythonLocation }}/bin/python"
+        with:
+          project: 'moment_kinetics/'
+      # The following is copied and simplified from
+      # https://github.com/julia-actions/julia-runtest/blob/master/action.yml
+      # in order to pass customised arguments to `Pkg.test()`
       - run: |
-          pip3 install --user matplotlib
-      - uses: julia-actions/julia-runtest@v1
+          julia --check-bounds=yes --color=yes --depwarn=yes --project=moment_kinetics/ -e 'import Pkg; Pkg.test(; test_args=["--force-optional-dependencies"])'
+        shell: bash

.gitignore

@@ -2,11 +2,27 @@
 performance-tests/config.toml
 Manifest.toml
 LocalPreferences.toml
+/Project.toml
+/makie_post_processing/Project.toml
+/plots_post_processing/Project.toml
 /julia.env
-bin
-machines/shared/compile_dependencies.sh
-machines/shared/machine_setup_stage_two.jl
-machines/artifacts
+/.julia
+/bin
+/machines/shared/compile_dependencies.sh
+/machines/artifacts
+/machines/generic-batch
+/precompile-makie-post-processing-submit.sh
+/precompile-plots-post-processing-submit.sh
+/precompile-submit.sh
+/submit-restart.sh
+/submit-run.sh
+/submit-precompile-and-restart.sh
+/submit-precompile-and-run.sh
+/.julia_default.txt
+/.julia_directory_default.txt
+/.this_machine_name.txt
 moment_kinetics.so
+makie_postproc.so
+plots_postproc.so
 precompile-temp
-post_processing_input.toml
+post_processing_input.toml