From 887e0fd5fa84a76a635358cb948f9c796f6b7d2b Mon Sep 17 00:00:00 2001 From: Abbas Ballout Date: Mon, 25 Nov 2024 15:17:13 +0100 Subject: [PATCH 1/5] MPI multiphase --- .../SpatialDiscretization.f90 | 23 ++++++++++++++----- Solver/src/libs/mesh/HexMesh.f90 | 4 +++- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/Solver/src/MultiphaseSolver/SpatialDiscretization.f90 b/Solver/src/MultiphaseSolver/SpatialDiscretization.f90 index 46519ccbf..6c99e293e 100644 --- a/Solver/src/MultiphaseSolver/SpatialDiscretization.f90 +++ b/Solver/src/MultiphaseSolver/SpatialDiscretization.f90 @@ -222,7 +222,6 @@ SUBROUTINE ComputeTimeDerivative( mesh, particles, time, mode, HO_Elements) real(kind=RP) :: sqrtRho, invMa2 class(Element), pointer :: e - !$omp parallel shared(mesh, time) ! !/////////////////////////////////////////////////// @@ -347,6 +346,7 @@ SUBROUTINE ComputeTimeDerivative( mesh, particles, time, mode, HO_Elements) #ifdef _HAS_MPI_ !$omp single call mesh % UpdateMPIFacesSolution(NCOMP) + call mesh % GatherMPIFacesSolution(NCOMP) !$omp end single #endif end select @@ -371,6 +371,7 @@ SUBROUTINE ComputeTimeDerivative( mesh, particles, time, mode, HO_Elements) #ifdef _HAS_MPI_ !$omp single call mesh % UpdateMPIFacesGradients(NCOMP) + call mesh % GatherMPIFacesGradients(NCOMP) !$omp end single #endif ! @@ -416,6 +417,7 @@ SUBROUTINE ComputeTimeDerivative( mesh, particles, time, mode, HO_Elements) #ifdef _HAS_MPI_ !$omp single call mesh % UpdateMPIFacesSolution(NCONS) + call mesh % GatherMPIFacesSolution(NCONS) !$omp end single #endif ! @@ -494,6 +496,14 @@ SUBROUTINE ComputeTimeDerivative( mesh, particles, time, mode, HO_Elements) !$omp end do call ViscousDiscretization % LiftGradients( NCONS, NCONS, mesh , time , mGradientVariables) + +#ifdef _HAS_MPI_ +!$omp single + ! Not sure about the position of this w.r.t the MPI directly above + call mesh % UpdateMPIFacesGradients(NCONS) + call mesh % GatherMPIFacesGradients(NCONS) +!$omp end single +#endif ! ! ----------------------- ! Compute time derivative @@ -574,6 +584,7 @@ SUBROUTINE ComputeTimeDerivative( mesh, particles, time, mode, HO_Elements) !$omp end do end select !$omp end parallel + ! END SUBROUTINE ComputeTimeDerivative ! @@ -736,9 +747,9 @@ subroutine ComputeNSTimeDerivative( mesh , t ) end do ; end do ; end do - do k = 0, e % Nxyz(3) ; do j = 0, e % Nxyz(2) ; do i = 0, e % Nxyz(1) - e % storage % QDot(:,i,j,k) = e % storage % QDot(:,i,j,k) / e % geom % jacobian(i,j,k) - end do ; end do ; end do + ! do k = 0, e % Nxyz(3) ; do j = 0, e % Nxyz(2) ; do i = 0, e % Nxyz(1) + ! e % storage % QDot(:,i,j,k) = e % storage % QDot(:,i,j,k) / e % geom % jacobian(i,j,k) + ! end do ; end do ; end do end associate end do !$omp end do @@ -1176,7 +1187,7 @@ subroutine ComputeLaplacian( mesh , t) associate(e => mesh % elements(eID)) if ( e % hasSharedFaces ) cycle call Laplacian_FacesContribution(e, t, mesh) - + do k = 0, e % Nxyz(3) ; do j = 0, e % Nxyz(2) ; do i = 0, e % Nxyz(1) e % storage % QDot(:,i,j,k) = e % storage % QDot(:,i,j,k) / e % geom % jacobian(i,j,k) end do ; end do ; end do @@ -1217,7 +1228,7 @@ subroutine ComputeLaplacian( mesh , t) do eID = 1, size(mesh % elements) associate(e => mesh % elements(eID)) if ( .not. e % hasSharedFaces ) cycle - call TimeDerivative_FacesContribution(e, t, mesh) + call Laplacian_FacesContribution(e, t, mesh) do k = 0, e % Nxyz(3) ; do j = 0, e % Nxyz(2) ; do i = 0, e % Nxyz(1) e % storage % QDot(:,i,j,k) = e % storage % QDot(:,i,j,k) / e % geom % jacobian(i,j,k) diff --git a/Solver/src/libs/mesh/HexMesh.f90 b/Solver/src/libs/mesh/HexMesh.f90 index 079f84ecb..a3f34fc06 100644 --- a/Solver/src/libs/mesh/HexMesh.f90 +++ b/Solver/src/libs/mesh/HexMesh.f90 @@ -2246,8 +2246,10 @@ subroutine HexMesh_SetConnectivitiesAndLinkFaces(self,nodes,facesList) call ConstructMPIFacesStorage(self % MPIfaces, NCONS, NGRAD, MPI_NDOFS) #elif defined(INCNS) call ConstructMPIFacesStorage(self % MPIfaces, NCONS, NCONS, MPI_NDOFS) -#elif defined(CAHNHILLIARD) +#elif defined(CAHNHILLIARD) && !defined(MULTIPHASE) call ConstructMPIFacesStorage(self % MPIfaces, NCOMP, NCOMP, MPI_NDOFS) +#elif defined(MULTIPHASE) + call ConstructMPIFacesStorage(self % MPIfaces, NCONS, NCONS, MPI_NDOFS) #elif defined(ACOUSTIC) call ConstructMPIFacesStorage(self % MPIfaces, NCONS, NCONS, MPI_NDOFS) #endif From abdd77dc02542182fbe46422a74961e0c7639e1f Mon Sep 17 00:00:00 2001 From: Dhueper Date: Fri, 8 Nov 2024 12:38:05 +0100 Subject: [PATCH 2/5] BR2 and tests fixed --- Solver/src/libs/discretization/EllipticBR2.f90 | 6 +++--- .../NavierStokes/CylinderBR2/SETUP/ProblemFile.f90 | 14 +++++++------- .../CylinderDucros/SETUP/ProblemFile.f90 | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Solver/src/libs/discretization/EllipticBR2.f90 b/Solver/src/libs/discretization/EllipticBR2.f90 index 3ff7fe73c..d5d0bdda9 100644 --- a/Solver/src/libs/discretization/EllipticBR2.f90 +++ b/Solver/src/libs/discretization/EllipticBR2.f90 @@ -430,9 +430,9 @@ subroutine BR2_ComputeGradientFaceIntegrals( self, nGradEqn, e, mesh) unStar => mesh % faces(e % faceIDs(EBOTTOM)) % storage(e % faceSide(EBOTTOM)) % unStar ) do k = 0, e%Nxyz(3) ; do j = 0, e%Nxyz(2) ; do i = 0, e%Nxyz(1) - U_x(:,i,j) = U_x(:,i,j) - self % eta * unStar(:,1,i,j) * bv_z(k,LEFT) * invjac(i,i,j) - U_y(:,i,j) = U_y(:,i,j) - self % eta * unStar(:,2,i,j) * bv_z(k,LEFT) * invjac(i,i,j) - U_z(:,i,j) = U_z(:,i,j) - self % eta * unStar(:,3,i,j) * bv_z(k,LEFT) * invjac(i,i,j) + U_x(:,i,j) = U_x(:,i,j) - self % eta * unStar(:,1,i,j) * bv_z(k,LEFT) * invjac(i,j,k) + U_y(:,i,j) = U_y(:,i,j) - self % eta * unStar(:,2,i,j) * bv_z(k,LEFT) * invjac(i,j,k) + U_z(:,i,j) = U_z(:,i,j) - self % eta * unStar(:,3,i,j) * bv_z(k,LEFT) * invjac(i,j,k) end do ; end do ; end do end associate diff --git a/Solver/test/NavierStokes/CylinderBR2/SETUP/ProblemFile.f90 b/Solver/test/NavierStokes/CylinderBR2/SETUP/ProblemFile.f90 index 4bcbbe653..08dca477f 100644 --- a/Solver/test/NavierStokes/CylinderBR2/SETUP/ProblemFile.f90 +++ b/Solver/test/NavierStokes/CylinderBR2/SETUP/ProblemFile.f90 @@ -552,16 +552,16 @@ SUBROUTINE UserDefinedFinalize(mesh, time, iter, maxResidual & ! #if defined(NAVIERSTOKES) INTEGER :: iterations(3:7) = [100, 0, 0, 0, 0] - real(kind=RP), parameter :: residuals(5) = [ 8.94947477740774_RP, & - 18.0524814828053_RP, & - 0.188804475468846_RP, & - 24.2331142737927_RP, & - 244.034603817743_RP ] + real(kind=RP), parameter :: residuals(5) = [ 8.9494751074667516_RP, & + 18.052481444063439_RP, & + 0.1887988263729878_RP, & + 24.233109718227368_RP, & + 244.03459342403502_RP ] real(kind=RP), parameter :: wake_u = 8.381270411983929E-009_RP - real(kind=RP), parameter :: cd = 34.3031214698872_RP - real(kind=RP), parameter :: cl = -5.536320494302416E-003_RP + real(kind=RP), parameter :: cd = 34.303121634815788_RP + real(kind=RP), parameter :: cl = -5.536315782160184E-003_RP N = mesh % elements(1) % Nxyz(1) ! This works here because all the elements have the same order in all directions CALL initializeSharedAssertionsManager diff --git a/Solver/test/NavierStokes/CylinderDucros/SETUP/ProblemFile.f90 b/Solver/test/NavierStokes/CylinderDucros/SETUP/ProblemFile.f90 index bc3fe2b11..e703fc8db 100644 --- a/Solver/test/NavierStokes/CylinderDucros/SETUP/ProblemFile.f90 +++ b/Solver/test/NavierStokes/CylinderDucros/SETUP/ProblemFile.f90 @@ -524,7 +524,7 @@ SUBROUTINE UserDefinedFinalize(mesh, time, iter, maxResidual & ! Local variables ! --------------- ! - CHARACTER(LEN=29) :: testName = "Re 200 Cylinder with Ducros Skewsymmetric and BR2" + CHARACTER(LEN=29) :: testName = "Re 200 Cylinder with Ducros Skewsymmetric and BR1" REAL(KIND=RP) :: maxError REAL(KIND=RP), ALLOCATABLE :: QExpected(:,:,:,:) INTEGER :: eID From b4565486035813ae692d8d12a493cfdcb7f9ba67 Mon Sep 17 00:00:00 2001 From: loganoz Date: Thu, 21 Nov 2024 14:12:59 +0100 Subject: [PATCH 3/5] Add new CI for testing --- .github/workflows/CI_sequential 5.yml | 49 +++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 .github/workflows/CI_sequential 5.yml diff --git a/.github/workflows/CI_sequential 5.yml b/.github/workflows/CI_sequential 5.yml new file mode 100644 index 000000000..1bf1f451e --- /dev/null +++ b/.github/workflows/CI_sequential 5.yml @@ -0,0 +1,49 @@ +# This is a basic workflow to help you get started with Actions + +name: CI SEQ TEST fortran-lang + +######################################################################## +# Controls when the action will run. +on: + # Triggers the workflow on push or pull request events but only for the master branch + push: + branches: [ master ] + pull_request: + branches: [ master ] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + test: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + toolchain: + - {compiler: gcc, version: 13} + - {compiler: intel, version: '2023.2'} + - {compiler: intel-classic, version: '2021.10'} + - {compiler: nvidia-hpc, version: '23.11'} + include: + - os: ubuntu-latest + toolchain: {compiler: gcc, version: 12} + exclude: + - os: macos-latest + toolchain: {compiler: intel, version: '2023.2'} + - os: macos-latest + toolchain: {compiler: nvidia-hpc, version: '23.11'} + - os: windows-latest + toolchain: {compiler: nvidia-hpc, version: '23.11'} + + steps: + - uses: fortran-lang/setup-fortran@v1 + id: setup-fortran + with: + compiler: ${{ matrix.toolchain.compiler }} + version: ${{ matrix.toolchain.version }} + + - run: | + ${{ env.FC }} ... # environment vars FC, CC, and CXX are set + ${{ steps.setup-fortran.outputs.fc }} ... # outputs work too \ No newline at end of file From ace37a7572a506709a6edd44fa2903a8a4785b9c Mon Sep 17 00:00:00 2001 From: loganoz Date: Thu, 21 Nov 2024 14:32:00 +0100 Subject: [PATCH 4/5] Update CI_sequential 1.yml --- .github/workflows/CI_sequential 1.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/CI_sequential 1.yml b/.github/workflows/CI_sequential 1.yml index 97966e23a..5f7629f8e 100644 --- a/.github/workflows/CI_sequential 1.yml +++ b/.github/workflows/CI_sequential 1.yml @@ -38,7 +38,7 @@ jobs: - compiler: ifort mkl: 'NO' env: - INTEL_COMPILER_DIR : /opt/intel/oneapi/compiler/2024.1 + INTEL_COMPILER_DIR : /opt/intel/oneapi/compiler/2023.2.0 # Steps represent a sequence of tasks that will be executed as part of the job @@ -76,8 +76,10 @@ jobs: - name: Install Intel oneAPI # UNCOMMENT TO USE CACHED IFORT # if: (steps.cache-intel-compilers.outputs.cache-hit != 'true') - run: sudo apt-get install intel-oneapi-compiler-fortran intel-oneapi-compiler-dpcpp-cpp intel-oneapi-mpi intel-oneapi-mpi-devel intel-oneapi-mkl-devel ninja-build - # Runs a single command using the runners shell + # run: sudo apt-get install intel-oneapi-compiler-fortran intel-oneapi-compiler-dpcpp-cpp intel-oneapi-mpi intel-oneapi-mpi-devel intel-oneapi-mkl-devel ninja-build + run: sudo apt-get install intel-oneapi-compiler-fortran-2023.2.0 intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-2023.2.0 intel-oneapi-mpi intel-oneapi-mpi-devel intel-oneapi-mkl-devel ninja-build + + # Runs a single command using the runners shell ##- name: Install gfortran ## run: | ## sudo add-apt-repository ppa:ubuntu-toolchain-r/test From db1bdb141b74da5998635b164820643bbbf082a4 Mon Sep 17 00:00:00 2001 From: loganoz Date: Thu, 21 Nov 2024 14:47:12 +0100 Subject: [PATCH 5/5] Update CI so they load old version of oneapi (2023.2.0) --- .github/workflows/CI_parallel.yml | 5 +-- .github/workflows/CI_sequential 2.yml | 5 +-- .github/workflows/CI_sequential 3.yml | 5 +-- .github/workflows/CI_sequential 4.yml | 5 +-- .github/workflows/CI_sequential 5.yml | 49 --------------------------- 5 files changed, 12 insertions(+), 57 deletions(-) delete mode 100644 .github/workflows/CI_sequential 5.yml diff --git a/.github/workflows/CI_parallel.yml b/.github/workflows/CI_parallel.yml index 7971bb707..a5f794cd2 100644 --- a/.github/workflows/CI_parallel.yml +++ b/.github/workflows/CI_parallel.yml @@ -33,7 +33,7 @@ jobs: env: METIS_HOME: /home/runner/work/horses3d/horses3d/metis-5.1.0/build/Linux-x86_64 - INTEL_COMPILER_DIR : /opt/intel/oneapi/compiler/2024.1 + INTEL_COMPILER_DIR : /opt/intel/oneapi/compiler/2023.2.0 # Steps represent a sequence of tasks that will be executed as part of the job steps: @@ -68,7 +68,8 @@ jobs: - name: Install Intel oneAPI # UNCOMMENT TO USE CACHED IFORT # if: (steps.cache-intel-compilers.outputs.cache-hit != 'true') - run: sudo apt-get install intel-oneapi-compiler-fortran intel-oneapi-compiler-dpcpp-cpp intel-oneapi-mpi intel-oneapi-mpi-devel intel-oneapi-mkl-devel ninja-build + # run: sudo apt-get install intel-oneapi-compiler-fortran intel-oneapi-compiler-dpcpp-cpp intel-oneapi-mpi intel-oneapi-mpi-devel intel-oneapi-mkl-devel ninja-build + run: sudo apt-get install intel-oneapi-compiler-fortran-2023.2.0 intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-2023.2.0 intel-oneapi-mpi intel-oneapi-mpi-devel intel-oneapi-mkl-devel ninja-build # - name: cache-metis # id: cache-metis diff --git a/.github/workflows/CI_sequential 2.yml b/.github/workflows/CI_sequential 2.yml index 884b95bdc..42e9365a5 100644 --- a/.github/workflows/CI_sequential 2.yml +++ b/.github/workflows/CI_sequential 2.yml @@ -43,7 +43,7 @@ jobs: - compiler: ifort hdf5: 'YES' env: - INTEL_COMPILER_DIR : /opt/intel/oneapi/compiler/2024.1 + INTEL_COMPILER_DIR : /opt/intel/oneapi/compiler/2023.2.0 # Steps represent a sequence of tasks that will be executed as part of the job steps: @@ -82,7 +82,8 @@ jobs: - name: Install Intel oneAPI # UNCOMMENT TO USE CACHED IFORT # if: (steps.cache-intel-compilers.outputs.cache-hit != 'true') - run: sudo apt-get install intel-oneapi-compiler-fortran intel-oneapi-compiler-dpcpp-cpp intel-oneapi-mpi intel-oneapi-mpi-devel intel-oneapi-mkl-devel ninja-build + # run: sudo apt-get install intel-oneapi-compiler-fortran intel-oneapi-compiler-dpcpp-cpp intel-oneapi-mpi intel-oneapi-mpi-devel intel-oneapi-mkl-devel ninja-build + run: sudo apt-get install intel-oneapi-compiler-fortran-2023.2.0 intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-2023.2.0 intel-oneapi-mpi intel-oneapi-mpi-devel intel-oneapi-mkl-devel ninja-build # Runs a single command using the runners shell ##- name: Install gfortran ## run: | diff --git a/.github/workflows/CI_sequential 3.yml b/.github/workflows/CI_sequential 3.yml index 6e92cfc15..b24e38f90 100644 --- a/.github/workflows/CI_sequential 3.yml +++ b/.github/workflows/CI_sequential 3.yml @@ -38,7 +38,7 @@ jobs: - compiler: ifort mkl: 'NO' env: - INTEL_COMPILER_DIR : /opt/intel/oneapi/compiler/2024.1 + INTEL_COMPILER_DIR : /opt/intel/oneapi/compiler/2023.2.0 # Steps represent a sequence of tasks that will be executed as part of the job steps: @@ -75,7 +75,8 @@ jobs: - name: Install Intel oneAPI # UNCOMMENT TO USE CACHED IFORT # if: (steps.cache-intel-compilers.outputs.cache-hit != 'true') - run: sudo apt-get install intel-oneapi-compiler-fortran intel-oneapi-compiler-dpcpp-cpp intel-oneapi-mpi intel-oneapi-mpi-devel intel-oneapi-mkl-devel ninja-build + # run: sudo apt-get install intel-oneapi-compiler-fortran intel-oneapi-compiler-dpcpp-cpp intel-oneapi-mpi intel-oneapi-mpi-devel intel-oneapi-mkl-devel ninja-build + run: sudo apt-get install intel-oneapi-compiler-fortran-2023.2.0 intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-2023.2.0 intel-oneapi-mpi intel-oneapi-mpi-devel intel-oneapi-mkl-devel ninja-build # Runs a single command using the runners shell ##- name: Install gfortran ## run: | diff --git a/.github/workflows/CI_sequential 4.yml b/.github/workflows/CI_sequential 4.yml index eac5b99ae..57657770c 100644 --- a/.github/workflows/CI_sequential 4.yml +++ b/.github/workflows/CI_sequential 4.yml @@ -38,7 +38,7 @@ jobs: - compiler: ifort mkl: 'NO' env: - INTEL_COMPILER_DIR : /opt/intel/oneapi/compiler/2024.1 + INTEL_COMPILER_DIR : /opt/intel/oneapi/compiler/2023.2.0 # Steps represent a sequence of tasks that will be executed as part of the job steps: @@ -75,7 +75,8 @@ jobs: - name: Install Intel oneAPI # UNCOMMENT TO USE CACHED IFORT # if: (steps.cache-intel-compilers.outputs.cache-hit != 'true') - run: sudo apt-get install intel-oneapi-compiler-fortran intel-oneapi-compiler-dpcpp-cpp intel-oneapi-mpi intel-oneapi-mpi-devel intel-oneapi-mkl-devel ninja-build + # run: sudo apt-get install intel-oneapi-compiler-fortran intel-oneapi-compiler-dpcpp-cpp intel-oneapi-mpi intel-oneapi-mpi-devel intel-oneapi-mkl-devel ninja-build + run: sudo apt-get install intel-oneapi-compiler-fortran-2023.2.0 intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-2023.2.0 intel-oneapi-mpi intel-oneapi-mpi-devel intel-oneapi-mkl-devel ninja-build # Runs a single command using the runners shell ##- name: Install gfortran ## run: | diff --git a/.github/workflows/CI_sequential 5.yml b/.github/workflows/CI_sequential 5.yml deleted file mode 100644 index 1bf1f451e..000000000 --- a/.github/workflows/CI_sequential 5.yml +++ /dev/null @@ -1,49 +0,0 @@ -# This is a basic workflow to help you get started with Actions - -name: CI SEQ TEST fortran-lang - -######################################################################## -# Controls when the action will run. -on: - # Triggers the workflow on push or pull request events but only for the master branch - push: - branches: [ master ] - pull_request: - branches: [ master ] - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -jobs: - test: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - toolchain: - - {compiler: gcc, version: 13} - - {compiler: intel, version: '2023.2'} - - {compiler: intel-classic, version: '2021.10'} - - {compiler: nvidia-hpc, version: '23.11'} - include: - - os: ubuntu-latest - toolchain: {compiler: gcc, version: 12} - exclude: - - os: macos-latest - toolchain: {compiler: intel, version: '2023.2'} - - os: macos-latest - toolchain: {compiler: nvidia-hpc, version: '23.11'} - - os: windows-latest - toolchain: {compiler: nvidia-hpc, version: '23.11'} - - steps: - - uses: fortran-lang/setup-fortran@v1 - id: setup-fortran - with: - compiler: ${{ matrix.toolchain.compiler }} - version: ${{ matrix.toolchain.version }} - - - run: | - ${{ env.FC }} ... # environment vars FC, CC, and CXX are set - ${{ steps.setup-fortran.outputs.fc }} ... # outputs work too \ No newline at end of file