diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 4548f37061..63464f0437 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -14,7 +14,7 @@ jobs: strategy: matrix: python: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11', '3.12'] - modules_tool: [Lmod-8.1.14, modules-tcl-1.147, modules-3.2.10, modules-4.1.4] + modules_tool: [Lmod-8.1.14, modules-tcl-1.147, modules-3.2.10, modules-4.5.3] module_syntax: [Lua, Tcl] # exclude some configuration for non-Lmod modules tool: # - don't test with Lua module syntax (only supported in Lmod) @@ -24,7 +24,7 @@ jobs: module_syntax: Lua - modules_tool: modules-3.2.10 module_syntax: Lua - - modules_tool: modules-4.1.4 + - modules_tool: modules-4.5.3 module_syntax: Lua - modules_tool: modules-tcl-1.147 python: 3.7 @@ -50,17 +50,17 @@ jobs: python: '3.11' - modules_tool: modules-3.2.10 python: '3.12' - - modules_tool: modules-4.1.4 + - modules_tool: modules-4.5.3 python: 3.7 - - modules_tool: modules-4.1.4 + - modules_tool: modules-4.5.3 python: 3.8 - - modules_tool: modules-4.1.4 + - modules_tool: modules-4.5.3 python: 3.9 - - modules_tool: modules-4.1.4 + - modules_tool: modules-4.5.3 python: '3.10' - - modules_tool: modules-4.1.4 + - modules_tool: modules-4.5.3 python: '3.11' - - modules_tool: modules-4.1.4 + - modules_tool: modules-4.5.3 python: '3.12' fail-fast: false steps: diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 8050832107..ec28f4fb93 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -6,6 +6,63 @@ These release notes can also be consulted at http://easybuild.readthedocs.org/en The latest version of easybuild-easyblocks provides 259 software-specific easyblocks and 43 generic easyblocks. +v4.9.3 (14 September 2024) +-------------------------- + +update/bugfix release +- minor updates, including: + - update custom easyblock for Tensorflow for versions 2.14 + 2.15 (#3303) + - add support for versions >= 2024a to MCR easyblock (#3369) + - update custom easyblock for Bazel for versions >= 7.0 (#3370) + - don't consider lib/release for impi >= 2021.11 (#3375) + - update ORCA easyblock for version 6 (#3395) + - update custom easyblock for psmpi to support the renaming of a couple of options in 5.10.0-1 (#3420) + - update sanity check in OpenFOAM easyblock, since there's no `modifyMesh` in OpenFOAM 12 (#3434) + - fix sanity check for Geant4 >= v11.2 (#3439) +- minor enhancements, including: + - enhance psmpi easyblock to activate CUDA support when it is a dependency + make check for static libs in MPICH easyblock optional (#2787) + - make a project environment and manage LOAD_PATH for JuliaPackage (#3239) + - set build type for PyTorch explicitely (#3332) + - add `sanity_check_test_inputs` custom easyconfig parameter, mapping for zen4, and support building of non-stable versions to LAMMPS easyblock (#3336) + - add RISCV64 support and update config options of new versions of Extrae (#3339) + - set `$ESMFMKFILE` environment variable in environment module for ESMF (#3368) + - use build dir for big files/folders while building TensorFlow (#3371) + - update `PythonPackage` easyblock to allow installation of Python packages with `$PIP_REQUIRE_VIRTUALENV` set + move temporary pip folder into build dir (#3374) + - add RISC-V support to Boost easyblock (#3376) + - add support for generating `.gem` files from `.gemspec` files and support for `preinstallopts` in the `RubyGem` easyblock (#3381) + - simplify install step in custom easyblock for `Tkinter` (#3382) + - add MSA and specially PMIx support for ParaStationMPI (#3383) + - update custom easyblock for PETSc to consider `include/suitesparse` subdirectory for SuiteSparse headers (#3391) + - enhance custom easyblock for GCC to use `with-arch` option for nvptx with 13.1+ (#3396) + - support revisions in crates extraction of cargo packages (#3405) + - set `$R_LIBS_USER` in `RPackage` easyblock to avoid picking up on R packages installed in home directory (#3407) + - update custom easyblock for QuantumESPRESSO to be aware of MPI tests being disabled in EasyBuild configuration (#3412) + - disable the version check of pip in Python packages by defining `$PIP_DISABLE_PIP_VERSION_CHECK` (#3427) + - make sure user packages are not used in sanity check of PythonBundle (#3435) + - enhance OpenFOAM easyblock to also build the plugins for OpenFOAM >= v2406 (#3436) + - enhance binutils easyblock to explicitely pass msgpack configure option (#3438) +- various bug fixes, including: + - patch `ctypes` for Python installations when filtering `LD_LIBRARY_PATH` + fix path to `ldconfig` when using alternate sysroot (#3352) + - also consider 'normalised' package name with underscore rather than dash in EasyBuild easyblock (#3358) + - use PRRTE MCA environment variable for oversubscription in OpenMPI easyblock (#3360) + - fix import of `parse` in `openssl_wrapper.py` for Python 2.7 (#3364) + - make sure that OpenFOAM's wmake can find MPFR and GMP if CGAL >= 5.0 (header-only) is used (#3366) + - remove dummy license server info from ANSYS (#3377) + - explicitly call `csh` in custom easyblock for WPS (#3384) + - use relative paths to object files when compiling shared libraries in the OpenFOAM easyblock (#3388) + - change extra `$PATH` entry for OCaml to use `opam/default` rather than `opam/system` (#3390) + - force `--without-unwind` for ARM and add `--with-libz` option for all architectures in custom easyblock for Extrae (#3393) + - fix `--sanity-check-only` for impi (#3403) + - fix crash in Cargo easyblock when no crates are specified (#3404) + - fix double initialization of `Cargo` by `CargoPythonPackage` by removing incorrect custom `__init__` implementation + fix use of `super()` in PALM easyblock (since that doesn't work with Python 2.7) (#3406) + - fix typo in log message in `PythonPackage` easyblock (#3408) + - make sure to raise an error if `pick_python_cmd` returns `False` for Python bundles/packages (#3430) + - don't wipe build environment before building opam in install step of OCaml easyblock (#3443) +- other changes: + - merge of the `ConfigureMake` and `CMakeMake` versions of the easyblock for QuantumESPRESSO (#3338) + - remove redundant backslashes in GEANT easyblock (#3394) + + v4.9.2 (12 June 2024) --------------------- diff --git a/easybuild/easyblocks/b/binutils.py b/easybuild/easyblocks/b/binutils.py index 42aa454e7c..49c9647591 100644 --- a/easybuild/easyblocks/b/binutils.py +++ b/easybuild/easyblocks/b/binutils.py @@ -148,6 +148,15 @@ def configure_step(self): else: libs.append(libz_path) + msgpackroot = get_software_root('msgpack-c') + if LooseVersion(self.version) >= LooseVersion('2.39'): + if msgpackroot: + self.cfg.update('configopts', '--with-msgpack') + else: + self.cfg.update('configopts', '--without-msgpack') + elif msgpackroot: + raise EasyBuildError('msgpack is only supported since binutils 2.39. Remove the dependency!') + env.setvar('LIBS', ' '.join(libs)) # explicitly configure binutils to use / as sysroot diff --git a/easybuild/easyblocks/g/gcc.py b/easybuild/easyblocks/g/gcc.py index d7f0a47752..c1f0fd3b89 100644 --- a/easybuild/easyblocks/g/gcc.py +++ b/easybuild/easyblocks/g/gcc.py @@ -367,6 +367,70 @@ def prep_extra_src_dirs(self, stage, target_prefix=None): 'versions': versions } + def map_nvptx_capability(self): + """ + Convert PTX ISA architecture passed via EasyBuild configs to a version which is understood by GCC. + Valid architecture strings include 'sm_30', 'sm_35', 'sm_53', 'sm_70', 'sm_75', 'sm_80'. + (as of GCC 14.1.0). Some additional architectures may be mapped. + See: https://github.com/gcc-mirror/gcc/commit/de0ef04419e90eacf0d1ddb265552a1b08c18d4b + + As this list is updated regularly, try to parse the GCC source file (gcc/config/nvptx/nvptx.opt) + and extract the supported architectures and their mapping. Based on the result, determine the lowest + architecture required to support all 'cuda_compute_capabilities' and return this value. + """ + cuda_cc_list = build_option('cuda_compute_capabilities') or self.cfg['cuda_compute_capabilities'] + architecture_mappings_file = os.path.join(self.cfg['start_dir'], 'gcc', 'config', 'nvptx', 'nvptx.opt') + architecture_mappings_flag = "march-map=" + architecture_mappings_replacement = "misa=," + + # Determine which compute capabilities are configured. If there are none, return immediately. + if cuda_cc_list is None: + return None + cuda_sm_list = [f"sm_{cc.replace('.', '')}" for cc in cuda_cc_list] + + if not os.path.exists(architecture_mappings_file): + warn_msg = f"Tried to parse nvptx.opt but file {architecture_mappings_file} was not found. " \ + "Please check the path and update the EasyBlock if necessary!" + self.log.warning(warn_msg) + return None + + # We want to read the mappings found in the GCC sources and create a map for this. + # We're searching for the following pattern: + # march-map=sm_32 + # Target RejectNegative Alias(misa=,sm_30) + gcc_architecture_mappings = {} + file_content = read_file(architecture_mappings_file).splitlines() + for line_idx, line in enumerate(file_content): + line = line.strip() + if line.startswith(architecture_mappings_flag): + key = line.split('=')[1] + # Mapped architecture can be found in the following line + line = file_content[line_idx + 1] + if architecture_mappings_replacement not in line: + warn_msg = "Tried to parse nvptx.opt but failed to extract mapped architectures! " \ + f"Expected to find substring '{architecture_mappings_replacement}' in " \ + f"line {line_idx + 1} but found '{line}'. Choosing default of GCC." + self.log.warning(warn_msg) + # Bail out, since results of mapping cannot be trusted + return None + value = line.split(architecture_mappings_replacement)[1].rstrip(')') + gcc_architecture_mappings[key] = value + self.log.info(f"Available architecture mappings in GCC {self.version}:\n {str(gcc_architecture_mappings)}") + + # Map compute capabilities to GCC ones + # If no compute capability can be mapped, stick to default of GCC and return None + gcc_cc = [gcc_architecture_mappings[cc] if cc in gcc_architecture_mappings else None for cc in cuda_sm_list] + self.log.info(f"Mapped architectures: {str(cuda_sm_list)} -> {str(gcc_cc)}") + if any(cc is None for cc in gcc_cc): + self.log.info("At least one architecture could not be mapped. Choosing default of GCC.") + return None + + # Get the lowest architecture mapping and return it + sorted_gcc_cc = sorted(gcc_cc) + self.log.info("Choosing first architecture in sorted list as default nvptx " + f"architecture: {str(sorted_gcc_cc)}") + return sorted_gcc_cc[0] + def run_configure_cmd(self, cmd): """ Run a configure command, with some extra checking (e.g. for unrecognized options). @@ -545,6 +609,10 @@ def configure_step(self): self.create_dir("build-nvptx-gcc") target = 'nvptx-none' self.cfg.update('configopts', "--enable-newlib-io-long-long") + if LooseVersion(self.version) >= LooseVersion('13.1.0'): + cuda_cc = self.map_nvptx_capability() + if cuda_cc: + self.cfg.update('configopts', f'--with-arch={cuda_cc}') else: # compile AMD GCN target compiler self.create_dir("build-amdgcn-gcc") diff --git a/easybuild/easyblocks/g/geant4.py b/easybuild/easyblocks/g/geant4.py index 14bb729382..659200d989 100644 --- a/easybuild/easyblocks/g/geant4.py +++ b/easybuild/easyblocks/g/geant4.py @@ -34,8 +34,9 @@ import os -from easybuild.framework.easyconfig import CUSTOM from easybuild.easyblocks.generic.cmakemake import CMakeMake +from easybuild.framework.easyconfig import CUSTOM +from easybuild.tools import LooseVersion class EB_Geant4(CMakeMake): @@ -100,8 +101,16 @@ def sanity_check_step(self): Custom sanity check for Geant4 """ bin_files = ["bin/geant4-config", "bin/geant4.sh", "bin/geant4.csh"] - lib_files = ["lib64/libG4%s.so" % x for x in ['analysis', 'event', 'GMocren', 'materials', - 'persistency', 'readout', 'Tree', 'VRML']] + libs = ['analysis', 'event', 'GMocren', 'materials', 'readout', 'Tree', 'VRML'] + + # G4Persistency library was split up in Geant v11.2, + # see https://geant4.web.cern.ch/download/release-notes/notes-v11.2.0.html + if LooseVersion(self.version) >= LooseVersion('11.2'): + libs.extend(['gdml', 'geomtext', 'mctruth', 'geomtext']) + else: + libs.append('persistency') + + lib_files = ["lib64/libG4%s.so" % x for x in libs] include_dir = 'include/Geant4' custom_paths = { diff --git a/easybuild/easyblocks/generic/bundle.py b/easybuild/easyblocks/generic/bundle.py index 7a748a262b..437cd707d4 100644 --- a/easybuild/easyblocks/generic/bundle.py +++ b/easybuild/easyblocks/generic/bundle.py @@ -84,6 +84,10 @@ def __init__(self, *args, **kwargs): if self.cfg['patches']: raise EasyBuildError("List of patches for bundle itself must be empty, found %s", self.cfg['patches']) + # copy EasyConfig instance before we make changes to it + # (like adding component sources to top-level sources easyconfig parameter) + self.cfg = self.cfg.copy() + # disable templating to avoid premature resolving of template values self.cfg.enable_templating = False diff --git a/easybuild/easyblocks/generic/cargo.py b/easybuild/easyblocks/generic/cargo.py index 9bd6821224..31eb8e16c9 100644 --- a/easybuild/easyblocks/generic/cargo.py +++ b/easybuild/easyblocks/generic/cargo.py @@ -131,34 +131,35 @@ def __init__(self, *args, **kwargs): env.setvar('RUST_LOG', 'DEBUG') env.setvar('RUST_BACKTRACE', '1') - # Populate sources from "crates" list of tuples (only once) - if self.cfg['crates']: - # Move 'crates' list from easyconfig parameter to property, - # to avoid that creates are processed into 'sources' easyconfig parameter again - # when easyblock is initialized again using the same parsed easyconfig - # (for example when check_sha256_checksums function is called, like in easyconfigs test suite) - self.crates = self.cfg['crates'] - self.cfg['crates'] = [] - sources = [] - for crate_info in self.crates: - if len(crate_info) == 2: - sources.append({ - 'download_filename': self.crate_download_filename(*crate_info), - 'filename': self.crate_src_filename(*crate_info), - 'source_urls': [CRATESIO_SOURCE], - 'alt_location': 'crates.io', - }) - else: - crate, version, repo, rev = crate_info - url, repo_name = repo.rsplit('/', maxsplit=1) - if repo_name.endswith('.git'): - repo_name = repo_name[:-4] - sources.append({ - 'git_config': {'url': url, 'repo_name': repo_name, 'commit': rev}, - 'filename': self.crate_src_filename(crate, version), - }) + # Populate sources from "crates" list of tuples + sources = [] + for crate_info in self.crates: + if len(crate_info) == 2: + sources.append({ + 'download_filename': self.crate_download_filename(*crate_info), + 'filename': self.crate_src_filename(*crate_info), + 'source_urls': [CRATESIO_SOURCE], + 'alt_location': 'crates.io', + }) + else: + crate, version, repo, rev = crate_info + url, repo_name = repo.rsplit('/', maxsplit=1) + if repo_name.endswith('.git'): + repo_name = repo_name[:-4] + sources.append({ + 'git_config': {'url': url, 'repo_name': repo_name, 'commit': rev}, + 'filename': self.crate_src_filename(crate, version), + }) + + # copy EasyConfig instance before we make changes to it + self.cfg = self.cfg.copy() - self.cfg.update('sources', sources) + self.cfg.update('sources', sources) + + @property + def crates(self): + """Return the crates as defined in the EasyConfig""" + return self.cfg['crates'] def extract_step(self): """ @@ -314,13 +315,14 @@ def generate_crate_list(sourcedir): if dep['source'] == 'registry+https://github.com/rust-lang/crates.io-index': crates.append((name, version)) else: - # Lock file has revision#revision in the url + # Lock file has #revision in the url url, rev = dep['source'].rsplit('#', maxsplit=1) for prefix in ('registry+', 'git+'): if url.startswith(prefix): url = url[len(prefix):] - # Remove branch name if present + # Remove branch name and revision URL parameters if present url = re.sub(r'\?branch=\w+$', '', url) + url = re.sub(r'\?rev=%s+$' % rev, '', url) crates.append((name, version, url, rev)) else: other_crates.append((name, version)) diff --git a/easybuild/easyblocks/generic/cargopythonpackage.py b/easybuild/easyblocks/generic/cargopythonpackage.py index 9d27f67181..fa85cad411 100644 --- a/easybuild/easyblocks/generic/cargopythonpackage.py +++ b/easybuild/easyblocks/generic/cargopythonpackage.py @@ -47,11 +47,6 @@ def extra_options(extra_vars=None): return extra_vars - def __init__(self, *args, **kwargs): - """Constructor for CargoPythonPackage easyblock.""" - Cargo.__init__(self, *args, **kwargs) - PythonPackage.__init__(self, *args, **kwargs) - def extract_step(self): """Specifically use the overloaded variant from Cargo as is populates vendored sources with checksums.""" return Cargo.extract_step(self) diff --git a/easybuild/easyblocks/generic/juliabundle.py b/easybuild/easyblocks/generic/juliabundle.py index f63ea651bc..7f2c7080ce 100644 --- a/easybuild/easyblocks/generic/juliabundle.py +++ b/easybuild/easyblocks/generic/juliabundle.py @@ -94,6 +94,6 @@ def sanity_check_step(self, *args, **kwargs): super(JuliaBundle, self).sanity_check_step(custom_paths=custom_paths) def make_module_extra(self, *args, **kwargs): - """Custom module environement from JuliaPackage""" + """Custom module environment from JuliaPackage""" mod = super(JuliaBundle, self).make_module_extra(*args, **kwargs) return mod diff --git a/easybuild/easyblocks/generic/juliapackage.py b/easybuild/easyblocks/generic/juliapackage.py index 088092e1bf..04398eced0 100644 --- a/easybuild/easyblocks/generic/juliapackage.py +++ b/easybuild/easyblocks/generic/juliapackage.py @@ -66,7 +66,7 @@ class JuliaPackage(ExtensionEasyBlock): """ Builds and installs Julia Packages. - Julia environement setup during installation: + Julia environment setup during installation: - initialize new Julia environment in 'environments' subdir in installation directory - remove paths in user depot '~/.julia' from DEPOT_PATH and LOAD_PATH - put installation directory as top DEPOT_PATH, the target depot for installations with Pkg diff --git a/easybuild/easyblocks/generic/pythonbundle.py b/easybuild/easyblocks/generic/pythonbundle.py index 480f0acc44..09315cae9d 100644 --- a/easybuild/easyblocks/generic/pythonbundle.py +++ b/easybuild/easyblocks/generic/pythonbundle.py @@ -106,6 +106,15 @@ def prepare_step(self, *args, **kwargs): python_cmd = pick_python_cmd(req_maj_ver=req_py_majver, req_min_ver=req_py_minver) + # If pick_python_cmd didn't find a (system) Python command, we should raise an error + if python_cmd: + self.log.info("Python command being used: %s", python_cmd) + else: + raise EasyBuildError( + "Failed to pick Python command that satisfies requirements in the easyconfig " + "(req_py_majver = %s, req_py_minver = %s)", req_py_majver, req_py_minver + ) + self.all_pylibdirs = get_pylibdirs(python_cmd=python_cmd) self.pylibdir = self.all_pylibdirs[0] @@ -152,6 +161,18 @@ def make_module_extra(self, *args, **kwargs): return txt + def load_module(self, *args, **kwargs): + """ + Make sure that $PYTHONNOUSERSITE is defined after loading module file for this software.""" + + super(PythonBundle, self).load_module(*args, **kwargs) + + # Don't add user site directory to sys.path (equivalent to python -s), + # to avoid that any Python packages installed in $HOME/.local/lib affect the sanity check. + # Required here to ensure that it is defined for sanity check commands of the bundle + # because the environment is reset to the initial environment right before loading the module + env.setvar('PYTHONNOUSERSITE', '1', verbose=False) + def sanity_check_step(self, *args, **kwargs): """Custom sanity check for bundle of Python package.""" diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 6233b654eb..08194780c1 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -430,6 +430,8 @@ def __init__(self, *args, **kwargs): # Users or sites may require using a virtualenv for user installations # We need to disable this to be able to install into the modules env.setvar('PIP_REQUIRE_VIRTUALENV', 'false') + # Don't let pip connect to PYPI to check for a new version + env.setvar('PIP_DISABLE_PIP_VERSION_CHECK', 'true') def determine_install_command(self): """ @@ -510,13 +512,23 @@ def prepare_python(self): # if using system Python, go hunting for a 'python' command that satisfies the requirements python = pick_python_cmd(req_maj_ver=req_py_majver, req_min_ver=req_py_minver) - if python: + # Check if we have Python by now. If not, and if self.require_python, raise a sensible error + if python: + self.python_cmd = python + self.log.info("Python command being used: %s", self.python_cmd) + elif self.require_python: + if req_py_majver is not None or req_py_minver is not None: + raise EasyBuildError( + "Failed to pick Python command that satisfies requirements in the easyconfig " + "(req_py_majver = %s, req_py_minver = %s)", req_py_majver, req_py_minver + ) + else: + raise EasyBuildError("Failed to pick Python command to use") + else: + self.log.warning("No Python command found!") + else: self.python_cmd = python self.log.info("Python command being used: %s", self.python_cmd) - elif self.require_python: - raise EasyBuildError("Failed to pick Python command to use") - else: - self.log.warning("No Python command found!") if self.python_cmd: # set Python lib directories @@ -971,7 +983,7 @@ def sanity_check_step(self, *args, **kwargs): env.setvar('PYTHONNOUSERSITE', '1', verbose=False) if self.cfg.get('download_dep_fail', True): - self.log.info("Detection of downloaded depenencies enabled, checking output of installation command...") + self.log.info("Detection of downloaded depdenencies enabled, checking output of installation command...") patterns = [ 'Downloading .*/packages/.*', # setuptools r'Collecting .*', # pip diff --git a/easybuild/easyblocks/generic/rpackage.py b/easybuild/easyblocks/generic/rpackage.py index 9579857f0b..943d7d7ad1 100644 --- a/easybuild/easyblocks/generic/rpackage.py +++ b/easybuild/easyblocks/generic/rpackage.py @@ -41,6 +41,7 @@ from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.extensioneasyblock import ExtensionEasyBlock from easybuild.tools.build_log import EasyBuildError, print_warning +from easybuild.tools.environment import setvar from easybuild.tools.filetools import mkdir, copy_file from easybuild.tools.run import run_shell_cmd, parse_log_for_error @@ -275,6 +276,11 @@ def prepare_r_ext_install(self): :return: Shell command to run + string to pass to stdin. """ + # set $R_LIBS_USER to non-existing path in build directory, + # to avoid picking up on R packages installed in home directory of current user + # (from ~/R/x86_64-pc-linux-gnu-library/) + setvar('R_LIBS_USER', os.path.join(self.builddir, 'r_libs')) + # determine location if isinstance(self.master, EB_R): # extension is being installed as part of an R installation/module diff --git a/easybuild/easyblocks/i/impi.py b/easybuild/easyblocks/i/impi.py index c77a33579d..ef61bf4087 100644 --- a/easybuild/easyblocks/i/impi.py +++ b/easybuild/easyblocks/i/impi.py @@ -34,6 +34,7 @@ @author: Alex Domingo (Vrije Universiteit Brussel) """ import os +import tempfile from easybuild.tools import LooseVersion import easybuild.tools.toolchain as toolchain @@ -244,6 +245,16 @@ def sanity_check_step(self): if build_option('mpi_tests'): if impi_ver >= LooseVersion('2017'): # Add minimal test program to sanity checks + if build_option('sanity_check_only'): + # When only running the sanity check we need to manually make sure that + # variables for compilers and parallelism have been set + self.set_parallel() + self.prepare_step(start_dir=False) + + impi_testexe = os.path.join(tempfile.mkdtemp(), 'mpi_test') + else: + impi_testexe = os.path.join(self.builddir, 'mpi_test') + if impi_ver >= LooseVersion('2021'): impi_testsrc = os.path.join(self.installdir, self.get_versioned_subdir('mpi')) if impi_ver >= LooseVersion('2021.11'): @@ -252,7 +263,6 @@ def sanity_check_step(self): else: impi_testsrc = os.path.join(self.installdir, 'test', 'test.c') - impi_testexe = os.path.join(self.builddir, 'mpi_test') self.log.info("Adding minimal MPI test program to sanity checks: %s", impi_testsrc) # Build test program with appropriate compiler from current toolchain diff --git a/easybuild/easyblocks/l/lammps.py b/easybuild/easyblocks/l/lammps.py index bd61cbdac5..e80f2259cd 100644 --- a/easybuild/easyblocks/l/lammps.py +++ b/easybuild/easyblocks/l/lammps.py @@ -221,6 +221,7 @@ def extra_options(**kwargs): 'kokkos': [True, "Enable kokkos build.", CUSTOM], 'kokkos_arch': [None, "Set kokkos processor arch manually, if auto-detection doesn't work.", CUSTOM], 'user_packages': [None, "List user packages (without prefix PKG_ or USER-PKG_).", CUSTOM], + 'sanity_check_test_inputs': [None, "List of tests for sanity-check.", CUSTOM], }) extra_vars['separate_build_dir'][0] = True return extra_vars @@ -233,6 +234,7 @@ def update_kokkos_cpu_mapping(self): if LooseVersion(self.cur_version) >= LooseVersion(translate_lammps_version('21sep2021')): self.kokkos_cpu_mapping['a64fx'] = 'A64FX' + self.kokkos_cpu_mapping['zen4'] = 'ZEN3' def prepare_step(self, *args, **kwargs): """Custom prepare step for LAMMPS.""" @@ -469,7 +471,7 @@ def install_step(self): mkdir(site_packages, parents=True) - self.lammpsdir = os.path.join(self.builddir, '%s-stable_%s' % (self.name.lower(), self.version)) + self.lammpsdir = os.path.join(self.builddir, '%s-*_%s' % (self.name.lower(), self.version)) self.python_dir = os.path.join(self.lammpsdir, 'python') # The -i flag is added through a patch to the lammps source file python/install.py @@ -492,11 +494,14 @@ def sanity_check_step(self, *args, **kwargs): # Output files need to go somewhere (and has to work for --module-only as well) execution_dir = tempfile.mkdtemp() - check_files = [ - 'atm', 'balance', 'colloid', 'crack', 'dipole', 'friction', - 'hugoniostat', 'indent', 'melt', 'min', 'msst', - 'nemd', 'obstacle', 'pour', 'voronoi', - ] + if self.cfg['sanity_check_test_inputs']: + sanity_check_test_inputs = self.cfg['sanity_check_test_inputs'] + else: + sanity_check_test_inputs = [ + 'atm', 'balance', 'colloid', 'crack', 'dipole', 'friction', + 'hugoniostat', 'indent', 'melt', 'min', 'msst', + 'nemd', 'obstacle', 'pour', 'voronoi', + ] custom_commands = [ # LAMMPS test - you need to call specific test file on path @@ -504,7 +509,7 @@ def sanity_check_step(self, *args, **kwargs): # Examples are part of the install with paths like (installdir)/examples/filename/in.filename os.path.join(self.installdir, "examples", "%s" % check_file, "in.%s" % check_file) # And this should be done for every file specified above - for check_file in check_files + for check_file in sanity_check_test_inputs ] # mpirun command needs an l.finalize() in the sanity check from LAMMPS 29Sep2021 diff --git a/easybuild/easyblocks/o/ocaml.py b/easybuild/easyblocks/o/ocaml.py index 35cfe06b17..2a9b85c62a 100644 --- a/easybuild/easyblocks/o/ocaml.py +++ b/easybuild/easyblocks/o/ocaml.py @@ -101,8 +101,6 @@ def install_step(self): """ super(EB_OCaml, self).install_step() - fake_mod_data = self.load_fake_module(purge=True) - try: all_dirs = os.listdir(self.builddir) except OSError as err: @@ -110,6 +108,9 @@ def install_step(self): opam_dirs = [d for d in all_dirs if d.startswith('opam')] if len(opam_dirs) == 1: + # load temporary module so OCaml installation is available for building & installing opam + fake_mod_data = self.load_fake_module() + opam_dir = os.path.join(self.builddir, opam_dirs[0]) self.log.info("Found unpacked OPAM sources at %s, so installing it.", opam_dir) self.with_opam = True @@ -122,11 +123,11 @@ def install_step(self): opam_init_cmd = mk_opam_init_cmd(root=os.path.join(self.installdir, OPAM_SUBDIR)) run_shell_cmd(opam_init_cmd) + + self.clean_up_fake_module(fake_mod_data) else: self.log.warning("OPAM sources not found in %s: %s", self.builddir, all_dirs) - self.clean_up_fake_module(fake_mod_data) - def prepare_for_extensions(self): """Set default class and filter for OCaml packages.""" # build and install additional packages with OCamlPackage easyblock @@ -146,12 +147,21 @@ def sanity_check_step(self): binaries.append('bin/opam') dirs.append(OPAM_SUBDIR) + extension_names = [ext_name for ext_name, _ in self.cfg['exts_list']] + + custom_commands = ["ocaml --help"] + if 'ocamlfind' in extension_names: + custom_commands.append("ocamlfind list") + + if 'dune' in extension_names: + custom_commands.append("dune --version") + custom_paths = { 'files': binaries, 'dirs': dirs, } - super(EB_OCaml, self).sanity_check_step(custom_paths=custom_paths) + super(EB_OCaml, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) def make_module_req_guess(self): """Custom extra paths/variables to define in generated module for OCaml.""" @@ -160,7 +170,7 @@ def make_module_req_guess(self): guesses.update({ 'CAML_LD_LIBRARY_PATH': ['lib'], 'OPAMROOT': [OPAM_SUBDIR], - 'PATH': ['bin', os.path.join(OPAM_SUBDIR, 'system', 'bin')], + 'PATH': ['bin', os.path.join(OPAM_SUBDIR, 'default', 'bin')], }) return guesses diff --git a/easybuild/easyblocks/o/openfoam.py b/easybuild/easyblocks/o/openfoam.py index 0c1d42cd87..efeb53a126 100644 --- a/easybuild/easyblocks/o/openfoam.py +++ b/easybuild/easyblocks/o/openfoam.py @@ -368,6 +368,12 @@ def build_step(self): if self.looseversion > LooseVersion('1606'): # use Allwmake -log option if possible since this can be useful during builds, but also afterwards cmd += ' -log' + + if self.looseversion >= LooseVersion('2406'): + # Also build the plugins + cmd += ' && %s %s -log' % (self.cfg['prebuildopts'], + os.path.join(self.builddir, self.openfoamdir, 'Allwmake-plugins')) + run_shell_cmd(cmd_tmpl % cmd) def det_psubdir(self): @@ -476,6 +482,12 @@ def sanity_check_step(self): if self.looseversion < LooseVersion("11"): tools.append("buoyantFoam") tools.append("reactingFoam") + # modifyMesh is no longer there in OpenFOAM >= 12 + if self.is_dot_org and self.looseversion >= LooseVersion("12"): + tools.remove("modifyMesh") + if self.looseversion >= LooseVersion('2406'): + # built from the plugins + tools.append("cartesianMesh") bins = [os.path.join(self.openfoamdir, "bin", x) for x in ["paraFoam"]] + \ [os.path.join(toolsdir, x) for x in tools] diff --git a/easybuild/easyblocks/o/orca.py b/easybuild/easyblocks/o/orca.py index 1788e2dd85..490702779a 100644 --- a/easybuild/easyblocks/o/orca.py +++ b/easybuild/easyblocks/o/orca.py @@ -85,14 +85,21 @@ def install_step(self): (['auto*', 'orca*', 'otool*'], 'bin'), (['*.pdf'], 'share'), ] - # Version 5 extra files - if LooseVersion(self.version) >= LooseVersion('5.0.0'): - compoundmethods = (['ORCACompoundMethods'], 'bin') - files_to_copy.append(compoundmethods) - # Shared builds have additional libraries - libs_to_copy = (['liborca*'], 'lib') - if all([glob.glob(p) for p in libs_to_copy[0]]): - files_to_copy.append(libs_to_copy) + + # Version 6 extra files + if LooseVersion(self.version) >= LooseVersion('6.0.0'): + files_to_copy.extend(['datasets', 'lib', (['CompoundScripts'], 'bin')]) + + else: + # Version 5 extra files + if LooseVersion(self.version) >= LooseVersion('5.0.0'): + compoundmethods = (['ORCACompoundMethods'], 'bin') + files_to_copy.append(compoundmethods) + + # Shared builds have additional libraries + libs_to_copy = (['liborca*'], 'lib') + if all([glob.glob(p) for p in libs_to_copy[0]]): + files_to_copy.append(libs_to_copy) self.cfg['files_to_copy'] = files_to_copy diff --git a/easybuild/easyblocks/p/palm.py b/easybuild/easyblocks/p/palm.py index d8509a110c..b1290e3e0c 100644 --- a/easybuild/easyblocks/p/palm.py +++ b/easybuild/easyblocks/p/palm.py @@ -39,7 +39,7 @@ class EB_PALM(EasyBlock): def __init__(self, *args, **kwargs): """Initialise PALM easyblock.""" - super().__init__(*args, **kwargs) + super(EB_PALM, self).__init__(*args, **kwargs) def configure_step(self): """No configuration procedure for PALM.""" @@ -73,4 +73,4 @@ def sanity_check_step(self): 'files': [os.path.join(self.installdir, 'bin', 'palmrun')], 'dirs': [], } - super().sanity_check_step(custom_paths=custom_paths) + super(EB_PALM, self).sanity_check_step(custom_paths=custom_paths) diff --git a/easybuild/easyblocks/p/petsc.py b/easybuild/easyblocks/p/petsc.py index 6d70c06401..bb8f9e30da 100644 --- a/easybuild/easyblocks/p/petsc.py +++ b/easybuild/easyblocks/p/petsc.py @@ -261,7 +261,12 @@ def configure_step(self): ss_libs = [x if x != "SLIP_LU" else "SPEX" for x in ss_libs] suitesparse_inc = os.path.join(suitesparse, "include") - inc_spec = "-include=[%s]" % suitesparse_inc + suitesparse_incs = [suitesparse_inc] + # SuiteSparse can install its headers into a subdirectory of the include directory instead. + suitesparse_inc_subdir = os.path.join(suitesparse_inc, 'suitesparse') + if os.path.exists(suitesparse_inc_subdir): + suitesparse_incs.append(suitesparse_inc_subdir) + inc_spec = "-include=[%s]" % ','.join(suitesparse_incs) suitesparse_libs = [os.path.join(suitesparse, "lib", "lib%s.so" % x.replace("_", "").lower()) for x in ss_libs] diff --git a/easybuild/easyblocks/p/psmpi.py b/easybuild/easyblocks/p/psmpi.py index 82e9c33443..9d91dd09d3 100644 --- a/easybuild/easyblocks/p/psmpi.py +++ b/easybuild/easyblocks/p/psmpi.py @@ -92,7 +92,10 @@ def configure_step(self): if self.cfg['msa']: self.log.info("Enabling MSA-Awareness...") - self.cfg.update('configopts', ' --with-msa-awareness') + if LooseVersion(self.version) >= LooseVersion('5.10.0-1'): + self.cfg.update('configopts', ' --enable-msa-awareness') + else: + self.cfg.update('configopts', ' --with-msa-awareness') # Set confset comp_fam = self.toolchain.comp_family() @@ -104,7 +107,10 @@ def configure_step(self): # Enable threading, if necessary if self.cfg['threaded']: - self.cfg.update('configopts', ' --with-threading') + if LooseVersion(self.version) >= LooseVersion('5.10.0-1'): + self.cfg.update('configopts', ' --enable-threading') + else: + self.cfg.update('configopts', ' --with-threading') # Add extra mpich options, if any if self.cfg['mpich_opts'] is not None: diff --git a/easybuild/easyblocks/q/quantumespresso.py b/easybuild/easyblocks/q/quantumespresso.py index 3f0014b41f..eacad2a994 100644 --- a/easybuild/easyblocks/q/quantumespresso.py +++ b/easybuild/easyblocks/q/quantumespresso.py @@ -42,6 +42,7 @@ from easybuild.framework.easyconfig import CUSTOM, EasyConfig from easybuild.tools import LooseVersion from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.config import build_option from easybuild.tools.filetools import copy_dir, copy_file from easybuild.tools.modules import get_software_root, get_software_version from easybuild.tools.run import run_shell_cmd @@ -347,6 +348,9 @@ def test_step(self): Test the compilation using Quantum ESPRESSO's test suite. ctest -j NCONCURRENT (NCONCURRENT = max (1, PARALLEL / NPROCS)) """ + if not build_option('mpi_tests'): + self.log.info("Skipping testing of QuantumESPRESSO since MPI testing is disabled") + return thr = self.cfg.get('test_suite_threshold', 0.97) concurrent = max(1, self.cfg.get('parallel', 1) // self._test_nprocs) @@ -1028,6 +1032,9 @@ def test_step(self): Test the compilation using Quantum ESPRESSO's test suite. cd test-suite && make run-tests NPROCS=XXX (XXX <= 4) """ + if not build_option('mpi_tests'): + self.log.info("Skipping testing of QuantumESPRESSO since MPI testing is disabled") + return thr = self.cfg.get('test_suite_threshold', 0.9) stot = 0