From 539af08cece2eaca471569276575e8822315b422 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Thu, 7 Nov 2024 12:41:17 +0100 Subject: [PATCH 01/50] gromacs: make it possible to disable single precision as build target. --- easybuild/easyblocks/g/gromacs.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/g/gromacs.py b/easybuild/easyblocks/g/gromacs.py index b73f36ce18..b48f60cc0f 100644 --- a/easybuild/easyblocks/g/gromacs.py +++ b/easybuild/easyblocks/g/gromacs.py @@ -63,6 +63,8 @@ def extra_options(): extra_vars.update({ 'double_precision': [None, "Build with double precision enabled (-DGMX_DOUBLE=ON), " + "default is to build double precision unless CUDA is enabled", CUSTOM], + 'single_precision': [None, "Build with single precision enabled (-DGMX_DOUBLE=OFF), " + + "default is to build single precision", CUSTOM], 'mpisuffix': ['_mpi', "Suffix to append to MPI-enabled executables (only for GROMACS < 4.6)", CUSTOM], 'mpiexec': ['mpirun', "MPI executable to use when running tests", CUSTOM], 'mpiexec_numproc_flag': ['-np', "Flag to introduce the number of MPI tasks when running tests", CUSTOM], @@ -744,10 +746,15 @@ def run_all_steps(self, *args, **kwargs): 'mpi': 'install' } - precisions = ['single'] + precisions = [] + if self.cfg.get('single_precision') is None or self.cfg.get('single_precision'): + precisions.append('single') if self.cfg.get('double_precision') is None or self.cfg.get('double_precision'): precisions.append('double') + if precisions == []: + raise EasyBuildError("No precision selected. At least one of single/double_precision must be unset or True") + mpitypes = ['nompi'] if self.toolchain.options.get('usempi', None): mpitypes.append('mpi') From 1d17e414e2db7c6f1cc05ade46749332bbe6e467 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 25 Oct 2023 11:33:07 +0200 Subject: [PATCH 02/50] Error out on unknown configure args --- easybuild/easyblocks/generic/configuremake.py | 30 +++++++++++++++++-- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/easybuild/easyblocks/generic/configuremake.py b/easybuild/easyblocks/generic/configuremake.py index f84f4dad45..33fbea7b5b 100644 --- a/easybuild/easyblocks/generic/configuremake.py +++ b/easybuild/easyblocks/generic/configuremake.py @@ -45,11 +45,12 @@ from easybuild.easyblocks import VERSION as EASYBLOCKS_VERSION from easybuild.framework.easyblock import EasyBlock from easybuild.framework.easyconfig import CUSTOM -from easybuild.tools.build_log import print_warning -from easybuild.tools.config import source_paths, build_option +from easybuild.tools.build_log import print_warning, EasyBuildError +from easybuild.tools.config import source_paths, build_option, ERROR, IGNORE, WARN from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, adjust_permissions, compute_checksum, download_file from easybuild.tools.filetools import read_file, remove_file -from easybuild.tools.run import run_shell_cmd +from easybuild.tools.run import extract_errors_from_log, run_shell_cmd +from easybuild.tools.utilities import nub # string that indicates that a configure script was generated by Autoconf # note: bytes string since this constant is used to check the contents of 'configure' which is read as bytes @@ -195,6 +196,11 @@ def extra_options(extra_vars=None): 'tar_config_opts': [False, "Override tar settings as determined by configure.", CUSTOM], 'test_cmd': [None, "Test command to use ('runtest' value is appended, default: '%s')" % DEFAULT_TEST_CMD, CUSTOM], + 'unrecognized_configure_options': [ERROR, + "Action to do when unrecognized arguments passed to ./configure are" + " detected, defaults to aborting the build. Can be set to '" + WARN + + "' or '" + IGNORE + "' (NOT RECOMMENDED! It might hide actual errors" + " e.g. misspelling of intended or changed options)", CUSTOM], }) return extra_vars @@ -325,6 +331,24 @@ def configure_step(self, cmd_prefix=''): res = run_shell_cmd(cmd) + action = self.cfg['unrecognized_configure_options'] + valid_actions = (ERROR, WARN, IGNORE) + # Always verify the EC param + if action not in valid_actions: + raise EasyBuildError('Invalid value for `unrecognized_configure_options`: %s. Must be one of: ', + action, ', '.join(valid_actions)) + if action != IGNORE: + unrecognized_options_str = 'configure: WARNING: unrecognized options:' + unrecognized_options = extract_errors_from_log(res.output, unrecognized_options_str)[1] + # Keep only unique options (remove the warning string and strip whitespace) + unrecognized_options = nub(x.split(unrecognized_options_str)[-1].strip() for x in unrecognized_options) + if unrecognized_options: + msg = 'Found unrecognized configure options: ' + '; '.join(unrecognized_options) + if action == WARN: + print_warning(msg) + else: + raise EasyBuildError(msg) + return res.output def build_step(self, verbose=None, path=None): From 2739cf0a4f720797140a748fbbc51c126c29d9df Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Mon, 11 Nov 2024 07:46:10 +0100 Subject: [PATCH 03/50] gromacs: make single precision default value True. --- easybuild/easyblocks/g/gromacs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/g/gromacs.py b/easybuild/easyblocks/g/gromacs.py index b48f60cc0f..4450448b86 100644 --- a/easybuild/easyblocks/g/gromacs.py +++ b/easybuild/easyblocks/g/gromacs.py @@ -63,7 +63,7 @@ def extra_options(): extra_vars.update({ 'double_precision': [None, "Build with double precision enabled (-DGMX_DOUBLE=ON), " + "default is to build double precision unless CUDA is enabled", CUSTOM], - 'single_precision': [None, "Build with single precision enabled (-DGMX_DOUBLE=OFF), " + + 'single_precision': [True, "Build with single precision enabled (-DGMX_DOUBLE=OFF), " + "default is to build single precision", CUSTOM], 'mpisuffix': ['_mpi', "Suffix to append to MPI-enabled executables (only for GROMACS < 4.6)", CUSTOM], 'mpiexec': ['mpirun', "MPI executable to use when running tests", CUSTOM], @@ -747,7 +747,7 @@ def run_all_steps(self, *args, **kwargs): } precisions = [] - if self.cfg.get('single_precision') is None or self.cfg.get('single_precision'): + if self.cfg.get('single_precision'): precisions.append('single') if self.cfg.get('double_precision') is None or self.cfg.get('double_precision'): precisions.append('double') From 2e6170594f564c918262779931f045886a80ac52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Andr=C3=A9=20Reuter?= Date: Tue, 12 Nov 2024 23:48:20 +0100 Subject: [PATCH 04/50] QuantumESPRESSO: Let internal EasyBlock not create a log file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This fixes the increased memory usage when running the EasyConfig test suite as log files would stay open. Signed-off-by: Jan André Reuter --- easybuild/easyblocks/q/quantumespresso.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easybuild/easyblocks/q/quantumespresso.py b/easybuild/easyblocks/q/quantumespresso.py index 162abde86b..d5607d1581 100644 --- a/easybuild/easyblocks/q/quantumespresso.py +++ b/easybuild/easyblocks/q/quantumespresso.py @@ -28,6 +28,7 @@ @author: Kenneth Hoste (Ghent University) @author: Ake Sandgren (HPC2N, Umea University) @author: Davide Grassano (CECAM, EPFL) +@author: Jan Reuter (Juelich Supercomputing Centre) """ import fileinput @@ -84,6 +85,8 @@ def __init__(self, ec, *args, **kwargs): # Required to avoid CMakeMake default extra_opts to override the ConfigMake ones new_ec = EasyConfig(ec.path, extra_options=eb.extra_options()) + # Disable log file for nested EasyBlock + kwargs['logfile'] = self.logfile self.ebclass = eb(new_ec, *args, **kwargs) class EB_QuantumESPRESSOcmake(CMakeMake): From 8c30f6ba11b500b69776d98447216008d14b67c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Fri, 22 Nov 2024 16:53:36 +0100 Subject: [PATCH 05/50] Move sitecustomize.py into site-packages --- easybuild/easyblocks/p/python.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/easybuild/easyblocks/p/python.py b/easybuild/easyblocks/p/python.py index 01355b23e5..f7372845a8 100644 --- a/easybuild/easyblocks/p/python.py +++ b/easybuild/easyblocks/p/python.py @@ -142,9 +142,6 @@ def __init__(self, *args, **kwargs): self.pyshortver = '.'.join(self.version.split('.')[:2]) - # Used for EBPYTHONPREFIXES handler script - self.pythonpath = os.path.join(log_path(), 'python') - ext_defaults = { # Use PYPI_SOURCE as the default for source_urls of extensions. 'source_urls': [PYPI_SOURCE], @@ -504,7 +501,8 @@ def install_step(self): symlink('pip' + self.pyshortver, pip_binary_path, use_abspath_source=False) if self.cfg.get('ebpythonprefixes'): - write_file(os.path.join(self.installdir, self.pythonpath, 'sitecustomize.py'), SITECUSTOMIZE) + site_packages_path = os.path.join('lib', 'python' + self.pyshortver, 'site-packages') + write_file(os.path.join(self.installdir, site_packages_path, 'sitecustomize.py'), SITECUSTOMIZE) # symlink lib/python*/lib-dynload to lib64/python*/lib-dynload if it doesn't exist; # see https://github.com/easybuilders/easybuild-easyblocks/issues/1957 @@ -640,7 +638,16 @@ def make_module_extra(self, *args, **kwargs): """Add path to sitecustomize.py to $PYTHONPATH""" txt = super(EB_Python, self).make_module_extra() + # Legacy support for existing installations doing "--rebuild --module-only" if self.cfg.get('ebpythonprefixes'): - txt += self.module_generator.prepend_paths(PYTHONPATH, self.pythonpath) + new_dir = os.path.join('lib', 'python' + self.pyshortver, 'site-packages') + old_dir = os.path.join(log_path(), 'python') + if not os.path.exists(os.path.join(self.installdir, new_dir, 'sitecustomize.py')): + if not os.path.exists(os.path.join(self.installdir, old_dir, 'sitecustomize.py')): + raise EasyBuildError("sitecustomize.py is missing from installation.") + else: + txt += self.module_generator.prepend_paths(PYTHONPATH, os.path.join(self.installdir, old_dir)) + + return txt From f573a3978502e315554a097f68833fa53ca6adf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Fri, 22 Nov 2024 17:45:42 +0100 Subject: [PATCH 06/50] Remove error since it breaks CI --- easybuild/easyblocks/p/python.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/easybuild/easyblocks/p/python.py b/easybuild/easyblocks/p/python.py index f7372845a8..f4f66dfbb8 100644 --- a/easybuild/easyblocks/p/python.py +++ b/easybuild/easyblocks/p/python.py @@ -643,11 +643,6 @@ def make_module_extra(self, *args, **kwargs): new_dir = os.path.join('lib', 'python' + self.pyshortver, 'site-packages') old_dir = os.path.join(log_path(), 'python') if not os.path.exists(os.path.join(self.installdir, new_dir, 'sitecustomize.py')): - if not os.path.exists(os.path.join(self.installdir, old_dir, 'sitecustomize.py')): - raise EasyBuildError("sitecustomize.py is missing from installation.") - else: - txt += self.module_generator.prepend_paths(PYTHONPATH, os.path.join(self.installdir, old_dir)) - - + txt += self.module_generator.prepend_paths(PYTHONPATH, os.path.join(self.installdir, old_dir)) return txt From db0e35b05346ab964182121e71fee220ba7c6157 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Fri, 22 Nov 2024 17:51:03 +0100 Subject: [PATCH 07/50] Fix using relative path --- easybuild/easyblocks/p/python.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/p/python.py b/easybuild/easyblocks/p/python.py index f4f66dfbb8..0d2b61d106 100644 --- a/easybuild/easyblocks/p/python.py +++ b/easybuild/easyblocks/p/python.py @@ -643,6 +643,6 @@ def make_module_extra(self, *args, **kwargs): new_dir = os.path.join('lib', 'python' + self.pyshortver, 'site-packages') old_dir = os.path.join(log_path(), 'python') if not os.path.exists(os.path.join(self.installdir, new_dir, 'sitecustomize.py')): - txt += self.module_generator.prepend_paths(PYTHONPATH, os.path.join(self.installdir, old_dir)) + txt += self.module_generator.prepend_paths(PYTHONPATH, old_dir) return txt From 6d6f56f07e4b45b8ae66e8b2d430cd198f5425fd Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 23 Nov 2024 16:35:28 +0000 Subject: [PATCH 08/50] remove use of deprecated parse_log_for_error in rpackage --- easybuild/easyblocks/generic/rpackage.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/generic/rpackage.py b/easybuild/easyblocks/generic/rpackage.py index c7aabcc36a..b2b5221ce8 100644 --- a/easybuild/easyblocks/generic/rpackage.py +++ b/easybuild/easyblocks/generic/rpackage.py @@ -43,7 +43,7 @@ from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.environment import setvar from easybuild.tools.filetools import mkdir, copy_file -from easybuild.tools.run import run_shell_cmd, parse_log_for_error +from easybuild.tools.run import run_shell_cmd def make_R_install_option(opt, values, cmdline=False): @@ -183,8 +183,16 @@ def check_install_output(self, output): """ Check output of installation command, and clean up installation if needed. """ - errors = parse_log_for_error(output, regExp="^ERROR:") + reg = re.compile("^ERROR:", re.I) + + errors = [] + for line in output.split('\n'): + r = reg.search(line) + if r: + errors.append([line, r.groups()]) + if errors: + self.log.info("R package %s failed with error:\n%s", self.name, '\n'.join([x[0] for x in res])) self.handle_installation_errors() cmd = "R -q --no-save" stdin = """ From ddb6c94e3b9eb90b650591aefdcf77f8d71a3bad Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 23 Nov 2024 16:36:30 +0000 Subject: [PATCH 09/50] oops --- easybuild/easyblocks/generic/rpackage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/generic/rpackage.py b/easybuild/easyblocks/generic/rpackage.py index b2b5221ce8..1fcca38db8 100644 --- a/easybuild/easyblocks/generic/rpackage.py +++ b/easybuild/easyblocks/generic/rpackage.py @@ -192,7 +192,7 @@ def check_install_output(self, output): errors.append([line, r.groups()]) if errors: - self.log.info("R package %s failed with error:\n%s", self.name, '\n'.join([x[0] for x in res])) + self.log.info("R package %s failed with error:\n%s", self.name, '\n'.join([x[0] for x in errors])) self.handle_installation_errors() cmd = "R -q --no-save" stdin = """ From 2a963bb1b623b42164c9c0125edf2a8bedd100e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 26 Nov 2024 17:16:56 +0100 Subject: [PATCH 10/50] force use of bash for Allwmake scripts --- easybuild/easyblocks/o/openfoam.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/o/openfoam.py b/easybuild/easyblocks/o/openfoam.py index 88ae377417..fd9d892c7a 100644 --- a/easybuild/easyblocks/o/openfoam.py +++ b/easybuild/easyblocks/o/openfoam.py @@ -340,7 +340,7 @@ def build_step(self): cleancmd = "wcleanAll" # make directly in install directory - cmd_tmpl = "%(precmd)s && %(cleancmd)s && %(prebuildopts)s %(makecmd)s" % { + cmd_tmpl = "%(precmd)s && %(cleancmd)s && %(prebuildopts)s bash %(makecmd)s" % { 'precmd': precmd, 'cleancmd': cleancmd, 'prebuildopts': self.cfg['prebuildopts'], @@ -371,7 +371,7 @@ def build_step(self): if self.looseversion >= LooseVersion('2406'): # Also build the plugins - cmd += ' && %s %s -log' % (self.cfg['prebuildopts'], + cmd += ' && %s bash %s -log' % (self.cfg['prebuildopts'], os.path.join(self.builddir, self.openfoamdir, 'Allwmake-plugins')) run_cmd(cmd_tmpl % cmd, log_all=True, simple=True, log_output=True) From e4a3ff1932350d575dffc7597435609fad6dd691 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 26 Nov 2024 17:18:58 +0100 Subject: [PATCH 11/50] fix indentation --- easybuild/easyblocks/o/openfoam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/o/openfoam.py b/easybuild/easyblocks/o/openfoam.py index fd9d892c7a..4ffa5e98d6 100644 --- a/easybuild/easyblocks/o/openfoam.py +++ b/easybuild/easyblocks/o/openfoam.py @@ -372,7 +372,7 @@ def build_step(self): if self.looseversion >= LooseVersion('2406'): # Also build the plugins cmd += ' && %s bash %s -log' % (self.cfg['prebuildopts'], - os.path.join(self.builddir, self.openfoamdir, 'Allwmake-plugins')) + os.path.join(self.builddir, self.openfoamdir, 'Allwmake-plugins')) run_cmd(cmd_tmpl % cmd, log_all=True, simple=True, log_output=True) From b0f4de0982a09e3af40689641eb0dbc578b3f654 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 28 Nov 2024 09:40:35 +0100 Subject: [PATCH 12/50] fix detection of math library in numpy build --- easybuild/easyblocks/n/numpy.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/easybuild/easyblocks/n/numpy.py b/easybuild/easyblocks/n/numpy.py index 968cefa919..a29dda4ded 100644 --- a/easybuild/easyblocks/n/numpy.py +++ b/easybuild/easyblocks/n/numpy.py @@ -204,6 +204,17 @@ def get_libs_for_mkl(varname): 'includes': includes, } + numpy_version = LooseVersion(self.version) + + if numpy_version < '1.26': + # NumPy detects the required math by trying to link a minimal code containing a call to `log(0.)`. + # The first try is without any libraries, which works with `gcc -fno-math-errno` (our optimization default) + # because the call gets removed due to not having any effect. So it concludes that `-lm` is not required. + # This then fails to detect availability of functions such as `acosh` which do not get removed in the same + # way and so less exact replacements are used instead which e.g. fail the tests on PPC. + # This variable makes it try `-lm` first and is supported until the Meson backend is used in 1.26+. + env.setvar('MATHLIB', 'm') + super(EB_numpy, self).configure_step() if LooseVersion(self.version) < LooseVersion('1.21'): From 259051b07ac70b65ea2815cb717c4df2550ba8c5 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 28 Nov 2024 10:52:49 +0100 Subject: [PATCH 13/50] Remove `numpy_version` variable again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jan André Reuter --- easybuild/easyblocks/n/numpy.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/easybuild/easyblocks/n/numpy.py b/easybuild/easyblocks/n/numpy.py index a29dda4ded..e1b92eca55 100644 --- a/easybuild/easyblocks/n/numpy.py +++ b/easybuild/easyblocks/n/numpy.py @@ -204,9 +204,7 @@ def get_libs_for_mkl(varname): 'includes': includes, } - numpy_version = LooseVersion(self.version) - - if numpy_version < '1.26': + if LooseVersion(self.version) < LooseVersion('1.26'): # NumPy detects the required math by trying to link a minimal code containing a call to `log(0.)`. # The first try is without any libraries, which works with `gcc -fno-math-errno` (our optimization default) # because the call gets removed due to not having any effect. So it concludes that `-lm` is not required. From d1c3e886ad9aea87e3d15fbb5eb8c7302e0a32f5 Mon Sep 17 00:00:00 2001 From: Stefan Wolfsheimer Date: Fri, 29 Nov 2024 07:57:16 +0100 Subject: [PATCH 14/50] WRF: set NETCFF env variable --- easybuild/easyblocks/w/wrf.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/w/wrf.py b/easybuild/easyblocks/w/wrf.py index 3c9c76f18e..76463fcb8f 100644 --- a/easybuild/easyblocks/w/wrf.py +++ b/easybuild/easyblocks/w/wrf.py @@ -95,8 +95,10 @@ def configure_step(self): wrfdir = os.path.join(self.builddir, self.wrfsubdir) - # define $NETCDF* for netCDF dependency (used when creating WRF module file) - set_netcdf_env_vars(self.log) + set_netcdf_env_vars(self.log) + netcdf_fortran = get_software_root('NETCDFMINFORTRAN') + if netcdf_fortran: + env.setvar('NETCDFF', netcdf_fortran) # HDF5 (optional) dependency hdf5 = get_software_root('HDF5') From df15a8daa45a22694b301528842e327aa68ddafc Mon Sep 17 00:00:00 2001 From: Stefan Wolfsheimer Date: Fri, 29 Nov 2024 09:48:54 +0100 Subject: [PATCH 15/50] wps.py: set NETCDFF_DIR --- easybuild/easyblocks/w/wps.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easybuild/easyblocks/w/wps.py b/easybuild/easyblocks/w/wps.py index 6549d493d0..73f6359899 100644 --- a/easybuild/easyblocks/w/wps.py +++ b/easybuild/easyblocks/w/wps.py @@ -106,6 +106,9 @@ def configure_step(self): wrfdir = os.path.join(wrf, det_wrf_subdir(get_software_version('WRF'))) else: raise EasyBuildError("WRF module not loaded?") + netcdf_fortran = get_software_root('NETCDFMINFORTRAN') + if netcdf_fortran: + env.setvar('NETCDFF_DIR', netcdf_fortran) self.compile_script = 'compile' From 2c6f0e3e95bddc12e9f947c5bb9275876f1bfcfb Mon Sep 17 00:00:00 2001 From: Stefan Wolfsheimer Date: Fri, 29 Nov 2024 13:52:37 +0100 Subject: [PATCH 16/50] fix PEP8 --- easybuild/easyblocks/w/wrf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/w/wrf.py b/easybuild/easyblocks/w/wrf.py index 76463fcb8f..696237f551 100644 --- a/easybuild/easyblocks/w/wrf.py +++ b/easybuild/easyblocks/w/wrf.py @@ -95,7 +95,7 @@ def configure_step(self): wrfdir = os.path.join(self.builddir, self.wrfsubdir) - set_netcdf_env_vars(self.log) + set_netcdf_env_vars(self.log) netcdf_fortran = get_software_root('NETCDFMINFORTRAN') if netcdf_fortran: env.setvar('NETCDFF', netcdf_fortran) From 5d1651491a5c5136bdabb3da9f792a70ca946212 Mon Sep 17 00:00:00 2001 From: Stefan Wolfsheimer Date: Fri, 29 Nov 2024 18:01:25 +0100 Subject: [PATCH 17/50] revert change to wrf.py --- easybuild/easyblocks/w/wrf.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/easybuild/easyblocks/w/wrf.py b/easybuild/easyblocks/w/wrf.py index 696237f551..3c9c76f18e 100644 --- a/easybuild/easyblocks/w/wrf.py +++ b/easybuild/easyblocks/w/wrf.py @@ -95,10 +95,8 @@ def configure_step(self): wrfdir = os.path.join(self.builddir, self.wrfsubdir) + # define $NETCDF* for netCDF dependency (used when creating WRF module file) set_netcdf_env_vars(self.log) - netcdf_fortran = get_software_root('NETCDFMINFORTRAN') - if netcdf_fortran: - env.setvar('NETCDFF', netcdf_fortran) # HDF5 (optional) dependency hdf5 = get_software_root('HDF5') From ee216590a83315ab7e6deaddcdb1030502a3e9fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sat, 30 Nov 2024 13:43:01 +0100 Subject: [PATCH 18/50] pass the preferred host compiler to the CUDA compiler --- easybuild/easyblocks/generic/cmakemake.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/easybuild/easyblocks/generic/cmakemake.py b/easybuild/easyblocks/generic/cmakemake.py index 797a504fb2..b5bd33f250 100644 --- a/easybuild/easyblocks/generic/cmakemake.py +++ b/easybuild/easyblocks/generic/cmakemake.py @@ -328,6 +328,11 @@ def configure_step(self, srcdir=None, builddir=None): # ensure CMake uses EB python, not system or virtualenv python options.update(get_cmake_python_config_dict()) + # pass the preferred host compiler to the CUDA compiler + cuda_root = get_software_root('CUDA') + if cuda_root: + self.cfg.update('preconfigopts', 'CUDAHOSTCXX=%s' % which(os.getenv('CXX', 'g++'))) + if not self.cfg.get('allow_system_boost', False): boost_root = get_software_root('Boost') if boost_root: From 0857b9fbcab2f975408aa59623d128aa75e00d5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 2 Dec 2024 14:15:07 +0100 Subject: [PATCH 19/50] also set CMAKE_CUDA_COMPILER and CMAKE_CUDA_ARCHITECTURES --- easybuild/easyblocks/generic/cmakemake.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/generic/cmakemake.py b/easybuild/easyblocks/generic/cmakemake.py index b5bd33f250..296e6f26fe 100644 --- a/easybuild/easyblocks/generic/cmakemake.py +++ b/easybuild/easyblocks/generic/cmakemake.py @@ -328,10 +328,18 @@ def configure_step(self, srcdir=None, builddir=None): # ensure CMake uses EB python, not system or virtualenv python options.update(get_cmake_python_config_dict()) - # pass the preferred host compiler to the CUDA compiler + # pass the preferred host compiler, CUDA compiler, and CUDA architectures to the CUDA compiler cuda_root = get_software_root('CUDA') if cuda_root: - self.cfg.update('preconfigopts', 'CUDAHOSTCXX=%s' % which(os.getenv('CXX', 'g++'))) + options['CMAKE_CUDA_HOST_COMPILER'] = which(os.getenv('CXX', 'g++')) + options['CMAKE_CUDA_COMPILER'] = 'nvcc' + cuda_cc = build_option('cuda_compute_capabilities') or self.cfg['cuda_compute_capabilities'] + if cuda_cc: + options['CMAKE_CUDA_ARCHITECTURES'] = '"%s"' % ';'.join([cc.replace('.', '') for cc in cuda_cc]) + else: + raise EasyBuildError('List of CUDA compute capabilities must be specified, either via ' + 'cuda_compute_capabilities easyconfig parameter or via ' + '--cuda-compute-capabilities') if not self.cfg.get('allow_system_boost', False): boost_root = get_software_root('Boost') From 90495ed23d26b3d5fd8162bf5d7b4c073a0682fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 2 Dec 2024 14:48:16 +0100 Subject: [PATCH 20/50] use absolute path for nvcc Co-authored-by: ocaisa --- easybuild/easyblocks/generic/cmakemake.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/generic/cmakemake.py b/easybuild/easyblocks/generic/cmakemake.py index 296e6f26fe..3f8e591cd1 100644 --- a/easybuild/easyblocks/generic/cmakemake.py +++ b/easybuild/easyblocks/generic/cmakemake.py @@ -332,7 +332,7 @@ def configure_step(self, srcdir=None, builddir=None): cuda_root = get_software_root('CUDA') if cuda_root: options['CMAKE_CUDA_HOST_COMPILER'] = which(os.getenv('CXX', 'g++')) - options['CMAKE_CUDA_COMPILER'] = 'nvcc' + options['CMAKE_CUDA_COMPILER'] = which('nvcc') cuda_cc = build_option('cuda_compute_capabilities') or self.cfg['cuda_compute_capabilities'] if cuda_cc: options['CMAKE_CUDA_ARCHITECTURES'] = '"%s"' % ';'.join([cc.replace('.', '') for cc in cuda_cc]) From 14c889fcb56a1982be4f06357f3837229d1f9993 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Andr=C3=A9=20Reuter?= Date: Fri, 15 Nov 2024 08:28:42 +0100 Subject: [PATCH 21/50] Reintroduce #3472 after framework changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jan André Reuter --- easybuild/easyblocks/generic/bundle.py | 45 ++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/easybuild/easyblocks/generic/bundle.py b/easybuild/easyblocks/generic/bundle.py index b8abecf8fc..5f098f83d7 100644 --- a/easybuild/easyblocks/generic/bundle.py +++ b/easybuild/easyblocks/generic/bundle.py @@ -31,6 +31,7 @@ @author: Pieter De Baets (Ghent University) @author: Jens Timmerman (Ghent University) @author: Jasper Grimm (University of York) +@author: Jan Andre Reuter (Juelich Supercomputing Centre) """ import copy import os @@ -71,8 +72,8 @@ def __init__(self, *args, **kwargs): self.altroot = None self.altversion = None - # list of EasyConfig instances for components - self.comp_cfgs = [] + # list of EasyConfig instances and their EasyBlocks for components + self.comp_instances = [] # list of EasyConfig instances of components for which to run sanity checks self.comp_cfgs_sanity_check = [] @@ -198,7 +199,7 @@ def __init__(self, *args, **kwargs): if comp_cfg['patches']: self.cfg.update('patches', comp_cfg['patches']) - self.comp_cfgs.append(comp_cfg) + self.comp_instances.append((comp_cfg, comp_cfg.easyblock(comp_cfg))) self.cfg.update('checksums', checksums_patches) @@ -217,7 +218,7 @@ def check_checksums(self): """ checksum_issues = super(Bundle, self).check_checksums() - for comp in self.comp_cfgs: + for comp, _ in self.comp_instances: checksum_issues.extend(self.check_checksums_for(comp, sub="of component %s" % comp['name'])) return checksum_issues @@ -247,14 +248,12 @@ def build_step(self): def install_step(self): """Install components, if specified.""" comp_cnt = len(self.cfg['components']) - for idx, cfg in enumerate(self.comp_cfgs): + for idx, (cfg, comp) in enumerate(self.comp_instances): print_msg("installing bundle component %s v%s (%d/%d)..." % (cfg['name'], cfg['version'], idx + 1, comp_cnt)) self.log.info("Installing component %s v%s using easyblock %s", cfg['name'], cfg['version'], cfg.easyblock) - comp = cfg.easyblock(cfg) - # correct build/install dirs comp.builddir = self.builddir comp.install_subdir, comp.installdir = self.install_subdir, self.installdir @@ -324,6 +323,38 @@ def install_step(self): # close log for this component comp.close_log() + def make_module_req_guess(self): + """ + Set module requirements from all comppnents, e.g. $PATH, etc. + During the install step, we only set these requirements temporarily. + Later on when building the module, those paths are not considered. + Therefore, iterate through all the components again and gather + the requirements. + + Do not remove duplicates or check for existance of folders, + as this is done in the generic EasyBlock while creating + the modulefile already. + """ + # Start with the paths from the generic EasyBlock. + # If not added here, they might be missing entirely and fail sanity checks. + final_reqs = super(Bundle, self).make_module_req_guess() + + for cfg, comp in self.comp_instances: + self.log.info("Gathering module paths for component %s v%s", cfg['name'], cfg['version']) + reqs = comp.make_module_req_guess() + + try: + for key, value in sorted(reqs.items()): + if isinstance(reqs, string_type): + value = [value] + final_reqs.setdefault(key, []) + final_reqs[key] += value + except AttributeError: + raise EasyBuildError("Cannot process module requirements of bundle component %s v%s", + cfg['name'], cfg['version']) + + return final_reqs + def make_module_extra(self, *args, **kwargs): """Set extra stuff in module file, e.g. $EBROOT*, $EBVERSION*, etc.""" if not self.altroot and not self.altversion: From fd3cd657cdfc8b209f6660c324a456c575d90080 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Andr=C3=A9=20Reuter?= Date: Fri, 15 Nov 2024 08:33:16 +0100 Subject: [PATCH 22/50] Update clang_aomp for bundle changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jan André Reuter --- easybuild/easyblocks/c/clang_aomp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/c/clang_aomp.py b/easybuild/easyblocks/c/clang_aomp.py index 6f046cdb28..7e8181faba 100644 --- a/easybuild/easyblocks/c/clang_aomp.py +++ b/easybuild/easyblocks/c/clang_aomp.py @@ -145,7 +145,7 @@ def configure_step(self): raise EasyBuildError("Could not find 'ROCm-Device-Libs' source directory in %s", self.builddir) num_comps = len(self.cfg['components']) - for idx, comp in enumerate(self.comp_cfgs): + for idx, (comp, _) in enumerate(self.comp_instances): name = comp['name'] msg = "configuring bundle component %s %s (%d/%d)..." % (name, comp['version'], idx + 1, num_comps) print_msg(msg) From 43ec3552236795f8cd92742e8736dea2d75161f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Andr=C3=A9=20Reuter?= Date: Fri, 15 Nov 2024 08:33:28 +0100 Subject: [PATCH 23/50] Transfer logfile from bundle to components MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This fixes the increased memory usage when running the EasyConfig test suite as log files would stay open. Signed-off-by: Jan André Reuter --- easybuild/easyblocks/generic/bundle.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/easybuild/easyblocks/generic/bundle.py b/easybuild/easyblocks/generic/bundle.py index 5f098f83d7..dd17b82a83 100644 --- a/easybuild/easyblocks/generic/bundle.py +++ b/easybuild/easyblocks/generic/bundle.py @@ -199,7 +199,7 @@ def __init__(self, *args, **kwargs): if comp_cfg['patches']: self.cfg.update('patches', comp_cfg['patches']) - self.comp_instances.append((comp_cfg, comp_cfg.easyblock(comp_cfg))) + self.comp_instances.append((comp_cfg, comp_cfg.easyblock(comp_cfg, logfile=self.logfile))) self.cfg.update('checksums', checksums_patches) @@ -320,9 +320,6 @@ def install_step(self): new_val = path env.setvar(envvar, new_val) - # close log for this component - comp.close_log() - def make_module_req_guess(self): """ Set module requirements from all comppnents, e.g. $PATH, etc. From bce45f57918edf9d839fc4bb3c6090d8c39ae7cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Andr=C3=A9=20Reuter?= Date: Wed, 4 Dec 2024 10:19:29 +0100 Subject: [PATCH 24/50] Bundle: Address review comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jan André Reuter --- easybuild/easyblocks/generic/bundle.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/easybuild/easyblocks/generic/bundle.py b/easybuild/easyblocks/generic/bundle.py index dd17b82a83..2571df7f32 100644 --- a/easybuild/easyblocks/generic/bundle.py +++ b/easybuild/easyblocks/generic/bundle.py @@ -322,15 +322,15 @@ def install_step(self): def make_module_req_guess(self): """ - Set module requirements from all comppnents, e.g. $PATH, etc. + Set module requirements from all components, e.g. $PATH, etc. During the install step, we only set these requirements temporarily. Later on when building the module, those paths are not considered. Therefore, iterate through all the components again and gather the requirements. - Do not remove duplicates or check for existance of folders, + Do not remove duplicates or check for existence of folders, as this is done in the generic EasyBlock while creating - the modulefile already. + the module file already. """ # Start with the paths from the generic EasyBlock. # If not added here, they might be missing entirely and fail sanity checks. @@ -342,10 +342,10 @@ def make_module_req_guess(self): try: for key, value in sorted(reqs.items()): - if isinstance(reqs, string_type): + if isinstance(value, string_type): value = [value] final_reqs.setdefault(key, []) - final_reqs[key] += value + final_reqs[key].append(value) except AttributeError: raise EasyBuildError("Cannot process module requirements of bundle component %s v%s", cfg['name'], cfg['version']) From 4ebf4a36a1ba66e032cec324b9c00c587ae0839a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Andr=C3=A9=20Reuter?= Date: Wed, 4 Dec 2024 15:32:17 +0100 Subject: [PATCH 25/50] Bundle: Add comment to try-except block in make_module_req_guess() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jan André Reuter --- easybuild/easyblocks/generic/bundle.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easybuild/easyblocks/generic/bundle.py b/easybuild/easyblocks/generic/bundle.py index 2571df7f32..caaad68cd2 100644 --- a/easybuild/easyblocks/generic/bundle.py +++ b/easybuild/easyblocks/generic/bundle.py @@ -340,6 +340,9 @@ def make_module_req_guess(self): self.log.info("Gathering module paths for component %s v%s", cfg['name'], cfg['version']) reqs = comp.make_module_req_guess() + # Try-except block to fail with an easily understandable error message. + # This should only trigger when an EasyBlock returns non-dict module requirements + # for make_module_req_guess() which should then be fixed in the components EasyBlock. try: for key, value in sorted(reqs.items()): if isinstance(value, string_type): From a9d5ea1da8703a540727f6f4e0e5cbccf5fc7cb7 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 4 Dec 2024 17:36:39 +0000 Subject: [PATCH 26/50] Rename `post_install_step` to `post_processing_step` Matches https://github.com/easybuilders/easybuild-framework/pull/4715 --- easybuild/easyblocks/a/aedt.py | 2 +- easybuild/easyblocks/a/aocc.py | 4 ++-- easybuild/easyblocks/a/aomp.py | 4 ++-- easybuild/easyblocks/c/clang.py | 4 ++-- easybuild/easyblocks/c/crispr_dav.py | 2 +- easybuild/easyblocks/c/cuda.py | 4 ++-- easybuild/easyblocks/d/dualsphysics.py | 4 ++-- easybuild/easyblocks/e/easybuildmeta.py | 4 ++-- easybuild/easyblocks/f/fftwmpi.py | 4 ++-- easybuild/easyblocks/g/gcc.py | 4 ++-- easybuild/easyblocks/generic/binary.py | 4 ++-- easybuild/easyblocks/generic/systemcompiler.py | 2 +- easybuild/easyblocks/generic/systemmpi.py | 2 +- easybuild/easyblocks/h/hadoop.py | 2 +- easybuild/easyblocks/i/imkl.py | 4 ++-- easybuild/easyblocks/i/imkl_fftw.py | 4 ++-- easybuild/easyblocks/i/impi.py | 4 ++-- easybuild/easyblocks/j/java.py | 4 ++-- easybuild/easyblocks/m/mathematica.py | 4 ++-- easybuild/easyblocks/m/metagenome_atlas.py | 2 +- easybuild/easyblocks/p/perl.py | 4 ++-- 21 files changed, 36 insertions(+), 36 deletions(-) diff --git a/easybuild/easyblocks/a/aedt.py b/easybuild/easyblocks/a/aedt.py index a381e2fd24..27685a3115 100644 --- a/easybuild/easyblocks/a/aedt.py +++ b/easybuild/easyblocks/a/aedt.py @@ -71,7 +71,7 @@ def install_step(self): ]) run_shell_cmd("./Linux/AnsysEM/Disk1/InstData/setup.exe %s" % options) - def post_install_step(self): + def post_processing_step(self): """Disable OS check and set LC_ALL/LANG for runtime""" if not self.subdir: self._set_subdir() diff --git a/easybuild/easyblocks/a/aocc.py b/easybuild/easyblocks/a/aocc.py index 24da9f3c5e..b42c12d063 100644 --- a/easybuild/easyblocks/a/aocc.py +++ b/easybuild/easyblocks/a/aocc.py @@ -214,7 +214,7 @@ def install_step(self): super(EB_AOCC, self).install_step() - def post_install_step(self): + def post_processing_step(self): """ For AOCC <5.0.0: Create wrappers for the compilers to make sure compilers picks up GCCcore as GCC toolchain. @@ -245,7 +245,7 @@ def post_install_step(self): self._create_compiler_config_files(compilers_to_add_config_files) self._create_compiler_wrappers(compilers_to_wrap) - super(EB_AOCC, self).post_install_step() + super(EB_AOCC, self).post_processing_step() def sanity_check_step(self): """Custom sanity check for AOCC, based on sanity check for Clang.""" diff --git a/easybuild/easyblocks/a/aomp.py b/easybuild/easyblocks/a/aomp.py index 2a1f0472d8..f62c124b19 100644 --- a/easybuild/easyblocks/a/aomp.py +++ b/easybuild/easyblocks/a/aomp.py @@ -133,8 +133,8 @@ def configure_step(self): # Only build selected components self.cfg['installopts'] = 'select ' + ' '.join(components) - def post_install_step(self): - super(EB_AOMP, self).post_install_step() + def post_processing_step(self): + super(EB_AOMP, self).post_processing_step() # The install script will create a symbolic link as the install # directory, this creates problems for EB as it won't remove the # symlink. To remedy this we remove the link here and rename the actual diff --git a/easybuild/easyblocks/c/clang.py b/easybuild/easyblocks/c/clang.py index ddefa7cbff..3ee17819e3 100644 --- a/easybuild/easyblocks/c/clang.py +++ b/easybuild/easyblocks/c/clang.py @@ -612,9 +612,9 @@ def install_step(self): except OSError as err: raise EasyBuildError("Failed to copy static analyzer dirs to install dir: %s", err) - def post_install_step(self): + def post_processing_step(self): """Install python bindings.""" - super(EB_Clang, self).post_install_step() + super(EB_Clang, self).post_processing_step() # copy Python bindings here in post-install step so that it is not done more than once in multi_deps context if self.cfg['python_bindings']: diff --git a/easybuild/easyblocks/c/crispr_dav.py b/easybuild/easyblocks/c/crispr_dav.py index e85da91da2..edf0e5e431 100644 --- a/easybuild/easyblocks/c/crispr_dav.py +++ b/easybuild/easyblocks/c/crispr_dav.py @@ -46,7 +46,7 @@ def __init__(self, *args, **kwargs): super(EB_CRISPR_minus_DAV, self).__init__(*args, **kwargs) self.cfg['extract_sources'] = True - def post_install_step(self): + def post_processing_step(self): """Update configuration files with correct paths to dependencies and files in installation.""" # getting paths of deps + files we will work with diff --git a/easybuild/easyblocks/c/cuda.py b/easybuild/easyblocks/c/cuda.py index b66563045e..3fa3a3a949 100644 --- a/easybuild/easyblocks/c/cuda.py +++ b/easybuild/easyblocks/c/cuda.py @@ -218,7 +218,7 @@ def install_step(self): self.log.debug("Running patch %s", patch['name']) run_shell_cmd("/bin/sh " + patch['path'] + " --accept-eula --silent --installdir=" + self.installdir) - def post_install_step(self): + def post_processing_step(self): """ Create wrappers for the specified host compilers, generate the appropriate stub symlinks, and create version independent pkgconfig files @@ -288,7 +288,7 @@ def create_wrapper(wrapper_name, wrapper_comp): symlink(pc_file, link, use_abspath_source=False) change_dir(cwd) - super(EB_CUDA, self).post_install_step() + super(EB_CUDA, self).post_processing_step() def sanity_check_step(self): """Custom sanity check for CUDA.""" diff --git a/easybuild/easyblocks/d/dualsphysics.py b/easybuild/easyblocks/d/dualsphysics.py index 5fc794a59e..964312b5c3 100644 --- a/easybuild/easyblocks/d/dualsphysics.py +++ b/easybuild/easyblocks/d/dualsphysics.py @@ -90,9 +90,9 @@ def install_step(self): ] super(EB_DualSPHysics, self).install_step() - def post_install_step(self): + def post_processing_step(self): """Custom post-installation step: ensure rpath is patched into binaries/libraries if configured.""" - super(EB_DualSPHysics, self).post_install_step() + super(EB_DualSPHysics, self).post_processing_step() if build_option('rpath'): # only the compiled binary (e.g. DualSPHysics5.0CPU_linux64) is rpath'd, the precompiled libraries diff --git a/easybuild/easyblocks/e/easybuildmeta.py b/easybuild/easyblocks/e/easybuildmeta.py index f0d24e79c7..88ad7a742f 100644 --- a/easybuild/easyblocks/e/easybuildmeta.py +++ b/easybuild/easyblocks/e/easybuildmeta.py @@ -136,10 +136,10 @@ def install_step(self): except OSError as err: raise EasyBuildError("Failed to install EasyBuild packages: %s", err) - def post_install_step(self): + def post_processing_step(self): """Remove setuptools.pth file that hard includes a system-wide (site-packages) path, if it is there.""" - super(EB_EasyBuildMeta, self).post_install_step() + super(EB_EasyBuildMeta, self).post_processing_step() setuptools_pth = os.path.join(self.installdir, self.pylibdir, 'setuptools.pth') if os.path.exists(setuptools_pth): diff --git a/easybuild/easyblocks/f/fftwmpi.py b/easybuild/easyblocks/f/fftwmpi.py index cb285a576b..00059e1e39 100644 --- a/easybuild/easyblocks/f/fftwmpi.py +++ b/easybuild/easyblocks/f/fftwmpi.py @@ -59,7 +59,7 @@ def prepare_step(self, *args, **kwargs): if not fftw_root: raise EasyBuildError("Required FFTW dependency is missing!") - def post_install_step(self): + def post_processing_step(self): """Custom post install step for FFTW.MPI""" # remove everything except include files that are already in non-MPI FFTW dependency. @@ -68,7 +68,7 @@ def post_install_step(self): glob.glob(os.path.join(self.installdir, 'lib*/pkgconfig')) + glob.glob(os.path.join(self.installdir, 'lib*/cmake')) + [os.path.join(self.installdir, p) for p in ['bin', 'share']]) - super(EB_FFTW_period_MPI, self).post_install_step() + super(EB_FFTW_period_MPI, self).post_processing_step() def sanity_check_step(self): """Custom sanity check for FFTW.MPI: check if all libraries/headers for MPI interfaces are there.""" diff --git a/easybuild/easyblocks/g/gcc.py b/easybuild/easyblocks/g/gcc.py index bb0ba20ef4..e23f260572 100644 --- a/easybuild/easyblocks/g/gcc.py +++ b/easybuild/easyblocks/g/gcc.py @@ -966,11 +966,11 @@ def install_step(self, *args, **kwargs): else: super(EB_GCC, self).install_step(*args, **kwargs) - def post_install_step(self, *args, **kwargs): + def post_processing_step(self, *args, **kwargs): """ Post-processing after installation: add symlinks for cc, c++, f77, f95 """ - super(EB_GCC, self).post_install_step(*args, **kwargs) + super(EB_GCC, self).post_processing_step(*args, **kwargs) # Add symlinks for cc/c++/f77/f95. bindir = os.path.join(self.installdir, 'bin') diff --git a/easybuild/easyblocks/generic/binary.py b/easybuild/easyblocks/generic/binary.py index c241946620..c0cd005266 100644 --- a/easybuild/easyblocks/generic/binary.py +++ b/easybuild/easyblocks/generic/binary.py @@ -131,7 +131,7 @@ def install_step(self): raise EasyBuildError("Incorrect value type for install_cmds, should be list or tuple: ", install_cmds) - def post_install_step(self): + def post_processing_step(self): """Copy installation to actual installation directory in case of a staged installation.""" if self.cfg.get('staged_install', False): staged_installdir = self.installdir @@ -145,7 +145,7 @@ def post_install_step(self): raise EasyBuildError("Failed to move staged install from %s to %s: %s", staged_installdir, self.installdir, err) - super(Binary, self).post_install_step() + super(Binary, self).post_processing_step() def sanity_check_rpath(self): """Skip the rpath sanity check, this is binary software""" diff --git a/easybuild/easyblocks/generic/systemcompiler.py b/easybuild/easyblocks/generic/systemcompiler.py index a95991ddda..c901cc86c7 100644 --- a/easybuild/easyblocks/generic/systemcompiler.py +++ b/easybuild/easyblocks/generic/systemcompiler.py @@ -270,7 +270,7 @@ def make_module_extra(self, *args, **kwargs): extras = super(SystemCompiler, self).make_module_extra(*args, **kwargs) return extras - def post_install_step(self, *args, **kwargs): + def post_processing_step(self, *args, **kwargs): """Do nothing.""" pass diff --git a/easybuild/easyblocks/generic/systemmpi.py b/easybuild/easyblocks/generic/systemmpi.py index c98381cc40..ee72758bca 100644 --- a/easybuild/easyblocks/generic/systemmpi.py +++ b/easybuild/easyblocks/generic/systemmpi.py @@ -221,7 +221,7 @@ def make_installdir(self, dontcreate=None): """Custom implementation of make installdir: do nothing, do not touch system MPI directories and files.""" pass - def post_install_step(self): + def post_processing_step(self): """Do nothing.""" pass diff --git a/easybuild/easyblocks/h/hadoop.py b/easybuild/easyblocks/h/hadoop.py index cf59603333..dc5069c6db 100644 --- a/easybuild/easyblocks/h/hadoop.py +++ b/easybuild/easyblocks/h/hadoop.py @@ -76,7 +76,7 @@ def install_step(self): else: super(EB_Hadoop, self).install_step() - def post_install_step(self): + def post_processing_step(self): """After the install, copy the extra native libraries into place.""" for native_library, lib_path in self.cfg['extra_native_libs']: lib_root = get_software_root(native_library) diff --git a/easybuild/easyblocks/i/imkl.py b/easybuild/easyblocks/i/imkl.py index 8234626f27..eebbb6081f 100644 --- a/easybuild/easyblocks/i/imkl.py +++ b/easybuild/easyblocks/i/imkl.py @@ -348,11 +348,11 @@ def build_mkl_flexiblas(self, flexiblasdir): if res.exit_code: raise EasyBuildError("Building FlexiBLAS-compatible library (cmd: %s) failed", cmd) - def post_install_step(self): + def post_processing_step(self): """ Install group libraries and interfaces (if desired). """ - super(EB_imkl, self).post_install_step() + super(EB_imkl, self).post_processing_step() # extract examples examples_subdir = os.path.join(self.installdir, self.mkl_basedir, self.examples_subdir) diff --git a/easybuild/easyblocks/i/imkl_fftw.py b/easybuild/easyblocks/i/imkl_fftw.py index 18cd02246b..ba9cfc39d9 100644 --- a/easybuild/easyblocks/i/imkl_fftw.py +++ b/easybuild/easyblocks/i/imkl_fftw.py @@ -64,9 +64,9 @@ def make_module_extra(self): # bypass extra module variables for imkl return super(EB_imkl, self).make_module_extra() - def post_install_step(self): + def post_processing_step(self): """Custom post install step for imkl-FFTW""" - # bypass post_install_step of imkl easyblock + # bypass post_processing_step of imkl easyblock pass def sanity_check_step(self): diff --git a/easybuild/easyblocks/i/impi.py b/easybuild/easyblocks/i/impi.py index ef61bf4087..429ea36139 100644 --- a/easybuild/easyblocks/i/impi.py +++ b/easybuild/easyblocks/i/impi.py @@ -164,9 +164,9 @@ def install_step(self): else: raise EasyBuildError("Rebuild of libfabric is requested, but ofi_internal is set to False.") - def post_install_step(self): + def post_processing_step(self): """Custom post install step for IMPI, fix broken env scripts after moving installed files.""" - super(EB_impi, self).post_install_step() + super(EB_impi, self).post_processing_step() impiver = LooseVersion(self.version) diff --git a/easybuild/easyblocks/j/java.py b/easybuild/easyblocks/j/java.py index a835f671e2..441c5e8501 100644 --- a/easybuild/easyblocks/j/java.py +++ b/easybuild/easyblocks/j/java.py @@ -87,12 +87,12 @@ def install_step(self): else: PackedBinary.install_step(self) - def post_install_step(self): + def post_processing_step(self): """ Custom post-installation step: - ensure correct glibc is used when installing into custom sysroot and using RPATH """ - super(EB_Java, self).post_install_step() + super(EB_Java, self).post_processing_step() # patch binaries and libraries when using alternate sysroot in combination with RPATH sysroot = build_option('sysroot') diff --git a/easybuild/easyblocks/m/mathematica.py b/easybuild/easyblocks/m/mathematica.py index cc6b68608b..b993851ede 100644 --- a/easybuild/easyblocks/m/mathematica.py +++ b/easybuild/easyblocks/m/mathematica.py @@ -109,7 +109,7 @@ def install_step(self): if orig_display is not None: os.environ['DISPLAY'] = orig_display - def post_install_step(self): + def post_processing_step(self): """Activate installation by using activation key, if provided.""" if self.cfg['activation_key']: # activation key is printed by using '$ActivationKey' in Mathematica, so no reason to keep it 'secret' @@ -126,7 +126,7 @@ def post_install_step(self): else: self.log.info("No activation key provided, so skipping activation of the installation.") - super(EB_Mathematica, self).post_install_step() + super(EB_Mathematica, self).post_processing_step() def sanity_check_step(self): """Custom sanity check for Mathematica.""" diff --git a/easybuild/easyblocks/m/metagenome_atlas.py b/easybuild/easyblocks/m/metagenome_atlas.py index 2391595cee..1899522dd2 100644 --- a/easybuild/easyblocks/m/metagenome_atlas.py +++ b/easybuild/easyblocks/m/metagenome_atlas.py @@ -39,7 +39,7 @@ class EB_Metagenome_Atlas(PythonPackage): Support for building/installing Metagenome-Atlas. """ - def post_install_step(self): + def post_processing_step(self): """Create snakemake config files""" # https://metagenome-atlas.readthedocs.io/en/latest/usage/getting_started.html#set-up-of-cluster-execution diff --git a/easybuild/easyblocks/p/perl.py b/easybuild/easyblocks/p/perl.py index 159f755273..52a72950a4 100644 --- a/easybuild/easyblocks/p/perl.py +++ b/easybuild/easyblocks/p/perl.py @@ -155,7 +155,7 @@ def prepare_for_extensions(self): # from specified sysroot rather than from host OS setvar('OPENSSL_PREFIX', sysroot) - def post_install_step(self, *args, **kwargs): + def post_processing_step(self, *args, **kwargs): """ Custom post-installation step for Perl: avoid excessive long shebang lines in Perl scripts. """ @@ -177,7 +177,7 @@ def post_install_step(self, *args, **kwargs): # specify pattern for paths (relative to install dir) of files for which shebang should be patched self.cfg['fix_perl_shebang_for'] = 'bin/*' - super(EB_Perl, self).post_install_step(*args, **kwargs) + super(EB_Perl, self).post_processing_step(*args, **kwargs) def sanity_check_step(self): """Custom sanity check for Perl.""" From 4b0522acf511f0e67edad4635b473d054ca40155 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Andr=C3=A9=20Reuter?= Date: Wed, 4 Dec 2024 20:41:02 +0100 Subject: [PATCH 27/50] Bundle: Replace append with extend when copying reqs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jan André Reuter --- easybuild/easyblocks/generic/bundle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/generic/bundle.py b/easybuild/easyblocks/generic/bundle.py index caaad68cd2..62228fd5a0 100644 --- a/easybuild/easyblocks/generic/bundle.py +++ b/easybuild/easyblocks/generic/bundle.py @@ -348,7 +348,7 @@ def make_module_req_guess(self): if isinstance(value, string_type): value = [value] final_reqs.setdefault(key, []) - final_reqs[key].append(value) + final_reqs[key].extend(value) except AttributeError: raise EasyBuildError("Cannot process module requirements of bundle component %s v%s", cfg['name'], cfg['version']) From 9bb5841bdf5afa3cfe496ccc2e4b49b3a9e8d947 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Fri, 6 Dec 2024 19:10:24 +0000 Subject: [PATCH 28/50] use re.findall --- easybuild/easyblocks/generic/rpackage.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/easybuild/easyblocks/generic/rpackage.py b/easybuild/easyblocks/generic/rpackage.py index 1fcca38db8..bb7e6715eb 100644 --- a/easybuild/easyblocks/generic/rpackage.py +++ b/easybuild/easyblocks/generic/rpackage.py @@ -183,16 +183,10 @@ def check_install_output(self, output): """ Check output of installation command, and clean up installation if needed. """ - reg = re.compile("^ERROR:", re.I) - - errors = [] - for line in output.split('\n'): - r = reg.search(line) - if r: - errors.append([line, r.groups()]) + errors = re.findall(r"^ERROR:.*", output, flags=re.I|re.M) if errors: - self.log.info("R package %s failed with error:\n%s", self.name, '\n'.join([x[0] for x in errors])) + self.log.info("R package %s failed with error:\n%s", self.name, '\n'.join(errors)) self.handle_installation_errors() cmd = "R -q --no-save" stdin = """ From 63719f37a41025031512795595d633a649266ce9 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Fri, 6 Dec 2024 19:11:05 +0000 Subject: [PATCH 29/50] hound --- easybuild/easyblocks/generic/rpackage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/generic/rpackage.py b/easybuild/easyblocks/generic/rpackage.py index bb7e6715eb..6ca009de95 100644 --- a/easybuild/easyblocks/generic/rpackage.py +++ b/easybuild/easyblocks/generic/rpackage.py @@ -183,7 +183,7 @@ def check_install_output(self, output): """ Check output of installation command, and clean up installation if needed. """ - errors = re.findall(r"^ERROR:.*", output, flags=re.I|re.M) + errors = re.findall(r"^ERROR:.*", output, flags=re.I | re.M) if errors: self.log.info("R package %s failed with error:\n%s", self.name, '\n'.join(errors)) From b6fedfe480eb3d6642b500aa8f86ed1734ce9bce Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Fri, 13 Dec 2024 16:53:25 +0100 Subject: [PATCH 30/50] show path of output file for matlab --- easybuild/easyblocks/m/matlab.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/m/matlab.py b/easybuild/easyblocks/m/matlab.py index 56475545cd..8cf8aef6e8 100644 --- a/easybuild/easyblocks/m/matlab.py +++ b/easybuild/easyblocks/m/matlab.py @@ -206,8 +206,8 @@ def install_step(self): regex.pattern, cmd, out) with open(self.outputfile) as f: if regex.search(f.read()): - raise EasyBuildError("Found error pattern '%s' in output file of installer", - regex.pattern) + raise EasyBuildError("Found error pattern '%s' in output file of installer at %s", + regex.pattern, self.outputfile) def sanity_check_step(self): """Custom sanity check for MATLAB.""" From f536db32b965c302cd58b6d8445d633f959957f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Sun, 15 Dec 2024 17:07:07 +0100 Subject: [PATCH 31/50] Remove backwards compat for module only rebuilds. Also uses property to clean up small code duplication --- easybuild/easyblocks/p/python.py | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/easybuild/easyblocks/p/python.py b/easybuild/easyblocks/p/python.py index 0d2b61d106..8cb78b2bcf 100644 --- a/easybuild/easyblocks/p/python.py +++ b/easybuild/easyblocks/p/python.py @@ -478,6 +478,10 @@ def build_step(self, *args, **kwargs): super(EB_Python, self).build_step(*args, **kwargs) + @property + def site_packages_path(self): + return os.path.join('lib', 'python' + self.pyshortver, 'site-packages') + def install_step(self): """Extend make install to make sure that the 'python' command is present.""" @@ -501,8 +505,7 @@ def install_step(self): symlink('pip' + self.pyshortver, pip_binary_path, use_abspath_source=False) if self.cfg.get('ebpythonprefixes'): - site_packages_path = os.path.join('lib', 'python' + self.pyshortver, 'site-packages') - write_file(os.path.join(self.installdir, site_packages_path, 'sitecustomize.py'), SITECUSTOMIZE) + write_file(os.path.join(self.installdir, self.site_packages_path, 'sitecustomize.py'), SITECUSTOMIZE) # symlink lib/python*/lib-dynload to lib64/python*/lib-dynload if it doesn't exist; # see https://github.com/easybuilders/easybuild-easyblocks/issues/1957 @@ -523,8 +526,7 @@ def install_step(self): def _sanity_check_ebpythonprefixes(self): """Check that EBPYTHONPREFIXES works""" temp_prefix = tempfile.mkdtemp(suffix='-tmp-prefix') - site_packages_path = os.path.join('lib', 'python' + self.pyshortver, 'site-packages') - temp_site_packages_path = os.path.join(temp_prefix, site_packages_path) + temp_site_packages_path = os.path.join(temp_prefix, self.site_packages_path) mkdir(temp_site_packages_path, parents=True) # Must exist res = run_shell_cmd("%s=%s python -c 'import sys; print(sys.path)'" % (EBPYTHONPREFIXES, temp_prefix)) out = res.output.strip() @@ -532,7 +534,7 @@ def _sanity_check_ebpythonprefixes(self): if not out.startswith('[') or not out.endswith(']'): raise EasyBuildError("Unexpected output for sys.path: %s", out) paths = eval(out) - base_site_packages_path = os.path.join(self.installdir, site_packages_path) + base_site_packages_path = os.path.join(self.installdir, self.site_packages_path) try: base_prefix_idx = paths.index(base_site_packages_path) except ValueError: @@ -633,16 +635,3 @@ def sanity_check_step(self): raise EasyBuildError("Expected to find exactly one _tkinter*.so: %s", tkinter_so_hits) super(EB_Python, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) - - def make_module_extra(self, *args, **kwargs): - """Add path to sitecustomize.py to $PYTHONPATH""" - txt = super(EB_Python, self).make_module_extra() - - # Legacy support for existing installations doing "--rebuild --module-only" - if self.cfg.get('ebpythonprefixes'): - new_dir = os.path.join('lib', 'python' + self.pyshortver, 'site-packages') - old_dir = os.path.join(log_path(), 'python') - if not os.path.exists(os.path.join(self.installdir, new_dir, 'sitecustomize.py')): - txt += self.module_generator.prepend_paths(PYTHONPATH, old_dir) - - return txt From a2bc105836cb14e2c3fc31e57d42d025c90d6c8a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 18 Dec 2024 08:51:44 +0100 Subject: [PATCH 32/50] use "options" rather than "arguments" in help text for unrecognized_configure_options custom easyconfig parameter --- easybuild/easyblocks/generic/configuremake.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/generic/configuremake.py b/easybuild/easyblocks/generic/configuremake.py index 33fbea7b5b..adbc12d385 100644 --- a/easybuild/easyblocks/generic/configuremake.py +++ b/easybuild/easyblocks/generic/configuremake.py @@ -197,7 +197,7 @@ def extra_options(extra_vars=None): 'test_cmd': [None, "Test command to use ('runtest' value is appended, default: '%s')" % DEFAULT_TEST_CMD, CUSTOM], 'unrecognized_configure_options': [ERROR, - "Action to do when unrecognized arguments passed to ./configure are" + "Action to do when unrecognized options passed to ./configure are" " detected, defaults to aborting the build. Can be set to '" + WARN + "' or '" + IGNORE + "' (NOT RECOMMENDED! It might hide actual errors" " e.g. misspelling of intended or changed options)", CUSTOM], From 51b25e537df326b383b6d85ac70a5672c4046455 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Wed, 18 Dec 2024 09:13:19 +0100 Subject: [PATCH 33/50] Drop unused imports --- easybuild/easyblocks/p/python.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/p/python.py b/easybuild/easyblocks/p/python.py index 8cb78b2bcf..824a44dd11 100644 --- a/easybuild/easyblocks/p/python.py +++ b/easybuild/easyblocks/p/python.py @@ -45,7 +45,7 @@ from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.easyconfig.templates import PYPI_SOURCE from easybuild.tools.build_log import EasyBuildError, print_warning -from easybuild.tools.config import build_option, ERROR, log_path, PYTHONPATH, EBPYTHONPREFIXES +from easybuild.tools.config import build_option, ERROR, EBPYTHONPREFIXES from easybuild.tools.modules import get_software_libdir, get_software_root, get_software_version from easybuild.tools.filetools import apply_regex_substitutions, change_dir, mkdir from easybuild.tools.filetools import read_file, remove_dir, symlink, write_file From 6c04a20b7bed3a2531751b4152dfd2169348686c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 18 Dec 2024 15:21:55 +0100 Subject: [PATCH 34/50] remove unused custom easyblock for TAU --- easybuild/easyblocks/t/tau.py | 280 ---------------------------------- 1 file changed, 280 deletions(-) delete mode 100644 easybuild/easyblocks/t/tau.py diff --git a/easybuild/easyblocks/t/tau.py b/easybuild/easyblocks/t/tau.py deleted file mode 100644 index 2cb7598656..0000000000 --- a/easybuild/easyblocks/t/tau.py +++ /dev/null @@ -1,280 +0,0 @@ -## -# Copyright 2009-2024 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -EasyBuild support for building and installing TAU, implemented as an easyblock - -@author Kenneth Hoste (Ghent University) -@author Markus Geimer (Juelich Supercomputing Centre) -@author Bernd Mohr (Juelich Supercomputing Centre) -""" -import os - -from easybuild.easyblocks.generic.configuremake import ConfigureMake -from easybuild.easyblocks.pdt import find_arch_dir -from easybuild.framework.easyconfig import CUSTOM -from easybuild.tools import toolchain -from easybuild.tools.build_log import EasyBuildError, print_msg -from easybuild.tools.filetools import symlink -from easybuild.tools.modules import get_software_root, get_software_version -from easybuild.tools.systemtools import get_shared_lib_ext - - -KNOWN_BACKENDS = { - 'scalasca': 'Scalasca', - 'scorep': 'Score-P', - 'vampirtrace': 'Vampirtrace', -} - - -class EB_TAU(ConfigureMake): - """Support for building/installing TAU.""" - - @staticmethod - def extra_options(): - """Custom easyconfig parameters for TAU.""" - backends = "Extra TAU backends to build and install; possible values: %s" % ','.join(sorted(KNOWN_BACKENDS)) - extra_vars = { - 'extra_backends': [None, backends, CUSTOM], - 'tau_makefile': ['Makefile.tau-papi-mpi-pdt', "Name of Makefile to use in $TAU_MAKEFILE", CUSTOM], - } - return ConfigureMake.extra_options(extra_vars) - - def __init__(self, *args, **kwargs): - """Initialize TAU easyblock.""" - super(EB_TAU, self).__init__(*args, **kwargs) - - self.variant_index = 0 - self.cc, self.cxx, self.fortran = None, None, None - self.mpi_inc_dir, self.mpi_lib_dir = None, None - self.opt_pkgs_opts = None - self.variant_labels = None - - def run_all_steps(self, *args, **kwargs): - """ - Put configure options in place for the different selected backends of TAU, - for the MPI, OpenMP, and hybrid variants. - """ - if self.cfg['extra_backends'] is None: - self.cfg['extra_backends'] = [] - - # make sure selected extra backends are known - unknown_backends = [] - for backend in self.cfg['extra_backends']: - if backend not in KNOWN_BACKENDS: - unknown_backends.append(backend) - if unknown_backends: - raise EasyBuildError("Encountered unknown backends: %s", ', '.join(unknown_backends)) - - # compiler options - comp_opts = "-cc=%(cc)s -c++=%(cxx)s -fortran=%(fortran)s" - - # variant-specific options - openmp_opts = " -opari" - mpi_opts = " -mpiinc=%(mpi_inc_dir)s -mpilib=%(mpi_lib_dir)s" - - # options for optional packages - opt_pkgs_opts = " %(opt_pkgs_opts)s" - - # backend option - backend_opt = " %(backend_opt)s" - - # compose templates - mpi_tmpl = comp_opts + mpi_opts + opt_pkgs_opts + backend_opt - openmp_tmpl = comp_opts + openmp_opts + opt_pkgs_opts + backend_opt - hybrid_tmpl = comp_opts + openmp_opts + mpi_opts + opt_pkgs_opts + backend_opt - - # number of iterations: # backends + 1 (for basic) - iter_cnt = len(self.cfg['extra_backends']) + 1 - - # define list of configure options to iterate over - if self.cfg['configopts']: - raise EasyBuildError("Specifying additional configure options for TAU is not supported (yet)") - - self.cfg['configopts'] = [mpi_tmpl, openmp_tmpl, hybrid_tmpl] * iter_cnt - self.log.debug("List of configure options to iterate over: %s", self.cfg['configopts']) - - # custom prefix option for configure command - self.cfg['prefix_opt'] = '-prefix=' - - # installation command is 'make install clean' - self.cfg.update('installopts', 'clean') - - return super(EB_TAU, self).run_all_steps(*args, **kwargs) - - def prepare_step(self, *args, **kwargs): - """Custom prepare step for Tau: check required dependencies and collect information on them.""" - super(EB_TAU, self).prepare_step(*args, **kwargs) - - # install prefixes for selected backends - self.backend_opts = {'tau': ''} - for backend_name, dep in KNOWN_BACKENDS.items(): - root = get_software_root(dep) - if backend_name in self.cfg['extra_backends']: - if root: - self.backend_opts[backend_name] = "-%s=%s" % (backend_name, root) - else: - raise EasyBuildError("%s is listed in extra_backends, but not available as a dependency", dep) - elif root: - raise EasyBuildError("%s included as dependency, but '%s' not in extra_backends", dep, backend_name) - - # make sure Scalasca v1.x is used as a dependency (if it's there) - if 'scalasca' in self.backend_opts and get_software_version('Scalasca').split('.')[0] != '1': - raise EasyBuildError("Scalasca v1.x must be used when scalasca backend is enabled") - - # determine values for compiler flags to use - known_compilers = { - toolchain.CLANGGCC: ['clang', 'clang++', 'gfortran'], - toolchain.GCC: ['gcc', 'g++', 'gfortran'], - toolchain.INTELCOMP: ['icc', 'icpc', 'intel'], - } - comp_fam = self.toolchain.comp_family() - if comp_fam in known_compilers: - self.cc, self.cxx, self.fortran = known_compilers[comp_fam] - - # determine values for MPI flags - self.mpi_inc_dir, self.mpi_lib_dir = os.getenv('MPI_INC_DIR'), os.getenv('MPI_LIB_DIR') - - # determine value for optional packages option template - self.opt_pkgs_opts = '' - for dep, opt in [('OTF', 'otf'), ('PAPI', 'papi'), ('PDT', 'pdt'), ('binutils', 'bfd')]: - root = get_software_root(dep) - if root: - self.opt_pkgs_opts += ' -%s=%s' % (opt, root) - - # determine list of labels, based on selected (extra) backends, variants and optional packages - self.variant_labels = [] - backend_labels = ['', '-epilog-scalasca-trace', '-scorep', '-vampirtrace-trace'] - for backend, backend_label in zip(['tau'] + sorted(KNOWN_BACKENDS.keys()), backend_labels): - if backend in ['tau'] + self.cfg['extra_backends']: - for pref, suff in [('-mpi', ''), ('', '-openmp-opari'), ('-mpi', '-openmp-opari')]: - - variant_label = 'tau' - # For non-GCC builds, the compiler name is encoded in the variant - if self.cxx and self.cxx != 'g++': - variant_label += '-' + self.cxx - if get_software_root('PAPI'): - variant_label += '-papi' - variant_label += pref - if get_software_root('PDT'): - variant_label += '-pdt' - variant_label += suff + backend_label - - self.variant_labels.append(variant_label) - - def make_installdir(self): - """Skip make install dir 'step', install dir is already created in prepare_step.""" - pass - - def configure_step(self): - """Custom configuration procedure for TAU: template configuration options before using them.""" - if self.cc is None or self.cxx is None or self.fortran is None: - raise EasyBuildError("Compiler family not supported yet: %s", self.toolchain.comp_family()) - - if self.mpi_inc_dir is None or self.mpi_lib_dir is None: - raise EasyBuildError("Failed to determine MPI include/library paths, no MPI available in toolchain?") - - # make sure selected default TAU makefile will be available - avail_makefiles = ['Makefile.' + x for x in self.variant_labels] - if self.cfg['tau_makefile'] not in avail_makefiles: - raise EasyBuildError("Specified tau_makefile %s will not be available (only: %s)", - self.cfg['tau_makefile'], avail_makefiles) - - # inform which backend/variant is being handled - backend = (['tau'] + self.cfg['extra_backends'])[self.variant_index // 3] - variant = ['mpi', 'openmp', 'hybrid'][self.variant_index % 3] - print_msg("starting with %s backend (%s variant)" % (backend, variant), log=self.log, silent=self.silent) - - self.cfg['configopts'] = self.cfg['configopts'] % { - 'backend_opt': self.backend_opts[backend], - 'cc': self.cc, - 'cxx': self.cxx, - 'fortran': self.fortran, - 'mpi_inc_dir': self.mpi_inc_dir, - 'mpi_lib_dir': self.mpi_lib_dir, - 'opt_pkgs_opts': self.opt_pkgs_opts, - } - - for key in ['preconfigopts', 'configopts', 'prebuildopts', 'preinstallopts']: - self.log.debug("%s for TAU (variant index: %s): %s", key, self.variant_index, self.cfg[key]) - - # Configure creates required subfolders in installdir, so create first (but only once, during first iteration) - if self.iter_idx == 0: - super(EB_TAU, self).make_installdir() - - super(EB_TAU, self).configure_step() - - self.variant_index += 1 - - def build_step(self): - """No custom build procedure for TAU.""" - pass - - def install_step(self): - """Create symlinks into arch-specific directories""" - super(EB_TAU, self).install_step() - # Link arch-specific directories into prefix - arch_dir = find_arch_dir(self.installdir) - self.log.info('Found %s as architecture specific directory. Creating symlinks...', arch_dir) - for subdir in ('bin', 'lib'): - src = os.path.join(arch_dir, subdir) - dst = os.path.join(self.installdir, subdir) - if os.path.lexists(dst): - self.log.info('Skipping creation of symlink %s as it already exists', dst) - else: - symlink(os.path.relpath(src, self.installdir), dst, use_abspath_source=False) - - def sanity_check_step(self): - """Custom sanity check for TAU.""" - custom_paths = { - 'files': - [os.path.join('bin', 'pprof'), os.path.join('include', 'TAU.h'), - os.path.join('lib', 'libTAU.%s' % get_shared_lib_ext())] + - [os.path.join('lib', 'lib%s.a' % x) for x in self.variant_labels] + - [os.path.join('lib', 'Makefile.' + x) for x in self.variant_labels], - 'dirs': [], - } - super(EB_TAU, self).sanity_check_step(custom_paths=custom_paths) - - def make_module_extra(self): - """Custom extra module file entries for TAU.""" - txt = super(EB_TAU, self).make_module_extra() - - txt += self.module_generator.prepend_paths('TAU_MF_DIR', 'lib') - - tau_makefile = os.path.join(self.installdir, 'lib', self.cfg['tau_makefile']) - txt += self.module_generator.set_environment('TAU_MAKEFILE', tau_makefile) - - # default measurement settings - tau_vars = { - 'TAU_CALLPATH': '1', - 'TAU_CALLPATH_DEPTH': '10', - 'TAU_COMM_MATRIX': '1', - 'TAU_PROFILE': '1', - 'TAU_TRACE': '0', - } - for key in tau_vars: - txt += self.module_generator.set_environment(key, tau_vars[key]) - - return txt From 9cf1f2f52ec4474fbf5bfeff9ad3cc22e9edf956 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 18 Dec 2024 15:30:18 +0100 Subject: [PATCH 35/50] clean up custom easyblock for Paraver, only support Paravar >= v4.7 --- easybuild/easyblocks/p/paraver.py | 90 +++++-------------------------- 1 file changed, 13 insertions(+), 77 deletions(-) diff --git a/easybuild/easyblocks/p/paraver.py b/easybuild/easyblocks/p/paraver.py index 4e4b9c6390..6c1f8c3971 100644 --- a/easybuild/easyblocks/p/paraver.py +++ b/easybuild/easyblocks/p/paraver.py @@ -34,46 +34,19 @@ from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake -from easybuild.tools.build_log import EasyBuildError, print_msg -from easybuild.tools.filetools import change_dir +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.modules import get_software_libdir, get_software_root class EB_Paraver(ConfigureMake): """Support for building/installing Paraver.""" - def run_all_steps(self, *args, **kwargs): - """ - Put configure/build/install options in place for the 3 different components of Paraver. - Each component lives in a separate subdirectory. - """ - if LooseVersion(self.version) < LooseVersion('4.7'): - - # leverage support for iterated installation for older Paraver versions - self.components = ['ptools_common_files', 'paraver-kernel', 'wxparaver'] - self.current_component = 0 # index in list above - - # initiate configopts with empty list - self.cfg['configopts'] = [] - - # first phase: build and install ptools - # no specific configure options for the ptools component (but configopts list element must be there) - self.cfg.update('configopts', ['']) - - # second phase: build and install paraver-kernel - self.cfg.update('configopts', ["--with-boost=%(boost)s --with-ptools-common-files=%(installdir)s"]) + def __init__(self, *args, **kwargs): + """Constructor for custom easyblock for Paraver.""" + super(EB_Paraver, self).__init__(*args, **kwargs) - # third phase: build and install wxparaver - wxparaver_configopts = ' '.join([ - '--with-boost=%(boost)s', - '--with-wxpropgrid=%(wxpropgrid)s', - '--with-paraver=%(installdir)s', - ]) - self.cfg.update('configopts', [wxparaver_configopts]) - else: - self.components, self.current_component = None, None - - return super(EB_Paraver, self).run_all_steps(*args, **kwargs) + if LooseVersion(self.version) < LooseVersion('4.7'): + raise EasyBuildError("Custom easyblock for Paraver only supports Paraver versions >= 4.7") def configure_step(self): """Custom configuration procedure for Paraver: template configuration options before using them.""" @@ -88,8 +61,6 @@ def configure_step(self): wxwidgets = get_software_root('wxWidgets') if wxwidgets: wx_config = os.path.join(wxwidgets, 'bin', 'wx-config') - elif LooseVersion(self.version) >= LooseVersion('4.7'): - raise EasyBuildError("wxWidgets is not available as a dependency") # determine value to pass to --with-wxpropgrid (library name) wxpropgrid = None @@ -107,53 +78,18 @@ def configure_step(self): else: self.log.info("wxPropertyGrid not included as dependency, assuming that's OK...") - if LooseVersion(self.version) < LooseVersion('4.7'): - component = self.components[self.current_component] - change_dir(component) - self.log.info("Customized start directory for component %s: %s", component, os.getcwd()) - - print_msg("starting with component %s" % component, log=self.log) - - self.cfg['configopts'] = self.cfg['configopts'] % { - 'boost': boost_root, - 'installdir': self.installdir, - 'wxpropgrid': wxpropgrid, - } - else: - self.cfg.update('configopts', '--with-boost=%s' % boost_root) - self.cfg.update('configopts', '--with-paraver=%s' % self.installdir) - self.cfg.update('configopts', '--with-wx-config=%s' % wx_config) - # wxPropertyGrid is not required with recent wxWidgets - if wxpropgrid: - self.cfg.update('configopts', '--with-wxpropgrid=%s' % wxpropgrid) + self.cfg.update('configopts', '--with-boost=%s' % boost_root) + self.cfg.update('configopts', '--with-paraver=%s' % self.installdir) + self.cfg.update('configopts', '--with-wx-config=%s' % wx_config) + # wxPropertyGrid is not required with recent wxWidgets + if wxpropgrid: + self.cfg.update('configopts', '--with-wxpropgrid=%s' % wxpropgrid) super(EB_Paraver, self).configure_step() def build_step(self): """Custom build procedure for Paraver: skip 'make' for recent versions.""" - - if LooseVersion(self.version) < LooseVersion('4.7'): - super(EB_Paraver, self).build_step() - - def install_step(self): - """Custom installation procedure for Paraver: put symlink in place for library subdirectory.""" - super(EB_Paraver, self).install_step() - - if LooseVersion(self.version) < LooseVersion('4.7'): - # link lib to lib64 if needed - # this is a workaround for an issue with libtool which sometimes creates lib64 rather than lib - if self.components[self.current_component] == self.components[0]: - lib64dir = os.path.join(self.installdir, 'lib64') - libdir = os.path.join(self.installdir, 'lib') - - if os.path.exists(lib64dir): - try: - self.log.debug("Symlinking %s to %s", lib64dir, libdir) - os.symlink(lib64dir, libdir) - except OSError as err: - raise EasyBuildError("Symlinking lib64 to lib failed: %s" % err) - - self.current_component += 1 + pass def sanity_check_step(self): """Custom sanity check for Paraver.""" From b875ce7e9f8f0cdba5360a48b0e73139fa29d326 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 18 Dec 2024 15:52:00 +0100 Subject: [PATCH 36/50] fix docstring for `build_step` method in custom easyblock for Paraver Co-authored-by: Simon Branford <4967+branfosj@users.noreply.github.com> --- easybuild/easyblocks/p/paraver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/p/paraver.py b/easybuild/easyblocks/p/paraver.py index 6c1f8c3971..a7f0a453a4 100644 --- a/easybuild/easyblocks/p/paraver.py +++ b/easybuild/easyblocks/p/paraver.py @@ -88,7 +88,7 @@ def configure_step(self): super(EB_Paraver, self).configure_step() def build_step(self): - """Custom build procedure for Paraver: skip 'make' for recent versions.""" + """No build ('make') required for recent versions.""" pass def sanity_check_step(self): From ee1ee1473d45b16e2f63d97f9864ac0f5db017e0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 18 Dec 2024 15:53:10 +0100 Subject: [PATCH 37/50] add back raising of an error if wxWidgets is not a dependency for Paraver --- easybuild/easyblocks/p/paraver.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easybuild/easyblocks/p/paraver.py b/easybuild/easyblocks/p/paraver.py index a7f0a453a4..83631eb1d1 100644 --- a/easybuild/easyblocks/p/paraver.py +++ b/easybuild/easyblocks/p/paraver.py @@ -61,6 +61,8 @@ def configure_step(self): wxwidgets = get_software_root('wxWidgets') if wxwidgets: wx_config = os.path.join(wxwidgets, 'bin', 'wx-config') + else: + raise EasyBuildError("wxWidgets is not available as a dependency") # determine value to pass to --with-wxpropgrid (library name) wxpropgrid = None From f0254270237f4034b07b1cd12b24085914bf587a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 18 Dec 2024 16:16:25 +0100 Subject: [PATCH 38/50] take into account that custom easyblock for Paraver requires v4.7+ in easyblocks tests --- test/easyblocks/init_easyblocks.py | 3 +++ test/easyblocks/module.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/test/easyblocks/init_easyblocks.py b/test/easyblocks/init_easyblocks.py index 68331218bd..0514ef8f34 100644 --- a/test/easyblocks/init_easyblocks.py +++ b/test/easyblocks/init_easyblocks.py @@ -237,6 +237,9 @@ def innertest(self): elif easyblock_fn == 'openssl_wrapper.py': # easyblock to create OpenSSL wrapper expects an OpenSSL version innertest = make_inner_test(easyblock, version='1.1') + elif easyblock_fn == 'paraver.py': + # custom easyblock for Paraver requires version >= 4.7 + innertest = make_inner_test(easyblock, version='4.8') elif easyblock_fn == 'torchvision.py': # torchvision easyblock requires that PyTorch is listed as dependency innertest = make_inner_test(easyblock, name='torchvision', deps=[('PyTorch', '1.12.1')]) diff --git a/test/easyblocks/module.py b/test/easyblocks/module.py index 7b9c6d612f..173581f7de 100644 --- a/test/easyblocks/module.py +++ b/test/easyblocks/module.py @@ -464,6 +464,9 @@ def innertest(self): elif eb_fn == 'openssl_wrapper.py': # easyblock to create OpenSSL wrapper expects an OpenSSL version innertest = make_inner_test(easyblock, name='OpenSSL-wrapper', version='1.1') + elif eb_fn == 'paraver.py': + # custom easyblock for Paraver requires version >= 4.7 + innertest = make_inner_test(easyblock, name='Paraver', version='4.8') elif eb_fn == 'torchvision.py': # torchvision easyblock requires that PyTorch is listed as dependency extra_txt = "dependencies = [('PyTorch', '1.12.1')]" From c0b5b5b4c7e697b7f813b47b22a3da4bb9042b7e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 18 Dec 2024 16:56:37 +0100 Subject: [PATCH 39/50] bump version to 5.0.0beta1 --- easybuild/easyblocks/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/__init__.py b/easybuild/easyblocks/__init__.py index dbd03779f7..5eec3996b5 100644 --- a/easybuild/easyblocks/__init__.py +++ b/easybuild/easyblocks/__init__.py @@ -42,7 +42,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = '5.0.0.dev0' +VERSION = '5.0.0beta1' UNKNOWN = 'UNKNOWN' From 8b6fda4df38d93f33265f57f73a3bd608ef5955f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 19 Dec 2024 08:41:32 +0100 Subject: [PATCH 40/50] be more careful when checking install_target value in PythonPackage easyblock, to avoid trouble with unresolved template values like '%(parallel)s' --- easybuild/easyblocks/generic/pythonpackage.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 4b76d7e14b..921b6e3ff8 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -546,12 +546,13 @@ def determine_install_command(self): else: self.use_setup_py = True self.install_cmd = SETUP_PY_INSTALL_CMD + install_target = self.cfg.get_ref('install_target') - if self.cfg['install_target'] == EASY_INSTALL_TARGET: + if install_target == EASY_INSTALL_TARGET: self.install_cmd += " %(loc)s" self.py_installopts.append('--no-deps') if self.cfg.get('zipped_egg', False): - if self.cfg['install_target'] == EASY_INSTALL_TARGET: + if install_target == EASY_INSTALL_TARGET: self.py_installopts.append('--zip-ok') else: raise EasyBuildError("Installing zipped eggs requires using easy_install or pip") From 88d0a81f6a059a401f7d5a0a7eb619eff1a83787 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 19 Dec 2024 08:42:11 +0100 Subject: [PATCH 41/50] be more careful with determining value of 'sources' and 'source_urls' in constructor of Bundle easyblock, to avoid trouble with unresolved template values like '%(builddir)s' --- easybuild/easyblocks/generic/bundle.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/easybuild/easyblocks/generic/bundle.py b/easybuild/easyblocks/generic/bundle.py index 437cd707d4..c15c0bb44d 100644 --- a/easybuild/easyblocks/generic/bundle.py +++ b/easybuild/easyblocks/generic/bundle.py @@ -164,24 +164,26 @@ def __init__(self, *args, **kwargs): comp_cfg.enable_templating = True # 'sources' is strictly required - if comp_cfg['sources']: + comp_sources = comp_cfg.get_ref('sources') + if comp_sources: # If per-component source URLs are provided, attach them directly to the relevant sources - if comp_cfg['source_urls']: - for source in comp_cfg['sources']: + comp_source_urls = comp_cfg.get_ref('source_urls') + if comp_source_urls: + for source in comp_sources: if isinstance(source, str): - self.cfg.update('sources', [{'filename': source, 'source_urls': comp_cfg['source_urls']}]) + self.cfg.update('sources', [{'filename': source, 'source_urls': comp_source_urls[:]}]) elif isinstance(source, dict): # Update source_urls in the 'source' dict to use the one for the components # (if it doesn't already exist) if 'source_urls' not in source: - source['source_urls'] = comp_cfg['source_urls'] + source['source_urls'] = comp_source_urls[:] self.cfg.update('sources', [source]) else: raise EasyBuildError("Source %s for component %s is neither a string nor a dict, cannot " "process it.", source, comp_cfg['name']) else: # add component sources to list of sources - self.cfg.update('sources', comp_cfg['sources']) + self.cfg.update('sources', comp_sources) else: raise EasyBuildError("No sources specification for component %s v%s", comp_name, comp_version) From 1973cbb0fe8c7f94d6e131e66e3808cb93813ec9 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 19 Dec 2024 09:43:16 +0100 Subject: [PATCH 42/50] be even more careful when determining easyconfig parameter values in Bundle constructor, to avoid trouble with unresolved templates --- easybuild/easyblocks/generic/bundle.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/easybuild/easyblocks/generic/bundle.py b/easybuild/easyblocks/generic/bundle.py index c15c0bb44d..1c41ab9f82 100644 --- a/easybuild/easyblocks/generic/bundle.py +++ b/easybuild/easyblocks/generic/bundle.py @@ -79,9 +79,9 @@ def __init__(self, *args, **kwargs): check_for_sources = getattr(self, 'check_for_sources', True) # list of sources for bundle itself *must* be empty (unless overridden by subclass) if check_for_sources: - if self.cfg['sources']: + if self.cfg.get_ref('sources'): raise EasyBuildError("List of sources for bundle itself must be empty, found %s", self.cfg['sources']) - if self.cfg['patches']: + if self.cfg.get_ref('patches'): raise EasyBuildError("List of patches for bundle itself must be empty, found %s", self.cfg['patches']) # copy EasyConfig instance before we make changes to it @@ -187,29 +187,31 @@ def __init__(self, *args, **kwargs): else: raise EasyBuildError("No sources specification for component %s v%s", comp_name, comp_version) - if comp_cfg['checksums']: - src_cnt = len(comp_cfg['sources']) + comp_checksums = comp_cfg.get_ref('checksums') + if comp_checksums: + src_cnt = len(comp_sources) # add per-component checksums for sources to list of checksums - self.cfg.update('checksums', comp_cfg['checksums'][:src_cnt]) + self.cfg.update('checksums', comp_checksums[:src_cnt]) # add per-component checksums for patches to list of checksums for patches - checksums_patches.extend(comp_cfg['checksums'][src_cnt:]) + checksums_patches.extend(comp_checksums[src_cnt:]) - if comp_cfg['patches']: - self.cfg.update('patches', comp_cfg['patches']) + comp_patches = comp_cfg.get_ref('patches') + if comp_patches: + self.cfg.update('patches', comp_patches) self.comp_cfgs.append(comp_cfg) self.cfg.update('checksums', checksums_patches) - self.cfg.enable_templating = True - # restore general sanity checks if using component-specific sanity checks if self.cfg['sanity_check_components'] or self.cfg['sanity_check_all_components']: self.cfg['sanity_check_paths'] = self.backup_sanity_paths self.cfg['sanity_check_commands'] = self.backup_sanity_cmds + self.cfg.enable_templating = True + def check_checksums(self): """ Check whether a SHA256 checksum is available for all sources & patches (incl. extensions). From 46e847c8d0c2cd8b0fbf80024d76e42114557371 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 19 Dec 2024 10:33:22 +0100 Subject: [PATCH 43/50] temporarily disable expectation that all template value can be resolved in Bundle constructor --- easybuild/easyblocks/generic/bundle.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/easybuild/easyblocks/generic/bundle.py b/easybuild/easyblocks/generic/bundle.py index 1c41ab9f82..196b0e5f8b 100644 --- a/easybuild/easyblocks/generic/bundle.py +++ b/easybuild/easyblocks/generic/bundle.py @@ -160,14 +160,18 @@ def __init__(self, *args, **kwargs): for key in comp_specs: comp_cfg[key] = comp_specs[key] - # enable resolving of templates for component-specific EasyConfig instance + # enable resolving of templates for component-specific EasyConfig instance, + # but don't require that all template values can be resolved at this point; + # this is important to ensure that template values like %(name)s and %(version)s + # are correctly resolved with the component name/version before values are copied over to self.cfg comp_cfg.enable_templating = True + comp_cfg.expect_resolved_template_values = False # 'sources' is strictly required - comp_sources = comp_cfg.get_ref('sources') + comp_sources = comp_cfg['sources'] if comp_sources: # If per-component source URLs are provided, attach them directly to the relevant sources - comp_source_urls = comp_cfg.get_ref('source_urls') + comp_source_urls = comp_cfg['source_urls'] if comp_source_urls: for source in comp_sources: if isinstance(source, str): @@ -187,7 +191,7 @@ def __init__(self, *args, **kwargs): else: raise EasyBuildError("No sources specification for component %s v%s", comp_name, comp_version) - comp_checksums = comp_cfg.get_ref('checksums') + comp_checksums = comp_cfg['checksums'] if comp_checksums: src_cnt = len(comp_sources) @@ -197,10 +201,12 @@ def __init__(self, *args, **kwargs): # add per-component checksums for patches to list of checksums for patches checksums_patches.extend(comp_checksums[src_cnt:]) - comp_patches = comp_cfg.get_ref('patches') + comp_patches = comp_cfg['patches'] if comp_patches: self.cfg.update('patches', comp_patches) + comp_cfg.expect_resolved_template_values = True + self.comp_cfgs.append(comp_cfg) self.cfg.update('checksums', checksums_patches) @@ -274,6 +280,7 @@ def install_step(self): comp.src = [] # find match entries in self.src for this component + comp.cfg.expect_resolved_template_values = False for source in comp.cfg['sources']: if isinstance(source, str): comp_src_fn = source @@ -297,6 +304,7 @@ def install_step(self): # location of first unpacked source is used to determine where to apply patch(es) comp.src[-1]['finalpath'] = comp.cfg['start_dir'] + comp.cfg.expect_resolved_template_values = True # check if sanity checks are enabled for the component if self.cfg['sanity_check_all_components'] or comp.cfg['name'] in self.cfg['sanity_check_components']: From 346ecb46ae30e331b0c645400ac27365a607867f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 19 Dec 2024 10:34:01 +0100 Subject: [PATCH 44/50] be more careful when getting value for 'buildcmd' custom easyconfig parameter in PythonPackage easyblock, to avoid error due to '%(python)s' template not being resolvable --- easybuild/easyblocks/generic/pythonpackage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 921b6e3ff8..855a0c533e 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -818,7 +818,7 @@ def configure_step(self): def build_step(self): """Build Python package using setup.py""" - build_cmd = self.cfg['buildcmd'] + build_cmd = self.cfg.get_ref('buildcmd') if self.use_setup_py: if get_software_root('CMake'): From ea31440918e1493be54fea6318112f431662997d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 19 Dec 2024 17:33:53 +0100 Subject: [PATCH 45/50] inject custom '%(python)s' template value before getting value of 'buildcmd' custom easyconfig parameter in PythonPackage easyblock --- easybuild/easyblocks/generic/pythonpackage.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 855a0c533e..875158df6d 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -818,7 +818,11 @@ def configure_step(self): def build_step(self): """Build Python package using setup.py""" - build_cmd = self.cfg.get_ref('buildcmd') + + # inject extra '%(python)s' template value before getting value of 'buildcmd' custom easyconfig parameter + self.cfg.template_values['python'] = self.python_cmd + build_cmd = self.cfg['buildcmd'] + if self.use_setup_py: if get_software_root('CMake'): @@ -829,10 +833,9 @@ def build_step(self): if not build_cmd: build_cmd = 'build' # Default value for setup.py - build_cmd = '%(python)s setup.py ' + build_cmd + build_cmd = self.python_cmd + ' setup.py ' + build_cmd if build_cmd: - build_cmd = build_cmd % {'python': self.python_cmd} cmd = ' '.join([self.cfg['prebuildopts'], build_cmd, self.cfg['buildopts']]) res = run_shell_cmd(cmd) From 51ce29903ddeab2c50d3b94449e390450578d957 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Thu, 19 Dec 2024 19:45:45 +0100 Subject: [PATCH 46/50] replace concatenation with f-string in PythonPackage easyblock --- easybuild/easyblocks/generic/pythonpackage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 875158df6d..e9ea708309 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -833,7 +833,7 @@ def build_step(self): if not build_cmd: build_cmd = 'build' # Default value for setup.py - build_cmd = self.python_cmd + ' setup.py ' + build_cmd + build_cmd = f"{self.python_cmd} setup.py {build_cmd}" if build_cmd: cmd = ' '.join([self.cfg['prebuildopts'], build_cmd, self.cfg['buildopts']]) From 36fc898c556e89a388a9e45c940ad4693552bcb6 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Fri, 20 Dec 2024 13:04:12 +0000 Subject: [PATCH 47/50] remove use of deprecated `extract_errors_from_log` in `configuremake` --- easybuild/easyblocks/generic/configuremake.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/easyblocks/generic/configuremake.py b/easybuild/easyblocks/generic/configuremake.py index adbc12d385..af62797129 100644 --- a/easybuild/easyblocks/generic/configuremake.py +++ b/easybuild/easyblocks/generic/configuremake.py @@ -49,7 +49,7 @@ from easybuild.tools.config import source_paths, build_option, ERROR, IGNORE, WARN from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, adjust_permissions, compute_checksum, download_file from easybuild.tools.filetools import read_file, remove_file -from easybuild.tools.run import extract_errors_from_log, run_shell_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.utilities import nub # string that indicates that a configure script was generated by Autoconf @@ -338,8 +338,8 @@ def configure_step(self, cmd_prefix=''): raise EasyBuildError('Invalid value for `unrecognized_configure_options`: %s. Must be one of: ', action, ', '.join(valid_actions)) if action != IGNORE: - unrecognized_options_str = 'configure: WARNING: unrecognized options:' - unrecognized_options = extract_errors_from_log(res.output, unrecognized_options_str)[1] + unrecognized_options_str = "^configure: WARNING: unrecognized options:" + unrecognized_options = re.findall(rf"{unrecognized_options_str}.*", res.output, flags=re.I | re.M) # Keep only unique options (remove the warning string and strip whitespace) unrecognized_options = nub(x.split(unrecognized_options_str)[-1].strip() for x in unrecognized_options) if unrecognized_options: From f875f97de0ad3da2228589572384ff680afe81b9 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Fri, 20 Dec 2024 13:25:41 +0000 Subject: [PATCH 48/50] removing unneeded change --- easybuild/easyblocks/generic/configuremake.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/generic/configuremake.py b/easybuild/easyblocks/generic/configuremake.py index af62797129..c20563228b 100644 --- a/easybuild/easyblocks/generic/configuremake.py +++ b/easybuild/easyblocks/generic/configuremake.py @@ -338,8 +338,8 @@ def configure_step(self, cmd_prefix=''): raise EasyBuildError('Invalid value for `unrecognized_configure_options`: %s. Must be one of: ', action, ', '.join(valid_actions)) if action != IGNORE: - unrecognized_options_str = "^configure: WARNING: unrecognized options:" - unrecognized_options = re.findall(rf"{unrecognized_options_str}.*", res.output, flags=re.I | re.M) + unrecognized_options_str = 'configure: WARNING: unrecognized options:' + unrecognized_options = re.findall(rf"^{unrecognized_options_str}.*", res.output, flags=re.I | re.M) # Keep only unique options (remove the warning string and strip whitespace) unrecognized_options = nub(x.split(unrecognized_options_str)[-1].strip() for x in unrecognized_options) if unrecognized_options: From 329aa6d46122a89284b2556bc0031eefd018675d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 26 Dec 2024 19:37:22 +0100 Subject: [PATCH 49/50] use str rather than string_type in Bundle easyblock --- easybuild/easyblocks/generic/bundle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/generic/bundle.py b/easybuild/easyblocks/generic/bundle.py index 2d35acf916..4e1d9e9d3f 100644 --- a/easybuild/easyblocks/generic/bundle.py +++ b/easybuild/easyblocks/generic/bundle.py @@ -356,7 +356,7 @@ def make_module_req_guess(self): # for make_module_req_guess() which should then be fixed in the components EasyBlock. try: for key, value in sorted(reqs.items()): - if isinstance(value, string_type): + if isinstance(value, str): value = [value] final_reqs.setdefault(key, []) final_reqs[key].extend(value) From 58162cdaeae453857d34dacd57be5284adac1b85 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Mon, 30 Dec 2024 14:02:47 +0000 Subject: [PATCH 50/50] imports without the single folder letter --- easybuild/easyblocks/i/icc.py | 2 +- easybuild/easyblocks/i/intel_compilers.py | 2 +- easybuild/easyblocks/m/mamba.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/easyblocks/i/icc.py b/easybuild/easyblocks/i/icc.py index 0debf99be8..d9d31b75cb 100644 --- a/easybuild/easyblocks/i/icc.py +++ b/easybuild/easyblocks/i/icc.py @@ -40,7 +40,7 @@ from easybuild.easyblocks.generic.intelbase import IntelBase, ACTIVATION_NAME_2012, COMP_ALL from easybuild.easyblocks.generic.intelbase import LICENSE_FILE_NAME_2012 -from easybuild.easyblocks.t.tbb import get_tbb_gccprefix +from easybuild.easyblocks.tbb import get_tbb_gccprefix from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import get_shared_lib_ext diff --git a/easybuild/easyblocks/i/intel_compilers.py b/easybuild/easyblocks/i/intel_compilers.py index cfc50d6080..5b9c2dc3a6 100644 --- a/easybuild/easyblocks/i/intel_compilers.py +++ b/easybuild/easyblocks/i/intel_compilers.py @@ -31,7 +31,7 @@ from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.intelbase import IntelBase -from easybuild.easyblocks.t.tbb import get_tbb_gccprefix +from easybuild.easyblocks.tbb import get_tbb_gccprefix from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.run import run_shell_cmd diff --git a/easybuild/easyblocks/m/mamba.py b/easybuild/easyblocks/m/mamba.py index 232011e137..99947ffe4a 100644 --- a/easybuild/easyblocks/m/mamba.py +++ b/easybuild/easyblocks/m/mamba.py @@ -31,7 +31,7 @@ import os -from easybuild.easyblocks.a.anaconda import EB_Anaconda +from easybuild.easyblocks.anaconda import EB_Anaconda class EB_Mamba(EB_Anaconda):