diff --git a/easybuild/easyblocks/__init__.py b/easybuild/easyblocks/__init__.py index dbd03779f7..5eec3996b5 100644 --- a/easybuild/easyblocks/__init__.py +++ b/easybuild/easyblocks/__init__.py @@ -42,7 +42,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = '5.0.0.dev0' +VERSION = '5.0.0beta1' UNKNOWN = 'UNKNOWN' diff --git a/easybuild/easyblocks/a/aedt.py b/easybuild/easyblocks/a/aedt.py index a381e2fd24..27685a3115 100644 --- a/easybuild/easyblocks/a/aedt.py +++ b/easybuild/easyblocks/a/aedt.py @@ -71,7 +71,7 @@ def install_step(self): ]) run_shell_cmd("./Linux/AnsysEM/Disk1/InstData/setup.exe %s" % options) - def post_install_step(self): + def post_processing_step(self): """Disable OS check and set LC_ALL/LANG for runtime""" if not self.subdir: self._set_subdir() diff --git a/easybuild/easyblocks/a/aocc.py b/easybuild/easyblocks/a/aocc.py index 24da9f3c5e..b42c12d063 100644 --- a/easybuild/easyblocks/a/aocc.py +++ b/easybuild/easyblocks/a/aocc.py @@ -214,7 +214,7 @@ def install_step(self): super(EB_AOCC, self).install_step() - def post_install_step(self): + def post_processing_step(self): """ For AOCC <5.0.0: Create wrappers for the compilers to make sure compilers picks up GCCcore as GCC toolchain. @@ -245,7 +245,7 @@ def post_install_step(self): self._create_compiler_config_files(compilers_to_add_config_files) self._create_compiler_wrappers(compilers_to_wrap) - super(EB_AOCC, self).post_install_step() + super(EB_AOCC, self).post_processing_step() def sanity_check_step(self): """Custom sanity check for AOCC, based on sanity check for Clang.""" diff --git a/easybuild/easyblocks/a/aomp.py b/easybuild/easyblocks/a/aomp.py index 2a1f0472d8..f62c124b19 100644 --- a/easybuild/easyblocks/a/aomp.py +++ b/easybuild/easyblocks/a/aomp.py @@ -133,8 +133,8 @@ def configure_step(self): # Only build selected components self.cfg['installopts'] = 'select ' + ' '.join(components) - def post_install_step(self): - super(EB_AOMP, self).post_install_step() + def post_processing_step(self): + super(EB_AOMP, self).post_processing_step() # The install script will create a symbolic link as the install # directory, this creates problems for EB as it won't remove the # symlink. To remedy this we remove the link here and rename the actual diff --git a/easybuild/easyblocks/c/clang.py b/easybuild/easyblocks/c/clang.py index ddefa7cbff..3ee17819e3 100644 --- a/easybuild/easyblocks/c/clang.py +++ b/easybuild/easyblocks/c/clang.py @@ -612,9 +612,9 @@ def install_step(self): except OSError as err: raise EasyBuildError("Failed to copy static analyzer dirs to install dir: %s", err) - def post_install_step(self): + def post_processing_step(self): """Install python bindings.""" - super(EB_Clang, self).post_install_step() + super(EB_Clang, self).post_processing_step() # copy Python bindings here in post-install step so that it is not done more than once in multi_deps context if self.cfg['python_bindings']: diff --git a/easybuild/easyblocks/c/clang_aomp.py b/easybuild/easyblocks/c/clang_aomp.py index 0221c6c5a8..7237eac9c2 100644 --- a/easybuild/easyblocks/c/clang_aomp.py +++ b/easybuild/easyblocks/c/clang_aomp.py @@ -145,7 +145,7 @@ def configure_step(self): raise EasyBuildError("Could not find 'ROCm-Device-Libs' source directory in %s", self.builddir) num_comps = len(self.cfg['components']) - for idx, comp in enumerate(self.comp_cfgs): + for idx, (comp, _) in enumerate(self.comp_instances): name = comp['name'] msg = "configuring bundle component %s %s (%d/%d)..." % (name, comp['version'], idx + 1, num_comps) print_msg(msg) diff --git a/easybuild/easyblocks/c/crispr_dav.py b/easybuild/easyblocks/c/crispr_dav.py index e85da91da2..edf0e5e431 100644 --- a/easybuild/easyblocks/c/crispr_dav.py +++ b/easybuild/easyblocks/c/crispr_dav.py @@ -46,7 +46,7 @@ def __init__(self, *args, **kwargs): super(EB_CRISPR_minus_DAV, self).__init__(*args, **kwargs) self.cfg['extract_sources'] = True - def post_install_step(self): + def post_processing_step(self): """Update configuration files with correct paths to dependencies and files in installation.""" # getting paths of deps + files we will work with diff --git a/easybuild/easyblocks/c/cuda.py b/easybuild/easyblocks/c/cuda.py index b66563045e..3fa3a3a949 100644 --- a/easybuild/easyblocks/c/cuda.py +++ b/easybuild/easyblocks/c/cuda.py @@ -218,7 +218,7 @@ def install_step(self): self.log.debug("Running patch %s", patch['name']) run_shell_cmd("/bin/sh " + patch['path'] + " --accept-eula --silent --installdir=" + self.installdir) - def post_install_step(self): + def post_processing_step(self): """ Create wrappers for the specified host compilers, generate the appropriate stub symlinks, and create version independent pkgconfig files @@ -288,7 +288,7 @@ def create_wrapper(wrapper_name, wrapper_comp): symlink(pc_file, link, use_abspath_source=False) change_dir(cwd) - super(EB_CUDA, self).post_install_step() + super(EB_CUDA, self).post_processing_step() def sanity_check_step(self): """Custom sanity check for CUDA.""" diff --git a/easybuild/easyblocks/d/dualsphysics.py b/easybuild/easyblocks/d/dualsphysics.py index 5fc794a59e..964312b5c3 100644 --- a/easybuild/easyblocks/d/dualsphysics.py +++ b/easybuild/easyblocks/d/dualsphysics.py @@ -90,9 +90,9 @@ def install_step(self): ] super(EB_DualSPHysics, self).install_step() - def post_install_step(self): + def post_processing_step(self): """Custom post-installation step: ensure rpath is patched into binaries/libraries if configured.""" - super(EB_DualSPHysics, self).post_install_step() + super(EB_DualSPHysics, self).post_processing_step() if build_option('rpath'): # only the compiled binary (e.g. DualSPHysics5.0CPU_linux64) is rpath'd, the precompiled libraries diff --git a/easybuild/easyblocks/e/easybuildmeta.py b/easybuild/easyblocks/e/easybuildmeta.py index f0d24e79c7..88ad7a742f 100644 --- a/easybuild/easyblocks/e/easybuildmeta.py +++ b/easybuild/easyblocks/e/easybuildmeta.py @@ -136,10 +136,10 @@ def install_step(self): except OSError as err: raise EasyBuildError("Failed to install EasyBuild packages: %s", err) - def post_install_step(self): + def post_processing_step(self): """Remove setuptools.pth file that hard includes a system-wide (site-packages) path, if it is there.""" - super(EB_EasyBuildMeta, self).post_install_step() + super(EB_EasyBuildMeta, self).post_processing_step() setuptools_pth = os.path.join(self.installdir, self.pylibdir, 'setuptools.pth') if os.path.exists(setuptools_pth): diff --git a/easybuild/easyblocks/f/fftwmpi.py b/easybuild/easyblocks/f/fftwmpi.py index cb285a576b..00059e1e39 100644 --- a/easybuild/easyblocks/f/fftwmpi.py +++ b/easybuild/easyblocks/f/fftwmpi.py @@ -59,7 +59,7 @@ def prepare_step(self, *args, **kwargs): if not fftw_root: raise EasyBuildError("Required FFTW dependency is missing!") - def post_install_step(self): + def post_processing_step(self): """Custom post install step for FFTW.MPI""" # remove everything except include files that are already in non-MPI FFTW dependency. @@ -68,7 +68,7 @@ def post_install_step(self): glob.glob(os.path.join(self.installdir, 'lib*/pkgconfig')) + glob.glob(os.path.join(self.installdir, 'lib*/cmake')) + [os.path.join(self.installdir, p) for p in ['bin', 'share']]) - super(EB_FFTW_period_MPI, self).post_install_step() + super(EB_FFTW_period_MPI, self).post_processing_step() def sanity_check_step(self): """Custom sanity check for FFTW.MPI: check if all libraries/headers for MPI interfaces are there.""" diff --git a/easybuild/easyblocks/g/gcc.py b/easybuild/easyblocks/g/gcc.py index bb0ba20ef4..e23f260572 100644 --- a/easybuild/easyblocks/g/gcc.py +++ b/easybuild/easyblocks/g/gcc.py @@ -966,11 +966,11 @@ def install_step(self, *args, **kwargs): else: super(EB_GCC, self).install_step(*args, **kwargs) - def post_install_step(self, *args, **kwargs): + def post_processing_step(self, *args, **kwargs): """ Post-processing after installation: add symlinks for cc, c++, f77, f95 """ - super(EB_GCC, self).post_install_step(*args, **kwargs) + super(EB_GCC, self).post_processing_step(*args, **kwargs) # Add symlinks for cc/c++/f77/f95. bindir = os.path.join(self.installdir, 'bin') diff --git a/easybuild/easyblocks/g/gromacs.py b/easybuild/easyblocks/g/gromacs.py index e1b49b557d..bb7c5137e6 100644 --- a/easybuild/easyblocks/g/gromacs.py +++ b/easybuild/easyblocks/g/gromacs.py @@ -63,6 +63,8 @@ def extra_options(): extra_vars.update({ 'double_precision': [None, "Build with double precision enabled (-DGMX_DOUBLE=ON), " + "default is to build double precision unless CUDA is enabled", CUSTOM], + 'single_precision': [True, "Build with single precision enabled (-DGMX_DOUBLE=OFF), " + + "default is to build single precision", CUSTOM], 'mpisuffix': ['_mpi', "Suffix to append to MPI-enabled executables (only for GROMACS < 4.6)", CUSTOM], 'mpiexec': ['mpirun', "MPI executable to use when running tests", CUSTOM], 'mpiexec_numproc_flag': ['-np', "Flag to introduce the number of MPI tasks when running tests", CUSTOM], @@ -745,10 +747,15 @@ def run_all_steps(self, *args, **kwargs): 'mpi': 'install' } - precisions = ['single'] + precisions = [] + if self.cfg.get('single_precision'): + precisions.append('single') if self.cfg.get('double_precision') is None or self.cfg.get('double_precision'): precisions.append('double') + if precisions == []: + raise EasyBuildError("No precision selected. At least one of single/double_precision must be unset or True") + mpitypes = ['nompi'] if self.toolchain.options.get('usempi', None): mpitypes.append('mpi') diff --git a/easybuild/easyblocks/generic/binary.py b/easybuild/easyblocks/generic/binary.py index c241946620..c0cd005266 100644 --- a/easybuild/easyblocks/generic/binary.py +++ b/easybuild/easyblocks/generic/binary.py @@ -131,7 +131,7 @@ def install_step(self): raise EasyBuildError("Incorrect value type for install_cmds, should be list or tuple: ", install_cmds) - def post_install_step(self): + def post_processing_step(self): """Copy installation to actual installation directory in case of a staged installation.""" if self.cfg.get('staged_install', False): staged_installdir = self.installdir @@ -145,7 +145,7 @@ def post_install_step(self): raise EasyBuildError("Failed to move staged install from %s to %s: %s", staged_installdir, self.installdir, err) - super(Binary, self).post_install_step() + super(Binary, self).post_processing_step() def sanity_check_rpath(self): """Skip the rpath sanity check, this is binary software""" diff --git a/easybuild/easyblocks/generic/bundle.py b/easybuild/easyblocks/generic/bundle.py index 437cd707d4..4e1d9e9d3f 100644 --- a/easybuild/easyblocks/generic/bundle.py +++ b/easybuild/easyblocks/generic/bundle.py @@ -31,6 +31,7 @@ @author: Pieter De Baets (Ghent University) @author: Jens Timmerman (Ghent University) @author: Jasper Grimm (University of York) +@author: Jan Andre Reuter (Juelich Supercomputing Centre) """ import copy import os @@ -70,8 +71,8 @@ def __init__(self, *args, **kwargs): self.altroot = None self.altversion = None - # list of EasyConfig instances for components - self.comp_cfgs = [] + # list of EasyConfig instances and their EasyBlocks for components + self.comp_instances = [] # list of EasyConfig instances of components for which to run sanity checks self.comp_cfgs_sanity_check = [] @@ -79,9 +80,9 @@ def __init__(self, *args, **kwargs): check_for_sources = getattr(self, 'check_for_sources', True) # list of sources for bundle itself *must* be empty (unless overridden by subclass) if check_for_sources: - if self.cfg['sources']: + if self.cfg.get_ref('sources'): raise EasyBuildError("List of sources for bundle itself must be empty, found %s", self.cfg['sources']) - if self.cfg['patches']: + if self.cfg.get_ref('patches'): raise EasyBuildError("List of patches for bundle itself must be empty, found %s", self.cfg['patches']) # copy EasyConfig instance before we make changes to it @@ -160,54 +161,64 @@ def __init__(self, *args, **kwargs): for key in comp_specs: comp_cfg[key] = comp_specs[key] - # enable resolving of templates for component-specific EasyConfig instance + # enable resolving of templates for component-specific EasyConfig instance, + # but don't require that all template values can be resolved at this point; + # this is important to ensure that template values like %(name)s and %(version)s + # are correctly resolved with the component name/version before values are copied over to self.cfg comp_cfg.enable_templating = True + comp_cfg.expect_resolved_template_values = False # 'sources' is strictly required - if comp_cfg['sources']: + comp_sources = comp_cfg['sources'] + if comp_sources: # If per-component source URLs are provided, attach them directly to the relevant sources - if comp_cfg['source_urls']: - for source in comp_cfg['sources']: + comp_source_urls = comp_cfg['source_urls'] + if comp_source_urls: + for source in comp_sources: if isinstance(source, str): - self.cfg.update('sources', [{'filename': source, 'source_urls': comp_cfg['source_urls']}]) + self.cfg.update('sources', [{'filename': source, 'source_urls': comp_source_urls[:]}]) elif isinstance(source, dict): # Update source_urls in the 'source' dict to use the one for the components # (if it doesn't already exist) if 'source_urls' not in source: - source['source_urls'] = comp_cfg['source_urls'] + source['source_urls'] = comp_source_urls[:] self.cfg.update('sources', [source]) else: raise EasyBuildError("Source %s for component %s is neither a string nor a dict, cannot " "process it.", source, comp_cfg['name']) else: # add component sources to list of sources - self.cfg.update('sources', comp_cfg['sources']) + self.cfg.update('sources', comp_sources) else: raise EasyBuildError("No sources specification for component %s v%s", comp_name, comp_version) - if comp_cfg['checksums']: - src_cnt = len(comp_cfg['sources']) + comp_checksums = comp_cfg['checksums'] + if comp_checksums: + src_cnt = len(comp_sources) # add per-component checksums for sources to list of checksums - self.cfg.update('checksums', comp_cfg['checksums'][:src_cnt]) + self.cfg.update('checksums', comp_checksums[:src_cnt]) # add per-component checksums for patches to list of checksums for patches - checksums_patches.extend(comp_cfg['checksums'][src_cnt:]) + checksums_patches.extend(comp_checksums[src_cnt:]) - if comp_cfg['patches']: - self.cfg.update('patches', comp_cfg['patches']) + comp_patches = comp_cfg['patches'] + if comp_patches: + self.cfg.update('patches', comp_patches) - self.comp_cfgs.append(comp_cfg) + comp_cfg.expect_resolved_template_values = True - self.cfg.update('checksums', checksums_patches) + self.comp_instances.append((comp_cfg, comp_cfg.easyblock(comp_cfg, logfile=self.logfile))) - self.cfg.enable_templating = True + self.cfg.update('checksums', checksums_patches) # restore general sanity checks if using component-specific sanity checks if self.cfg['sanity_check_components'] or self.cfg['sanity_check_all_components']: self.cfg['sanity_check_paths'] = self.backup_sanity_paths self.cfg['sanity_check_commands'] = self.backup_sanity_cmds + self.cfg.enable_templating = True + def check_checksums(self): """ Check whether a SHA256 checksum is available for all sources & patches (incl. extensions). @@ -216,7 +227,7 @@ def check_checksums(self): """ checksum_issues = super(Bundle, self).check_checksums() - for comp in self.comp_cfgs: + for comp, _ in self.comp_instances: checksum_issues.extend(self.check_checksums_for(comp, sub="of component %s" % comp['name'])) return checksum_issues @@ -246,14 +257,12 @@ def build_step(self): def install_step(self): """Install components, if specified.""" comp_cnt = len(self.cfg['components']) - for idx, cfg in enumerate(self.comp_cfgs): + for idx, (cfg, comp) in enumerate(self.comp_instances): print_msg("installing bundle component %s v%s (%d/%d)..." % (cfg['name'], cfg['version'], idx + 1, comp_cnt)) self.log.info("Installing component %s v%s using easyblock %s", cfg['name'], cfg['version'], cfg.easyblock) - comp = cfg.easyblock(cfg) - # correct build/install dirs comp.builddir = self.builddir comp.install_subdir, comp.installdir = self.install_subdir, self.installdir @@ -270,6 +279,7 @@ def install_step(self): comp.src = [] # find match entries in self.src for this component + comp.cfg.expect_resolved_template_values = False for source in comp.cfg['sources']: if isinstance(source, str): comp_src_fn = source @@ -293,6 +303,7 @@ def install_step(self): # location of first unpacked source is used to determine where to apply patch(es) comp.src[-1]['finalpath'] = comp.cfg['start_dir'] + comp.cfg.expect_resolved_template_values = True # check if sanity checks are enabled for the component if self.cfg['sanity_check_all_components'] or comp.cfg['name'] in self.cfg['sanity_check_components']: @@ -320,8 +331,40 @@ def install_step(self): new_val = path env.setvar(envvar, new_val) - # close log for this component - comp.close_log() + def make_module_req_guess(self): + """ + Set module requirements from all components, e.g. $PATH, etc. + During the install step, we only set these requirements temporarily. + Later on when building the module, those paths are not considered. + Therefore, iterate through all the components again and gather + the requirements. + + Do not remove duplicates or check for existence of folders, + as this is done in the generic EasyBlock while creating + the module file already. + """ + # Start with the paths from the generic EasyBlock. + # If not added here, they might be missing entirely and fail sanity checks. + final_reqs = super(Bundle, self).make_module_req_guess() + + for cfg, comp in self.comp_instances: + self.log.info("Gathering module paths for component %s v%s", cfg['name'], cfg['version']) + reqs = comp.make_module_req_guess() + + # Try-except block to fail with an easily understandable error message. + # This should only trigger when an EasyBlock returns non-dict module requirements + # for make_module_req_guess() which should then be fixed in the components EasyBlock. + try: + for key, value in sorted(reqs.items()): + if isinstance(value, str): + value = [value] + final_reqs.setdefault(key, []) + final_reqs[key].extend(value) + except AttributeError: + raise EasyBuildError("Cannot process module requirements of bundle component %s v%s", + cfg['name'], cfg['version']) + + return final_reqs def make_module_extra(self, *args, **kwargs): """Set extra stuff in module file, e.g. $EBROOT*, $EBVERSION*, etc.""" diff --git a/easybuild/easyblocks/generic/cmakemake.py b/easybuild/easyblocks/generic/cmakemake.py index 5465bff70d..b9208d2d40 100644 --- a/easybuild/easyblocks/generic/cmakemake.py +++ b/easybuild/easyblocks/generic/cmakemake.py @@ -349,6 +349,19 @@ def configure_step(self, srcdir=None, builddir=None): # ensure CMake uses EB python, not system or virtualenv python options.update(get_cmake_python_config_dict()) + # pass the preferred host compiler, CUDA compiler, and CUDA architectures to the CUDA compiler + cuda_root = get_software_root('CUDA') + if cuda_root: + options['CMAKE_CUDA_HOST_COMPILER'] = which(os.getenv('CXX', 'g++')) + options['CMAKE_CUDA_COMPILER'] = which('nvcc') + cuda_cc = build_option('cuda_compute_capabilities') or self.cfg['cuda_compute_capabilities'] + if cuda_cc: + options['CMAKE_CUDA_ARCHITECTURES'] = '"%s"' % ';'.join([cc.replace('.', '') for cc in cuda_cc]) + else: + raise EasyBuildError('List of CUDA compute capabilities must be specified, either via ' + 'cuda_compute_capabilities easyconfig parameter or via ' + '--cuda-compute-capabilities') + if not self.cfg.get('allow_system_boost', False): boost_root = get_software_root('Boost') if boost_root: diff --git a/easybuild/easyblocks/generic/configuremake.py b/easybuild/easyblocks/generic/configuremake.py index f84f4dad45..c20563228b 100644 --- a/easybuild/easyblocks/generic/configuremake.py +++ b/easybuild/easyblocks/generic/configuremake.py @@ -45,11 +45,12 @@ from easybuild.easyblocks import VERSION as EASYBLOCKS_VERSION from easybuild.framework.easyblock import EasyBlock from easybuild.framework.easyconfig import CUSTOM -from easybuild.tools.build_log import print_warning -from easybuild.tools.config import source_paths, build_option +from easybuild.tools.build_log import print_warning, EasyBuildError +from easybuild.tools.config import source_paths, build_option, ERROR, IGNORE, WARN from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, adjust_permissions, compute_checksum, download_file from easybuild.tools.filetools import read_file, remove_file from easybuild.tools.run import run_shell_cmd +from easybuild.tools.utilities import nub # string that indicates that a configure script was generated by Autoconf # note: bytes string since this constant is used to check the contents of 'configure' which is read as bytes @@ -195,6 +196,11 @@ def extra_options(extra_vars=None): 'tar_config_opts': [False, "Override tar settings as determined by configure.", CUSTOM], 'test_cmd': [None, "Test command to use ('runtest' value is appended, default: '%s')" % DEFAULT_TEST_CMD, CUSTOM], + 'unrecognized_configure_options': [ERROR, + "Action to do when unrecognized options passed to ./configure are" + " detected, defaults to aborting the build. Can be set to '" + WARN + + "' or '" + IGNORE + "' (NOT RECOMMENDED! It might hide actual errors" + " e.g. misspelling of intended or changed options)", CUSTOM], }) return extra_vars @@ -325,6 +331,24 @@ def configure_step(self, cmd_prefix=''): res = run_shell_cmd(cmd) + action = self.cfg['unrecognized_configure_options'] + valid_actions = (ERROR, WARN, IGNORE) + # Always verify the EC param + if action not in valid_actions: + raise EasyBuildError('Invalid value for `unrecognized_configure_options`: %s. Must be one of: ', + action, ', '.join(valid_actions)) + if action != IGNORE: + unrecognized_options_str = 'configure: WARNING: unrecognized options:' + unrecognized_options = re.findall(rf"^{unrecognized_options_str}.*", res.output, flags=re.I | re.M) + # Keep only unique options (remove the warning string and strip whitespace) + unrecognized_options = nub(x.split(unrecognized_options_str)[-1].strip() for x in unrecognized_options) + if unrecognized_options: + msg = 'Found unrecognized configure options: ' + '; '.join(unrecognized_options) + if action == WARN: + print_warning(msg) + else: + raise EasyBuildError(msg) + return res.output def build_step(self, verbose=None, path=None): diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 4b76d7e14b..e9ea708309 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -546,12 +546,13 @@ def determine_install_command(self): else: self.use_setup_py = True self.install_cmd = SETUP_PY_INSTALL_CMD + install_target = self.cfg.get_ref('install_target') - if self.cfg['install_target'] == EASY_INSTALL_TARGET: + if install_target == EASY_INSTALL_TARGET: self.install_cmd += " %(loc)s" self.py_installopts.append('--no-deps') if self.cfg.get('zipped_egg', False): - if self.cfg['install_target'] == EASY_INSTALL_TARGET: + if install_target == EASY_INSTALL_TARGET: self.py_installopts.append('--zip-ok') else: raise EasyBuildError("Installing zipped eggs requires using easy_install or pip") @@ -817,7 +818,11 @@ def configure_step(self): def build_step(self): """Build Python package using setup.py""" + + # inject extra '%(python)s' template value before getting value of 'buildcmd' custom easyconfig parameter + self.cfg.template_values['python'] = self.python_cmd build_cmd = self.cfg['buildcmd'] + if self.use_setup_py: if get_software_root('CMake'): @@ -828,10 +833,9 @@ def build_step(self): if not build_cmd: build_cmd = 'build' # Default value for setup.py - build_cmd = '%(python)s setup.py ' + build_cmd + build_cmd = f"{self.python_cmd} setup.py {build_cmd}" if build_cmd: - build_cmd = build_cmd % {'python': self.python_cmd} cmd = ' '.join([self.cfg['prebuildopts'], build_cmd, self.cfg['buildopts']]) res = run_shell_cmd(cmd) diff --git a/easybuild/easyblocks/generic/rpackage.py b/easybuild/easyblocks/generic/rpackage.py index c7aabcc36a..6ca009de95 100644 --- a/easybuild/easyblocks/generic/rpackage.py +++ b/easybuild/easyblocks/generic/rpackage.py @@ -43,7 +43,7 @@ from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.environment import setvar from easybuild.tools.filetools import mkdir, copy_file -from easybuild.tools.run import run_shell_cmd, parse_log_for_error +from easybuild.tools.run import run_shell_cmd def make_R_install_option(opt, values, cmdline=False): @@ -183,8 +183,10 @@ def check_install_output(self, output): """ Check output of installation command, and clean up installation if needed. """ - errors = parse_log_for_error(output, regExp="^ERROR:") + errors = re.findall(r"^ERROR:.*", output, flags=re.I | re.M) + if errors: + self.log.info("R package %s failed with error:\n%s", self.name, '\n'.join(errors)) self.handle_installation_errors() cmd = "R -q --no-save" stdin = """ diff --git a/easybuild/easyblocks/generic/systemcompiler.py b/easybuild/easyblocks/generic/systemcompiler.py index a95991ddda..c901cc86c7 100644 --- a/easybuild/easyblocks/generic/systemcompiler.py +++ b/easybuild/easyblocks/generic/systemcompiler.py @@ -270,7 +270,7 @@ def make_module_extra(self, *args, **kwargs): extras = super(SystemCompiler, self).make_module_extra(*args, **kwargs) return extras - def post_install_step(self, *args, **kwargs): + def post_processing_step(self, *args, **kwargs): """Do nothing.""" pass diff --git a/easybuild/easyblocks/generic/systemmpi.py b/easybuild/easyblocks/generic/systemmpi.py index c98381cc40..ee72758bca 100644 --- a/easybuild/easyblocks/generic/systemmpi.py +++ b/easybuild/easyblocks/generic/systemmpi.py @@ -221,7 +221,7 @@ def make_installdir(self, dontcreate=None): """Custom implementation of make installdir: do nothing, do not touch system MPI directories and files.""" pass - def post_install_step(self): + def post_processing_step(self): """Do nothing.""" pass diff --git a/easybuild/easyblocks/h/hadoop.py b/easybuild/easyblocks/h/hadoop.py index cf59603333..dc5069c6db 100644 --- a/easybuild/easyblocks/h/hadoop.py +++ b/easybuild/easyblocks/h/hadoop.py @@ -76,7 +76,7 @@ def install_step(self): else: super(EB_Hadoop, self).install_step() - def post_install_step(self): + def post_processing_step(self): """After the install, copy the extra native libraries into place.""" for native_library, lib_path in self.cfg['extra_native_libs']: lib_root = get_software_root(native_library) diff --git a/easybuild/easyblocks/i/icc.py b/easybuild/easyblocks/i/icc.py index a773340fd3..167a6646f2 100644 --- a/easybuild/easyblocks/i/icc.py +++ b/easybuild/easyblocks/i/icc.py @@ -39,7 +39,7 @@ from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.intelbase import IntelBase, COMP_ALL -from easybuild.easyblocks.t.tbb import get_tbb_gccprefix +from easybuild.easyblocks.tbb import get_tbb_gccprefix from easybuild.tools.build_log import EasyBuildError from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import get_shared_lib_ext diff --git a/easybuild/easyblocks/i/imkl.py b/easybuild/easyblocks/i/imkl.py index 1ec95de9d5..87f7bcb92d 100644 --- a/easybuild/easyblocks/i/imkl.py +++ b/easybuild/easyblocks/i/imkl.py @@ -324,11 +324,11 @@ def build_mkl_flexiblas(self, flexiblasdir): if res.exit_code: raise EasyBuildError("Building FlexiBLAS-compatible library (cmd: %s) failed", cmd) - def post_install_step(self): + def post_processing_step(self): """ Install group libraries and interfaces (if desired). """ - super(EB_imkl, self).post_install_step() + super(EB_imkl, self).post_processing_step() # extract examples examples_subdir = os.path.join(self.installdir, self.mkl_basedir, self.examples_subdir) diff --git a/easybuild/easyblocks/i/imkl_fftw.py b/easybuild/easyblocks/i/imkl_fftw.py index 18cd02246b..ba9cfc39d9 100644 --- a/easybuild/easyblocks/i/imkl_fftw.py +++ b/easybuild/easyblocks/i/imkl_fftw.py @@ -64,9 +64,9 @@ def make_module_extra(self): # bypass extra module variables for imkl return super(EB_imkl, self).make_module_extra() - def post_install_step(self): + def post_processing_step(self): """Custom post install step for imkl-FFTW""" - # bypass post_install_step of imkl easyblock + # bypass post_processing_step of imkl easyblock pass def sanity_check_step(self): diff --git a/easybuild/easyblocks/i/impi.py b/easybuild/easyblocks/i/impi.py index 3f3388518e..8871a25043 100644 --- a/easybuild/easyblocks/i/impi.py +++ b/easybuild/easyblocks/i/impi.py @@ -122,9 +122,9 @@ def install_step(self): else: raise EasyBuildError("Rebuild of libfabric is requested, but ofi_internal is set to False.") - def post_install_step(self): + def post_processing_step(self): """Custom post install step for IMPI, fix broken env scripts after moving installed files.""" - super(EB_impi, self).post_install_step() + super(EB_impi, self).post_processing_step() impiver = LooseVersion(self.version) diff --git a/easybuild/easyblocks/i/intel_compilers.py b/easybuild/easyblocks/i/intel_compilers.py index cfc50d6080..5b9c2dc3a6 100644 --- a/easybuild/easyblocks/i/intel_compilers.py +++ b/easybuild/easyblocks/i/intel_compilers.py @@ -31,7 +31,7 @@ from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.intelbase import IntelBase -from easybuild.easyblocks.t.tbb import get_tbb_gccprefix +from easybuild.easyblocks.tbb import get_tbb_gccprefix from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.run import run_shell_cmd diff --git a/easybuild/easyblocks/j/java.py b/easybuild/easyblocks/j/java.py index a835f671e2..441c5e8501 100644 --- a/easybuild/easyblocks/j/java.py +++ b/easybuild/easyblocks/j/java.py @@ -87,12 +87,12 @@ def install_step(self): else: PackedBinary.install_step(self) - def post_install_step(self): + def post_processing_step(self): """ Custom post-installation step: - ensure correct glibc is used when installing into custom sysroot and using RPATH """ - super(EB_Java, self).post_install_step() + super(EB_Java, self).post_processing_step() # patch binaries and libraries when using alternate sysroot in combination with RPATH sysroot = build_option('sysroot') diff --git a/easybuild/easyblocks/m/mamba.py b/easybuild/easyblocks/m/mamba.py index 232011e137..99947ffe4a 100644 --- a/easybuild/easyblocks/m/mamba.py +++ b/easybuild/easyblocks/m/mamba.py @@ -31,7 +31,7 @@ import os -from easybuild.easyblocks.a.anaconda import EB_Anaconda +from easybuild.easyblocks.anaconda import EB_Anaconda class EB_Mamba(EB_Anaconda): diff --git a/easybuild/easyblocks/m/mathematica.py b/easybuild/easyblocks/m/mathematica.py index cc6b68608b..b993851ede 100644 --- a/easybuild/easyblocks/m/mathematica.py +++ b/easybuild/easyblocks/m/mathematica.py @@ -109,7 +109,7 @@ def install_step(self): if orig_display is not None: os.environ['DISPLAY'] = orig_display - def post_install_step(self): + def post_processing_step(self): """Activate installation by using activation key, if provided.""" if self.cfg['activation_key']: # activation key is printed by using '$ActivationKey' in Mathematica, so no reason to keep it 'secret' @@ -126,7 +126,7 @@ def post_install_step(self): else: self.log.info("No activation key provided, so skipping activation of the installation.") - super(EB_Mathematica, self).post_install_step() + super(EB_Mathematica, self).post_processing_step() def sanity_check_step(self): """Custom sanity check for Mathematica.""" diff --git a/easybuild/easyblocks/m/matlab.py b/easybuild/easyblocks/m/matlab.py index 97263b0b86..bcf2f3eb4c 100644 --- a/easybuild/easyblocks/m/matlab.py +++ b/easybuild/easyblocks/m/matlab.py @@ -205,8 +205,8 @@ def install_step(self): regex.pattern, cmd, res.output) with open(self.outputfile) as f: if regex.search(f.read()): - raise EasyBuildError("Found error pattern '%s' in output file of installer", - regex.pattern) + raise EasyBuildError("Found error pattern '%s' in output file of installer at %s", + regex.pattern, self.outputfile) def sanity_check_step(self): """Custom sanity check for MATLAB.""" diff --git a/easybuild/easyblocks/m/metagenome_atlas.py b/easybuild/easyblocks/m/metagenome_atlas.py index 2391595cee..1899522dd2 100644 --- a/easybuild/easyblocks/m/metagenome_atlas.py +++ b/easybuild/easyblocks/m/metagenome_atlas.py @@ -39,7 +39,7 @@ class EB_Metagenome_Atlas(PythonPackage): Support for building/installing Metagenome-Atlas. """ - def post_install_step(self): + def post_processing_step(self): """Create snakemake config files""" # https://metagenome-atlas.readthedocs.io/en/latest/usage/getting_started.html#set-up-of-cluster-execution diff --git a/easybuild/easyblocks/n/numpy.py b/easybuild/easyblocks/n/numpy.py index 50ee42c76c..4b8dbfe86c 100644 --- a/easybuild/easyblocks/n/numpy.py +++ b/easybuild/easyblocks/n/numpy.py @@ -204,6 +204,15 @@ def get_libs_for_mkl(varname): 'includes': includes, } + if LooseVersion(self.version) < LooseVersion('1.26'): + # NumPy detects the required math by trying to link a minimal code containing a call to `log(0.)`. + # The first try is without any libraries, which works with `gcc -fno-math-errno` (our optimization default) + # because the call gets removed due to not having any effect. So it concludes that `-lm` is not required. + # This then fails to detect availability of functions such as `acosh` which do not get removed in the same + # way and so less exact replacements are used instead which e.g. fail the tests on PPC. + # This variable makes it try `-lm` first and is supported until the Meson backend is used in 1.26+. + env.setvar('MATHLIB', 'm') + super(EB_numpy, self).configure_step() if LooseVersion(self.version) < LooseVersion('1.21'): diff --git a/easybuild/easyblocks/o/openfoam.py b/easybuild/easyblocks/o/openfoam.py index efeb53a126..52f31aa8df 100644 --- a/easybuild/easyblocks/o/openfoam.py +++ b/easybuild/easyblocks/o/openfoam.py @@ -340,7 +340,7 @@ def build_step(self): cleancmd = "wcleanAll" # make directly in install directory - cmd_tmpl = "%(precmd)s && %(cleancmd)s && %(prebuildopts)s %(makecmd)s" % { + cmd_tmpl = "%(precmd)s && %(cleancmd)s && %(prebuildopts)s bash %(makecmd)s" % { 'precmd': precmd, 'cleancmd': cleancmd, 'prebuildopts': self.cfg['prebuildopts'], @@ -371,8 +371,8 @@ def build_step(self): if self.looseversion >= LooseVersion('2406'): # Also build the plugins - cmd += ' && %s %s -log' % (self.cfg['prebuildopts'], - os.path.join(self.builddir, self.openfoamdir, 'Allwmake-plugins')) + cmd += ' && %s bash %s -log' % (self.cfg['prebuildopts'], + os.path.join(self.builddir, self.openfoamdir, 'Allwmake-plugins')) run_shell_cmd(cmd_tmpl % cmd) diff --git a/easybuild/easyblocks/p/paraver.py b/easybuild/easyblocks/p/paraver.py index 4e4b9c6390..83631eb1d1 100644 --- a/easybuild/easyblocks/p/paraver.py +++ b/easybuild/easyblocks/p/paraver.py @@ -34,46 +34,19 @@ from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake -from easybuild.tools.build_log import EasyBuildError, print_msg -from easybuild.tools.filetools import change_dir +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.modules import get_software_libdir, get_software_root class EB_Paraver(ConfigureMake): """Support for building/installing Paraver.""" - def run_all_steps(self, *args, **kwargs): - """ - Put configure/build/install options in place for the 3 different components of Paraver. - Each component lives in a separate subdirectory. - """ - if LooseVersion(self.version) < LooseVersion('4.7'): - - # leverage support for iterated installation for older Paraver versions - self.components = ['ptools_common_files', 'paraver-kernel', 'wxparaver'] - self.current_component = 0 # index in list above - - # initiate configopts with empty list - self.cfg['configopts'] = [] - - # first phase: build and install ptools - # no specific configure options for the ptools component (but configopts list element must be there) - self.cfg.update('configopts', ['']) - - # second phase: build and install paraver-kernel - self.cfg.update('configopts', ["--with-boost=%(boost)s --with-ptools-common-files=%(installdir)s"]) + def __init__(self, *args, **kwargs): + """Constructor for custom easyblock for Paraver.""" + super(EB_Paraver, self).__init__(*args, **kwargs) - # third phase: build and install wxparaver - wxparaver_configopts = ' '.join([ - '--with-boost=%(boost)s', - '--with-wxpropgrid=%(wxpropgrid)s', - '--with-paraver=%(installdir)s', - ]) - self.cfg.update('configopts', [wxparaver_configopts]) - else: - self.components, self.current_component = None, None - - return super(EB_Paraver, self).run_all_steps(*args, **kwargs) + if LooseVersion(self.version) < LooseVersion('4.7'): + raise EasyBuildError("Custom easyblock for Paraver only supports Paraver versions >= 4.7") def configure_step(self): """Custom configuration procedure for Paraver: template configuration options before using them.""" @@ -88,7 +61,7 @@ def configure_step(self): wxwidgets = get_software_root('wxWidgets') if wxwidgets: wx_config = os.path.join(wxwidgets, 'bin', 'wx-config') - elif LooseVersion(self.version) >= LooseVersion('4.7'): + else: raise EasyBuildError("wxWidgets is not available as a dependency") # determine value to pass to --with-wxpropgrid (library name) @@ -107,53 +80,18 @@ def configure_step(self): else: self.log.info("wxPropertyGrid not included as dependency, assuming that's OK...") - if LooseVersion(self.version) < LooseVersion('4.7'): - component = self.components[self.current_component] - change_dir(component) - self.log.info("Customized start directory for component %s: %s", component, os.getcwd()) - - print_msg("starting with component %s" % component, log=self.log) - - self.cfg['configopts'] = self.cfg['configopts'] % { - 'boost': boost_root, - 'installdir': self.installdir, - 'wxpropgrid': wxpropgrid, - } - else: - self.cfg.update('configopts', '--with-boost=%s' % boost_root) - self.cfg.update('configopts', '--with-paraver=%s' % self.installdir) - self.cfg.update('configopts', '--with-wx-config=%s' % wx_config) - # wxPropertyGrid is not required with recent wxWidgets - if wxpropgrid: - self.cfg.update('configopts', '--with-wxpropgrid=%s' % wxpropgrid) + self.cfg.update('configopts', '--with-boost=%s' % boost_root) + self.cfg.update('configopts', '--with-paraver=%s' % self.installdir) + self.cfg.update('configopts', '--with-wx-config=%s' % wx_config) + # wxPropertyGrid is not required with recent wxWidgets + if wxpropgrid: + self.cfg.update('configopts', '--with-wxpropgrid=%s' % wxpropgrid) super(EB_Paraver, self).configure_step() def build_step(self): - """Custom build procedure for Paraver: skip 'make' for recent versions.""" - - if LooseVersion(self.version) < LooseVersion('4.7'): - super(EB_Paraver, self).build_step() - - def install_step(self): - """Custom installation procedure for Paraver: put symlink in place for library subdirectory.""" - super(EB_Paraver, self).install_step() - - if LooseVersion(self.version) < LooseVersion('4.7'): - # link lib to lib64 if needed - # this is a workaround for an issue with libtool which sometimes creates lib64 rather than lib - if self.components[self.current_component] == self.components[0]: - lib64dir = os.path.join(self.installdir, 'lib64') - libdir = os.path.join(self.installdir, 'lib') - - if os.path.exists(lib64dir): - try: - self.log.debug("Symlinking %s to %s", lib64dir, libdir) - os.symlink(lib64dir, libdir) - except OSError as err: - raise EasyBuildError("Symlinking lib64 to lib failed: %s" % err) - - self.current_component += 1 + """No build ('make') required for recent versions.""" + pass def sanity_check_step(self): """Custom sanity check for Paraver.""" diff --git a/easybuild/easyblocks/p/perl.py b/easybuild/easyblocks/p/perl.py index 159f755273..52a72950a4 100644 --- a/easybuild/easyblocks/p/perl.py +++ b/easybuild/easyblocks/p/perl.py @@ -155,7 +155,7 @@ def prepare_for_extensions(self): # from specified sysroot rather than from host OS setvar('OPENSSL_PREFIX', sysroot) - def post_install_step(self, *args, **kwargs): + def post_processing_step(self, *args, **kwargs): """ Custom post-installation step for Perl: avoid excessive long shebang lines in Perl scripts. """ @@ -177,7 +177,7 @@ def post_install_step(self, *args, **kwargs): # specify pattern for paths (relative to install dir) of files for which shebang should be patched self.cfg['fix_perl_shebang_for'] = 'bin/*' - super(EB_Perl, self).post_install_step(*args, **kwargs) + super(EB_Perl, self).post_processing_step(*args, **kwargs) def sanity_check_step(self): """Custom sanity check for Perl.""" diff --git a/easybuild/easyblocks/p/python.py b/easybuild/easyblocks/p/python.py index 01355b23e5..824a44dd11 100644 --- a/easybuild/easyblocks/p/python.py +++ b/easybuild/easyblocks/p/python.py @@ -45,7 +45,7 @@ from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.easyconfig.templates import PYPI_SOURCE from easybuild.tools.build_log import EasyBuildError, print_warning -from easybuild.tools.config import build_option, ERROR, log_path, PYTHONPATH, EBPYTHONPREFIXES +from easybuild.tools.config import build_option, ERROR, EBPYTHONPREFIXES from easybuild.tools.modules import get_software_libdir, get_software_root, get_software_version from easybuild.tools.filetools import apply_regex_substitutions, change_dir, mkdir from easybuild.tools.filetools import read_file, remove_dir, symlink, write_file @@ -142,9 +142,6 @@ def __init__(self, *args, **kwargs): self.pyshortver = '.'.join(self.version.split('.')[:2]) - # Used for EBPYTHONPREFIXES handler script - self.pythonpath = os.path.join(log_path(), 'python') - ext_defaults = { # Use PYPI_SOURCE as the default for source_urls of extensions. 'source_urls': [PYPI_SOURCE], @@ -481,6 +478,10 @@ def build_step(self, *args, **kwargs): super(EB_Python, self).build_step(*args, **kwargs) + @property + def site_packages_path(self): + return os.path.join('lib', 'python' + self.pyshortver, 'site-packages') + def install_step(self): """Extend make install to make sure that the 'python' command is present.""" @@ -504,7 +505,7 @@ def install_step(self): symlink('pip' + self.pyshortver, pip_binary_path, use_abspath_source=False) if self.cfg.get('ebpythonprefixes'): - write_file(os.path.join(self.installdir, self.pythonpath, 'sitecustomize.py'), SITECUSTOMIZE) + write_file(os.path.join(self.installdir, self.site_packages_path, 'sitecustomize.py'), SITECUSTOMIZE) # symlink lib/python*/lib-dynload to lib64/python*/lib-dynload if it doesn't exist; # see https://github.com/easybuilders/easybuild-easyblocks/issues/1957 @@ -525,8 +526,7 @@ def install_step(self): def _sanity_check_ebpythonprefixes(self): """Check that EBPYTHONPREFIXES works""" temp_prefix = tempfile.mkdtemp(suffix='-tmp-prefix') - site_packages_path = os.path.join('lib', 'python' + self.pyshortver, 'site-packages') - temp_site_packages_path = os.path.join(temp_prefix, site_packages_path) + temp_site_packages_path = os.path.join(temp_prefix, self.site_packages_path) mkdir(temp_site_packages_path, parents=True) # Must exist res = run_shell_cmd("%s=%s python -c 'import sys; print(sys.path)'" % (EBPYTHONPREFIXES, temp_prefix)) out = res.output.strip() @@ -534,7 +534,7 @@ def _sanity_check_ebpythonprefixes(self): if not out.startswith('[') or not out.endswith(']'): raise EasyBuildError("Unexpected output for sys.path: %s", out) paths = eval(out) - base_site_packages_path = os.path.join(self.installdir, site_packages_path) + base_site_packages_path = os.path.join(self.installdir, self.site_packages_path) try: base_prefix_idx = paths.index(base_site_packages_path) except ValueError: @@ -635,12 +635,3 @@ def sanity_check_step(self): raise EasyBuildError("Expected to find exactly one _tkinter*.so: %s", tkinter_so_hits) super(EB_Python, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) - - def make_module_extra(self, *args, **kwargs): - """Add path to sitecustomize.py to $PYTHONPATH""" - txt = super(EB_Python, self).make_module_extra() - - if self.cfg.get('ebpythonprefixes'): - txt += self.module_generator.prepend_paths(PYTHONPATH, self.pythonpath) - - return txt diff --git a/easybuild/easyblocks/q/quantumespresso.py b/easybuild/easyblocks/q/quantumespresso.py index eacad2a994..64c4cee5d1 100644 --- a/easybuild/easyblocks/q/quantumespresso.py +++ b/easybuild/easyblocks/q/quantumespresso.py @@ -28,6 +28,7 @@ @author: Kenneth Hoste (Ghent University) @author: Ake Sandgren (HPC2N, Umea University) @author: Davide Grassano (CECAM, EPFL) +@author: Jan Reuter (Juelich Supercomputing Centre) """ import fileinput @@ -84,6 +85,8 @@ def __init__(self, ec, *args, **kwargs): # Required to avoid CMakeMake default extra_opts to override the ConfigMake ones new_ec = EasyConfig(ec.path, extra_options=eb.extra_options()) + # Disable log file for nested EasyBlock + kwargs['logfile'] = self.logfile self.ebclass = eb(new_ec, *args, **kwargs) class EB_QuantumESPRESSOcmake(CMakeMake): diff --git a/easybuild/easyblocks/t/tau.py b/easybuild/easyblocks/t/tau.py deleted file mode 100644 index 2cb7598656..0000000000 --- a/easybuild/easyblocks/t/tau.py +++ /dev/null @@ -1,280 +0,0 @@ -## -# Copyright 2009-2024 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -EasyBuild support for building and installing TAU, implemented as an easyblock - -@author Kenneth Hoste (Ghent University) -@author Markus Geimer (Juelich Supercomputing Centre) -@author Bernd Mohr (Juelich Supercomputing Centre) -""" -import os - -from easybuild.easyblocks.generic.configuremake import ConfigureMake -from easybuild.easyblocks.pdt import find_arch_dir -from easybuild.framework.easyconfig import CUSTOM -from easybuild.tools import toolchain -from easybuild.tools.build_log import EasyBuildError, print_msg -from easybuild.tools.filetools import symlink -from easybuild.tools.modules import get_software_root, get_software_version -from easybuild.tools.systemtools import get_shared_lib_ext - - -KNOWN_BACKENDS = { - 'scalasca': 'Scalasca', - 'scorep': 'Score-P', - 'vampirtrace': 'Vampirtrace', -} - - -class EB_TAU(ConfigureMake): - """Support for building/installing TAU.""" - - @staticmethod - def extra_options(): - """Custom easyconfig parameters for TAU.""" - backends = "Extra TAU backends to build and install; possible values: %s" % ','.join(sorted(KNOWN_BACKENDS)) - extra_vars = { - 'extra_backends': [None, backends, CUSTOM], - 'tau_makefile': ['Makefile.tau-papi-mpi-pdt', "Name of Makefile to use in $TAU_MAKEFILE", CUSTOM], - } - return ConfigureMake.extra_options(extra_vars) - - def __init__(self, *args, **kwargs): - """Initialize TAU easyblock.""" - super(EB_TAU, self).__init__(*args, **kwargs) - - self.variant_index = 0 - self.cc, self.cxx, self.fortran = None, None, None - self.mpi_inc_dir, self.mpi_lib_dir = None, None - self.opt_pkgs_opts = None - self.variant_labels = None - - def run_all_steps(self, *args, **kwargs): - """ - Put configure options in place for the different selected backends of TAU, - for the MPI, OpenMP, and hybrid variants. - """ - if self.cfg['extra_backends'] is None: - self.cfg['extra_backends'] = [] - - # make sure selected extra backends are known - unknown_backends = [] - for backend in self.cfg['extra_backends']: - if backend not in KNOWN_BACKENDS: - unknown_backends.append(backend) - if unknown_backends: - raise EasyBuildError("Encountered unknown backends: %s", ', '.join(unknown_backends)) - - # compiler options - comp_opts = "-cc=%(cc)s -c++=%(cxx)s -fortran=%(fortran)s" - - # variant-specific options - openmp_opts = " -opari" - mpi_opts = " -mpiinc=%(mpi_inc_dir)s -mpilib=%(mpi_lib_dir)s" - - # options for optional packages - opt_pkgs_opts = " %(opt_pkgs_opts)s" - - # backend option - backend_opt = " %(backend_opt)s" - - # compose templates - mpi_tmpl = comp_opts + mpi_opts + opt_pkgs_opts + backend_opt - openmp_tmpl = comp_opts + openmp_opts + opt_pkgs_opts + backend_opt - hybrid_tmpl = comp_opts + openmp_opts + mpi_opts + opt_pkgs_opts + backend_opt - - # number of iterations: # backends + 1 (for basic) - iter_cnt = len(self.cfg['extra_backends']) + 1 - - # define list of configure options to iterate over - if self.cfg['configopts']: - raise EasyBuildError("Specifying additional configure options for TAU is not supported (yet)") - - self.cfg['configopts'] = [mpi_tmpl, openmp_tmpl, hybrid_tmpl] * iter_cnt - self.log.debug("List of configure options to iterate over: %s", self.cfg['configopts']) - - # custom prefix option for configure command - self.cfg['prefix_opt'] = '-prefix=' - - # installation command is 'make install clean' - self.cfg.update('installopts', 'clean') - - return super(EB_TAU, self).run_all_steps(*args, **kwargs) - - def prepare_step(self, *args, **kwargs): - """Custom prepare step for Tau: check required dependencies and collect information on them.""" - super(EB_TAU, self).prepare_step(*args, **kwargs) - - # install prefixes for selected backends - self.backend_opts = {'tau': ''} - for backend_name, dep in KNOWN_BACKENDS.items(): - root = get_software_root(dep) - if backend_name in self.cfg['extra_backends']: - if root: - self.backend_opts[backend_name] = "-%s=%s" % (backend_name, root) - else: - raise EasyBuildError("%s is listed in extra_backends, but not available as a dependency", dep) - elif root: - raise EasyBuildError("%s included as dependency, but '%s' not in extra_backends", dep, backend_name) - - # make sure Scalasca v1.x is used as a dependency (if it's there) - if 'scalasca' in self.backend_opts and get_software_version('Scalasca').split('.')[0] != '1': - raise EasyBuildError("Scalasca v1.x must be used when scalasca backend is enabled") - - # determine values for compiler flags to use - known_compilers = { - toolchain.CLANGGCC: ['clang', 'clang++', 'gfortran'], - toolchain.GCC: ['gcc', 'g++', 'gfortran'], - toolchain.INTELCOMP: ['icc', 'icpc', 'intel'], - } - comp_fam = self.toolchain.comp_family() - if comp_fam in known_compilers: - self.cc, self.cxx, self.fortran = known_compilers[comp_fam] - - # determine values for MPI flags - self.mpi_inc_dir, self.mpi_lib_dir = os.getenv('MPI_INC_DIR'), os.getenv('MPI_LIB_DIR') - - # determine value for optional packages option template - self.opt_pkgs_opts = '' - for dep, opt in [('OTF', 'otf'), ('PAPI', 'papi'), ('PDT', 'pdt'), ('binutils', 'bfd')]: - root = get_software_root(dep) - if root: - self.opt_pkgs_opts += ' -%s=%s' % (opt, root) - - # determine list of labels, based on selected (extra) backends, variants and optional packages - self.variant_labels = [] - backend_labels = ['', '-epilog-scalasca-trace', '-scorep', '-vampirtrace-trace'] - for backend, backend_label in zip(['tau'] + sorted(KNOWN_BACKENDS.keys()), backend_labels): - if backend in ['tau'] + self.cfg['extra_backends']: - for pref, suff in [('-mpi', ''), ('', '-openmp-opari'), ('-mpi', '-openmp-opari')]: - - variant_label = 'tau' - # For non-GCC builds, the compiler name is encoded in the variant - if self.cxx and self.cxx != 'g++': - variant_label += '-' + self.cxx - if get_software_root('PAPI'): - variant_label += '-papi' - variant_label += pref - if get_software_root('PDT'): - variant_label += '-pdt' - variant_label += suff + backend_label - - self.variant_labels.append(variant_label) - - def make_installdir(self): - """Skip make install dir 'step', install dir is already created in prepare_step.""" - pass - - def configure_step(self): - """Custom configuration procedure for TAU: template configuration options before using them.""" - if self.cc is None or self.cxx is None or self.fortran is None: - raise EasyBuildError("Compiler family not supported yet: %s", self.toolchain.comp_family()) - - if self.mpi_inc_dir is None or self.mpi_lib_dir is None: - raise EasyBuildError("Failed to determine MPI include/library paths, no MPI available in toolchain?") - - # make sure selected default TAU makefile will be available - avail_makefiles = ['Makefile.' + x for x in self.variant_labels] - if self.cfg['tau_makefile'] not in avail_makefiles: - raise EasyBuildError("Specified tau_makefile %s will not be available (only: %s)", - self.cfg['tau_makefile'], avail_makefiles) - - # inform which backend/variant is being handled - backend = (['tau'] + self.cfg['extra_backends'])[self.variant_index // 3] - variant = ['mpi', 'openmp', 'hybrid'][self.variant_index % 3] - print_msg("starting with %s backend (%s variant)" % (backend, variant), log=self.log, silent=self.silent) - - self.cfg['configopts'] = self.cfg['configopts'] % { - 'backend_opt': self.backend_opts[backend], - 'cc': self.cc, - 'cxx': self.cxx, - 'fortran': self.fortran, - 'mpi_inc_dir': self.mpi_inc_dir, - 'mpi_lib_dir': self.mpi_lib_dir, - 'opt_pkgs_opts': self.opt_pkgs_opts, - } - - for key in ['preconfigopts', 'configopts', 'prebuildopts', 'preinstallopts']: - self.log.debug("%s for TAU (variant index: %s): %s", key, self.variant_index, self.cfg[key]) - - # Configure creates required subfolders in installdir, so create first (but only once, during first iteration) - if self.iter_idx == 0: - super(EB_TAU, self).make_installdir() - - super(EB_TAU, self).configure_step() - - self.variant_index += 1 - - def build_step(self): - """No custom build procedure for TAU.""" - pass - - def install_step(self): - """Create symlinks into arch-specific directories""" - super(EB_TAU, self).install_step() - # Link arch-specific directories into prefix - arch_dir = find_arch_dir(self.installdir) - self.log.info('Found %s as architecture specific directory. Creating symlinks...', arch_dir) - for subdir in ('bin', 'lib'): - src = os.path.join(arch_dir, subdir) - dst = os.path.join(self.installdir, subdir) - if os.path.lexists(dst): - self.log.info('Skipping creation of symlink %s as it already exists', dst) - else: - symlink(os.path.relpath(src, self.installdir), dst, use_abspath_source=False) - - def sanity_check_step(self): - """Custom sanity check for TAU.""" - custom_paths = { - 'files': - [os.path.join('bin', 'pprof'), os.path.join('include', 'TAU.h'), - os.path.join('lib', 'libTAU.%s' % get_shared_lib_ext())] + - [os.path.join('lib', 'lib%s.a' % x) for x in self.variant_labels] + - [os.path.join('lib', 'Makefile.' + x) for x in self.variant_labels], - 'dirs': [], - } - super(EB_TAU, self).sanity_check_step(custom_paths=custom_paths) - - def make_module_extra(self): - """Custom extra module file entries for TAU.""" - txt = super(EB_TAU, self).make_module_extra() - - txt += self.module_generator.prepend_paths('TAU_MF_DIR', 'lib') - - tau_makefile = os.path.join(self.installdir, 'lib', self.cfg['tau_makefile']) - txt += self.module_generator.set_environment('TAU_MAKEFILE', tau_makefile) - - # default measurement settings - tau_vars = { - 'TAU_CALLPATH': '1', - 'TAU_CALLPATH_DEPTH': '10', - 'TAU_COMM_MATRIX': '1', - 'TAU_PROFILE': '1', - 'TAU_TRACE': '0', - } - for key in tau_vars: - txt += self.module_generator.set_environment(key, tau_vars[key]) - - return txt diff --git a/easybuild/easyblocks/w/wps.py b/easybuild/easyblocks/w/wps.py index 1643582692..762105613c 100644 --- a/easybuild/easyblocks/w/wps.py +++ b/easybuild/easyblocks/w/wps.py @@ -106,6 +106,9 @@ def configure_step(self): wrfdir = os.path.join(wrf, det_wrf_subdir(get_software_version('WRF'))) else: raise EasyBuildError("WRF module not loaded?") + netcdf_fortran = get_software_root('NETCDFMINFORTRAN') + if netcdf_fortran: + env.setvar('NETCDFF_DIR', netcdf_fortran) self.compile_script = 'compile' diff --git a/test/easyblocks/init_easyblocks.py b/test/easyblocks/init_easyblocks.py index 6bb786d715..97f1bccc4e 100644 --- a/test/easyblocks/init_easyblocks.py +++ b/test/easyblocks/init_easyblocks.py @@ -241,6 +241,9 @@ def innertest(self): elif easyblock_fn == 'openssl_wrapper.py': # easyblock to create OpenSSL wrapper expects an OpenSSL version innertest = make_inner_test(easyblock, version='1.1') + elif easyblock_fn == 'paraver.py': + # custom easyblock for Paraver requires version >= 4.7 + innertest = make_inner_test(easyblock, version='4.8') elif easyblock_fn == 'torchvision.py': # torchvision easyblock requires that PyTorch is listed as dependency innertest = make_inner_test(easyblock, name='torchvision', deps=[('PyTorch', '1.12.1')]) diff --git a/test/easyblocks/module.py b/test/easyblocks/module.py index 6377260243..a4d2874305 100644 --- a/test/easyblocks/module.py +++ b/test/easyblocks/module.py @@ -468,6 +468,9 @@ def innertest(self): elif eb_fn == 'openssl_wrapper.py': # easyblock to create OpenSSL wrapper expects an OpenSSL version innertest = make_inner_test(easyblock, name='OpenSSL-wrapper', version='1.1') + elif eb_fn == 'paraver.py': + # custom easyblock for Paraver requires version >= 4.7 + innertest = make_inner_test(easyblock, name='Paraver', version='4.8') elif eb_fn == 'torchvision.py': # torchvision easyblock requires that PyTorch is listed as dependency extra_txt = "dependencies = [('PyTorch', '1.12.1')]"