Merge branch '5.0.x' into cleanup-old-intelbase

easybuild/easyblocks/__init__.py

@@ -42,7 +42,7 @@
 # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like
 #   UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0'
 # This causes problems further up the dependency chain...
-VERSION = '5.0.0.dev0'
+VERSION = '5.0.0beta1'
 UNKNOWN = 'UNKNOWN'
 
 

easybuild/easyblocks/a/aedt.py

@@ -71,7 +71,7 @@ def install_step(self):
         ])
         run_shell_cmd("./Linux/AnsysEM/Disk1/InstData/setup.exe %s" % options)
 
-    def post_install_step(self):
+    def post_processing_step(self):
         """Disable OS check and set LC_ALL/LANG for runtime"""
         if not self.subdir:
             self._set_subdir()

easybuild/easyblocks/a/aocc.py

@@ -214,7 +214,7 @@ def install_step(self):
 
         super(EB_AOCC, self).install_step()
 
-    def post_install_step(self):
+    def post_processing_step(self):
         """
         For AOCC <5.0.0:
         Create wrappers for the compilers to make sure compilers picks up GCCcore as GCC toolchain.
@@ -245,7 +245,7 @@ def post_install_step(self):
 
         self._create_compiler_config_files(compilers_to_add_config_files)
         self._create_compiler_wrappers(compilers_to_wrap)
-        super(EB_AOCC, self).post_install_step()
+        super(EB_AOCC, self).post_processing_step()
 
     def sanity_check_step(self):
         """Custom sanity check for AOCC, based on sanity check for Clang."""

easybuild/easyblocks/a/aomp.py

@@ -133,8 +133,8 @@ def configure_step(self):
         # Only build selected components
         self.cfg['installopts'] = 'select ' + ' '.join(components)
 
-    def post_install_step(self):
-        super(EB_AOMP, self).post_install_step()
+    def post_processing_step(self):
+        super(EB_AOMP, self).post_processing_step()
         # The install script will create a symbolic link as the install
         # directory, this creates problems for EB as it won't remove the
         # symlink. To remedy this we remove the link here and rename the actual

easybuild/easyblocks/c/clang.py

@@ -612,9 +612,9 @@ def install_step(self):
             except OSError as err:
                 raise EasyBuildError("Failed to copy static analyzer dirs to install dir: %s", err)
 
-    def post_install_step(self):
+    def post_processing_step(self):
         """Install python bindings."""
-        super(EB_Clang, self).post_install_step()
+        super(EB_Clang, self).post_processing_step()
 
         # copy Python bindings here in post-install step so that it is not done more than once in multi_deps context
         if self.cfg['python_bindings']:

easybuild/easyblocks/c/clang_aomp.py

@@ -145,7 +145,7 @@ def configure_step(self):
             raise EasyBuildError("Could not find 'ROCm-Device-Libs' source directory in %s", self.builddir)
 
         num_comps = len(self.cfg['components'])
-        for idx, comp in enumerate(self.comp_cfgs):
+        for idx, (comp, _) in enumerate(self.comp_instances):
             name = comp['name']
             msg = "configuring bundle component %s %s (%d/%d)..." % (name, comp['version'], idx + 1, num_comps)
             print_msg(msg)

easybuild/easyblocks/c/crispr_dav.py

@@ -46,7 +46,7 @@ def __init__(self, *args, **kwargs):
         super(EB_CRISPR_minus_DAV, self).__init__(*args, **kwargs)
         self.cfg['extract_sources'] = True
 
-    def post_install_step(self):
+    def post_processing_step(self):
         """Update configuration files with correct paths to dependencies and files in installation."""
 
         # getting paths of deps + files we will work with

easybuild/easyblocks/c/cuda.py

@@ -218,7 +218,7 @@ def install_step(self):
                 self.log.debug("Running patch %s", patch['name'])
                 run_shell_cmd("/bin/sh " + patch['path'] + " --accept-eula --silent --installdir=" + self.installdir)
 
-    def post_install_step(self):
+    def post_processing_step(self):
         """
         Create wrappers for the specified host compilers, generate the appropriate stub symlinks,
         and create version independent pkgconfig files
@@ -288,7 +288,7 @@ def create_wrapper(wrapper_name, wrapper_comp):
                 symlink(pc_file, link, use_abspath_source=False)
             change_dir(cwd)
 
-        super(EB_CUDA, self).post_install_step()
+        super(EB_CUDA, self).post_processing_step()
 
     def sanity_check_step(self):
         """Custom sanity check for CUDA."""

easybuild/easyblocks/d/dualsphysics.py

@@ -90,9 +90,9 @@ def install_step(self):
         ]
         super(EB_DualSPHysics, self).install_step()
 
-    def post_install_step(self):
+    def post_processing_step(self):
         """Custom post-installation step: ensure rpath is patched into binaries/libraries if configured."""
-        super(EB_DualSPHysics, self).post_install_step()
+        super(EB_DualSPHysics, self).post_processing_step()
 
         if build_option('rpath'):
             # only the compiled binary (e.g. DualSPHysics5.0CPU_linux64) is rpath'd, the precompiled libraries

easybuild/easyblocks/e/easybuildmeta.py

@@ -136,10 +136,10 @@ def install_step(self):
         except OSError as err:
             raise EasyBuildError("Failed to install EasyBuild packages: %s", err)
 
-    def post_install_step(self):
+    def post_processing_step(self):
         """Remove setuptools.pth file that hard includes a system-wide (site-packages) path, if it is there."""
 
-        super(EB_EasyBuildMeta, self).post_install_step()
+        super(EB_EasyBuildMeta, self).post_processing_step()
 
         setuptools_pth = os.path.join(self.installdir, self.pylibdir, 'setuptools.pth')
         if os.path.exists(setuptools_pth):

easybuild/easyblocks/f/fftwmpi.py

@@ -59,7 +59,7 @@ def prepare_step(self, *args, **kwargs):
         if not fftw_root:
             raise EasyBuildError("Required FFTW dependency is missing!")
 
-    def post_install_step(self):
+    def post_processing_step(self):
         """Custom post install step for FFTW.MPI"""
 
         # remove everything except include files that are already in non-MPI FFTW dependency.
@@ -68,7 +68,7 @@ def post_install_step(self):
                glob.glob(os.path.join(self.installdir, 'lib*/pkgconfig')) +
                glob.glob(os.path.join(self.installdir, 'lib*/cmake')) +
                [os.path.join(self.installdir, p) for p in ['bin', 'share']])
-        super(EB_FFTW_period_MPI, self).post_install_step()
+        super(EB_FFTW_period_MPI, self).post_processing_step()
 
     def sanity_check_step(self):
         """Custom sanity check for FFTW.MPI: check if all libraries/headers for MPI interfaces are there."""

easybuild/easyblocks/g/gcc.py

@@ -966,11 +966,11 @@ def install_step(self, *args, **kwargs):
         else:
             super(EB_GCC, self).install_step(*args, **kwargs)
 
-    def post_install_step(self, *args, **kwargs):
+    def post_processing_step(self, *args, **kwargs):
         """
         Post-processing after installation: add symlinks for cc, c++, f77, f95
         """
-        super(EB_GCC, self).post_install_step(*args, **kwargs)
+        super(EB_GCC, self).post_processing_step(*args, **kwargs)
 
         # Add symlinks for cc/c++/f77/f95.
         bindir = os.path.join(self.installdir, 'bin')

easybuild/easyblocks/g/gromacs.py

@@ -63,6 +63,8 @@ def extra_options():
         extra_vars.update({
             'double_precision': [None, "Build with double precision enabled (-DGMX_DOUBLE=ON), " +
                                  "default is to build double precision unless CUDA is enabled", CUSTOM],
+            'single_precision': [True, "Build with single precision enabled (-DGMX_DOUBLE=OFF), " +
+                                 "default is to build single precision", CUSTOM],
             'mpisuffix': ['_mpi', "Suffix to append to MPI-enabled executables (only for GROMACS < 4.6)", CUSTOM],
             'mpiexec': ['mpirun', "MPI executable to use when running tests", CUSTOM],
             'mpiexec_numproc_flag': ['-np', "Flag to introduce the number of MPI tasks when running tests", CUSTOM],
@@ -745,10 +747,15 @@ def run_all_steps(self, *args, **kwargs):
                 'mpi': 'install'
             }
 
-        precisions = ['single']
+        precisions = []
+        if self.cfg.get('single_precision'):
+            precisions.append('single')
         if self.cfg.get('double_precision') is None or self.cfg.get('double_precision'):
             precisions.append('double')
 
+        if precisions == []:
+            raise EasyBuildError("No precision selected. At least one of single/double_precision must be unset or True")
+
         mpitypes = ['nompi']
         if self.toolchain.options.get('usempi', None):
             mpitypes.append('mpi')

easybuild/easyblocks/generic/binary.py

@@ -131,7 +131,7 @@ def install_step(self):
                 raise EasyBuildError("Incorrect value type for install_cmds, should be list or tuple: ",
                                      install_cmds)
 
-    def post_install_step(self):
+    def post_processing_step(self):
         """Copy installation to actual installation directory in case of a staged installation."""
         if self.cfg.get('staged_install', False):
             staged_installdir = self.installdir
@@ -145,7 +145,7 @@ def post_install_step(self):
                 raise EasyBuildError("Failed to move staged install from %s to %s: %s",
                                      staged_installdir, self.installdir, err)
 
-        super(Binary, self).post_install_step()
+        super(Binary, self).post_processing_step()
 
     def sanity_check_rpath(self):
         """Skip the rpath sanity check, this is binary software"""

easybuild/easyblocks/generic/bundle.py

@@ -31,6 +31,7 @@
 @author: Pieter De Baets (Ghent University)
 @author: Jens Timmerman (Ghent University)
 @author: Jasper Grimm (University of York)
+@author: Jan Andre Reuter (Juelich Supercomputing Centre)
 """
 import copy
 import os
@@ -70,18 +71,18 @@ def __init__(self, *args, **kwargs):
         self.altroot = None
         self.altversion = None
 
-        # list of EasyConfig instances for components
-        self.comp_cfgs = []
+        # list of EasyConfig instances and their EasyBlocks for components
+        self.comp_instances = []
 
         # list of EasyConfig instances of components for which to run sanity checks
         self.comp_cfgs_sanity_check = []
 
         check_for_sources = getattr(self, 'check_for_sources', True)
         # list of sources for bundle itself *must* be empty (unless overridden by subclass)
         if check_for_sources:
-            if self.cfg['sources']:
+            if self.cfg.get_ref('sources'):
                 raise EasyBuildError("List of sources for bundle itself must be empty, found %s", self.cfg['sources'])
-            if self.cfg['patches']:
+            if self.cfg.get_ref('patches'):
                 raise EasyBuildError("List of patches for bundle itself must be empty, found %s", self.cfg['patches'])
 
         # copy EasyConfig instance before we make changes to it
@@ -160,54 +161,64 @@ def __init__(self, *args, **kwargs):
             for key in comp_specs:
                 comp_cfg[key] = comp_specs[key]
 
-            # enable resolving of templates for component-specific EasyConfig instance
+            # enable resolving of templates for component-specific EasyConfig instance,
+            # but don't require that all template values can be resolved at this point;
+            # this is important to ensure that template values like %(name)s and %(version)s
+            # are correctly resolved with the component name/version before values are copied over to self.cfg
             comp_cfg.enable_templating = True
+            comp_cfg.expect_resolved_template_values = False
 
             # 'sources' is strictly required
-            if comp_cfg['sources']:
+            comp_sources = comp_cfg['sources']
+            if comp_sources:
                 # If per-component source URLs are provided, attach them directly to the relevant sources
-                if comp_cfg['source_urls']:
-                    for source in comp_cfg['sources']:
+                comp_source_urls = comp_cfg['source_urls']
+                if comp_source_urls:
+                    for source in comp_sources:
                         if isinstance(source, str):
-                            self.cfg.update('sources', [{'filename': source, 'source_urls': comp_cfg['source_urls']}])
+                            self.cfg.update('sources', [{'filename': source, 'source_urls': comp_source_urls[:]}])
                         elif isinstance(source, dict):
                             # Update source_urls in the 'source' dict to use the one for the components
                             # (if it doesn't already exist)
                             if 'source_urls' not in source:
-                                source['source_urls'] = comp_cfg['source_urls']
+                                source['source_urls'] = comp_source_urls[:]
                             self.cfg.update('sources', [source])
                         else:
                             raise EasyBuildError("Source %s for component %s is neither a string nor a dict, cannot "
                                                  "process it.", source, comp_cfg['name'])
                 else:
                     # add component sources to list of sources
-                    self.cfg.update('sources', comp_cfg['sources'])
+                    self.cfg.update('sources', comp_sources)
             else:
                 raise EasyBuildError("No sources specification for component %s v%s", comp_name, comp_version)
 
-            if comp_cfg['checksums']:
-                src_cnt = len(comp_cfg['sources'])
+            comp_checksums = comp_cfg['checksums']
+            if comp_checksums:
+                src_cnt = len(comp_sources)
 
                 # add per-component checksums for sources to list of checksums
-                self.cfg.update('checksums', comp_cfg['checksums'][:src_cnt])
+                self.cfg.update('checksums', comp_checksums[:src_cnt])
 
                 # add per-component checksums for patches to list of checksums for patches
-                checksums_patches.extend(comp_cfg['checksums'][src_cnt:])
+                checksums_patches.extend(comp_checksums[src_cnt:])
 
-            if comp_cfg['patches']:
-                self.cfg.update('patches', comp_cfg['patches'])
+            comp_patches = comp_cfg['patches']
+            if comp_patches:
+                self.cfg.update('patches', comp_patches)
 
-            self.comp_cfgs.append(comp_cfg)
+            comp_cfg.expect_resolved_template_values = True
 
-        self.cfg.update('checksums', checksums_patches)
+            self.comp_instances.append((comp_cfg, comp_cfg.easyblock(comp_cfg, logfile=self.logfile)))
 
-        self.cfg.enable_templating = True
+        self.cfg.update('checksums', checksums_patches)
 
         # restore general sanity checks if using component-specific sanity checks
         if self.cfg['sanity_check_components'] or self.cfg['sanity_check_all_components']:
             self.cfg['sanity_check_paths'] = self.backup_sanity_paths
             self.cfg['sanity_check_commands'] = self.backup_sanity_cmds
 
+        self.cfg.enable_templating = True
+
     def check_checksums(self):
         """
         Check whether a SHA256 checksum is available for all sources & patches (incl. extensions).
@@ -216,7 +227,7 @@ def check_checksums(self):
         """
         checksum_issues = super(Bundle, self).check_checksums()
 
-        for comp in self.comp_cfgs:
+        for comp, _ in self.comp_instances:
             checksum_issues.extend(self.check_checksums_for(comp, sub="of component %s" % comp['name']))
 
         return checksum_issues
@@ -246,14 +257,12 @@ def build_step(self):
     def install_step(self):
         """Install components, if specified."""
         comp_cnt = len(self.cfg['components'])
-        for idx, cfg in enumerate(self.comp_cfgs):
+        for idx, (cfg, comp) in enumerate(self.comp_instances):
 
             print_msg("installing bundle component %s v%s (%d/%d)..." %
                       (cfg['name'], cfg['version'], idx + 1, comp_cnt))
             self.log.info("Installing component %s v%s using easyblock %s", cfg['name'], cfg['version'], cfg.easyblock)
 
-            comp = cfg.easyblock(cfg)
-
             # correct build/install dirs
             comp.builddir = self.builddir
             comp.install_subdir, comp.installdir = self.install_subdir, self.installdir
@@ -270,6 +279,7 @@ def install_step(self):
             comp.src = []
 
             # find match entries in self.src for this component
+            comp.cfg.expect_resolved_template_values = False
             for source in comp.cfg['sources']:
                 if isinstance(source, str):
                     comp_src_fn = source
@@ -293,6 +303,7 @@ def install_step(self):
 
                 # location of first unpacked source is used to determine where to apply patch(es)
                 comp.src[-1]['finalpath'] = comp.cfg['start_dir']
+            comp.cfg.expect_resolved_template_values = True
 
             # check if sanity checks are enabled for the component
             if self.cfg['sanity_check_all_components'] or comp.cfg['name'] in self.cfg['sanity_check_components']:
@@ -320,8 +331,40 @@ def install_step(self):
                             new_val = path
                         env.setvar(envvar, new_val)
 
-            # close log for this component
-            comp.close_log()
+    def make_module_req_guess(self):
+        """
+        Set module requirements from all components, e.g. $PATH, etc.
+        During the install step, we only set these requirements temporarily.
+        Later on when building the module, those paths are not considered.
+        Therefore, iterate through all the components again and gather
+        the requirements.
+
+        Do not remove duplicates or check for existence of folders,
+        as this is done in the generic EasyBlock while creating
+        the module file already.
+        """
+        # Start with the paths from the generic EasyBlock.
+        # If not added here, they might be missing entirely and fail sanity checks.
+        final_reqs = super(Bundle, self).make_module_req_guess()
+
+        for cfg, comp in self.comp_instances:
+            self.log.info("Gathering module paths for component %s v%s", cfg['name'], cfg['version'])
+            reqs = comp.make_module_req_guess()
+
+            # Try-except block to fail with an easily understandable error message.
+            # This should only trigger when an EasyBlock returns non-dict module requirements
+            # for make_module_req_guess() which should then be fixed in the components EasyBlock.
+            try:
+                for key, value in sorted(reqs.items()):
+                    if isinstance(value, str):
+                        value = [value]
+                    final_reqs.setdefault(key, [])
+                    final_reqs[key].extend(value)
+            except AttributeError:
+                raise EasyBuildError("Cannot process module requirements of bundle component %s v%s",
+                                     cfg['name'], cfg['version'])
+
+        return final_reqs
 
     def make_module_extra(self, *args, **kwargs):
         """Set extra stuff in module file, e.g. $EBROOT*, $EBVERSION*, etc."""