From 32b7445fb3831174ab47904060076172c4830cb6 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 7 Jun 2024 12:33:32 +0200 Subject: [PATCH 01/10] Patch ctypes for Python installations when filtering LD_LIBRARY_PATH --- easybuild/easyblocks/p/python.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/easybuild/easyblocks/p/python.py b/easybuild/easyblocks/p/python.py index 0f42c6e41f..9cb579113e 100644 --- a/easybuild/easyblocks/p/python.py +++ b/easybuild/easyblocks/p/python.py @@ -192,6 +192,19 @@ def patch_step(self, *args, **kwargs): # Ignore user site dir. -E ignores PYTHONNOUSERSITE, so we have to add -s apply_regex_substitutions('configure', [(r"(PYTHON_FOR_BUILD=.*-E)'", r"\1 -s'")]) + # If we filter out LD_LIBRARY_PATH (not unusual when using rpath), ctypes is not able to dynamically load + # libraries installed with EasyBuild (see https://github.com/EESSI/software-layer/issues/192). + # Allow ctypes to also use LIBRARY_PATH in this scenario (preferring the standard LD_LIBRARY_PATH) + filtered_env_vars = build_option('filter_env_vars') or [] + if 'LD_LIBRARY_PATH' in filtered_env_vars and 'LIBRARY_PATH' not in filtered_env_vars: + ctypes_util_py = os.path.join("Lib", "ctypes", "util.py") + orig_ld_libs = "libpath = os.environ.get('LD_LIBRARY_PATH')" + orig_ld_libs_regex = r'(\s*)' + re.escape(orig_ld_libs) + r'(\s*)' + updated_ld_libs = ("libpath = ':'.join(filter(None, " + "os.environ.get('LD_LIBRARY_PATH', '').split(':') + " + "os.environ.get('LIBRARY_PATH', '').split(':')))") + apply_regex_substitutions(ctypes_util_py, [(orig_ld_libs_regex, r'\1' + updated_ld_libs + r'\2')]) + # if we're installing Python with an alternate sysroot, # we need to patch setup.py which includes hardcoded paths like /usr/include and /lib64; # this fixes problems like not being able to build the _ssl module ("Could not build the ssl module") From f8c232158f24e7ab4e67ac83497cc6b995193f03 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 7 Jun 2024 15:04:16 +0200 Subject: [PATCH 02/10] Add test step by default for recent Python installations --- easybuild/easyblocks/p/python.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/easybuild/easyblocks/p/python.py b/easybuild/easyblocks/p/python.py index 9cb579113e..386e2f9945 100644 --- a/easybuild/easyblocks/p/python.py +++ b/easybuild/easyblocks/p/python.py @@ -450,6 +450,15 @@ def build_step(self, *args, **kwargs): super(EB_Python, self).build_step(*args, **kwargs) + def test_step(self): + """Test Python build via 'make test'.""" + # Turn on testing by default for recent Python versions + if LooseVersion(self.version) >= LooseVersion('3.10') and self.cfg['runtest'] is None: + self.cfg['runtest'] = 'test' + # Need to skip some troublesome tests + self.cfg['testopts'] = 'TESTOPTS="-x test_socket "' + super(EB_Python, self).test_step() + def install_step(self): """Extend make install to make sure that the 'python' command is present.""" From a771002d89ff81122fe98aaf3bb03b94bc6fd92a Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 7 Jun 2024 16:41:09 +0200 Subject: [PATCH 03/10] Also skip a curses test --- easybuild/easyblocks/p/python.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/p/python.py b/easybuild/easyblocks/p/python.py index 386e2f9945..a4d6a07702 100644 --- a/easybuild/easyblocks/p/python.py +++ b/easybuild/easyblocks/p/python.py @@ -456,7 +456,7 @@ def test_step(self): if LooseVersion(self.version) >= LooseVersion('3.10') and self.cfg['runtest'] is None: self.cfg['runtest'] = 'test' # Need to skip some troublesome tests - self.cfg['testopts'] = 'TESTOPTS="-x test_socket "' + self.cfg['testopts'] = 'TESTOPTS="-x test_socket test_curses "' super(EB_Python, self).test_step() def install_step(self): From 9af57500574af0ef30fe660de310da38670b9dd4 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 7 Jun 2024 16:59:50 +0200 Subject: [PATCH 04/10] Add comment on why we skip some tests --- easybuild/easyblocks/p/python.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easybuild/easyblocks/p/python.py b/easybuild/easyblocks/p/python.py index a4d6a07702..011ec035fd 100644 --- a/easybuild/easyblocks/p/python.py +++ b/easybuild/easyblocks/p/python.py @@ -456,6 +456,8 @@ def test_step(self): if LooseVersion(self.version) >= LooseVersion('3.10') and self.cfg['runtest'] is None: self.cfg['runtest'] = 'test' # Need to skip some troublesome tests + # (socket tests give a permission denied error, may be due to SELinux) + # (curses error is from test_background, just ignoring) self.cfg['testopts'] = 'TESTOPTS="-x test_socket test_curses "' super(EB_Python, self).test_step() From ac1ce4e0b0f94001b49014fb0ff70b94bba288fa Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 7 Jun 2024 17:49:39 +0200 Subject: [PATCH 05/10] Class is inherited so restrict application of new test step --- easybuild/easyblocks/p/python.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/p/python.py b/easybuild/easyblocks/p/python.py index 011ec035fd..56e1870c94 100644 --- a/easybuild/easyblocks/p/python.py +++ b/easybuild/easyblocks/p/python.py @@ -453,7 +453,8 @@ def build_step(self, *args, **kwargs): def test_step(self): """Test Python build via 'make test'.""" # Turn on testing by default for recent Python versions - if LooseVersion(self.version) >= LooseVersion('3.10') and self.cfg['runtest'] is None: + relevant_python = (self.name.lower() == 'python' and LooseVersion(self.version) >= LooseVersion('3.10')) + if relevant_python and self.cfg['runtest'] is None: self.cfg['runtest'] = 'test' # Need to skip some troublesome tests # (socket tests give a permission denied error, may be due to SELinux) From d07f920031f4431b388318895e308a9e7b20d9b2 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 10 Jun 2024 13:00:48 +0200 Subject: [PATCH 06/10] Adjust approach --- easybuild/easyblocks/p/python.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/easybuild/easyblocks/p/python.py b/easybuild/easyblocks/p/python.py index 56e1870c94..29597902a6 100644 --- a/easybuild/easyblocks/p/python.py +++ b/easybuild/easyblocks/p/python.py @@ -45,7 +45,7 @@ from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS from easybuild.tools.build_log import EasyBuildError, print_warning -from easybuild.tools.config import build_option, log_path +from easybuild.tools.config import build_option, ERROR, log_path from easybuild.tools.modules import get_software_libdir, get_software_root, get_software_version from easybuild.tools.filetools import apply_regex_substitutions, change_dir, mkdir from easybuild.tools.filetools import read_file, remove_dir, symlink, write_file @@ -194,16 +194,27 @@ def patch_step(self, *args, **kwargs): # If we filter out LD_LIBRARY_PATH (not unusual when using rpath), ctypes is not able to dynamically load # libraries installed with EasyBuild (see https://github.com/EESSI/software-layer/issues/192). - # Allow ctypes to also use LIBRARY_PATH in this scenario (preferring the standard LD_LIBRARY_PATH) + # ctypes is using GCC (and therefore LIBRARY_PATH) to figure out the full location but then only returns the + # soname, instead let's return the full path in this particular scenario filtered_env_vars = build_option('filter_env_vars') or [] if 'LD_LIBRARY_PATH' in filtered_env_vars and 'LIBRARY_PATH' not in filtered_env_vars: ctypes_util_py = os.path.join("Lib", "ctypes", "util.py") - orig_ld_libs = "libpath = os.environ.get('LD_LIBRARY_PATH')" - orig_ld_libs_regex = r'(\s*)' + re.escape(orig_ld_libs) + r'(\s*)' - updated_ld_libs = ("libpath = ':'.join(filter(None, " - "os.environ.get('LD_LIBRARY_PATH', '').split(':') + " - "os.environ.get('LIBRARY_PATH', '').split(':')))") - apply_regex_substitutions(ctypes_util_py, [(orig_ld_libs_regex, r'\1' + updated_ld_libs + r'\2')]) + orig_gcc_so_name = None + # Let's do this incrementally since we are going back in time + if LooseVersion(self.version) >= "3.9.1": + # From 3.9.1 to at least v3.12.4 there is only one match for this line + orig_gcc_so_name = "_get_soname(_findLib_gcc(name)) or _get_soname(_findLib_ld(name))" + if orig_gcc_so_name: + orig_gcc_so_name_regex = r'(\s*)' + re.escape(orig_gcc_so_name) + r'(\s*)' + updated_gcc_so_name = ( + "os.path.join(os.path.dirname(_findLib_gcc(name)), _get_soname(_findLib_gcc(name)))" + + " or _get_soname(_findLib_ld(name))" + ) + apply_regex_substitutions( + ctypes_util_py, + [(orig_gcc_so_name_regex, r'\1' + updated_gcc_so_name + r'\2')], + on_missing_match=ERROR + ) # if we're installing Python with an alternate sysroot, # we need to patch setup.py which includes hardcoded paths like /usr/include and /lib64; From 0cc765f297ac07d61a91c34aeb5f6a08c3b69766 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 10 Jun 2024 17:13:13 +0200 Subject: [PATCH 07/10] Split off the testing step to another PR --- easybuild/easyblocks/p/python.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/easybuild/easyblocks/p/python.py b/easybuild/easyblocks/p/python.py index 29597902a6..d701a5f1d2 100644 --- a/easybuild/easyblocks/p/python.py +++ b/easybuild/easyblocks/p/python.py @@ -461,18 +461,6 @@ def build_step(self, *args, **kwargs): super(EB_Python, self).build_step(*args, **kwargs) - def test_step(self): - """Test Python build via 'make test'.""" - # Turn on testing by default for recent Python versions - relevant_python = (self.name.lower() == 'python' and LooseVersion(self.version) >= LooseVersion('3.10')) - if relevant_python and self.cfg['runtest'] is None: - self.cfg['runtest'] = 'test' - # Need to skip some troublesome tests - # (socket tests give a permission denied error, may be due to SELinux) - # (curses error is from test_background, just ignoring) - self.cfg['testopts'] = 'TESTOPTS="-x test_socket test_curses "' - super(EB_Python, self).test_step() - def install_step(self): """Extend make install to make sure that the 'python' command is present.""" From 4818e6eb5a3bff486e8caab3ac43de743532b414 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 24 Jul 2024 15:04:31 +0200 Subject: [PATCH 08/10] Return full path when gcc or ld are used to find the library. That way, we don't rely on the SONAME being set (it isn't set for many libraries), nor do we rely on the SONAME being the same as the filename (which isn't necessarily true). Finally, it resolves the issue that find_library only returns the library name, and then if some python code opens this library dynamically, the runtime loader might find a different one (or none at all, e.g. because LD_LIBRARY_PATH is filtered) --- easybuild/easyblocks/p/python.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/easybuild/easyblocks/p/python.py b/easybuild/easyblocks/p/python.py index d701a5f1d2..1459e7b538 100644 --- a/easybuild/easyblocks/p/python.py +++ b/easybuild/easyblocks/p/python.py @@ -207,8 +207,7 @@ def patch_step(self, *args, **kwargs): if orig_gcc_so_name: orig_gcc_so_name_regex = r'(\s*)' + re.escape(orig_gcc_so_name) + r'(\s*)' updated_gcc_so_name = ( - "os.path.join(os.path.dirname(_findLib_gcc(name)), _get_soname(_findLib_gcc(name)))" - + " or _get_soname(_findLib_ld(name))" + "_findLib_gcc(name) or _findLib_ld(name)" ) apply_regex_substitutions( ctypes_util_py, From 8480c217cdbe2a65fca9921a5da3ccbf545aca14 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 24 Jul 2024 15:36:53 +0200 Subject: [PATCH 09/10] If we're installing with custom sysroot, we should replace a hardcoded path to ldconfig in Lib/ctypes/util.py --- easybuild/easyblocks/p/python.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/easybuild/easyblocks/p/python.py b/easybuild/easyblocks/p/python.py index 1459e7b538..ea971a6525 100644 --- a/easybuild/easyblocks/p/python.py +++ b/easybuild/easyblocks/p/python.py @@ -267,6 +267,22 @@ def patch_step(self, *args, **kwargs): apply_regex_substitutions(setup_py_fn, regex_subs) + # The path to ldconfig is hardcoded in cpython.util._findSoname_ldconfig(name) as /sbin/ldconfig. + # This is incorrect if a custom sysroot is used + if sysroot is not None: + # Have confirmed for all versions starting with this one that _findSoname_ldconfig hardcodes /sbin/ldconfig + if LooseVersion(self.version) >= "3.9.1": + orig_ld_config_call = "with subprocess.Popen(['/sbin/ldconfig', '-p']," + if orig_ld_config_call: + ctypes_util_py = os.path.join("Lib", "ctypes", "util.py") + orig_ld_config_call_regex = r'(\s*)' + re.escape(orig_ld_config_call) + r'(\s*)' + updated_ld_config_call = "with subprocess.Popen(['%s/sbin/ldconfig', '-p']," % sysroot + apply_regex_substitutions( + ctypes_util_py, + [(orig_ld_config_call_regex, r'\1' + updated_ld_config_call + r'\2')], + on_missing_match=ERROR + ) + def prepare_for_extensions(self): """ Set default class and filter for Python packages From 1ee17c0f7726c69e97442f53c65c5f041d65c94f Mon Sep 17 00:00:00 2001 From: ocaisa Date: Fri, 26 Jul 2024 14:22:24 +0200 Subject: [PATCH 10/10] Update python.py --- easybuild/easyblocks/p/python.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easybuild/easyblocks/p/python.py b/easybuild/easyblocks/p/python.py index ea971a6525..cede206825 100644 --- a/easybuild/easyblocks/p/python.py +++ b/easybuild/easyblocks/p/python.py @@ -206,6 +206,10 @@ def patch_step(self, *args, **kwargs): orig_gcc_so_name = "_get_soname(_findLib_gcc(name)) or _get_soname(_findLib_ld(name))" if orig_gcc_so_name: orig_gcc_so_name_regex = r'(\s*)' + re.escape(orig_gcc_so_name) + r'(\s*)' + # _get_soname() takes the full path as an argument and uses objdump to get the SONAME field from + # the shared object file. The presence or absence of the SONAME field in the ELF header of a shared + # library is influenced by how the library is compiled and linked. For manually built libraries we + # may be lacking this field, this approach also solves that problem. updated_gcc_so_name = ( "_findLib_gcc(name) or _findLib_ld(name)" )