Merge pull request #4953 from easybuilders/5.1.x

release EasyBuild v5.1.1

Author: boegel

.github/workflows/container_tests.yml

@@ -97,7 +97,7 @@ jobs:
           # see https://docs.easybuild.io/en/latest/Containers.html
           curl -OL https://raw.githubusercontent.com/easybuilders/easybuild-easyconfigs/develop/easybuild/easyconfigs/b/bzip2/bzip2-1.0.8.eb
           export EASYBUILD_CONTAINERPATH=$PWD
-          export EASYBUILD_CONTAINER_CONFIG='bootstrap=docker,from=ghcr.io/easybuilders/centos-7.9-python3-amd64'
+          export EASYBUILD_CONTAINER_CONFIG='bootstrap=docker,from=ghcr.io/easybuilders/rockylinux-8.10-amd64'
           eb bzip2-1.0.8.eb --containerize --experimental --container-build-image
           singularity exec bzip2-1.0.8.sif command -v bzip2 | grep '/app/software/bzip2/1.0.8/bin/bzip2' || (echo "Path to bzip2 '$which_bzip2' is not correct" && exit 1)
           singularity exec bzip2-1.0.8.sif bzip2 --help

.github/workflows/container_tests_apptainer.yml

@@ -91,7 +91,7 @@ jobs:
           # see https://docs.easybuild.io/en/latest/Containers.html
           curl -OL https://raw.githubusercontent.com/easybuilders/easybuild-easyconfigs/develop/easybuild/easyconfigs/b/bzip2/bzip2-1.0.8.eb
           export EASYBUILD_CONTAINERPATH=$PWD
-          export EASYBUILD_CONTAINER_CONFIG='bootstrap=docker,from=ghcr.io/easybuilders/centos-7.9-python3-amd64'
+          export EASYBUILD_CONTAINER_CONFIG='bootstrap=docker,from=ghcr.io/easybuilders/rockylinux-8.10-amd64'
           export EASYBUILD_CONTAINER_TYPE='apptainer'
           eb bzip2-1.0.8.eb --containerize --experimental --container-build-image
           apptainer exec bzip2-1.0.8.sif command -v bzip2 | grep '/app/software/bzip2/1.0.8/bin/bzip2' || (echo "Path to bzip2 '$which_bzip2' is not correct" && exit 1)

RELEASE_NOTES

@@ -4,6 +4,42 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://docs.easybuild.io/release-notes .
 
 
+v5.1.1 (6 July 2025)
+--------------------
+
+update/bugfix release
+
+- enhancements:
+  - don't allow using `--pr-commit-msg` when only adding new files with `--new-pr` (unless `--force` is used) (#4498)
+  - update `install-EasyBuild-develop.sh` script to allow installation without forking (#4899)
+  - add blue, cyan, purple as known colors that can be used in colorize function (#4907)
+  - trigger `post_run_shell_cmd_hook` before raising error if shell command failed + pass down full `RunShellCmdResult` instance (#4908)
+  - also pass value of `fail_on_error` and `hidden` options of `run_shell_cmd` call down to pre/post `run_shell_cmd` hook (#4911)
+  - add configuration option to ignore unversioned (0.0.0) Python packages (#4912)
+  - add `required` option to `EasyConfig.get_cuda_cc_template_value` method (#4913)
+  - add total iteration count to trace output (#4919, #4952)
+  - add support for pre/post `easyblock` hook (#4923, #4938)
+  - catch `HTTPException` when doing GitHub API request (#4926)
+- bug fixes:
+  - fix reporting of method name in `EasyBlock.run_step` (#4920)
+  - fix handling of broken symlinks in `filetools.remove` (#4921)
+  - keep symlinks when copying build dirs of failed installations (#4933)
+  - also copy patches of extensions to easybuild subdirectory of installation directory (#4939, #4946)
+  - also copy easyblocks of bundle components to reprod dir (#4944)
+  - take into account that forked repository may not have same name as origin `--sync-pr-with-develop` (#4947)
+- enhancements and fixes for test suite:
+  - add support for passing valid unittest options to test suite (#3790)
+  - enhance `LooseVersion` test to add cases with leading alphabetical characters (#4901)
+  - prefer using `importlib.metadata` over the deprecated `pkg_resources` in `det_pypkg_version` + switch container tests to Rocky Linux 8.10 (#4904)
+  - fix `TypeError` for Python version comparison in `test_fetch_sources_git` (#4934)
+  - fix broken tests for GitHub integration features (#4954)
+- other changes:
+  - change value of architectures constants RISCV32 and RISCV64 (#4915)
+  - remove some superflous Python 2 checks (#4917)
+  - remove use of unsupported `source` key in patch spec (#4924)
+  - remove super() arguments from mk_tmpl_easyblock_for (#4936)
+
+
 v5.1.0 (26 May 2025)
 --------------------
 

easybuild/_deprecated.py

@@ -31,6 +31,7 @@
 """
 import contextlib
 import functools
+import json
 import os
 import re
 import signal
@@ -96,6 +97,12 @@ def cache_aware_func(cmd, *args, **kwargs):
     return cache_aware_func
 
 
+def json_loads(body):
+    """Deprecated wrapper for json.loads"""
+    _log.deprecated("json_loads is deprecated, use json.loads", '6.0')
+    return json.loads(body)
+
+
 def get_output_from_process(proc, read_size=None, asynchronous=False, print_deprecation_warning=True):
     """
     Get output from running process (that was opened with subprocess.Popen).

easybuild/base/fancylogger.py

@@ -578,8 +578,7 @@ def logToFile(filename, enable=True, filehandler=None, name=None, max_bytes=MAX_
         'maxBytes': max_bytes,
         'backupCount': backup_count,
     }
-    if sys.version_info[0] >= 3:
-        handleropts['encoding'] = 'utf-8'
+    handleropts['encoding'] = 'utf-8'
     # logging to a file is going to create the file later on, so let's try to be helpful and create the path if needed
     directory = os.path.dirname(filename)
     if not os.path.exists(directory):

easybuild/framework/easyblock.py

@@ -98,9 +98,9 @@
 from easybuild.tools.filetools import is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir
 from easybuild.tools.filetools import remove_file, remove_lock, symlink, verify_checksum, weld_paths, write_file
 from easybuild.tools.hooks import (
-    BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, EXTRACT_STEP, FETCH_STEP, INSTALL_STEP, MODULE_STEP,
-    MODULE_WRITE, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP, PREPARE_STEP, READY_STEP,
-    SANITYCHECK_STEP, SINGLE_EXTENSION, TEST_STEP, TESTCASES_STEP, load_hooks, run_hook,
+    BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EASYBLOCK, EXTENSIONS_STEP, EXTRACT_STEP, FETCH_STEP, INSTALL_STEP,
+    MODULE_STEP, MODULE_WRITE, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP, PREPARE_STEP,
+    READY_STEP, SANITYCHECK_STEP, SINGLE_EXTENSION, TEST_STEP, TESTCASES_STEP, load_hooks, run_hook,
 )
 from easybuild.tools.run import RunShellCmdError, raise_run_shell_cmd_error, run_shell_cmd
 from easybuild.tools.jenkins import write_to_xml
@@ -311,6 +311,9 @@ def __init__(self, ec, logfile=None):
         # initialize logger
         self._init_log()
 
+        # number of iterations
+        self.iter_cnt = -1
+
         # try and use the specified group (if any)
         group_name = build_option('group')
         group_spec = self.cfg['group']
@@ -448,7 +451,7 @@ def get_checksum_for(self, checksums, filename=None, index=None):
         if checksum and chksum_input_git is not None:
             # ignore any checksum for given filename due to changes in https://github.com/python/cpython/issues/90021
             # tarballs made for git repos are not reproducible when created with Python < 3.9
-            if sys.version_info[0] >= 3 and sys.version_info[1] < 9:
+            if sys.version_info < (3, 9):
                 print_warning(
                     "Reproducible tarballs of Git repos are only possible when using Python 3.9+ to run EasyBuild. "
                     f"Skipping checksum verification of {chksum_input} since Python < 3.9 is used."
@@ -2387,7 +2390,8 @@ def handle_iterate_opts(self):
                     self.log.debug("Iterating opt %s: %s", opt, self.iter_opts[opt])
 
             if self.iter_opts:
-                print_msg("starting iteration #%s ..." % self.iter_idx, log=self.log, silent=self.silent)
+                print_msg(f"starting iteration {self.iter_idx + 1}/{self.iter_cnt} ...", log=self.log,
+                          silent=self.silent)
                 self.log.info("Current iteration index: %s", self.iter_idx)
 
             # pop first element from all iterative easyconfig parameters as next value to use
@@ -2429,7 +2433,7 @@ def det_iter_cnt(self):
 
         # we need to take into account that builddependencies is always a list
         # we're only iterating over it if it's a list of lists
-        builddeps = self.cfg['builddependencies']
+        builddeps = self.cfg.get_ref('builddependencies')
         if all(isinstance(x, list) for x in builddeps):
             iter_opt_counts.append(len(builddeps))
 
@@ -2830,8 +2834,6 @@ def patch_step(self, beginpath=None, patches=None):
             self.log.info("Applying patch %s" % patch['name'])
             trace_msg("applying patch %s" % patch['name'])
 
-            # patch source at specified index (first source if not specified)
-            srcind = patch.get('source', 0)
             # if patch level is specified, use that (otherwise let apply_patch derive patch level)
             level = patch.get('level', None)
             # determine suffix of source path to apply patch in (if any)
@@ -2840,16 +2842,14 @@ def patch_step(self, beginpath=None, patches=None):
             copy_patch = 'copy' in patch and 'sourcepath' not in patch
             options = patch.get('opts', None)  # Extra options for patch command
 
-            self.log.debug("Source index: %s; patch level: %s; source path suffix: %s; copy patch: %s; options: %s",
-                           srcind, level, srcpathsuffix, copy_patch, options)
+            self.log.debug("Patch level: %s; source path suffix: %s; copy patch: %s; options: %s",
+                           level, srcpathsuffix, copy_patch, options)
 
             if beginpath is None:
-                try:
-                    beginpath = self.src[srcind]['finalpath']
-                    self.log.debug("Determine begin path for patch %s: %s" % (patch['name'], beginpath))
-                except IndexError as err:
-                    raise EasyBuildError("Can't apply patch %s to source at index %s of list %s: %s",
-                                         patch['name'], srcind, self.src, err)
+                if not self.src:
+                    raise EasyBuildError("Can't apply patch %s to source if no sources are given", patch['name'])
+                beginpath = self.src[0]['finalpath']
+                self.log.debug("Determined begin path for patch %s: %s" % (patch['name'], beginpath))
             else:
                 self.log.debug("Using specified begin path for patch %s: %s" % (patch['name'], beginpath))
 
@@ -4662,18 +4662,19 @@ def run_step(self, step, step_methods):
         run_hook(step, self.hooks, pre_step_hook=True, args=[self])
 
         for step_method in step_methods:
+            # step_method is a lambda function that takes an EasyBlock instance as an argument,
+            # and returns the actual method
+            current_method = step_method(self)
             # Remove leading underscore from e.g. "_test_step"
-            method_name = '_'.join(step_method.__code__.co_names).lstrip('_')
+            method_name = current_method.__name__.lstrip('_')
             self.log.info("Running method %s part of step %s", method_name, step)
 
             if self.dry_run:
                 self.dry_run_msg("[%s method]", method_name)
 
                 # if an known possible error occurs, just report it and continue
                 try:
-                    # step_method is a lambda function that takes an EasyBlock instance as an argument,
-                    # and returns the actual method, so use () to execute it
-                    step_method(self)()
+                    current_method()
                 except Exception as err:
                     if build_option('extended_dry_run_ignore_errors'):
                         dry_run_warning("ignoring error %s" % err, silent=self.silent)
@@ -4682,9 +4683,7 @@ def run_step(self, step, step_methods):
                         raise
                 self.dry_run_msg('')
             else:
-                # step_method is a lambda function that takes an EasyBlock instance as an argument,
-                # and returns the actual method, so use () to execute it
-                step_method(self)()
+                current_method()
 
         run_hook(step, self.hooks, post_step_hook=True, args=[self])
 
@@ -4794,7 +4793,8 @@ def run_all_steps(self, run_test_cases):
         if self.cfg['stop'] == 'cfg':
             return True
 
-        steps = self.get_steps(run_test_cases=run_test_cases, iteration_count=self.det_iter_cnt())
+        self.iter_cnt = self.det_iter_cnt()
+        steps = self.get_steps(run_test_cases=run_test_cases, iteration_count=self.iter_cnt)
 
         # figure out how many steps will actually be run (not be skipped)
         step_cnt = 0
@@ -4927,7 +4927,7 @@ def copy_build_dirs_logs_failed_install(application_log, silent, app, easyconfig
             msg = f"Build directory of failed installation copied to {build_dirs_path}"
 
             def operation(src, dest):
-                copy_dir(src, dest, dirs_exist_ok=True)
+                copy_dir(src, dest, dirs_exist_ok=True, symlinks=True)
 
             operation_args.append((operation, [app.builddir], build_dirs_path, msg))
 
@@ -5001,6 +5001,8 @@ def build_and_install_one(ecdict, init_env):
         _log.debug("Skip set to %s" % skip)
         app.cfg['skip'] = skip
 
+    hooks = load_hooks(build_option('hooks'))
+
     # build easyconfig
     error_msg = '(no error)'
     exit_code = None
@@ -5022,6 +5024,8 @@ def build_and_install_one(ecdict, init_env):
             else:
                 enabled_write_permissions = False
 
+        run_hook(EASYBLOCK, hooks, pre_step_hook=True, args=[app])
+
         result = app.run_all_steps(run_test_cases=run_test_cases)
 
         if not dry_run:
@@ -5033,7 +5037,9 @@ def build_and_install_one(ecdict, init_env):
             except EasyBuildError as err:
                 _log.warning("Failed to create build environment dump for easyconfig %s: %s", reprod_spec, err)
 
-            # also add any extension easyblocks used during the build for reproducibility
+            # also add any component/extension easyblocks used during the build for reproducibility
+            if hasattr(app, 'comp_instances'):
+                copy_easyblocks_for_reprod([comp for cfg, comp in app.comp_instances], reprod_dir)
             if app.ext_instances:
                 copy_easyblocks_for_reprod(app.ext_instances, reprod_dir)
             # If not already done remove the granted write permissions if we did so
@@ -5120,8 +5126,12 @@ def ensure_writable_log_dir(log_dir):
                     block = det_full_ec_version(app.cfg) + ".block"
                     repo.add_easyconfig(ecdict['original_spec'], app.name, block, buildstats, currentbuildstats)
                 repo.add_easyconfig(spec, app.name, det_full_ec_version(app.cfg), buildstats, currentbuildstats)
-                for patch in app.patches:
-                    repo.add_patch(patch['path'], app.name)
+                patches = app.patches
+                for ext in app.exts:
+                    patches += ext.get('patches', [])
+                for patch in patches:
+                    if 'path' in patch:
+                        repo.add_patch(patch['path'], app.name)
                 repo.commit("Built %s" % app.full_mod_name)
                 del repo
             except EasyBuildError as err:
@@ -5143,10 +5153,14 @@ def ensure_writable_log_dir(log_dir):
                 _log.debug("Copied easyconfig file %s to %s", spec, newspec)
 
                 # copy patches
-                for patch in app.patches:
-                    target = os.path.join(new_log_dir, os.path.basename(patch['path']))
-                    copy_file(patch['path'], target)
-                    _log.debug("Copied patch %s to %s", patch['path'], target)
+                patches = app.patches
+                for ext in app.exts:
+                    patches += ext.get('patches', [])
+                for patch in patches:
+                    if 'path' in patch:
+                        target = os.path.join(new_log_dir, os.path.basename(patch['path']))
+                        copy_file(patch['path'], target)
+                        _log.debug("Copied patch %s to %s", patch['path'], target)
 
                 if build_option('read_only_installdir') and not app.cfg['stop']:
                     # take away user write permissions (again)
@@ -5200,6 +5214,8 @@ def ensure_writable_log_dir(log_dir):
     if not success:
         copy_build_dirs_logs_failed_install(application_log, silent, app, ecdict['ec'])
 
+    run_hook(EASYBLOCK, hooks, post_step_hook=True, args=[app])
+
     del app
 
     return (success, application_log, error_msg, exit_code)

easybuild/framework/easyconfig/easyconfig.py

@@ -1968,17 +1968,22 @@ def asdict(self):
                 res[key] = value
         return res
 
-    def get_cuda_cc_template_value(self, key):
+    def get_cuda_cc_template_value(self, key, required=True):
         """
         Get template value based on --cuda-compute-capabilities EasyBuild configuration option
         and cuda_compute_capabilities easyconfig parameter.
         Returns user-friendly error message in case neither are defined,
         or if an unknown key is used.
+
+        :param required: If False and the key is not found, return an empty string instead of raising an error.
         """
         if key.startswith('cuda_') and any(x == key for x in TEMPLATE_NAMES_DYNAMIC):
             try:
                 return self.template_values[key]
             except KeyError:
+                if not required:
+                    self.log.debug(f'Key {key} not found in template values, returning empty value')
+                    return ''
                 error_msg = "Template value '%s' is not defined!\n"
                 error_msg += "Make sure that either the --cuda-compute-capabilities EasyBuild configuration "
                 error_msg += "option is set, or that the cuda_compute_capabilities easyconfig parameter is defined."

easybuild/scripts/findPythonDeps.py

@@ -85,8 +85,7 @@ def can_run(cmd, *arguments):
 def run_shell_cmd(arguments, action_desc, capture_stderr=True, **kwargs):
     """Run the command and return the return code and output"""
     extra_args = kwargs or {}
-    if sys.version_info[0] >= 3:
-        extra_args['universal_newlines'] = True
+    extra_args['universal_newlines'] = True
     stderr = subprocess.STDOUT if capture_stderr else subprocess.PIPE
     p = subprocess.Popen(arguments, stdout=subprocess.PIPE, stderr=stderr, **extra_args)
     out, err = p.communicate()

easybuild/scripts/install-EasyBuild-develop.sh

@@ -13,6 +13,7 @@ print_usage()
     echo "Usage: $0 <github_username> <install_dir>"
     echo
     echo "    github_username:     username on GitHub for which the EasyBuild repositories should be cloned"
+    echo "                         Use 'easybuilders' if you don't want to use a fork"
     echo
     echo "    install_dir:         directory where all the EasyBuild files will be installed"
     echo
@@ -28,11 +29,13 @@ github_clone_branch()
     echo "=== Cloning ${GITHUB_USERNAME}/${REPO} ..."
     git clone --branch "${BRANCH}" "[email protected]:${GITHUB_USERNAME}/${REPO}.git"
 
-    echo "=== Adding and fetching EasyBuilders GitHub repository @ easybuilders/${REPO} ..."
-    cd "${REPO}"
-    git remote add "github_easybuilders" "[email protected]:easybuilders/${REPO}.git"
-    git fetch github_easybuilders
-    git branch --set-upstream-to "github_easybuilders/${BRANCH}" "${BRANCH}"
+    if [[ $GITHUB_USERNAME != "easybuilders" ]]; then
+        echo "=== Adding and fetching EasyBuilders GitHub repository @ easybuilders/${REPO} ..."
+        cd "${REPO}"
+        git remote add "github_easybuilders" "[email protected]:easybuilders/${REPO}.git"
+        git fetch github_easybuilders
+        git branch --set-upstream-to "github_easybuilders/${BRANCH}" "${BRANCH}"
+    fi
 }
 
 # Print the content of the module