Proposed changes to make EasyBuild plugin-able through entrypoints

easybuild/framework/easyconfig/easyconfig.py

@@ -65,6 +65,7 @@
 from easybuild.framework.easyconfig.templates import ALTERNATIVE_EASYCONFIG_TEMPLATES, DEPRECATED_EASYCONFIG_TEMPLATES
 from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS, TEMPLATE_NAMES_DYNAMIC, template_constant_dict
 from easybuild.tools import LooseVersion
+from easybuild.tools.entrypoints import EntrypointEasyblock
 from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_warning, print_msg
 from easybuild.tools.config import GENERIC_EASYBLOCK_PKG, LOCAL_VAR_NAMING_CHECK_ERROR, LOCAL_VAR_NAMING_CHECK_LOG
 from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN
@@ -2016,9 +2017,15 @@ def get_easyblock_class(easyblock, name=None, error_on_failed_import=True, error
                           class_name, modulepath)
                 cls = get_class_for(modulepath, class_name)
             else:
-                modulepath = get_module_path(easyblock)
-                cls = get_class_for(modulepath, class_name)
-                _log.info("Derived full easyblock module path for %s: %s" % (class_name, modulepath))
+                eb_from_eps = EntrypointEasyblock.get_loaded_entrypoints(name=easyblock)
+                if eb_from_eps:
+                    ep = eb_from_eps[0]
+                    cls = ep.wrapped
+                    _log.info("Obtained easyblock class '%s' from entrypoint '%s'", easyblock, str(ep))
+                else:
+                    modulepath = get_module_path(easyblock)
+                    cls = get_class_for(modulepath, class_name)
+                    _log.info("Derived full easyblock module path for %s: %s" % (class_name, modulepath))
         else:
             # if no easyblock specified, try to find if one exists
             if name is None:

easybuild/framework/easyconfig/tools.py

@@ -54,6 +54,7 @@
 from easybuild.framework.easyconfig.easyconfig import process_easyconfig
 from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check
 from easybuild.tools import LooseVersion
+from easybuild.tools.entrypoints import EntrypointEasyblock
 from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_error, print_msg, print_warning
 from easybuild.tools.config import build_option
 from easybuild.tools.environment import restore_env
@@ -799,6 +800,14 @@ def avail_easyblocks():
                     else:
                         raise EasyBuildError("Failed to determine easyblock class name for %s", easyblock_loc)
 
+    ept_eb_lst = EntrypointEasyblock.get_loaded_entrypoints()
+
+    for ept_eb in ept_eb_lst:
+        easyblocks[ept_eb.module] = {
+            'class': ept_eb.name,
+            'loc': ept_eb.file,
+        }
+
     return easyblocks
 
 

easybuild/tools/config.py

@@ -351,6 +351,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'upload_test_report',
         'update_modules_tool_cache',
         'use_ccache',
+        'use_entrypoints',
         'use_existing_modules',
         'use_f90cache',
         'wait_on_lock_limit',

easybuild/tools/entrypoints.py

@@ -0,0 +1,210 @@
+"""Python module to manage entry points for EasyBuild.
+
+Authors:
+
+* Davide Grassano (CECAM)
+"""
+import sys
+import importlib
+from easybuild.tools.config import build_option
+
+from easybuild.base import fancylogger
+from easybuild.tools.build_log import EasyBuildError
+from typing import TypeVar, List, Set, Any
+
+_T = TypeVar('_T')
+
+
+HAVE_ENTRY_POINTS = False
+HAVE_ENTRY_POINTS_CLS = False
+if sys.version_info >= (3, 8):
+    HAVE_ENTRY_POINTS = True
+    from importlib.metadata import entry_points, EntryPoint
+else:
+    EntryPoint = Any
+
+if sys.version_info >= (3, 10):
+    # Python >= 3.10 uses importlib.metadata.EntryPoints as a type for entry_points()
+    HAVE_ENTRY_POINTS_CLS = True
+
+
+_log = fancylogger.getLogger('entrypoints', fname=False)
+
+
+class EasybuildEntrypoint:
+    group = None
+    expected_type = None
+    registered = {}
+
+    def __init__(self):
+        if self.group is None:
+            raise EasyBuildError(
+                "Cannot use <EasybuildEntrypoint> drirectly. Please use a subclass that defines `group`",
+            )
+
+        self.wrapped = None
+        self.module = None
+        self.name = None
+        self.file = None
+
+    def __repr__(self):
+        return f"{self.__class__.__name__} <{self.module}:{self.name}>"
+
+    def __call__(self, wrap: _T) -> _T:
+        """Use an instance of this class as a decorator to register an entrypoint."""
+        if self.expected_type is not None:
+            check = False
+            try:
+                check = isinstance(wrap, self.expected_type) or issubclass(wrap, self.expected_type)
+            except Exception:
+                pass
+            if not check:
+                raise EasyBuildError(
+                    "Entrypoint '%s' expected type '%s', got '%s'",
+                    self.name, self.expected_type, type(wrap)
+                )
+        self.wrapped = wrap
+        self.module = getattr(wrap, '__module__', None)
+        self.name = getattr(wrap, '__name__', None)
+        if self.module:
+            mod = importlib.import_module(self.module)
+            self.file = getattr(mod, '__file__', None)
+
+        grp = self.registered.setdefault(self.group, set())
+
+        for ep in grp:
+            if ep.name == self.name and ep.module != self.module:
+                raise ValueError(
+                    "Entrypoint '%s' already registered in group '%s' by module '%s' vs '%s'",
+                    self.name, self.group, ep.module, self.module
+                )
+        grp.add(self)
+
+        self.validate()
+
+        _log.debug("Registered entrypoint: %s", self)
+
+        return wrap
+
+    @classmethod
+    def retrieve_entrypoints(cls) -> Set[EntryPoint]:
+        """"Get all entrypoints in this group."""
+        strict_python = True
+        use_eps = build_option('use_entrypoints', default=None)
+        if use_eps is None:
+            # Default True needed to work with commands like --list-toolchains that do not initialize the BuildOptions
+            use_eps = True
+            # Needed to work with older Python versions: do not raise errors when entry points are default enabled
+            strict_python = False
+        res = set()
+        if use_eps:
+            if not HAVE_ENTRY_POINTS:
+                if strict_python:
+                    msg = "`--use-entrypoints` requires importlib.metadata (Python >= 3.8)"
+                    _log.warning(msg)
+                    raise EasyBuildError(msg)
+                else:
+                    _log.debug("`get_group_entrypoints` called before BuildOptions initialized, with python < 3.8")
+            else:
+                if HAVE_ENTRY_POINTS_CLS:
+                    res = set(entry_points(group=cls.group))
+                else:
+                    res = set(entry_points().get(cls.group, []))
+
+        return res
+
+    @classmethod
+    def load_entrypoints(cls):
+        """Load all the entrypoints in this group. This is needed for the modules contining the entrypoints to be
+        actually imported in order to process the function decorators that will register them in the
+        `registered` dict."""
+        for ep in cls.retrieve_entrypoints():
+            try:
+                ep.load()
+            except Exception as e:
+                msg = f"Error loading entrypoint {ep}: {e}"
+                _log.warning(msg)
+                raise EasyBuildError(msg) from e
+
+    @classmethod
+    def get_loaded_entrypoints(cls: _T, name: str = None, **filter_params) -> List[_T]:
+        """Get all entrypoints in this group."""
+        cls.load_entrypoints()
+
+        entrypoints = []
+        for ep in cls.registered.get(cls.group, []):
+            cond = name is None or ep.name == name
+            for key, value in filter_params.items():
+                cond = cond and getattr(ep, key, None) == value
+            if cond:
+                entrypoints.append(ep)
+
+        return entrypoints
+
+    @staticmethod
+    def clear():
+        """Clear the registered entrypoints. Used for testing when the same entrypoint is loaded multiple times
+        from different temporary directories."""
+        EasybuildEntrypoint.registered.clear()
+
+    def validate(self):
+        """Validate the entrypoint."""
+        if self.module is None or self.name is None:
+            raise EasyBuildError("Entrypoint `%s` has no module or name associated", self.wrapped)
+
+
+class EntrypointHook(EasybuildEntrypoint):
+    """Class to represent a hook entrypoint."""
+    group = 'easybuild.hooks'
+
+    def __init__(self, step, pre_step=False, post_step=False, priority=0):
+        """Initialize the EntrypointHook."""
+        super().__init__()
+        self.step = step
+        self.pre_step = pre_step
+        self.post_step = post_step
+        self.priority = priority
+
+    def validate(self):
+        """Validate the hook entrypoint."""
+        from easybuild.tools.hooks import KNOWN_HOOKS, HOOK_SUFF, PRE_PREF, POST_PREF
+        super().validate()
+
+        if not callable(self.wrapped):
+            raise EasyBuildError("Hook entrypoint `%s` is not callable", self.wrapped)
+
+        prefix = ''
+        if self.pre_step:
+            prefix = PRE_PREF
+        elif self.post_step:
+            prefix = POST_PREF
+
+        hook_name = f'{prefix}{self.step}{HOOK_SUFF}'
+
+        if hook_name not in KNOWN_HOOKS:
+            msg = f"Attempting to register unknown hook '{hook_name}'"
+            _log.warning(msg)
+            raise EasyBuildError(msg)
+
+
+class EntrypointEasyblock(EasybuildEntrypoint):
+    """Class to represent an easyblock entrypoint."""
+    group = 'easybuild.easyblock'
+
+    def __init__(self):
+        super().__init__()
+        # Avoid circular imports by importing EasyBlock here
+        from easybuild.framework.easyblock import EasyBlock
+        self.expected_type = EasyBlock
+
+
+class EntrypointToolchain(EasybuildEntrypoint):
+    """Class to represent a toolchain entrypoint."""
+    group = 'easybuild.toolchain'
+
+    def __init__(self, prepend=False):
+        super().__init__()
+        # Avoid circular imports by importing Toolchain here
+        from easybuild.tools.toolchain.toolchain import Toolchain
+        self.expected_type = Toolchain
+        self.prepend = prepend

easybuild/tools/hooks.py

@@ -32,6 +32,8 @@
 import difflib
 import os
 
+from easybuild.tools.entrypoints import EntrypointHook
+
 from easybuild.base import fancylogger
 from easybuild.tools.build_log import EasyBuildError, print_msg
 from easybuild.tools.config import build_option
@@ -233,12 +235,9 @@ def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None,
     """
     hook = find_hook(label, hooks, pre_step_hook=pre_step_hook, post_step_hook=post_step_hook)
     res = None
+    args = args or []
+    kwargs = kwargs or {}
     if hook:
-        if args is None:
-            args = []
-        if kwargs is None:
-            kwargs = {}
-
         if pre_step_hook:
             label = 'pre-' + label
         elif post_step_hook:
@@ -251,4 +250,26 @@ def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None,
 
         _log.info("Running '%s' hook function (args: %s, keyword args: %s)...", hook.__name__, args, kwargs)
         res = hook(*args, **kwargs)
+
+    entrypoint_hooks = EntrypointHook.get_loaded_entrypoints(
+        step=label, pre_step=pre_step_hook, post_step=post_step_hook
+    )
+    if entrypoint_hooks:
+        msg = "Running entry point %s hook..." % label
+        if build_option('debug') and not build_option('silence_hook_trigger'):
+            print_msg(msg)
+        entrypoint_hooks.sort(
+            key=lambda x: (-x.priority, x.name),
+            )
+        for hook in entrypoint_hooks:
+            _log.info(
+                "Running entry point '%s' hook function (args: %s, keyword args: %s)...",
+                hook.name, args, kwargs
+            )
+            try:
+                res = hook.wrapped(*args, **kwargs)
+            except Exception as e:
+                _log.warning("Error running entry point '%s' hook: %s", hook.name, e)
+                raise EasyBuildError("Error running entry point '%s' hook: %s", hook.name, e) from e
+
     return res

easybuild/tools/options.py

@@ -110,6 +110,7 @@
 from easybuild.tools.systemtools import get_cpu_features, get_gpu_info, get_os_type, get_system_info
 from easybuild.tools.utilities import flatten
 from easybuild.tools.version import this_is_easybuild
+from easybuild.tools.entrypoints import EntrypointHook, EntrypointEasyblock, EntrypointToolchain
 
 
 try:
@@ -303,6 +304,9 @@ def basic_options(self):
             'stop': ("Stop the installation after certain step",
                      'choice', 'store_or_None', EXTRACT_STEP, 's', all_stops),
             'strict': ("Set strictness level", 'choice', 'store', WARN, strictness_options),
+            'use-entrypoints': (
+                "Use entry points for easyblocks, toolchains, and hooks", None, 'store_true', False,
+            ),
         })
 
         self.log.debug("basic_options: descr %s opts %s" % (descr, opts))
@@ -1634,6 +1638,19 @@ def det_location(opt, prefix=''):
 
         pretty_print_opts(opts_dict)
 
+        if build_option('use_entrypoints', default=True):
+            for prefix, cls in [
+                ('Hook', EntrypointHook),
+                ('Easyblock', EntrypointEasyblock),
+                ('Toolchain', EntrypointToolchain),
+            ]:
+                ept_list = cls.retrieve_entrypoints()
+                if ept_list:
+                    print()
+                    print("%ss from entrypoints (%d):" % (prefix, len(ept_list)))
+                    for ept in ept_list:
+                        print('-', ept)
+
 
 def parse_options(args=None, with_include=True):
     """wrapper function for option parsing"""

easybuild/tools/toolchain/toolchain.py

@@ -168,7 +168,7 @@ class Toolchain:
     CLASS_CONSTANTS_TO_RESTORE = None
     CLASS_CONSTANT_COPIES = {}
 
-    # class method
+    @classmethod
     def _is_toolchain_for(cls, name):
         """see if this class can provide support for toolchain named name"""
         # TODO report later in the initialization the found version
@@ -181,8 +181,6 @@ def _is_toolchain_for(cls, name):
             # is no name is supplied, check whether class can be used as a toolchain
             return bool(getattr(cls, 'NAME', None))
 
-    _is_toolchain_for = classmethod(_is_toolchain_for)
-
     def __init__(self, name=None, version=None, mns=None, class_constants=None, tcdeps=None, modtool=None,
                  hidden=False):
         """