From 2bace9aae71b4ec789cc9768c6c500da162a5bac Mon Sep 17 00:00:00 2001 From: "Jesse P. Johnson" Date: Wed, 9 Oct 2024 22:52:59 +0200 Subject: [PATCH 1/7] feat: load collection from entrypoint --- invoke/collection.py | 17 +++ invoke/loader.py | 119 ++++++++++++------- tests/_support/entry_point/pyproject.toml | 15 +++ tests/loader.py | 135 ++++++++-------------- 4 files changed, 161 insertions(+), 125 deletions(-) create mode 100644 tests/_support/entry_point/pyproject.toml diff --git a/invoke/collection.py b/invoke/collection.py index 23dcff928..7145de187 100644 --- a/invoke/collection.py +++ b/invoke/collection.py @@ -5,6 +5,7 @@ from .util import Lexicon, helpline from .config import merge_dicts, copy_dict +from .loader import EntryPointLoader from .parser import Context as ParserContext from .tasks import Task @@ -123,6 +124,9 @@ def _add_object(self, obj: Any, name: Optional[str] = None) -> None: raise TypeError("No idea how to insert {!r}!".format(type(obj))) method(obj, name=name) + def __str__(self) -> str: + return self.name or 'root' + def __repr__(self) -> str: task_names = list(self.tasks.keys()) collections = ["{}...".format(x) for x in self.collections.keys()] @@ -142,6 +146,19 @@ def __eq__(self, other: object) -> bool: def __bool__(self) -> bool: return bool(self.task_names) + @classmethod + def from_entry_point( + cls, group: str, name: str, **kwargs: Any + ) -> 'Collection': + """Load collection stack from entrypoint.""" + loader = EntryPointLoader(group=group) + collection = loader.load(name)[0] + if collection.name is None: + collection.name = name + if kwargs: + collection.configure(kwargs) + return collection + @classmethod def from_module( cls, diff --git a/invoke/loader.py b/invoke/loader.py index 801d16333..80a550b4d 100644 --- a/invoke/loader.py +++ b/invoke/loader.py @@ -1,12 +1,11 @@ import os import sys -from importlib.machinery import ModuleSpec +from importlib.metadata import entry_points from importlib.util import module_from_spec, spec_from_file_location from pathlib import Path -from types import ModuleType -from typing import Any, Optional, Tuple +from typing import Any, Iterable, Optional, Tuple, Union -from . import Config +from .config import Config from .exceptions import CollectionNotFound from .util import debug @@ -27,11 +26,9 @@ def __init__(self, config: Optional["Config"] = None) -> None: config options. Defaults to an anonymous ``Config()`` if none is given. """ - if config is None: - config = Config() - self.config = config + self.config = Config() if config is None else config - def find(self, name: str) -> Optional[ModuleSpec]: + def find(self, name: str) -> Optional[Union[Iterable[Any], Any]]: """ Implementation-specific finder method seeking collection ``name``. @@ -46,7 +43,7 @@ def find(self, name: str) -> Optional[ModuleSpec]: """ raise NotImplementedError - def load(self, name: Optional[str] = None) -> Tuple[ModuleType, str]: + def load(self, name: Optional[str] = None) -> Tuple[Any, str]: """ Load and return collection module identified by ``name``. @@ -65,40 +62,12 @@ def load(self, name: Optional[str] = None) -> Tuple[ModuleType, str]: .. versionadded:: 1.0 """ - if name is None: - name = self.config.tasks.collection_name - spec = self.find(name) - if spec and spec.loader and spec.origin: - # Typically either tasks.py or tasks/__init__.py - source_file = Path(spec.origin) - # Will be 'the dir tasks.py is in', or 'tasks/', in both cases this - # is what wants to be in sys.path for "from . import sibling" - enclosing_dir = source_file.parent - # Will be "the directory above the spot that 'import tasks' found", - # namely the parent of "your task tree", i.e. "where project level - # config files are looked for". So, same as enclosing_dir for - # tasks.py, but one more level up for tasks/__init__.py... - module_parent = enclosing_dir - if spec.parent: # it's a package, so we have to go up again - module_parent = module_parent.parent - # Get the enclosing dir on the path - enclosing_str = str(enclosing_dir) - if enclosing_str not in sys.path: - sys.path.insert(0, enclosing_str) - # Actual import - module = module_from_spec(spec) - sys.modules[spec.name] = module # so 'from . import xxx' works - spec.loader.exec_module(module) - # Return the module and the folder it was found in - return module, str(module_parent) - msg = "ImportError loading {!r}, raising ImportError" - debug(msg.format(name)) - raise ImportError + raise NotImplementedError class FilesystemLoader(Loader): """ - Loads Python files from the filesystem (e.g. ``tasks.py``.) + Loads Python files from the filesystem (e.g. ``tasks.py``). Searches recursively towards filesystem root from a given start point. @@ -120,7 +89,7 @@ def start(self) -> str: # Lazily determine default CWD if configured value is falsey return self._start or os.getcwd() - def find(self, name: str) -> Optional[ModuleSpec]: + def find(self, name: str) -> Optional[Any]: debug("FilesystemLoader find starting at {!r}".format(self.start)) spec = None module = "{}.py".format(name) @@ -152,3 +121,73 @@ def find(self, name: str) -> Optional[ModuleSpec]: debug(msg.format(name)) raise CollectionNotFound(name=name, start=self.start) return None + + def load(self, name: Optional[str] = None) -> Tuple[Any, str]: + if name is None: + name = self.config.tasks.collection_name + spec = self.find(name) + if spec and spec.loader and spec.origin: + # Typically either tasks.py or tasks/__init__.py + source_file = Path(spec.origin) + # Will be 'the dir tasks.py is in', or 'tasks/', in both cases this + # is what wants to be in sys.path for "from . import sibling" + enclosing_dir = source_file.parent + # Will be "the directory above the spot that 'import tasks' found", + # namely the parent of "your task tree", i.e. "where project level + # config files are looked for". So, same as enclosing_dir for + # tasks.py, but one more level up for tasks/__init__.py... + module_parent = enclosing_dir + if spec.parent: # it is a package, so we have to go up again + module_parent = module_parent.parent + # Get the enclosing dir on the path + enclosing_str = str(enclosing_dir) + if enclosing_str not in sys.path: + sys.path.insert(0, enclosing_str) + # Actual import + module = module_from_spec(spec) + sys.modules[spec.name] = module # so 'from . import xxx' works + spec.loader.exec_module(module) + # Return the module and the folder it was found in + return module, str(module_parent) + msg = "ImportError loading {!r}, raising ImportError" + debug(msg.format(name)) + raise ImportError + + +class EntryPointLoader(Loader): + """Load collections from entry point.""" + + def __init__(self, config: Optional[Config] = None, **kwargs: Any) -> None: + """Initialize entry point plugins for invoke.""" + self.group = kwargs.pop('group', None) + super().__init__(config, **kwargs) + + def find( + self, name: Optional[str] = None + ) -> Optional[Union[Iterable[Any], Any]]: + """Find entrypoints for invoke.""" + params = {} + if name: + params['name'] = name + if self.group: + params['group'] = self.group + modules = entry_points(**params) + if modules: + return modules + if self.group: + if name: + raise CollectionNotFound( + name=name or 'entry_point', start=self.group + ) + else: + raise ModuleNotFoundError( + 'no module matching %r found', + self.group, + ) + + def load(self, name: Optional[str] = None) -> Tuple[Any, str]: + """Load entrypoints for invoke.""" + modules = self.find(name) + for module in modules or []: + return (module.load(), os.getcwd()) + raise ImportError diff --git a/tests/_support/entry_point/pyproject.toml b/tests/_support/entry_point/pyproject.toml new file mode 100644 index 000000000..0dfbf0116 --- /dev/null +++ b/tests/_support/entry_point/pyproject.toml @@ -0,0 +1,15 @@ +[build-system] +requires = ['setuptools>=61.0.0', 'wheel'] +build-backend = 'setuptools.build_meta' + +[project] +name = "invoke-example" +version = "0.1.0" +description = "Example project using entry point with invoke." +requires-python = ">=3.7" +dependencies = [ + "invoke" +] + +[project.entry-points."invoke"] +test = "example.module:namespace" diff --git a/tests/loader.py b/tests/loader.py index 019ce1b37..838d12378 100644 --- a/tests/loader.py +++ b/tests/loader.py @@ -1,105 +1,28 @@ import os -import sys -from importlib.util import spec_from_file_location -from types import ModuleType -from pathlib import Path from pytest import raises -from invoke import Config -from invoke.loader import Loader, FilesystemLoader as FSLoader +from invoke import Config, Collection +from invoke.loader import FilesystemLoader, EntryPointLoader from invoke.exceptions import CollectionNotFound from _util import support -class _BasicLoader(Loader): - """ - Tests top level Loader behavior with basic finder stub. - - Used when we want to make sure we're testing Loader.load and not e.g. - FilesystemLoader's specific implementation. - """ - - def find(self, name): - path = os.path.join(support, name) - if os.path.exists(f"{path}.py"): - path = f"{path}.py" - elif os.path.exists(path): - path = os.path.join(path, "__init__.py") - spec = spec_from_file_location(name, path) - return spec - - -class Loader_: - def exhibits_default_config_object(self): - loader = _BasicLoader() - assert isinstance(loader.config, Config) - assert loader.config.tasks.collection_name == "tasks" - - def returns_module_and_location(self): - mod, path = _BasicLoader().load("namespacing") - assert isinstance(mod, ModuleType) - assert path == support - - def may_configure_config_via_constructor(self): - config = Config({"tasks": {"collection_name": "mytasks"}}) - loader = _BasicLoader(config=config) - assert loader.config.tasks.collection_name == "mytasks" - - def adds_module_parent_dir_to_sys_path(self): - # Crummy doesn't-explode test. - _BasicLoader().load("namespacing") - - def doesnt_duplicate_parent_dir_addition(self): - _BasicLoader().load("namespacing") - _BasicLoader().load("namespacing") - # If the bug is present, this will be 2 at least (and often more, since - # other tests will pollute it (!). - assert sys.path.count(support) == 1 - - def can_load_package(self): - loader = _BasicLoader() - # Load itself doesn't explode (tests 'from . import xxx' internally) - mod, enclosing_dir = loader.load("package") - # Properties of returned values look as expected - # (enclosing dir is always the one above the module-or-package) - assert enclosing_dir == support - assert mod.__file__ == str(Path(support) / "package" / "__init__.py") - - def load_name_defaults_to_config_tasks_collection_name(self): - "load() name defaults to config.tasks.collection_name" - - class MockLoader(_BasicLoader): - def find(self, name): - # Sanity - assert name == "simple_ns_list" - return super().find(name) - - config = Config({"tasks": {"collection_name": "simple_ns_list"}}) - loader = MockLoader(config=config) - # More sanity: expect simple_ns_list.py (not tasks.py) - mod, path = loader.load() - assert mod.__file__ == os.path.join(support, "simple_ns_list.py") - - -class FilesystemLoader_: +class FSLoader: def setup_method(self): - self.loader = FSLoader(start=support) + self.loader = FilesystemLoader(start=support) def discovery_start_point_defaults_to_cwd(self): - assert FSLoader().start == os.getcwd() - - def exposes_start_point_as_attribute(self): - assert FSLoader().start == os.getcwd() + assert FilesystemLoader().start == os.getcwd() def start_point_is_configurable_via_kwarg(self): start = "/tmp" - assert FSLoader(start=start).start == start + assert FilesystemLoader(start=start).start == start def start_point_is_configurable_via_config(self): config = Config({"tasks": {"search_root": "nowhere"}}) - assert FSLoader(config=config).start == "nowhere" + assert FilesystemLoader(config=config).start == "nowhere" def raises_CollectionNotFound_if_not_found(self): with raises(CollectionNotFound): @@ -117,7 +40,49 @@ def searches_towards_root_of_filesystem(self): directly = self.loader.load("foo") # Loaded while root is multiple dirs deeper than the .py deep = os.path.join(support, "ignoreme", "ignoremetoo") - indirectly = FSLoader(start=deep).load("foo") + indirectly = FilesystemLoader(start=deep).load("foo") assert directly[0].__file__ == indirectly[0].__file__ assert directly[0].__spec__ == indirectly[0].__spec__ assert directly[1] == indirectly[1] + + +def use_eploader_directly(): + basedir = os.path.dirname(__file__) + source_path = os.path.join(basedir, '_support', 'entry_point') + os.chdir(source_path) + loader = EntryPointLoader(group='invoke') + collection = loader.load('test')[0] + assert isinstance(collection, Collection) + assert 'mytask' in collection.tasks.keys() + assert collection.collections == {} + + +def use_eploader_from_collection(): + basedir = os.path.dirname(__file__) + source_path = os.path.join(basedir, '_support', 'entry_point') + os.chdir(source_path) + collection = Collection.from_entry_point(group='invoke', name='test') + assert isinstance(collection, Collection) + assert collection.name == 'test' + assert 'mytask' in collection.tasks.keys() + assert collection.collections == {} + + +def raises_ImportError_if_eploader_cannot_import_module(): + basedir = os.path.dirname(__file__) + source_path = os.path.join(basedir, '_support', 'entry_point') + os.chdir(source_path) + # Instead of masking with a CollectionNotFound + with raises(ModuleNotFoundError): + loader = EntryPointLoader(group='oops') + loader.find() + + +def raises_CollectionNotFound_is_eploader_cannot_find_collection(): + basedir = os.path.dirname(__file__) + source_path = os.path.join(basedir, '_support', 'entry_point') + os.chdir(source_path) + # Instead of masking with a CollectionNotFound + with raises(CollectionNotFound): + loader = EntryPointLoader(group='invoke') + loader.find(name='nope') From 49c6966a72872a5d91bfbe818e86902b3d641dea Mon Sep 17 00:00:00 2001 From: "Jesse P. Johnson" Date: Wed, 9 Oct 2024 23:17:37 +0200 Subject: [PATCH 2/7] tests: remove cruft for entry-point test setup --- dev-requirements.txt | 1 + tests/loader.py | 13 ------------- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 1bf0ad732..36b350062 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -10,6 +10,7 @@ alabaster==0.7.12 # Testing pytest-relaxed>=2 pytest-cov>=4 +./tests/_support/entry_point # Formatting # Flake8 5.x seems to have an odd importlib-metadata incompatibility? flake8>=4,<5 diff --git a/tests/loader.py b/tests/loader.py index 838d12378..56bc88f96 100644 --- a/tests/loader.py +++ b/tests/loader.py @@ -47,9 +47,6 @@ def searches_towards_root_of_filesystem(self): def use_eploader_directly(): - basedir = os.path.dirname(__file__) - source_path = os.path.join(basedir, '_support', 'entry_point') - os.chdir(source_path) loader = EntryPointLoader(group='invoke') collection = loader.load('test')[0] assert isinstance(collection, Collection) @@ -58,9 +55,6 @@ def use_eploader_directly(): def use_eploader_from_collection(): - basedir = os.path.dirname(__file__) - source_path = os.path.join(basedir, '_support', 'entry_point') - os.chdir(source_path) collection = Collection.from_entry_point(group='invoke', name='test') assert isinstance(collection, Collection) assert collection.name == 'test' @@ -69,9 +63,6 @@ def use_eploader_from_collection(): def raises_ImportError_if_eploader_cannot_import_module(): - basedir = os.path.dirname(__file__) - source_path = os.path.join(basedir, '_support', 'entry_point') - os.chdir(source_path) # Instead of masking with a CollectionNotFound with raises(ModuleNotFoundError): loader = EntryPointLoader(group='oops') @@ -79,10 +70,6 @@ def raises_ImportError_if_eploader_cannot_import_module(): def raises_CollectionNotFound_is_eploader_cannot_find_collection(): - basedir = os.path.dirname(__file__) - source_path = os.path.join(basedir, '_support', 'entry_point') - os.chdir(source_path) - # Instead of masking with a CollectionNotFound with raises(CollectionNotFound): loader = EntryPointLoader(group='invoke') loader.find(name='nope') From 870f908b9925b6b423d1c7a43f66f9107c94b837 Mon Sep 17 00:00:00 2001 From: "Jesse P. Johnson" Date: Sun, 13 Oct 2024 16:27:31 +0200 Subject: [PATCH 3/7] feat: update loader --- invoke/loader.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/invoke/loader.py b/invoke/loader.py index 80a550b4d..7513577dd 100644 --- a/invoke/loader.py +++ b/invoke/loader.py @@ -17,7 +17,9 @@ class Loader: .. versionadded:: 1.0 """ - def __init__(self, config: Optional["Config"] = None) -> None: + def __init__( + self, config: Optional["Config"] = None, **kwargs: Any + ) -> None: """ Set up a new loader with some `.Config`. @@ -32,9 +34,9 @@ def find(self, name: str) -> Optional[Union[Iterable[Any], Any]]: """ Implementation-specific finder method seeking collection ``name``. - Must return a ModuleSpec valid for use by `importlib`, which is - typically a name string followed by the contents of the 3-tuple - returned by `importlib.module_from_spec` (``name``, ``loader``, + Must return module valid for use by repsective ``load`` implementation, + which is typically a name string followed by the contents of the + 3-tuple returned by `importlib.module_from_spec` (``name``, ``loader``, ``origin``.) For a sample implementation, see `.FilesystemLoader`. @@ -164,7 +166,7 @@ def __init__(self, config: Optional[Config] = None, **kwargs: Any) -> None: def find( self, name: Optional[str] = None - ) -> Optional[Union[Iterable[Any], Any]]: + ) -> Union[Iterable[Any], Any]: """Find entrypoints for invoke.""" params = {} if name: @@ -174,20 +176,21 @@ def find( modules = entry_points(**params) if modules: return modules - if self.group: + elif self.group: if name: raise CollectionNotFound( name=name or 'entry_point', start=self.group ) else: raise ModuleNotFoundError( - 'no module matching %r found', + 'no entry point matching group %r found', self.group, ) + raise ModuleNotFoundError('no entry points found') def load(self, name: Optional[str] = None) -> Tuple[Any, str]: """Load entrypoints for invoke.""" modules = self.find(name) - for module in modules or []: + for module in modules: return (module.load(), os.getcwd()) raise ImportError From 1f55bf6f713aa73e5cec2ab5d3877633d6796ce5 Mon Sep 17 00:00:00 2001 From: "Jesse P. Johnson" Date: Sun, 13 Oct 2024 17:44:18 +0200 Subject: [PATCH 4/7] build: update test project for entry point --- tests/_support/entry_point/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/_support/entry_point/pyproject.toml b/tests/_support/entry_point/pyproject.toml index 0dfbf0116..e82a6bf74 100644 --- a/tests/_support/entry_point/pyproject.toml +++ b/tests/_support/entry_point/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ['setuptools>=61.0.0', 'wheel'] +requires = ['setuptools>=59.4.0', 'wheel'] build-backend = 'setuptools.build_meta' [project] From 44a411a6e9cafc7dd7acd0c42f61a45de0b486dd Mon Sep 17 00:00:00 2001 From: "Jesse P. Johnson" Date: Sun, 13 Oct 2024 17:53:18 +0200 Subject: [PATCH 5/7] build: update test project for entry point --- dev-requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev-requirements.txt b/dev-requirements.txt index 36b350062..f1178f40b 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -10,6 +10,8 @@ alabaster==0.7.12 # Testing pytest-relaxed>=2 pytest-cov>=4 +# XXX due to find_packages +importlib-metadata>=8.5.0;python_version=='3.6' ./tests/_support/entry_point # Formatting # Flake8 5.x seems to have an odd importlib-metadata incompatibility? From 3f6fb37aabe93e488e4b60ece990efbcf3fe5a74 Mon Sep 17 00:00:00 2001 From: "Jesse P. Johnson" Date: Sun, 13 Oct 2024 18:55:49 +0200 Subject: [PATCH 6/7] build: update test project for entry point --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index f1178f40b..05ed06add 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -11,7 +11,7 @@ alabaster==0.7.12 pytest-relaxed>=2 pytest-cov>=4 # XXX due to find_packages -importlib-metadata>=8.5.0;python_version=='3.6' +importlib-metadata>=8.5.0;python_version<='3.7' ./tests/_support/entry_point # Formatting # Flake8 5.x seems to have an odd importlib-metadata incompatibility? From 7b101b61f540513e4fde5b5e38a967177b34d484 Mon Sep 17 00:00:00 2001 From: "Jesse P. Johnson" Date: Sun, 13 Oct 2024 18:59:54 +0200 Subject: [PATCH 7/7] build: update test project for entry point --- dev-requirements.txt | 3 +-- tests/loader.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 05ed06add..7e43b2d5c 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -11,8 +11,7 @@ alabaster==0.7.12 pytest-relaxed>=2 pytest-cov>=4 # XXX due to find_packages -importlib-metadata>=8.5.0;python_version<='3.7' -./tests/_support/entry_point +./tests/_support/entry_point; python_version>='3.7' # Formatting # Flake8 5.x seems to have an odd importlib-metadata incompatibility? flake8>=4,<5 diff --git a/tests/loader.py b/tests/loader.py index 56bc88f96..17567c2bf 100644 --- a/tests/loader.py +++ b/tests/loader.py @@ -1,5 +1,7 @@ import os +import sys +import pytest from pytest import raises from invoke import Config, Collection @@ -46,6 +48,10 @@ def searches_towards_root_of_filesystem(self): assert directly[1] == indirectly[1] +@pytest.mark.skipif( + sys.version_info < (3, 7), + reason="requires python3.7 or higher", +) def use_eploader_directly(): loader = EntryPointLoader(group='invoke') collection = loader.load('test')[0] @@ -54,6 +60,10 @@ def use_eploader_directly(): assert collection.collections == {} +@pytest.mark.skipif( + sys.version_info < (3, 7), + reason="requires python3.7 or higher", +) def use_eploader_from_collection(): collection = Collection.from_entry_point(group='invoke', name='test') assert isinstance(collection, Collection) @@ -62,6 +72,10 @@ def use_eploader_from_collection(): assert collection.collections == {} +@pytest.mark.skipif( + sys.version_info < (3, 7), + reason="requires python3.7 or higher", +) def raises_ImportError_if_eploader_cannot_import_module(): # Instead of masking with a CollectionNotFound with raises(ModuleNotFoundError): @@ -69,6 +83,10 @@ def raises_ImportError_if_eploader_cannot_import_module(): loader.find() +@pytest.mark.skipif( + sys.version_info < (3, 7), + reason="requires python3.7 or higher", +) def raises_CollectionNotFound_is_eploader_cannot_find_collection(): with raises(CollectionNotFound): loader = EntryPointLoader(group='invoke')