diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 5e8b5a76e869..f0baca36352a 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -24359,7 +24359,9 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": unknown; + "application/json": { + [key: string]: unknown; + }; }; }; /** @description Request Error */ diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index 539375070e75..622b0d34748d 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -1217,22 +1217,6 @@ :Type: str -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``enable_tool_document_cache`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -:Description: - This option is deprecated, and the tool document cache will be - removed in the next release. Whether to enable the tool document - cache. This cache stores expanded XML strings. Enabling the tool - cache results in slightly faster startup times. The tool cache is - backed by a SQLite database, which cannot be stored on certain - network disks. The cache location is configurable with the - ``tool_cache_data_dir`` tag in tool config files. -:Default: ``false`` -:Type: bool - - ~~~~~~~~~~~~~~~~~~~~~~~~~ ``tool_search_index_dir`` ~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index d1b9aa8d46ec..a451a54c623d 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -788,7 +788,6 @@ def __init__(self, **kwargs) -> None: self.datatypes_registry.load_external_metadata_tool(self.toolbox) # Load history import/export tools. load_lib_tools(self.toolbox) - self.toolbox.persist_cache(register_postfork=True) # visualizations registry: associates resources with visualizations, controls how to render self.visualizations_registry = self._register_singleton( VisualizationsRegistry, diff --git a/lib/galaxy/app_unittest_utils/galaxy_mock.py b/lib/galaxy/app_unittest_utils/galaxy_mock.py index ade84e312247..eb02498b19e2 100644 --- a/lib/galaxy/app_unittest_utils/galaxy_mock.py +++ b/lib/galaxy/app_unittest_utils/galaxy_mock.py @@ -267,8 +267,6 @@ def __init__(self, **kwargs): self.version_major = "19.09" # set by MockDir - self.enable_tool_document_cache = False - self.tool_cache_data_dir = os.path.join(self.root, "tool_cache") self.external_chown_script = None self.check_job_script_integrity = False self.check_job_script_integrity_count = 0 diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample index 27f517a2e4b3..90cf214335b5 100644 --- a/lib/galaxy/config/sample/galaxy.yml.sample +++ b/lib/galaxy/config/sample/galaxy.yml.sample @@ -935,15 +935,6 @@ galaxy: # generated commands run in sh. #default_job_shell: /bin/bash - # This option is deprecated, and the tool document cache will be - # removed in the next release. Whether to enable the tool document - # cache. This cache stores expanded XML strings. Enabling the tool - # cache results in slightly faster startup times. The tool cache is - # backed by a SQLite database, which cannot be stored on certain - # network disks. The cache location is configurable with the - # ``tool_cache_data_dir`` tag in tool config files. - #enable_tool_document_cache: false - # Directory in which the toolbox search index is stored. The value of # this option will be resolved with respect to . #tool_search_index_dir: tool_search_index diff --git a/lib/galaxy/config/schemas/config_schema.yml b/lib/galaxy/config/schemas/config_schema.yml index e798a0a310d1..78d5e23b3a74 100644 --- a/lib/galaxy/config/schemas/config_schema.yml +++ b/lib/galaxy/config/schemas/config_schema.yml @@ -882,19 +882,6 @@ mapping: should be disabled. Containerized jobs always use /bin/sh - so more maximum portability tool authors should assume generated commands run in sh. - enable_tool_document_cache: - type: bool - default: false - required: false - desc: | - This option is deprecated, and the tool document cache will be removed - in the next release. - Whether to enable the tool document cache. This cache stores - expanded XML strings. Enabling the tool cache results in slightly faster startup - times. The tool cache is backed by a SQLite database, which cannot - be stored on certain network disks. The cache location is configurable - with the ``tool_cache_data_dir`` tag in tool config files. - tool_search_index_dir: type: str default: tool_search_index diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index 61c3b77ca100..985749157b97 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -185,7 +185,6 @@ social-auth-core==4.6.1 sortedcontainers==2.4.0 spython==0.3.14 sqlalchemy==2.0.40 -sqlitedict==2.1.0 sqlparse==0.5.3 starlette==0.46.2 starlette-context==0.4.0 diff --git a/lib/galaxy/managers/tools.py b/lib/galaxy/managers/tools.py index 0d4dcc96600e..74bba56778bc 100644 --- a/lib/galaxy/managers/tools.py +++ b/lib/galaxy/managers/tools.py @@ -53,10 +53,10 @@ def tool_payload_to_tool(app, tool_dict: Dict[str, Any]) -> Optional[Tool]: return tool -class DynamicToolManager(ModelManager[model.DynamicTool]): +class DynamicToolManager(ModelManager[DynamicTool]): """Manages dynamic tools stored in Galaxy's database.""" - model_class = model.DynamicTool + model_class = DynamicTool def ensure_can_use_unprivileged_tool(self, user: model.User): stmt = select( @@ -70,7 +70,7 @@ def ensure_can_use_unprivileged_tool(self, user: model.User): if not self.session().execute(stmt).scalar(): raise exceptions.InsufficientPermissionsException("User is not allowed to run unprivileged tools") - def get_tool_by_id_or_uuid(self, id_or_uuid: Union[int, str]): + def get_tool_by_id_or_uuid(self, id_or_uuid: Union[int, str]) -> Union[DynamicTool, None]: if isinstance(id_or_uuid, int): return self.get_tool_by_id(id_or_uuid) else: @@ -213,9 +213,10 @@ def deactivate_unprivileged_tool(self, user: model.User, dynamic_tool: DynamicTo session.execute(update_stmt) session.commit() - def deactivate(self, dynamic_tool): - self.update(dynamic_tool, {"active": False}) - return dynamic_tool + def deactivate(self, dynamic_tool: DynamicTool) -> DynamicTool: + assert isinstance(dynamic_tool.uuid, UUID) + del self.app.toolbox._tools_by_uuid[dynamic_tool.uuid] + return self.update(dynamic_tool, {"active": False}) class ToolFilterMixin: diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index d60c7816a690..bf3be71d4b85 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -341,7 +341,7 @@ def get_uuid(uuid: Optional[Union[UUID, str]] = None) -> UUID: return uuid if not uuid: return uuid4() - return UUID(str(uuid)) + return UUID(uuid) def to_json(sa_session, column, keys: List[str]): diff --git a/lib/galaxy/queue_worker.py b/lib/galaxy/queue_worker.py index e37153e6eba7..d56727236175 100644 --- a/lib/galaxy/queue_worker.py +++ b/lib/galaxy/queue_worker.py @@ -11,7 +11,10 @@ import threading import time from inspect import ismodule -from typing import TYPE_CHECKING +from typing import ( + Optional, + TYPE_CHECKING, +) from kombu import ( Consumer, @@ -33,10 +36,14 @@ log = logging.getLogger(__name__) if TYPE_CHECKING: - from galaxy.structured_app import MinimalManagerApp + from galaxy.app import UniverseApplication + from galaxy.structured_app import ( + MinimalManagerApp, + StructuredApp, + ) -def send_local_control_task(app, task, get_response=False, kwargs=None): +def send_local_control_task(app: "StructuredApp", task: str, get_response: bool = False, kwargs: Optional[dict] = None): """ This sends a message to the process-local control worker, which is useful for one-time asynchronous tasks like recalculating user disk usage. @@ -162,7 +169,7 @@ def reload_tool(app, **kwargs): log.error("Reload tool invoked without tool id.") -def reload_toolbox(app, save_integrated_tool_panel=True, **kwargs): +def reload_toolbox(app: "UniverseApplication", save_integrated_tool_panel: bool = True, **kwargs) -> None: reload_timer = util.ExecutionTimer() log.debug("Executing toolbox reload on '%s'", app.config.server_name) reload_count = app.toolbox._reload_count @@ -174,7 +181,7 @@ def reload_toolbox(app, save_integrated_tool_panel=True, **kwargs): log.debug("Toolbox reload %s", reload_timer) -def _get_new_toolbox(app, save_integrated_tool_panel=True): +def _get_new_toolbox(app: "UniverseApplication", save_integrated_tool_panel: bool = True) -> None: """ Generate a new toolbox, by constructing a toolbox from the config files, and then adding pre-existing data managers from the old toolbox to the new toolbox. @@ -190,7 +197,6 @@ def _get_new_toolbox(app, save_integrated_tool_panel=True): load_lib_tools(new_toolbox) [new_toolbox.register_tool(tool) for tool in new_toolbox.data_manager_tools.values()] app._toolbox = new_toolbox - app.toolbox.persist_cache() def reload_data_managers(app, **kwargs): diff --git a/lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py b/lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py index adf86824072b..8b56008e9b79 100644 --- a/lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py +++ b/lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -42,7 +42,6 @@ def add_to_shed_tool_config(self, shed_tool_conf_dict: Dict[str, Any], elem_list return old_toolbox = self.app.toolbox shed_tool_conf = shed_tool_conf_dict["config_filename"] - tool_cache_data_dir = shed_tool_conf_dict.get("tool_cache_data_dir") tool_path = shed_tool_conf_dict["tool_path"] config_elems = [] # Ideally shed_tool_conf.xml would be created before the repo is cloned and added to the DB, but this is called @@ -83,7 +82,7 @@ def add_to_shed_tool_config(self, shed_tool_conf_dict: Dict[str, Any], elem_list else: config_elems.append(elem_entry) # Persist the altered shed_tool_config file. - self.config_elems_to_xml_file(config_elems, shed_tool_conf, tool_path, tool_cache_data_dir) + self.config_elems_to_xml_file(config_elems, shed_tool_conf, tool_path) self.app.wait_for_toolbox_reload(old_toolbox) else: log.error(error_message) @@ -135,16 +134,13 @@ def add_to_tool_panel( self.app.toolbox.update_shed_config(shed_tool_conf_dict) self.add_to_shed_tool_config(shed_tool_conf_dict, elem_list) - def config_elems_to_xml_file(self, config_elems, config_filename, tool_path, tool_cache_data_dir=None) -> None: + def config_elems_to_xml_file(self, config_elems, config_filename, tool_path) -> None: """ Persist the current in-memory list of config_elems to a file named by the value of config_filename. """ try: - tool_cache_data_dir = f' tool_cache_data_dir="{tool_cache_data_dir}"' if tool_cache_data_dir else "" - root = parse_xml_string( - f'\n' - ) + root = parse_xml_string(f'\n') for elem in config_elems: root.append(elem) with RenamedTemporaryFile(config_filename, mode="w") as fh: diff --git a/lib/galaxy/tool_shed/unittest_utils/__init__.py b/lib/galaxy/tool_shed/unittest_utils/__init__.py index a58c3d6f6561..95f7a810c97b 100644 --- a/lib/galaxy/tool_shed/unittest_utils/__init__.py +++ b/lib/galaxy/tool_shed/unittest_utils/__init__.py @@ -2,10 +2,12 @@ from pathlib import Path from typing import ( Any, + cast, Dict, List, NamedTuple, Optional, + TYPE_CHECKING, Union, ) @@ -39,6 +41,10 @@ ) from galaxy.util.tool_shed.tool_shed_registry import Registry +if TYPE_CHECKING: + from galaxy.tools import Tool + from galaxy.util.path import StrPath + class ToolShedTarget(NamedTuple): url: str @@ -83,7 +89,7 @@ class TestTool: params_with_missing_data_table_entry: list = [] params_with_missing_index_file: list = [] - def __init__(self, config_file, tool_shed_repository, guid): + def __init__(self, config_file: "StrPath", tool_shed_repository, guid: str) -> None: self.config_file = config_file self.tool_shed_repository = tool_shed_repository self.guid = guid @@ -100,12 +106,14 @@ def lineage(self): class TestToolBox(AbstractToolBox): - def create_tool(self, config_file, tool_cache_data_dir=None, **kwds): - tool = TestTool(config_file, kwds["tool_shed_repository"], kwds["guid"]) + def create_tool(self, config_file: "StrPath", **kwds) -> "Tool": + tool = cast("Tool", TestTool(config_file, kwds["tool_shed_repository"], kwds["guid"])) tool._lineage = self._lineage_map.register(tool) # cleanup? return tool - def _get_tool_shed_repository(self, tool_shed, name, owner, installed_changeset_revision): + def _get_tool_shed_repository( + self, tool_shed: str, name: str, owner: str, installed_changeset_revision: Optional[str] + ): return get_installed_repository( self.app, tool_shed=tool_shed, diff --git a/lib/galaxy/tool_shed/util/repository_util.py b/lib/galaxy/tool_shed/util/repository_util.py index 315cd43426f1..99b322a0f34b 100644 --- a/lib/galaxy/tool_shed/util/repository_util.py +++ b/lib/galaxy/tool_shed/util/repository_util.py @@ -247,14 +247,14 @@ def get_absolute_path_to_file_in_repository(repo_files_dir, file_name): def get_installed_repository( app: "InstallationTarget", - tool_shed=None, - name=None, - owner=None, - changeset_revision=None, - installed_changeset_revision=None, - repository_id=None, - from_cache=False, -): + tool_shed: Optional[str] = None, + name: Optional[str] = None, + owner: Optional[str] = None, + changeset_revision: Optional[str] = None, + installed_changeset_revision: Optional[str] = None, + repository_id: Optional[int] = None, + from_cache: bool = False, +) -> ToolShedRepository: """ Return a tool shed repository database record defined by the combination of a toolshed, repository name, repository owner and either current or originally installed changeset_revision. diff --git a/lib/galaxy/tool_util/parser/interface.py b/lib/galaxy/tool_util/parser/interface.py index 116a90fbec91..6a1e59f43189 100644 --- a/lib/galaxy/tool_util/parser/interface.py +++ b/lib/galaxy/tool_util/parser/interface.py @@ -400,7 +400,7 @@ def parse_creator(self): return [] @property - def macro_paths(self): + def macro_paths(self) -> List[str]: return [] @property diff --git a/lib/galaxy/tool_util/parser/xml.py b/lib/galaxy/tool_util/parser/xml.py index 9651d774780e..a16a62a69a75 100644 --- a/lib/galaxy/tool_util/parser/xml.py +++ b/lib/galaxy/tool_util/parser/xml.py @@ -90,6 +90,7 @@ ) if TYPE_CHECKING: + from galaxy.util.path import StrPath from .output_objects import ToolOutputBase log = logging.getLogger(__name__) @@ -158,7 +159,9 @@ class XmlToolSource(ToolSource): language = "xml" - def __init__(self, xml_tree: ElementTree, source_path=None, macro_paths=None): + def __init__( + self, xml_tree: ElementTree, source_path: Optional["StrPath"] = None, macro_paths: Optional[List[str]] = None + ) -> None: self.xml_tree = xml_tree self.root = self.xml_tree.getroot() self._source_path = source_path @@ -683,7 +686,7 @@ def parse_help(self) -> Optional[HelpContent]: return HelpContent(format=help_format, content=content) @property - def macro_paths(self): + def macro_paths(self) -> List[str]: return self._macro_paths @property diff --git a/lib/galaxy/tool_util/toolbox/base.py b/lib/galaxy/tool_util/toolbox/base.py index d3bf951830ed..c8d21e355e27 100644 --- a/lib/galaxy/tool_util/toolbox/base.py +++ b/lib/galaxy/tool_util/toolbox/base.py @@ -68,8 +68,12 @@ from galaxy.model import ( DynamicTool, User, + Workflow, ) from galaxy.tools import Tool + from galaxy.tools.cache import ToolCache + from galaxy.util import Element + from galaxy.util.path import StrPath log = logging.getLogger(__name__) @@ -118,7 +122,7 @@ def has_tool(self, tool_id: str) -> bool: def get_tool(self, tool_id: str): return self.__toolbox.get_tool(tool_id) - def get_workflow(self, id: str): + def get_workflow(self, id: str) -> "Workflow": return self.__toolbox._workflows_by_id[id] def add_tool_to_tool_panel_view(self, tool, tool_panel_component) -> None: @@ -165,13 +169,13 @@ class AbstractToolBox(ManagesIntegratedToolPanelMixin): def __init__( self, - config_filenames, + config_filenames: List[str], tool_root_dir, app, view_sources=None, default_panel_view="default", - save_integrated_tool_panel=True, - ): + save_integrated_tool_panel: bool = True, + ) -> None: """ Create a toolbox from the config files named by `config_filenames`, using `tool_root_dir` as the base directory for finding individual tool config files. @@ -181,21 +185,21 @@ def __init__( # information about the tools defined in each shed-related # shed_tool_conf.xml file. self._dynamic_tool_confs = [] - self._tools_by_id = {} - self._tools_by_uuid = {} + self._tools_by_id: Dict[str, Tool] = {} + self._tools_by_uuid: Dict[UUID, Tool] = {} # Tool lineages can contain chains of related tools with different ids # so each will be present once in the above dictionary. The following # dictionary can instead hold multiple tools with different versions. - self._tool_versions_by_id = {} - self._tools_by_old_id = {} - self._workflows_by_id = {} + self._tool_versions_by_id: Dict[str, Dict[Union[str, None], Tool]] = {} + self._tools_by_old_id: Dict[str, List[Tool]] = {} + self._workflows_by_id: Dict[str, Workflow] = {} # Cache for tool's to_dict calls specific to toolbox. Invalidates on toolbox reload. - self._tool_to_dict_cache = {} - self._tool_to_dict_cache_admin = {} + self._tool_to_dict_cache: Dict[str, Dict[str, Any]] = {} + self._tool_to_dict_cache_admin: Dict[str, Dict[str, Any]] = {} # In-memory dictionary that defines the layout of the tool panel. self._tool_panel = ToolPanelElements() self._index = 0 - self.data_manager_tools = {} + self.data_manager_tools: Dict[str, Tool] = {} self._lineage_map = LineageMap(app) # Sets self._integrated_tool_panel and self._integrated_tool_panel_config_has_contents self._init_integrated_tool_panel(app.config) @@ -266,23 +270,23 @@ def _default_panel_view(self, trans): config_value = getattr(config, "default_panel_view", None) return config_value or self.__default_panel_view - def create_tool(self, config_file, tool_cache_data_dir=None, **kwds): + def create_tool(self, config_file: "StrPath", **kwds) -> "Tool": raise NotImplementedError() - def create_dynamic_tool(self, dynamic_tool: "DynamicTool"): + def create_dynamic_tool(self, dynamic_tool: "DynamicTool") -> "Tool": raise NotImplementedError() def can_load_config_file(self, config_filename): return True - def _load_workflow(self, workflow_id): + def _load_workflow(self, workflow_id: str) -> "Workflow": raise NotImplementedError() def tool_tag_manager(self): """Build a tool tag manager according to app's configuration and return it.""" raise NotImplementedError() - def _init_tools_from_configs(self, config_filenames): + def _init_tools_from_configs(self, config_filenames: List[str]) -> None: """Read through all tool config files and initialize tools in each with init_tools_from_config below. """ @@ -315,7 +319,7 @@ def _init_tools_from_configs(self, config_filenames): log.exception("Error loading tools defined in config %s", config_filename) log.debug("Reading tools from config files finished %s", execution_timer) - def _init_tools_from_config(self, config_filename): + def _init_tools_from_config(self, config_filename: str) -> None: """ Read the configuration file and load each tool. The following tags are currently supported: @@ -354,7 +358,6 @@ def _init_tools_from_config(self, config_filename): return raise tool_path = tool_conf_source.parse_tool_path() - tool_cache_data_dir = tool_conf_source.parse_tool_cache_data_dir() parsing_shed_tool_conf = tool_conf_source.is_shed_tool_conf() if parsing_shed_tool_conf: # Keep an in-memory list of xml elements to enable persistence of the changing tool config. @@ -372,7 +375,6 @@ def _init_tools_from_config(self, config_filename): self.load_item( item, tool_path=tool_path, - tool_cache_data_dir=tool_cache_data_dir, load_panel_dict=load_panel_dict, guid=item.get("guid"), index=index, @@ -384,12 +386,11 @@ def _init_tools_from_config(self, config_filename): shed_tool_conf_dict = dict( config_filename=config_filename, tool_path=tool_path, - tool_cache_data_dir=tool_cache_data_dir, config_elems=config_elems, ) self._dynamic_tool_confs.append(shed_tool_conf_dict) - def _get_tool_by_uuid(self, tool_uuid): + def _get_tool_by_uuid(self, tool_uuid: UUID) -> Union["Tool", None]: if tool_uuid in self._tools_by_uuid: return self._tools_by_uuid[tool_uuid] @@ -409,12 +410,13 @@ def panel_has_tool(self, tool, panel_view_id): panel_view_rendered = self._tool_panel_view_rendered[panel_view_id] return panel_view_rendered.has_item_recursive(tool) - def load_dynamic_tool(self, dynamic_tool: "DynamicTool"): + def load_dynamic_tool(self, dynamic_tool: "DynamicTool") -> Union["Tool", None]: if not dynamic_tool.active: return None tool = self.create_dynamic_tool(dynamic_tool) self.register_tool(tool) + assert isinstance(dynamic_tool.uuid, UUID) self._tools_by_uuid[dynamic_tool.uuid] = tool return tool @@ -427,7 +429,6 @@ def load_item( load_panel_dict=True, guid=None, index=None, - tool_cache_data_dir=None, ): with self.app._toolbox_lock: item = ensure_tool_conf_item(item) @@ -445,7 +446,6 @@ def load_item( load_panel_dict=load_panel_dict, guid=guid, index=index, - tool_cache_data_dir=tool_cache_data_dir, ) elif item_type == "workflow": self._load_workflow_tag_set( @@ -461,7 +461,6 @@ def load_item( tool_path=tool_path, load_panel_dict=load_panel_dict, index=index, - tool_cache_data_dir=tool_cache_data_dir, ) elif item_type == "label": self._load_label_tag_set( @@ -478,7 +477,6 @@ def load_item( tool_path, integrated_panel_dict, load_panel_dict=load_panel_dict, - tool_cache_data_dir=tool_cache_data_dir, ) def get_shed_config_dict_by_filename(self, filename) -> Optional[DynamicToolConfDict]: @@ -735,24 +733,29 @@ def get_tool( user: Optional["User"] = None, ) -> Union[Optional["Tool"], List["Tool"]]: """Attempt to locate a tool in the tool box. Note that `exact` only refers to the `tool_id`, not the `tool_version`.""" - if tool_uuid and user: - unprivileged_tool = self.get_unprivileged_tool_or_none(user, tool_uuid=tool_uuid) - if unprivileged_tool: - return unprivileged_tool + if tool_id is None: + if tool_uuid is None: + raise RequestParameterInvalidException( + "get_tool cannot be called with both tool_id and tool_uuid as None" + ) + if user: + unprivileged_tool = self.get_unprivileged_tool_or_none(user, tool_uuid=tool_uuid) + if unprivileged_tool: + return unprivileged_tool + tool_uuid = tool_uuid if isinstance(tool_uuid, UUID) else UUID(tool_uuid) + tool_from_uuid = self._get_tool_by_uuid(tool_uuid) + if tool_from_uuid is None: + raise ObjectNotFound(f"Failed to find a tool with uuid [{tool_uuid}]") + tool_id = tool_from_uuid.id + assert tool_id + if tool_version: tool_version = str(tool_version) if get_all_versions and exact: - raise AssertionError("Cannot specify get_tool with both get_all_versions and exact as True") - - if tool_id is None: - if tool_uuid is not None: - tool_from_uuid = self._get_tool_by_uuid(tool_uuid) - if tool_from_uuid is None: - raise ObjectNotFound(f"Failed to find a tool with uuid [{tool_uuid}]") - tool_id = tool_from_uuid.id - if tool_id is None: - raise AssertionError("get_tool called with tool_id as None") + raise RequestParameterInvalidException( + "get_tool cannot be called with both get_all_versions and exact as True" + ) if "/repos/" in tool_id: # test if tool came from a toolshed tool_id_without_tool_shed = tool_id.split("/repos/")[1] @@ -913,7 +916,6 @@ def _load_tool_tag_set( load_panel_dict, guid=None, index=None, - tool_cache_data_dir=None, ): try: path_template = item.get("file") @@ -927,12 +929,11 @@ def _load_tool_tag_set( can_load_into_panel_dict = True tool = self.load_tool_from_cache(concrete_path) - from_cache = tool - if from_cache: - if guid and tool.id != guid: - # In rare cases a tool shed tool is loaded into the cache without guid. - # In that case recreating the tool will correct the cached version. - from_cache = False + from_cache = tool is not None + if tool and guid and tool.id != guid: + # In rare cases a tool shed tool is loaded into the cache without guid. + # In that case recreating the tool will correct the cached version. + from_cache = False if guid and not from_cache: # tool was not in cache and is a tool shed tool tool_shed_repository = self.get_tool_repository_from_xml_item(item.elem, concrete_path) if tool_shed_repository: @@ -945,10 +946,9 @@ def _load_tool_tag_set( guid=guid, tool_shed_repository=tool_shed_repository, use_cached=False, - tool_cache_data_dir=tool_cache_data_dir, ) if not tool: # tool was not in cache and is not a tool shed tool. - tool = self.load_tool(concrete_path, use_cached=False, tool_cache_data_dir=tool_cache_data_dir) + tool = self.load_tool(concrete_path, use_cached=False) if string_as_bool(item.get("hidden", False)): tool.hidden = True key = f"tool_{str(tool.id)}" @@ -979,15 +979,24 @@ def _load_tool_tag_set( except Exception: log.exception("Error reading tool from path: %s", path) - def get_tool_repository_from_xml_item(self, elem, path): - tool_shed = elem.find("tool_shed").text - repository_name = elem.find("repository_name").text - repository_owner = elem.find("repository_owner").text + def get_tool_repository_from_xml_item(self, elem: "Element", path: str): + tool_shed_el = elem.find("tool_shed") + assert tool_shed_el is not None + tool_shed = tool_shed_el.text + assert tool_shed + repository_name_el = elem.find("repository_name") + assert repository_name_el is not None + repository_name = repository_name_el.text + assert repository_name + repository_owner_el = elem.find("repository_owner") + assert repository_owner_el is not None + repository_owner = repository_owner_el.text + assert repository_owner # The definition of `installed_changeset_revision` for a repository is that it has been cloned at # so if we load a tool it needs to be at a path that contains `installed_changeset_revision`. path_to_installed_changeset_revision = os.path.join(tool_shed, "repos", repository_owner, repository_name) if path_to_installed_changeset_revision in path: - installed_changeset_revision = path[ + installed_changeset_revision: Optional[str] = path[ path.index(path_to_installed_changeset_revision) + len(path_to_installed_changeset_revision) : ].split(os.path.sep)[1] else: @@ -995,6 +1004,7 @@ def get_tool_repository_from_xml_item(self, elem, path): if installed_changeset_revision_elem is None: # Backward compatibility issue - the tag used to be named 'changeset_revision'. installed_changeset_revision_elem = elem.find("changeset_revision") + assert installed_changeset_revision_elem is not None installed_changeset_revision = installed_changeset_revision_elem.text repository = self._get_tool_shed_repository( tool_shed=tool_shed, @@ -1009,6 +1019,7 @@ def get_tool_repository_from_xml_item(self, elem, path): ) log.warning(msg, repository_name, repository_owner) # Figure out path to repository on disk given the tool shed info and the path to the tool contained in the repo + assert installed_changeset_revision repository_path = os.path.join( tool_shed, "repos", repository_owner, repository_name, installed_changeset_revision ) @@ -1028,12 +1039,14 @@ def get_tool_repository_from_xml_item(self, elem, path): tsr_cache.add_local_repository(repository) return repository - def _get_tool_shed_repository(self, tool_shed, name, owner, installed_changeset_revision): + def _get_tool_shed_repository( + self, tool_shed: str, name: str, owner: str, installed_changeset_revision: Optional[str] + ): # Abstract class doesn't have a dependency on the database, for full Tool Shed # support the actual Galaxy ToolBox implements this method and returns a Tool Shed repository. return None - def __add_tool(self, tool, load_panel_dict, panel_dict): + def __add_tool(self, tool: "Tool", load_panel_dict, panel_dict) -> None: # Allow for the same tool to be loaded into multiple places in the # tool panel. We have to handle the case where the tool is contained # in a repository installed from the tool shed, and the Galaxy @@ -1068,7 +1081,7 @@ def _load_label_tag_set(self, item, panel_dict, integrated_panel_dict, load_pane panel_dict[key] = label integrated_panel_dict.update_or_append(index, key, label) - def _load_section_tag_set(self, item, tool_path, load_panel_dict, index=None, tool_cache_data_dir=None): + def _load_section_tag_set(self, item, tool_path, load_panel_dict, index=None): key = item.get("id") if key in self._tool_panel: section = self._tool_panel[key] @@ -1091,7 +1104,6 @@ def _load_section_tag_set(self, item, tool_path, load_panel_dict, index=None, to load_panel_dict=load_panel_dict, guid=sub_item.get("guid"), index=sub_index, - tool_cache_data_dir=tool_cache_data_dir, ) # Ensure each tool's section is stored @@ -1106,9 +1118,7 @@ def _load_section_tag_set(self, item, tool_path, load_panel_dict, index=None, to # Always load sections into the integrated_tool_panel. self._integrated_tool_panel.update_or_append(index, key, integrated_section) - def _load_tooldir_tag_set( - self, item, elems, tool_path, integrated_elems, load_panel_dict, tool_cache_data_dir=None - ): + def _load_tooldir_tag_set(self, item, elems, tool_path, integrated_elems, load_panel_dict): directory = os.path.join(tool_path, item.get("dir")) recursive = string_as_bool(item.get("recursive", True)) self.__watch_directory( @@ -1118,7 +1128,6 @@ def _load_tooldir_tag_set( load_panel_dict, recursive, force_watch=True, - tool_cache_data_dir=tool_cache_data_dir, ) def __watch_directory( @@ -1129,11 +1138,10 @@ def __watch_directory( load_panel_dict, recursive, force_watch=False, - tool_cache_data_dir=None, ): def quick_load(tool_file, async_load=True): try: - tool = self.load_tool(tool_file, tool_cache_data_dir) + tool = self.load_tool(tool_file) self.__add_tool(tool, load_panel_dict, elems) # Always load the tool into the integrated_panel_dict, or it will not be included in the integrated_tool_panel.xml file. key = f"tool_{str(tool.id)}" @@ -1172,18 +1180,22 @@ def quick_load(tool_file, async_load=True): self._tool_watcher.watch_directory(directory, quick_load) def load_tool( - self, config_file, guid=None, tool_shed_repository=None, use_cached=False, tool_cache_data_dir=None, **kwds - ): + self, + config_file: "StrPath", + guid=None, + tool_shed_repository=None, + use_cached: bool = False, + **kwds, + ) -> "Tool": """Load a single tool from the file named by `config_file` and return an instance of `Tool`.""" # Parse XML configuration file and get the root element - tool = None + tool: Optional[Tool] = None if use_cached: tool = self.load_tool_from_cache(config_file) if not tool or guid and guid != tool.guid: try: tool = self.create_tool( config_file, - tool_cache_data_dir=tool_cache_data_dir, tool_shed_repository=tool_shed_repository, guid=guid, **kwds, @@ -1208,13 +1220,13 @@ def watch_tool(self, tool): if self._tool_config_watcher: [self._tool_config_watcher.watch_file(macro_path) for macro_path in tool._macro_paths] - def add_tool_to_cache(self, tool, config_file): - tool_cache = getattr(self.app, "tool_cache", None) + def add_tool_to_cache(self, tool: "Tool", config_file: "StrPath") -> None: + tool_cache: Optional[ToolCache] = getattr(self.app, "tool_cache", None) if tool_cache: - self.app.tool_cache.cache_tool(config_file, tool) + tool_cache.cache_tool(config_file, tool) - def load_tool_from_cache(self, config_file, recover_tool=False): - tool_cache = getattr(self.app, "tool_cache", None) + def load_tool_from_cache(self, config_file: "StrPath", recover_tool: bool = False) -> Union["Tool", None]: + tool_cache: Optional[ToolCache] = getattr(self.app, "tool_cache", None) tool = None if tool_cache: if recover_tool: @@ -1235,8 +1247,9 @@ def load_hidden_tool(self, config_file, **kwds): self.register_tool(tool) return tool - def register_tool(self, tool): + def register_tool(self, tool: "Tool"): tool_id = tool.id + assert tool_id version = tool.version or None if tool_id not in self._tool_versions_by_id: self._tool_versions_by_id[tool_id] = {version: tool} @@ -1251,9 +1264,10 @@ def register_tool(self, tool): else: self._tools_by_id[tool_id] = tool old_id = tool.old_id - if old_id not in self._tools_by_old_id: - self._tools_by_old_id[old_id] = [] - self._tools_by_old_id[old_id].append(tool) + if old_id: + if old_id not in self._tools_by_old_id: + self._tools_by_old_id[old_id] = [] + self._tools_by_old_id[old_id].append(tool) def package_tool(self, trans, tool_id): """ @@ -1280,6 +1294,7 @@ def reload_tool_by_id(self, tool_id: str) -> Tuple[Union[str, Dict[str, str]], s status = "error" else: old_tool = self._tools_by_id[tool_id] + assert old_tool.config_file new_tool = self.load_tool(old_tool.config_file, use_cached=False) # The tool may have been installed from a tool shed, so set the tool shed attributes. # Since the tool version may have changed, we don't override it here. @@ -1295,6 +1310,7 @@ def reload_tool_by_id(self, tool_id: str) -> Tuple[Union[str, Dict[str, str]], s # (Re-)Register the reloaded tool, this will handle # _tools_by_id and _tool_versions_by_id self.register_tool(new_tool) + assert old_tool.id message = {"name": old_tool.name, "id": old_tool.id, "version": old_tool.version} status = "done" return message, status @@ -1313,8 +1329,9 @@ def remove_tool_by_id(self, tool_id, remove_from_panel=True): else: tool = self._tools_by_id[tool_id] del self._tools_by_id[tool_id] - self._tools_by_old_id[tool.old_id].remove(tool) - tool_cache = getattr(self.app, "tool_cache", None) + if tool.old_id: + self._tools_by_old_id[tool.old_id].remove(tool) + tool_cache: Optional[ToolCache] = getattr(self.app, "tool_cache", None) if tool_cache: tool_cache.expire_tool(tool_id) if remove_from_panel: @@ -1369,22 +1386,23 @@ def tool_panel_contents(self, trans, view=None, **kwds): if elt: yield elt - def get_tool_to_dict(self, trans, tool, tool_help=False): + def get_tool_to_dict(self, trans, tool: "Tool", tool_help: bool = False): """Return tool's to_dict. Use cache if present, store to cache otherwise. Note: The cached tool's to_dict is specific to the calls from toolbox. """ to_dict = None + assert tool.id if not trans.user_is_admin: if not tool_help: - to_dict = self._tool_to_dict_cache.get(tool.id, None) + to_dict = self._tool_to_dict_cache.get(tool.id) if not to_dict: to_dict = tool.to_dict(trans, link_details=True, tool_help=tool_help) if not tool_help: self._tool_to_dict_cache[tool.id] = to_dict else: if not tool_help: - to_dict = self._tool_to_dict_cache_admin.get(tool.id, None) + to_dict = self._tool_to_dict_cache_admin.get(tool.id) if not to_dict: to_dict = tool.to_dict(trans, link_details=True, tool_help=tool_help) if not tool_help: diff --git a/lib/galaxy/tool_util/toolbox/lineages/factory.py b/lib/galaxy/tool_util/toolbox/lineages/factory.py index 2a68c32b2ee4..1b32638180f0 100644 --- a/lib/galaxy/tool_util/toolbox/lineages/factory.py +++ b/lib/galaxy/tool_util/toolbox/lineages/factory.py @@ -1,11 +1,15 @@ from typing import ( Dict, Optional, + TYPE_CHECKING, ) from galaxy.util.tool_version import remove_version_from_guid from .interface import ToolLineage +if TYPE_CHECKING: + from galaxy.tools import Tool + class LineageMap: """Map each unique tool id to a lineage object.""" @@ -14,8 +18,9 @@ def __init__(self, app): self.lineage_map: Dict[str, ToolLineage] = {} self.app = app - def register(self, tool) -> ToolLineage: + def register(self, tool: "Tool") -> ToolLineage: tool_id = tool.id + assert tool_id versionless_tool_id = remove_version_from_guid(tool_id) lineage: ToolLineage if versionless_tool_id not in self.lineage_map: @@ -32,7 +37,7 @@ def register(self, tool) -> ToolLineage: self.lineage_map[tool_id] = lineage return self.lineage_map[tool_id] - def get(self, tool_id) -> Optional[ToolLineage]: + def get(self, tool_id: str) -> Optional[ToolLineage]: """ Get lineage for `tool_id`. @@ -55,13 +60,14 @@ def get(self, tool_id) -> Optional[ToolLineage]: tool = toolbox and toolbox._tools_by_id.get(tool_id) if tool: lineage = ToolLineage.from_tool(tool) - if lineage: self.lineage_map[tool_id] = lineage return self.lineage_map.get(tool_id) - def _get_versionless(self, tool_id) -> Optional[ToolLineage]: + def _get_versionless(self, tool_id: str) -> Optional[ToolLineage]: versionless_tool_id = remove_version_from_guid(tool_id) - return self.lineage_map.get(versionless_tool_id, None) + if not versionless_tool_id: + return None + return self.lineage_map.get(versionless_tool_id) __all__ = ("LineageMap",) diff --git a/lib/galaxy/tool_util/toolbox/lineages/interface.py b/lib/galaxy/tool_util/toolbox/lineages/interface.py index 71fdb97611d3..344a161c3ec0 100644 --- a/lib/galaxy/tool_util/toolbox/lineages/interface.py +++ b/lib/galaxy/tool_util/toolbox/lineages/interface.py @@ -3,6 +3,7 @@ Any, Dict, List, + TYPE_CHECKING, ) from sortedcontainers import SortedSet @@ -10,18 +11,21 @@ from galaxy.tool_util.version import parse_version from galaxy.util.tool_version import remove_version_from_guid +if TYPE_CHECKING: + from galaxy.tools import Tool + class ToolLineageVersion: """Represents a single tool in a lineage. If lineage is based around GUIDs that somehow encode the version (either using GUID or a simple tool id and a version).""" - def __init__(self, id, version): + def __init__(self, id: str, version: str) -> None: self.id = id self.version = version @property - def id_based(self): + def id_based(self) -> bool: """Return True if the lineage is defined by GUIDs (in this case the indexer of the tools (i.e. the ToolBox) should ignore the tool_version (because it is encoded in the GUID and managed @@ -29,7 +33,7 @@ def id_based(self): """ return self.version is None - def to_dict(self): + def to_dict(self) -> Dict[str, str]: return dict( id=self.id, version=self.version, @@ -44,7 +48,7 @@ class ToolLineage: lineages_by_id: Dict[str, "ToolLineage"] = {} lock = threading.Lock() - def __init__(self, tool_id, **kwds): + def __init__(self, tool_id: str) -> None: self.tool_id = tool_id self.tool_versions = SortedSet(key=parse_version) @@ -54,22 +58,22 @@ def tool_ids(self) -> List[str]: tool_id = versionless_tool_id or self.tool_id return [f"{tool_id}/{version}" for version in self.tool_versions] - @staticmethod - def from_tool(tool) -> "ToolLineage": + @classmethod + def from_tool(cls, tool: "Tool") -> "ToolLineage": tool_id = tool.id - lineages_by_id = ToolLineage.lineages_by_id - with ToolLineage.lock: + assert tool_id is not None + lineages_by_id = cls.lineages_by_id + with cls.lock: if tool_id not in lineages_by_id: lineages_by_id[tool_id] = ToolLineage(tool_id) lineage = lineages_by_id[tool_id] lineage.register_version(tool.version) return lineage - def register_version(self, tool_version) -> None: - assert tool_version is not None - self.tool_versions.add(str(tool_version)) + def register_version(self, tool_version: str) -> None: + self.tool_versions.add(tool_version) - def get_versions(self): + def get_versions(self) -> List[ToolLineageVersion]: """ Return an ordered list of lineages (ToolLineageVersion) in this chain, from oldest to newest. @@ -79,7 +83,7 @@ def get_versions(self): for tool_id, tool_version in zip(self.tool_ids, self.tool_versions) ] - def get_version_ids(self, reverse=False) -> List[str]: + def get_version_ids(self, reverse: bool = False) -> List[str]: if reverse: return list(reversed(self.tool_ids)) return self.tool_ids diff --git a/lib/galaxy/tool_util/toolbox/parser.py b/lib/galaxy/tool_util/toolbox/parser.py index 07b841f93f78..a87b5f82613f 100644 --- a/lib/galaxy/tool_util/toolbox/parser.py +++ b/lib/galaxy/tool_util/toolbox/parser.py @@ -48,9 +48,6 @@ def __init__(self, config_filename: StrPath): def parse_tool_path(self): return self.root.get("tool_path") - def parse_tool_cache_data_dir(self): - return self.root.get("tool_cache_data_dir") - def parse_items(self): return [ensure_tool_conf_item(_) for _ in self.root] @@ -72,9 +69,6 @@ def __init__(self, config_filename: StrPath): def parse_tool_path(self): return self.as_dict.get("tool_path") - def parse_tool_cache_data_dir(self): - return self.as_dict.get("tool_cache_data_dir") - def parse_items(self): return [ToolConfItem.from_dict(_) for _ in self.as_dict.get("items")] diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index acfa4090dfcf..e3d84f25226c 100644 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -127,7 +127,6 @@ from galaxy.tools.actions.data_manager import DataManagerToolAction from galaxy.tools.actions.data_source import DataSourceToolAction from galaxy.tools.actions.model_operations import ModelOperationToolAction -from galaxy.tools.cache import ToolDocumentCache from galaxy.tools.evaluation import global_tool_errors from galaxy.tools.execution_helpers import ToolExecutionCache from galaxy.tools.imp_exp import JobImportHistoryArchiveWrapper @@ -173,7 +172,6 @@ listify, Params, parse_xml_string, - parse_xml_string_to_etree, rst_to_html, string_as_bool, unicodify, @@ -235,6 +233,7 @@ from galaxy.model import ( DynamicTool, LibraryFolder, + Workflow, ) from galaxy.objectstore import ObjectStore from galaxy.schema.schema import JobState @@ -243,6 +242,7 @@ ToolOutputCollection, ) from galaxy.tool_util.provided_metadata import BaseToolProvidedMetadata + from galaxy.tool_util.toolbox.lineages.interface import ToolLineage from galaxy.tools.actions.metadata import SetMetadataToolAction from galaxy.tools.parameters import ToolInputsT @@ -483,10 +483,11 @@ class ToolBox(AbstractToolBox): app: "UniverseApplication" - def __init__(self, config_filenames, tool_root_dir, app, save_integrated_tool_panel: bool = True): + def __init__( + self, config_filenames: List[str], tool_root_dir, app, save_integrated_tool_panel: bool = True + ) -> None: self._reload_count = 0 self.tool_location_fetcher = ToolLocationFetcher() - self.cache_regions: Dict[str, ToolDocumentCache] = {} # This is here to deal with the old default value, which doesn't make # sense in an "installed Galaxy" world. # FIXME: ./ @@ -542,21 +543,6 @@ def load_builtin_converters(self): tool.hidden = False section.elems.append_tool(tool) - def persist_cache(self, register_postfork: bool = False): - """ - Persists any modified tool cache files to disk. - - Set ``register_postfork`` to stop database thread queue, - close database connection and register re-open function - that re-opens the database after forking. - """ - for region in self.cache_regions.values(): - if not region.disabled: - region.persist() - if register_postfork: - region.close() - self.app.application_stack.register_postfork_function(region.reopen_ro) - def can_load_config_file(self, config_filename): if config_filename == self.app.config.shed_tool_config_file and not self.app.config.is_set( "shed_tool_config_file" @@ -586,36 +572,16 @@ def tools_by_id(self): # Deprecated method, TODO - eliminate calls to this in test/. return self._tools_by_id - def get_cache_region(self, tool_cache_data_dir: Optional[str]): - if self.app.config.enable_tool_document_cache and tool_cache_data_dir: - if tool_cache_data_dir not in self.cache_regions: - self.cache_regions[tool_cache_data_dir] = ToolDocumentCache(cache_dir=tool_cache_data_dir) - return self.cache_regions[tool_cache_data_dir] - - def create_tool(self, config_file: str, tool_cache_data_dir: Optional[str] = None, **kwds): - cache = self.get_cache_region(tool_cache_data_dir) - if config_file.endswith(".xml") and cache and not cache.disabled: - tool_document = cache.get(config_file) - if tool_document: - tool_source = self.get_expanded_tool_source( - config_file=config_file, - xml_tree=parse_xml_string_to_etree(tool_document["document"]), - macro_paths=tool_document["macro_paths"], - ) - else: - tool_source = self.get_expanded_tool_source(config_file) - cache.set(config_file, tool_source) - else: - tool_source = self.get_expanded_tool_source(config_file) + def create_tool(self, config_file: StrPath, **kwds) -> "Tool": + tool_source = self.get_expanded_tool_source(config_file) return self._create_tool_from_source(tool_source, config_file=config_file, **kwds) - def get_expanded_tool_source(self, config_file, **kwargs): + def get_expanded_tool_source(self, config_file: StrPath) -> ToolSource: try: return get_tool_source( config_file, enable_beta_formats=getattr(self.app.config, "enable_beta_tool_formats", False), tool_location_fetcher=self.tool_location_fetcher, - **kwargs, ) except Exception as e: # capture and log parsing errors @@ -636,7 +602,13 @@ def get_unprivileged_tool_or_none(self, user: model.User, tool_uuid: Union[UUID, return None def dynamic_tool_to_tool(self, dynamic_tool: Optional["DynamicTool"]) -> Optional["Tool"]: - if not dynamic_tool or not dynamic_tool.active or (tool_representation := dynamic_tool.value) is None: + if not dynamic_tool: + return None + if not dynamic_tool.active: + log.debug("Tool %s is not active", dynamic_tool.uuid) + return None + if (tool_representation := dynamic_tool.value) is None: + log.debug("Tool %s has empty representation", dynamic_tool.uuid) return None if "name" not in tool_representation: tool_representation["name"] = f"dynamic tool {dynamic_tool.uuid}" @@ -709,7 +681,9 @@ def _path_template_kwds(self): "model_tools_path": MODEL_TOOLS_PATH, } - def _get_tool_shed_repository(self, tool_shed, name, owner, installed_changeset_revision): + def _get_tool_shed_repository( + self, tool_shed: str, name: str, owner: str, installed_changeset_revision: Optional[str] + ): # Abstract toolbox doesn't have a dependency on the database, so # override _get_tool_shed_repository here to provide this information. @@ -741,7 +715,7 @@ def _init_dependency_manager(self): default_tool_dependency_dir=default_tool_dependency_dir, ) - def _load_workflow(self, workflow_id): + def _load_workflow(self, workflow_id: str) -> "Workflow": """ Return an instance of 'Workflow' identified by `id`, which is encoded in the tool panel. @@ -1077,13 +1051,13 @@ def __init__( self.guid = guid self.old_id: Optional[str] = None self.python_template_version: Optional[Version] = None - self._lineage = None + self._lineage: Optional[ToolLineage] = None self.dependencies: List = [] # populate toolshed repository info, if available self.populate_tool_shed_info(tool_shed_repository) # add tool resource parameters self.populate_resource_parameters(tool_source) - self.tool_errors = None + self.tool_errors: Optional[str] = None # Parse XML element containing configuration self.tool_source = tool_source self.outputs: Dict[str, ToolOutputBase] = {} @@ -1109,11 +1083,6 @@ def __init__( if self.app.name == "galaxy": self.job_search = self.app.job_search - def remove_from_cache(self): - if source_path := self.tool_source.source_path: - for region in self.app.toolbox.cache_regions.values(): - region.delete(source_path) - @property def history_manager(self): return self.app.history_manager diff --git a/lib/galaxy/tools/cache.py b/lib/galaxy/tools/cache.py index ca7f30456ce0..df0faf7a721a 100644 --- a/lib/galaxy/tools/cache.py +++ b/lib/galaxy/tools/cache.py @@ -1,133 +1,23 @@ -import json import logging import os -import shutil -import sqlite3 -import tempfile -import zlib from threading import Lock from typing import ( Dict, + List, Optional, + Set, + TYPE_CHECKING, + Union, ) -from sqlitedict import SqliteDict - from galaxy.util import unicodify from galaxy.util.hash_util import md5_hash_file -log = logging.getLogger(__name__) - -CURRENT_TOOL_CACHE_VERSION = 0 - - -def encoder(obj): - return sqlite3.Binary(zlib.compress(json.dumps(obj).encode("utf-8"))) - - -def decoder(obj): - return json.loads(zlib.decompress(bytes(obj)).decode("utf-8")) - - -class ToolDocumentCache: - def __init__(self, cache_dir): - self.cache_dir = cache_dir - if not os.path.exists(self.cache_dir): - os.makedirs(self.cache_dir) - self.cache_file = os.path.join(self.cache_dir, "cache.sqlite") - self.writeable_cache_file = None - self._cache = None - self.disabled = False - self._get_cache(create_if_necessary=True) - - def close(self): - self._cache and self._cache.close() - - def _get_cache(self, flag="r", create_if_necessary=False): - try: - if create_if_necessary and not os.path.exists(self.cache_file): - # Create database if necessary using 'c' flag - self._cache = SqliteDict(self.cache_file, flag="c", encode=encoder, decode=decoder, autocommit=False) - if flag == "r": - self._cache.flag = flag - else: - cache_file = self.writeable_cache_file.name if self.writeable_cache_file else self.cache_file - self._cache = SqliteDict(cache_file, flag=flag, encode=encoder, decode=decoder, autocommit=False) - except (sqlite3.OperationalError, RuntimeError): - log.warning("Tool document cache unavailable") - self._cache = None - self.disabled = True - - @property - def cache_file_is_writeable(self): - return os.access(self.cache_file, os.W_OK) - - def reopen_ro(self): - self._get_cache(flag="r") - self.writeable_cache_file = None +if TYPE_CHECKING: + from galaxy.tools import Tool + from galaxy.util.path import StrPath - def get(self, config_file): - try: - tool_document = self._cache.get(config_file) - except sqlite3.OperationalError: - log.debug("Tool document cache unavailable") - return None - if not tool_document: - return None - if tool_document.get("tool_cache_version") != CURRENT_TOOL_CACHE_VERSION: - return None - if self.cache_file_is_writeable: - for path, modtime in tool_document["paths_and_modtimes"].items(): - if os.path.getmtime(path) != modtime: - return None - return tool_document - - def _make_writable(self): - if not self.writeable_cache_file: - self.writeable_cache_file = tempfile.NamedTemporaryFile( - dir=self.cache_dir, suffix="cache.sqlite.tmp", delete=False - ) - if os.path.exists(self.cache_file): - shutil.copy(self.cache_file, self.writeable_cache_file.name) - self._get_cache(flag="c") - - def persist(self): - if self.writeable_cache_file: - self._cache.commit() - os.rename(self.writeable_cache_file.name, self.cache_file) - self.reopen_ro() - - def set(self, config_file, tool_source): - try: - if self.cache_file_is_writeable: - self._make_writable() - to_persist = { - "document": tool_source.to_string(), - "macro_paths": tool_source.macro_paths, - "paths_and_modtimes": tool_source.paths_and_modtimes(), - "tool_cache_version": CURRENT_TOOL_CACHE_VERSION, - } - try: - self._cache[config_file] = to_persist - except RuntimeError: - log.debug("Tool document cache not writeable") - except sqlite3.OperationalError: - log.debug("Tool document cache unavailable") - - def delete(self, config_file): - if self.cache_file_is_writeable: - self._make_writable() - try: - del self._cache[config_file] - except (KeyError, RuntimeError): - pass - - def __del__(self): - if self.writeable_cache_file: - try: - os.unlink(self.writeable_cache_file.name) - except Exception: - pass +log = logging.getLogger(__name__) class ToolCache: @@ -136,39 +26,35 @@ class ToolCache: toolbox. """ - def __init__(self): + def __init__(self) -> None: self._lock = Lock() self._hash_by_tool_paths: Dict[str, ToolHash] = {} - self._tools_by_path = {} - self._tool_paths_by_id = {} - self._macro_paths_by_id = {} - self._new_tool_ids = set() - self._removed_tool_ids = set() - self._removed_tools_by_path = {} + self._tools_by_path: Dict[str, Tool] = {} + self._tool_paths_by_id: Dict[str, StrPath] = {} + self._new_tool_ids: Set[str] = set() + self._removed_tool_ids: Set[str] = set() + self._removed_tools_by_path: Dict[str, Tool] = {} self._hashes_initialized = False - def assert_hashes_initialized(self): + def assert_hashes_initialized(self) -> None: if not self._hashes_initialized: for tool_hash in self._hash_by_tool_paths.values(): tool_hash.hash # noqa: B018 self._hashes_initialized = True - def cleanup(self): + def cleanup(self) -> List[str]: """ Remove uninstalled tools from tool cache if they are not on disk anymore or if their content has changed. Returns list of tool_ids that have been removed. """ - removed_tool_ids = [] + removed_tool_ids: List[str] = [] try: with self._lock: - persist_tool_document_cache = False paths_to_cleanup = { (path, tool) for path, tool in self._tools_by_path.items() if self._should_cleanup(path) } for config_filename, tool in paths_to_cleanup: - tool.remove_from_cache() - persist_tool_document_cache = True del self._hash_by_tool_paths[config_filename] if os.path.exists(config_filename): # This tool has probably been broken while editing on disk @@ -184,8 +70,6 @@ def cleanup(self): self._removed_tool_ids.add(tool_id) if tool_id in self._new_tool_ids: self._new_tool_ids.remove(tool_id) - if persist_tool_document_cache: - tool.app.toolbox.persist_cache() except Exception as e: log.debug("Exception while checking tools to remove from cache: %s", unicodify(e)) # If by chance the file is being removed while calculating the hash or modtime @@ -194,13 +78,13 @@ def cleanup(self): log.debug(f"Removed the following tools from cache: {removed_tool_ids}") return removed_tool_ids - def _should_cleanup(self, config_filename): + def _should_cleanup(self, config_filename: str) -> bool: """Return True if `config_filename` does not exist or if modtime and hash have changes, else return False.""" try: new_mtime = os.path.getmtime(config_filename) tool_hash = self._hash_by_tool_paths.get(config_filename) - if tool_hash and tool_hash.modtime_less_than(new_mtime): - if not tool_hash.hash_equals(md5_hash_file(config_filename)): + if tool_hash and tool_hash.modtime < new_mtime: + if not tool_hash.hash == md5_hash_file(config_filename): return True else: # No change of content, so not necessary to calculate the md5 checksum every time @@ -208,51 +92,49 @@ def _should_cleanup(self, config_filename): tool = self._tools_by_path[config_filename] for macro_path in tool._macro_paths: new_mtime = os.path.getmtime(macro_path) - if self._hash_by_tool_paths.get(macro_path).modtime < new_mtime: + if (macro_hash := self._hash_by_tool_paths.get(str(macro_path))) and macro_hash.modtime < new_mtime: return True except FileNotFoundError: return True return False - def get_tool(self, config_filename): + def get_tool(self, config_filename: "StrPath") -> Union["Tool", None]: """Get the tool at `config_filename` from the cache if the tool is up to date.""" - return self._tools_by_path.get(config_filename, None) + return self._tools_by_path.get(str(config_filename)) - def get_removed_tool(self, config_filename): - return self._removed_tools_by_path.get(config_filename) + def get_removed_tool(self, config_filename: "StrPath") -> Union["Tool", None]: + return self._removed_tools_by_path.get(str(config_filename)) - def get_tool_by_id(self, tool_id): + def get_tool_by_id(self, tool_id: str) -> Union["Tool", None]: """Get the tool with the id `tool_id` from the cache if the tool is up to date.""" - return self.get_tool(self._tool_paths_by_id.get(tool_id)) + if tool_path := self._tool_paths_by_id.get(tool_id): + return self.get_tool(tool_path) + return None - def expire_tool(self, tool_id): + def expire_tool(self, tool_id: str) -> None: with self._lock: if tool_id in self._tool_paths_by_id: - config_filename = self._tool_paths_by_id[tool_id] + config_filename = str(self._tool_paths_by_id[tool_id]) del self._hash_by_tool_paths[config_filename] del self._tool_paths_by_id[tool_id] del self._tools_by_path[config_filename] if tool_id in self._new_tool_ids: self._new_tool_ids.remove(tool_id) - def cache_tool(self, config_filename, tool): + def cache_tool(self, config_filename: "StrPath", tool: "Tool") -> None: tool_id = str(tool.id) # We defer hashing of the config file if we haven't called assert_hashes_initialized. # This allows startup to occur without having to read in and hash all tool and macro files lazy_hash = not self._hashes_initialized with self._lock: - self._hash_by_tool_paths[config_filename] = ToolHash(config_filename, lazy_hash=lazy_hash) + self._hash_by_tool_paths[str(config_filename)] = ToolHash(config_filename, lazy_hash=lazy_hash) self._tool_paths_by_id[tool_id] = config_filename - self._tools_by_path[config_filename] = tool + self._tools_by_path[str(config_filename)] = tool self._new_tool_ids.add(tool_id) for macro_path in tool._macro_paths: - self._hash_by_tool_paths[macro_path] = ToolHash(macro_path, lazy_hash=lazy_hash) - if tool_id not in self._macro_paths_by_id: - self._macro_paths_by_id[tool_id] = {macro_path} - else: - self._macro_paths_by_id[tool_id].add(macro_path) + self._hash_by_tool_paths[str(macro_path)] = ToolHash(macro_path, lazy_hash=lazy_hash) - def reset_status(self): + def reset_status(self) -> None: """ Reset tracking of new and newly disabled tools. """ @@ -263,29 +145,15 @@ def reset_status(self): class ToolHash: - def __init__(self, path: str, modtime: Optional[float] = None, lazy_hash: bool = False): + def __init__(self, path: "StrPath", modtime: Optional[float] = None, lazy_hash: bool = False) -> None: self.path = path - self._modtime = modtime or os.path.getmtime(path) - self._tool_hash = None + self.modtime = modtime or os.path.getmtime(path) + self._tool_hash: Optional[str] = None if not lazy_hash: self.hash # noqa: B018 - def modtime_less_than(self, other_modtime: float): - return self._modtime < other_modtime - - def hash_equals(self, other_hash: Optional[str]): - return self.hash == other_hash - - @property - def modtime(self) -> float: - return self._modtime - - @modtime.setter - def modtime(self, new_value: float): - self._modtime = new_value - @property - def hash(self): + def hash(self) -> Union[str, None]: if self._tool_hash is None: self._tool_hash = md5_hash_file(self.path) return self._tool_hash diff --git a/lib/galaxy/tools/data_manager/manager.py b/lib/galaxy/tools/data_manager/manager.py index 8da9f8e97f9e..caf817067a06 100644 --- a/lib/galaxy/tools/data_manager/manager.py +++ b/lib/galaxy/tools/data_manager/manager.py @@ -172,6 +172,7 @@ def _load_from_element(self, elem: Element, tool_path: Optional[str]) -> None: tool_elem is not None ), f"Error loading tool for data manager. Make sure that a tool_file attribute or a tool tag set has been defined:\n{util.xml_to_string(elem)}" path = tool_elem.get("file") + assert path is not None, f"A tool file path could not be determined:\n{util.xml_to_string(elem)}" tool_guid = tool_elem.get("guid") # need to determine repository info so that dependencies will work correctly tool_shed_repository = self.data_managers.app.toolbox.get_tool_repository_from_xml_item(tool_elem, path) @@ -187,7 +188,6 @@ def _load_from_element(self, elem: Element, tool_path: Optional[str]) -> None: shed_conf = self.data_managers.app.toolbox.get_shed_config_dict_by_filename(shed_conf_file) if shed_conf: tool_path = shed_conf.get("tool_path", tool_path) - assert path is not None, f"A tool file path could not be determined:\n{util.xml_to_string(elem)}" assert tool_path, "A tool root path is required" self._load_tool( os.path.join(tool_path, path), @@ -221,6 +221,7 @@ def _load_tool( tool_shed_repository=tool_shed_repository, use_cached=True, ) + assert tool.id self.data_managers.app.toolbox.data_manager_tools[tool.id] = tool self.tool = tool return tool diff --git a/lib/galaxy/tools/search/__init__.py b/lib/galaxy/tools/search/__init__.py index 9fa0c24baff4..835adc1ec4b9 100644 --- a/lib/galaxy/tools/search/__init__.py +++ b/lib/galaxy/tools/search/__init__.py @@ -32,6 +32,7 @@ from typing import ( Dict, List, + TYPE_CHECKING, Union, ) @@ -63,6 +64,13 @@ unicodify, ) +if TYPE_CHECKING: + from galaxy.tools import ( + Tool, + ToolBox, + ) + from galaxy.tools.cache import ToolCache + log = logging.getLogger(__name__) CanConvertToFloat = Union[str, int, float] @@ -90,8 +98,8 @@ class ToolBoxSearch: Search is delegated off to ToolPanelViewSearch for each panel object. """ - def __init__(self, toolbox, index_dir: str, index_help: bool = True): - panel_searches = {} + def __init__(self, toolbox: "ToolBox", index_dir: str, index_help: bool = True) -> None: + panel_searches: Dict[str, ToolPanelViewSearch] = {} for panel_view in toolbox.panel_views(): panel_view_id = panel_view.id panel_index_dir = os.path.join(index_dir, panel_view_id) @@ -108,7 +116,7 @@ def __init__(self, toolbox, index_dir: str, index_help: bool = True): # reindexing if the index count is equal to the toolbox reload count. self.index_count = -1 - def build_index(self, tool_cache, toolbox, index_help: bool = True) -> None: + def build_index(self, tool_cache: "ToolCache", toolbox: "ToolBox", index_help: bool = True) -> None: self.index_count += 1 for panel_search in self.panel_searches.values(): panel_search.build_index(tool_cache, toolbox, index_help=index_help) @@ -202,7 +210,7 @@ def _index_setup(self) -> index.Index: """Get or create a reference to the index.""" return get_or_create_index(self.index_dir, self.schema) - def build_index(self, tool_cache, toolbox, index_help: bool = True) -> None: + def build_index(self, tool_cache: "ToolCache", toolbox: "ToolBox", index_help: bool = True) -> None: """Prepare search index for tools loaded in toolbox. Use `tool_cache` to determine which tools need indexing and which @@ -234,7 +242,7 @@ def build_index(self, tool_cache, toolbox, index_help: bool = True) -> None: log.debug("Toolbox index of panel %s finished %s", self.panel_view_id, execution_timer) - def _get_tools_to_remove(self, tool_cache) -> list: + def _get_tools_to_remove(self, tool_cache: "ToolCache") -> list: """Return list of tool IDs to be removed from index.""" tool_ids_to_remove = (self.indexed_tool_ids - set(tool_cache._tool_paths_by_id.keys())).union( tool_cache._removed_tool_ids @@ -252,9 +260,9 @@ def _get_tools_to_remove(self, tool_cache) -> list: return list(tool_ids_to_remove) - def _get_tool_list(self, toolbox, tool_cache) -> list: + def _get_tool_list(self, toolbox: "ToolBox", tool_cache: "ToolCache") -> List["Tool"]: """Return list of tools to add and remove from index.""" - tools_to_index = [] + tools_to_index: List[Tool] = [] for tool_id in tool_cache._new_tool_ids - self.indexed_tool_ids: tool = toolbox.get_tool(tool_id) @@ -267,6 +275,8 @@ def _get_tool_list(self, toolbox, tool_cache) -> list: tool = tool_cache.get_tool_by_id(tool_version.id) if tool and not tool.hidden: break + else: + continue else: continue tools_to_index.append(tool) diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index 287b7563dccb..f79a8c74b4ca 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -99,6 +99,9 @@ def find(self, path: str, namespaces: Optional[Mapping[str, str]] = None) -> Uni def findall(self, path: str, namespaces: Optional[Mapping[str, str]] = None) -> List[Self]: # type: ignore[override] return cast(List[Self], super().findall(path, namespaces)) + def iterfind(self, path: str, namespaces: Optional[Mapping[str, str]] = None) -> Iterator[Self]: + return cast(Iterator[Self], super().iterfind(path, namespaces)) + def SubElement(parent: Element, tag: str, attrib: Optional[Dict[str, str]] = None, **extra) -> Element: return cast(Element, etree.SubElement(parent, tag, attrib, **extra)) @@ -196,6 +199,14 @@ def str_removeprefix(s: str, prefix: str): return s +@overload +def remove_protocol_from_url(url: None) -> None: ... + + +@overload +def remove_protocol_from_url(url: str) -> str: ... + + def remove_protocol_from_url(url): """Supplied URL may be null, if not ensure http:// or https:// etc... is stripped off. diff --git a/lib/galaxy/util/tool_shed/common_util.py b/lib/galaxy/util/tool_shed/common_util.py index b65eab240f8f..0e654b954dbb 100644 --- a/lib/galaxy/util/tool_shed/common_util.py +++ b/lib/galaxy/util/tool_shed/common_util.py @@ -266,10 +266,7 @@ def remove_protocol_and_user_from_clone_url(repository_clone_url: str) -> str: return tmp_url.rstrip("/") -def remove_protocol_from_tool_shed_url(tool_shed_url: str) -> str: - """Return a partial Tool Shed URL, eliminating the protocol if it exists.""" - return util.remove_protocol_from_url(tool_shed_url) - +remove_protocol_from_tool_shed_url = util.remove_protocol_from_url __all__ = ( "accumulate_tool_dependencies", diff --git a/lib/galaxy/util/tool_version.py b/lib/galaxy/util/tool_version.py index 2cb22be093ed..b647eb694198 100644 --- a/lib/galaxy/util/tool_version.py +++ b/lib/galaxy/util/tool_version.py @@ -1,4 +1,7 @@ -def remove_version_from_guid(guid): +from typing import Union + + +def remove_version_from_guid(guid: str) -> Union[str, None]: """ Removes version from toolshed-derived tool_id(=guid). """ diff --git a/lib/galaxy/util/xml_macros.py b/lib/galaxy/util/xml_macros.py index a00f4e23f8b5..09c8a9503302 100644 --- a/lib/galaxy/util/xml_macros.py +++ b/lib/galaxy/util/xml_macros.py @@ -1,23 +1,33 @@ import os from copy import deepcopy from typing import ( + Callable, Dict, + Iterable, List, Optional, Tuple, + TYPE_CHECKING, + TypeVar, + Union, ) from galaxy.util import ( - Element, - ElementTree, parse_xml, + unicodify, ) -from galaxy.util.path import StrPath -REQUIRED_PARAMETER = object() +if TYPE_CHECKING: + from galaxy.util import ( + Element, + ElementTree, + ) + from galaxy.util.path import StrPath +MacrosDictT = Dict[str, List["Element"]] -def load_with_references(path: StrPath) -> Tuple[ElementTree, Optional[List[str]]]: + +def load_with_references(path: "StrPath") -> Tuple["ElementTree", Optional[List[str]]]: """Load XML documentation from file system and preprocesses XML macros. Return the XML representation of the expanded tree and paths to @@ -30,20 +40,24 @@ def load_with_references(path: StrPath) -> Tuple[ElementTree, Optional[List[str] if macros_el is None: return tree, [] - macros: Dict[str, List[Element]] = {} + macros: MacrosDictT = {} macro_paths = _import_macros(macros_el, path, macros) macros_el.clear() # Collect tokens - tokens = {} + tokens: Dict[str, str] = {} for m in macros.get("token", []): - tokens[m.get("name")] = m.text or "" + token_name = m.get("name") + assert token_name + tokens[token_name] = m.text or "" tokens = expand_nested_tokens(tokens) # Expand xml macros - macro_dict = {} + macro_dict: Dict[str, XmlMacroDef] = {} for m in macros.get("xml", []): - macro_dict[m.get("name")] = XmlMacroDef(m) + macro_name = m.get("name") + assert macro_name + macro_dict[macro_name] = XmlMacroDef(m) _expand_macros([root], macro_dict, tokens) # reinsert template macro which are used during tool execution @@ -53,25 +67,23 @@ def load_with_references(path: StrPath) -> Tuple[ElementTree, Optional[List[str] return tree, macro_paths -def load(path: StrPath) -> ElementTree: +def load(path: "StrPath") -> "ElementTree": tree, _ = load_with_references(path) return tree -def template_macro_params(root): +def template_macro_params(root: "Element") -> Dict[str, Union[str, None]]: """ Look for template macros and populate param_dict (for cheetah) with these. """ - param_dict = {} macros_el = _macros_el(root) - macro_dict = _macros_of_type(macros_el, "template", lambda el: el.text) - for key, value in macro_dict.items(): - param_dict[key] = value - return param_dict + if macros_el is not None: + return _macros_of_type(macros_el, "template", lambda el: el.text) + return {} -def raw_xml_tree(path: StrPath) -> ElementTree: +def raw_xml_tree(path: "StrPath") -> "ElementTree": """Load raw (no macro expansion) tree representation of XML represented at the specified path. """ @@ -79,37 +91,43 @@ def raw_xml_tree(path: StrPath) -> ElementTree: return tree -def imported_macro_paths(root): +def imported_macro_paths(root: "Element") -> List[str]: macros_el = _macros_el(root) + if macros_el is None: + return [] return _imported_macro_paths_from_el(macros_el) -def _import_macros(macros_el, path, macros) -> Optional[List[str]]: +def _import_macros(macros_el: "Element", path: "StrPath", macros: MacrosDictT) -> Optional[List[str]]: """ root the parsed XML tree path the path to the main xml document """ xml_base_dir = os.path.dirname(path) - if macros_el is not None: - macro_paths = _load_macros(macros_el, xml_base_dir, macros) - # _xml_set_children(macros_el, macro_els) - return macro_paths - return None + macro_paths = _load_macros(macros_el, xml_base_dir, macros) + # _xml_set_children(macros_el, macro_els) + return macro_paths -def _macros_el(root): +def _macros_el(root: "Element") -> Union["Element", None]: return root.find("macros") -def _macros_of_type(macros_el, type, el_func): - if macros_el is None: - return {} +T = TypeVar("T") + + +def _macros_of_type(macros_el: "Element", type: str, el_func: Callable[["Element"], T]) -> Dict[str, T]: macro_els = macros_el.findall("macro") - filtered_els = [(macro_el.get("name"), el_func(macro_el)) for macro_el in macro_els if macro_el.get("type") == type] - return dict(filtered_els) + ret: Dict[str, T] = {} + for macro_el in macro_els: + if macro_el.get("type") == type: + macro_name = macro_el.get("name") + assert macro_name + ret[macro_name] = el_func(macro_el) + return ret -def expand_nested_tokens(tokens): +def expand_nested_tokens(tokens: Dict[str, str]) -> Dict[str, str]: for token_name in tokens.keys(): for current_token_name, current_token_value in tokens.items(): if token_name in current_token_value: @@ -119,62 +137,67 @@ def expand_nested_tokens(tokens): return tokens -def _expand_tokens(elements, tokens): - if not tokens or elements is None: +def _expand_tokens(elements: Iterable["Element"], tokens: Dict[str, str]) -> None: + if not tokens: return for element in elements: _expand_tokens_for_el(element, tokens) -def _expand_tokens_for_el(element, tokens): +def _expand_tokens_for_el(element: "Element", tokens: Dict[str, str]) -> None: """ expand tokens in element and (recursively) in its children replacements of text attributes and attribute values are possible """ - value = element.text - if value: - new_value = _expand_tokens_str(element.text, tokens) - if new_value is not value: + element_text = element.text + if element_text: + new_value = _expand_tokens_str(element_text, tokens) + if new_value is not element_text: element.text = new_value for key, value in element.attrib.items(): - new_value = _expand_tokens_str(value, tokens) + new_value = _expand_tokens_str(unicodify(value), tokens) if new_value is not value: element.attrib[key] = new_value - new_key = _expand_tokens_str(key, tokens) + new_key = _expand_tokens_str(unicodify(key), tokens) if new_key is not key: element.attrib[new_key] = element.attrib[key] del element.attrib[key] # recursively expand in childrens - _expand_tokens(list(element), tokens) + _expand_tokens(element.__iter__(), tokens) -def _expand_tokens_str(s, tokens): +def _expand_tokens_str(s: str, tokens: Dict[str, str]) -> str: for key, value in tokens.items(): if key in s: s = s.replace(key, value) return s -def _expand_macros(elements, macros, tokens, visited=None): +def _expand_macros( + elements: Iterable["Element"], + macros: Dict[str, "XmlMacroDef"], + tokens: Dict[str, str], + visited: Optional[List[str]] = None, +) -> None: if not macros and not tokens: return if visited is None: - v = [] - else: - v = visited + visited = [] for element in elements: while True: expand_el = element.find(".//expand") if expand_el is None: break - _expand_macro(expand_el, macros, tokens, v) + _expand_macro(expand_el, macros, tokens, visited) -def _expand_macro(expand_el, macros, tokens, visited): +def _expand_macro( + expand_el: "Element", macros: Dict[str, "XmlMacroDef"], tokens: Dict[str, str], visited: List[str] +) -> None: macro_name = expand_el.get("macro") assert macro_name is not None, "Attempted to expand macro with no 'macro' attribute defined." # check for cycles in the nested macro expansion @@ -185,22 +208,22 @@ def _expand_macro(expand_el, macros, tokens, visited): assert macro_name in macros, f"No macro named {macro_name} found, known macros are {', '.join(macros.keys())}." macro_def = macros[macro_name] - expanded_elements = deepcopy(macro_def.element) - _expand_yield_statements(expanded_elements, expand_el) + macro_el = deepcopy(macro_def.element) + _expand_yield_statements(macro_el, expand_el) macro_tokens = macro_def.macro_tokens(expand_el) if macro_tokens: - _expand_tokens(expanded_elements, macro_tokens) + _expand_tokens(macro_el.__iter__(), macro_tokens) # Recursively expand contained macros. - _expand_macros(expanded_elements, macros, tokens, visited) - _xml_replace(expand_el, expanded_elements) + _expand_macros(macro_el.__iter__(), macros, tokens, visited) + _xml_replace(expand_el, macro_el.__iter__()) del visited[-1] -def _expand_yield_statements(macro_def, expand_el): +def _expand_yield_statements(macro_el: "Element", expand_el: "Element") -> None: """ - Modifies the macro_def element by replacing + Modifies the macro_el element by replacing 1. all named yield tags by the content of the corresponding token tags - token tags need to be direct children of the expand - processed in order of definition of the token tags @@ -208,40 +231,38 @@ def _expand_yield_statements(macro_def, expand_el): """ # replace named yields for token_el in expand_el.findall("./token"): - name = token_el.attrib.get("name", None) + name = token_el.attrib.get("name") assert name is not None, "Found unnamed token" + str(token_el.attrib) - yield_els = list(macro_def.findall(f".//yield[@name='{name}']")) + yield_els = list(macro_el.findall(f".//yield[@name='{name}']")) assert len(yield_els) > 0, f"No named yield found for named token {name}" - token_el_children = list(token_el) for yield_el in yield_els: - _xml_replace(yield_el, token_el_children) + _xml_replace(yield_el, token_el.__iter__()) # replace unnamed yields - yield_els = list(macro_def.findall(".//yield")) + yield_els = list(macro_el.findall(".//yield")) expand_el_children = [c for c in expand_el if c.tag != "token"] for yield_el in yield_els: _xml_replace(yield_el, expand_el_children) -def _load_macros(macros_el, xml_base_dir, macros) -> List[str]: +def _load_macros(macros_el: "Element", xml_base_dir: str, macros: MacrosDictT) -> List[str]: # Import macros from external files. macro_paths = _load_imported_macros(macros_el, xml_base_dir, macros) # Load all directly defined macros. - _load_embedded_macros(macros_el, xml_base_dir, macros) + _load_embedded_macros(macros_el, macros) return macro_paths -def _load_embedded_macros(macros_el, xml_base_dir, macros): - if macros_el is None: - return +def _load_embedded_macros(macros_el: "Element", macros: MacrosDictT) -> None: # attribute typed macro for macro in macros_el.iterfind("macro"): if "type" not in macro.attrib: macro.attrib["type"] = "xml" + macro_type = unicodify(macro.attrib["type"]) try: - macros[macro.attrib["type"]].append(macro) + macros[macro_type].append(macro) except KeyError: - macros[macro.attrib["type"]] = [macro] + macros[macro_type] = [macro] # type shortcuts ( is a shortcut for . @@ -255,7 +276,7 @@ def _load_embedded_macros(macros_el, xml_base_dir, macros): macros[tag] = [macro_el] -def _load_imported_macros(macros_el, xml_base_dir, macros) -> List[str]: +def _load_imported_macros(macros_el: "Element", xml_base_dir: str, macros: MacrosDictT) -> List[str]: macro_paths = [] for tool_relative_import_path in _imported_macro_paths_from_el(macros_el): @@ -266,28 +287,27 @@ def _load_imported_macros(macros_el, xml_base_dir, macros) -> List[str]: return macro_paths -def _imported_macro_paths_from_el(macros_el): +def _imported_macro_paths_from_el(macros_el: "Element") -> List[str]: imported_macro_paths = [] - macro_import_els = [] - if macros_el is not None: - macro_import_els = macros_el.findall("import") - for macro_import_el in macro_import_els: + for macro_import_el in macros_el.findall("import"): raw_import_path = macro_import_el.text + assert raw_import_path imported_macro_paths.append(raw_import_path) return imported_macro_paths -def _load_macro_file(path: StrPath, xml_base_dir, macros) -> List[str]: +def _load_macro_file(path: "StrPath", xml_base_dir: str, macros: MacrosDictT) -> List[str]: tree = parse_xml(path, strip_whitespace=False) root = tree.getroot() return _load_macros(root, xml_base_dir, macros) -def _xml_replace(query, targets): +def _xml_replace(query: "Element", targets: Iterable["Element"]) -> None: parent_el = query.find("..") + assert parent_el is not None matching_index = -1 # for index, el in enumerate(parent_el.iter('.')): ## Something like this for newer implementation - for index, el in enumerate(list(parent_el)): + for index, el in enumerate(parent_el): if el == query: matching_index = index break @@ -308,47 +328,46 @@ class XmlMacroDef: - the quote character, default '@' - parameter name - parameter names can be given as comma separated list using the - `token` attribute or as attributes `token_XXX` (where `XXX` is the name). + Parameter names can be given as comma separated list using the + `tokens` attribute or as attributes `token_XXX` (where `XXX` is the name). The former option should be used to specify required attributes of the - macro and the latter for optional attributes if the macro (the value of + macro and the latter for optional attributes of the macro (the value of `token_XXX is used as default value). TODO: `token_quote` forbids `"quote"` as character name of optional parameters """ - def __init__(self, el): + def __init__(self, el: "Element") -> None: self.element = el - parameters = {} - tokens = [] - token_quote = "@" + tokens: Dict[str, Union[str, None]] = {} + self.token_quote = "@" for key, value in el.attrib.items(): + key = unicodify(key) + value = unicodify(value) if key == "token_quote": - token_quote = value + self.token_quote = value if key == "tokens": for token in value.split(","): - tokens.append((token, REQUIRED_PARAMETER)) + tokens[token] = None # here None means that the token is a required parameter elif key.startswith("token_"): token = key[len("token_") :] - tokens.append((token, value)) - for name, default in tokens: - parameters[name] = (token_quote, default) - self.parameters = parameters + tokens[token] = value + self.tokens = tokens - def macro_tokens(self, expand_el): + def macro_tokens(self, expand_el: "Element") -> Dict[str, str]: """ get a dictionary mapping token names to values. The names are the parameter names surrounded by the quote character. Values are taken from the expand_el if absent default values of optional parameters are used. """ - tokens = {} - for key, (wrap_char, default_val) in self.parameters.items(): + tokens: Dict[str, str] = {} + for key, default_val in self.tokens.items(): token_value = expand_el.attrib.get(key, default_val) - if token_value is REQUIRED_PARAMETER: + if token_value is None: raise ValueError(f"Failed to expand macro - missing required parameter [{key}].") - token_name = f"{wrap_char}{key.upper()}{wrap_char}" + token_name = f"{self.token_quote}{key.upper()}{self.token_quote}" tokens[token_name] = token_value return tokens diff --git a/lib/galaxy/webapps/galaxy/api/dynamic_tools.py b/lib/galaxy/webapps/galaxy/api/dynamic_tools.py index 78906252bf42..b810f3730b95 100644 --- a/lib/galaxy/webapps/galaxy/api/dynamic_tools.py +++ b/lib/galaxy/webapps/galaxy/api/dynamic_tools.py @@ -1,6 +1,8 @@ import logging from datetime import datetime from typing import ( + Any, + Dict, List, Optional, Union, @@ -148,13 +150,15 @@ def create(self, payload: DynamicToolPayload): return dynamic_tool.to_dict() @router.delete("/api/dynamic_tools/{dynamic_tool_id}", require_admin=True) - def delete(self, dynamic_tool_id: DatabaseIdOrUUID): + def delete(self, dynamic_tool_id: DatabaseIdOrUUID) -> Dict[str, Any]: """ DELETE /api/dynamic_tools/{encoded_dynamic_tool_id|tool_uuid} Deactivate the specified dynamic tool. Deactivated tools will not be loaded into the toolbox. """ - dynamic_tool = dynamic_tool = self.dynamic_tools_manager.get_tool_by_id_or_uuid(dynamic_tool_id) + dynamic_tool = self.dynamic_tools_manager.get_tool_by_id_or_uuid(dynamic_tool_id) + if dynamic_tool is None: + raise ObjectNotFound() updated_dynamic_tool = self.dynamic_tools_manager.deactivate(dynamic_tool) return updated_dynamic_tool.to_dict() diff --git a/lib/galaxy/webapps/galaxy/api/users.py b/lib/galaxy/webapps/galaxy/api/users.py index 905d0f925416..2c70f4f6eaea 100644 --- a/lib/galaxy/webapps/galaxy/api/users.py +++ b/lib/galaxy/webapps/galaxy/api/users.py @@ -11,6 +11,7 @@ Dict, List, Optional, + TYPE_CHECKING, Union, ) @@ -94,6 +95,9 @@ from galaxy.webapps.galaxy.api.common import UserIdPathParam from galaxy.webapps.galaxy.services.users import UsersService +if TYPE_CHECKING: + from galaxy.work.context import SessionRequestContext + log = logging.getLogger(__name__) router = Router(tags=["users"]) @@ -169,7 +173,7 @@ class FastAPIUsers: ) def recalculate_disk_usage( self, - trans: ProvidesUserContext = DependsOnTrans, + trans: "SessionRequestContext" = DependsOnTrans, ): """This route will be removed in a future version. @@ -191,7 +195,7 @@ def recalculate_disk_usage( def recalculate_disk_usage_by_user_id( self, user_id: UserIdPathParam, - trans: ProvidesUserContext = DependsOnTrans, + trans: "SessionRequestContext" = DependsOnTrans, ): result = self.service.recalculate_disk_usage(trans, user_id) return Response(status_code=status.HTTP_204_NO_CONTENT) if result is None else result diff --git a/lib/galaxy/webapps/galaxy/services/users.py b/lib/galaxy/webapps/galaxy/services/users.py index 1d5705b34bac..77a02c2ddd39 100644 --- a/lib/galaxy/webapps/galaxy/services/users.py +++ b/lib/galaxy/webapps/galaxy/services/users.py @@ -1,6 +1,7 @@ from typing import ( List, Optional, + TYPE_CHECKING, Union, ) @@ -40,6 +41,9 @@ ) from galaxy.webapps.galaxy.services.roles import role_to_model +if TYPE_CHECKING: + from galaxy.work.context import SessionRequestContext + class UsersService(ServiceBase): """Common interface/service logic for interactions with users in the context of the API. @@ -66,7 +70,7 @@ def __init__( def recalculate_disk_usage( self, - trans: ProvidesUserContext, + trans: "SessionRequestContext", user_id: int, ): if trans.anonymous: diff --git a/mypy.ini b/mypy.ini index dad391937877..683bea771773 100644 --- a/mypy.ini +++ b/mypy.ini @@ -19,6 +19,22 @@ warn_unused_ignores = True platform = linux # green list - work on growing these please! +[mypy-galaxy.tool_util.toolbox.lineages.interface] +disallow_any_generics = True +disallow_subclassing_any = True +disallow_untyped_calls = True +disallow_untyped_decorators = True +disallow_untyped_defs = True +warn_return_any = True + +[mypy-galaxy.tools.cache] +disallow_any_generics = True +disallow_subclassing_any = True +disallow_untyped_calls = True +disallow_untyped_decorators = True +disallow_untyped_defs = True +warn_return_any = True + [mypy-galaxy.util.compression_utils] disallow_any_generics = True disallow_subclassing_any = True @@ -43,6 +59,14 @@ disallow_untyped_decorators = True disallow_untyped_defs = True warn_return_any = True +[mypy-galaxy.util.xml_macros] +disallow_any_generics = True +disallow_subclassing_any = True +disallow_untyped_calls = True +disallow_untyped_decorators = True +disallow_untyped_defs = True +warn_return_any = True + [mypy-galaxy.managers.secured] disallow_any_generics = True disallow_subclassing_any = True @@ -511,8 +535,6 @@ check_untyped_defs = False check_untyped_defs = False [mypy-galaxy.config.script] check_untyped_defs = False -[mypy-galaxy.tools.cache] -check_untyped_defs = False [mypy-galaxy.tool_shed.cache] check_untyped_defs = False [mypy-galaxy.tool_util.deps.containers] diff --git a/packages/app/setup.cfg b/packages/app/setup.cfg index 867b7bd8a5ab..2c879f7ff913 100644 --- a/packages/app/setup.cfg +++ b/packages/app/setup.cfg @@ -73,7 +73,6 @@ install_requires = regex requests SQLAlchemy>=2.0.37,<2.1,!=2.0.41 - sqlitedict starlette svgwrite typing-extensions diff --git a/pyproject.toml b/pyproject.toml index 1e180d4a28c3..1e5f51bb3a01 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,7 +94,6 @@ dependencies = [ "social-auth-core>=4.5.0", # to drop dependency on abandoned python-jose "sortedcontainers", "SQLAlchemy>=2.0.37,<2.1,!=2.0.41", # https://github.com/sqlalchemy/sqlalchemy/issues/12019 , https://github.com/sqlalchemy/sqlalchemy/issues/12600 - "sqlitedict", "sqlparse", "starlette", "starlette-context",