From 1b1e8fe77a4f3e75f51bd7cce3516332d92c6b97 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Wed, 16 Apr 2025 11:25:02 +0200 Subject: [PATCH 01/69] Add functions api. New commit to clean up db migration --- .../api_schemas_api_server/functions.py | 44 + .../functions_wb_schema.py | 132 ++ .../models/functions_models_db.py | 170 +++ .../webserver/functions/functions.py | 22 - .../functions/functions_rpc_interface.py | 187 +++ services/api-server/openapi.json | 1289 ++++++++++++++++- .../simcore_service_api_server/api/root.py | 19 +- .../api/routes/functions.py | 17 - .../api/routes/functions_routes.py | 453 ++++++ .../api/routes/solvers.py | 2 +- .../api/routes/studies_jobs.py | 8 +- .../models/schemas/functions_api_schema.py | 85 ++ .../services_rpc/wb_api_server.py | 106 +- .../functions/_controller_rpc.py | 21 - .../functions/_functions_controller_rpc.py | 262 ++++ .../functions/_functions_repository.py | 206 +++ .../functions/_service.py | 4 +- .../functions/plugin.py | 4 +- 18 files changed, 2954 insertions(+), 77 deletions(-) create mode 100644 packages/models-library/src/models_library/api_schemas_api_server/functions.py create mode 100644 packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py create mode 100644 packages/postgres-database/src/simcore_postgres_database/models/functions_models_db.py delete mode 100644 packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions.py create mode 100644 packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py delete mode 100644 services/api-server/src/simcore_service_api_server/api/routes/functions.py create mode 100644 services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py create mode 100644 services/api-server/src/simcore_service_api_server/models/schemas/functions_api_schema.py delete mode 100644 services/web/server/src/simcore_service_webserver/functions/_controller_rpc.py create mode 100644 services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py create mode 100644 services/web/server/src/simcore_service_webserver/functions/_functions_repository.py diff --git a/packages/models-library/src/models_library/api_schemas_api_server/functions.py b/packages/models-library/src/models_library/api_schemas_api_server/functions.py new file mode 100644 index 00000000000..44678efd539 --- /dev/null +++ b/packages/models-library/src/models_library/api_schemas_api_server/functions.py @@ -0,0 +1,44 @@ +from typing import Annotated, Any, Literal, TypeAlias + +from models_library import projects +from pydantic import BaseModel, Field + +FunctionID: TypeAlias = projects.ProjectID + + +class FunctionSchema(BaseModel): + schema_dict: dict[str, Any] | None # JSON Schema + + +class FunctionInputSchema(FunctionSchema): ... + + +class FunctionOutputSchema(FunctionSchema): ... + + +class Function(BaseModel): + uid: FunctionID | None = None + title: str | None = None + description: str | None = None + input_schema: FunctionInputSchema | None = None + output_schema: FunctionOutputSchema | None = None + + # @classmethod + # def compose_resource_name(cls, function_key) -> api_resources.RelativeResourceName: + # return api_resources.compose_resource_name("functions", function_key) + + +class StudyFunction(Function): + function_type: Literal["study"] = "study" + study_url: str + + +class PythonCodeFunction(Function): + function_type: Literal["python_code"] = "python_code" + code_url: str + + +FunctionUnion: TypeAlias = Annotated[ + StudyFunction | PythonCodeFunction, + Field(discriminator="function_type"), +] diff --git a/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py b/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py new file mode 100644 index 00000000000..4391a77a658 --- /dev/null +++ b/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py @@ -0,0 +1,132 @@ +from enum import Enum +from typing import Annotated, Any, Literal, TypeAlias +from uuid import UUID + +from models_library import projects +from pydantic import BaseModel, Field + +from ..projects import ProjectID + +FunctionID: TypeAlias = projects.ProjectID +FunctionJobID: TypeAlias = projects.ProjectID +FileID: TypeAlias = UUID + +InputTypes: TypeAlias = FileID | float | int | bool | str | list | None + + +class FunctionSchema(BaseModel): + schema_dict: dict[str, Any] | None # JSON Schema + + +class FunctionInputSchema(FunctionSchema): ... + + +class FunctionOutputSchema(FunctionSchema): ... + + +class FunctionClass(str, Enum): + project = "project" + python_code = "python_code" + + +FunctionClassSpecificData: TypeAlias = dict[str, Any] +FunctionJobClassSpecificData: TypeAlias = FunctionClassSpecificData + + +# TODO, use InputTypes here, but api is throwing weird errors and asking for dict for elements # noqa: FIX002 +FunctionInputs: TypeAlias = dict[str, Any] | None + +FunctionInputsList: TypeAlias = list[FunctionInputs] + +FunctionOutputs: TypeAlias = dict[str, Any] | None + + +class FunctionBase(BaseModel): + uid: FunctionID | None = None + title: str | None = None + description: str | None = None + function_class: FunctionClass + input_schema: FunctionInputSchema | None = None + output_schema: FunctionOutputSchema | None = None + + +class FunctionDB(BaseModel): + uuid: FunctionJobID | None = None + title: str | None = None + description: str | None = None + function_class: FunctionClass + input_schema: FunctionInputSchema | None = None + output_schema: FunctionOutputSchema | None = None + class_specific_data: FunctionClassSpecificData + + +class FunctionJobDB(BaseModel): + uuid: FunctionJobID | None = None + function_uuid: FunctionID + title: str | None = None + inputs: FunctionInputs | None = None + outputs: FunctionOutputs | None = None + class_specific_data: FunctionJobClassSpecificData + function_class: FunctionClass + + +class ProjectFunction(FunctionBase): + function_class: Literal[FunctionClass.project] = FunctionClass.project + project_id: ProjectID + + +class PythonCodeFunction(FunctionBase): + function_class: Literal[FunctionClass.python_code] = FunctionClass.python_code + code_url: str + + +Function: TypeAlias = Annotated[ + ProjectFunction | PythonCodeFunction, + Field(discriminator="function_class"), +] + +FunctionJobCollectionID: TypeAlias = projects.ProjectID + + +class FunctionJobBase(BaseModel): + uid: FunctionJobID | None = None + title: str | None = None + description: str | None = None + function_uid: FunctionID + inputs: FunctionInputs | None = None + outputs: FunctionOutputs | None = None + function_class: FunctionClass + + +class ProjectFunctionJob(FunctionJobBase): + function_class: Literal[FunctionClass.project] = FunctionClass.project + project_job_id: ProjectID + + +class PythonCodeFunctionJob(FunctionJobBase): + function_class: Literal[FunctionClass.python_code] = FunctionClass.python_code + code_url: str + + +FunctionJob: TypeAlias = Annotated[ + ProjectFunctionJob | PythonCodeFunctionJob, + Field(discriminator="function_class"), +] + + +class FunctionJobStatus(BaseModel): + status: str + + +class FunctionJobCollection(BaseModel): + """Model for a collection of function jobs""" + + id: FunctionJobCollectionID + title: str | None + description: str | None + job_ids: list[FunctionJobID] + status: str + + +class FunctionJobCollectionStatus(BaseModel): + status: list[str] diff --git a/packages/postgres-database/src/simcore_postgres_database/models/functions_models_db.py b/packages/postgres-database/src/simcore_postgres_database/models/functions_models_db.py new file mode 100644 index 00000000000..e8a8ba6f2ec --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/models/functions_models_db.py @@ -0,0 +1,170 @@ +"""Functions table + +- List of functions served by the simcore platform +""" + +import uuid + +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import UUID + +from ._common import RefActions +from .base import metadata + +functions = sa.Table( + "functions", + metadata, + sa.Column( + "uuid", + UUID(as_uuid=True), + primary_key=True, + index=True, + default=uuid.uuid4, + doc="Unique id of the function", + ), + sa.Column( + "title", + sa.String, + doc="Name of the function", + ), + sa.Column( + "function_class", + sa.String, + doc="Class of the function", + ), + sa.Column( + "description", + sa.String, + doc="Description of the function", + ), + sa.Column( + "input_schema", + sa.JSON, + doc="Input schema of the function", + ), + sa.Column( + "output_schema", + sa.JSON, + doc="Output schema of the function", + ), + sa.Column( + "system_tags", + sa.JSON, + nullable=True, + doc="System-level tags of the function", + ), + sa.Column( + "user_tags", + sa.JSON, + nullable=True, + doc="User-level tags of the function", + ), + sa.Column( + "class_specific_data", + sa.JSON, + nullable=True, + doc="Fields specific for a function class", + ), + sa.PrimaryKeyConstraint("uuid", name="functions_pk"), +) + +function_jobs = sa.Table( + "function_jobs", + metadata, + sa.Column( + "uuid", + UUID(as_uuid=True), + primary_key=True, + index=True, + default=uuid.uuid4, + doc="Unique id of the function job", + ), + sa.Column( + "title", + sa.String, + doc="Name of the function job", + ), + sa.Column( + "function_uuid", + sa.ForeignKey( + functions.c.uuid, + onupdate=RefActions.CASCADE, + ondelete=RefActions.CASCADE, + name="fk_functions_to_function_jobs_to_function_uuid", + ), + nullable=False, + index=True, + doc="Unique identifier of the function", + ), + sa.Column( + "function_class", + sa.String, + doc="Class of the function", + ), + sa.Column( + "status", + sa.String, + doc="Status of the function job", + ), + sa.Column( + "inputs", + sa.JSON, + doc="Inputs of the function job", + ), + sa.Column( + "outputs", + sa.JSON, + doc="Outputs of the function job", + ), + sa.Column( + "class_specific_data", + sa.JSON, + nullable=True, + doc="Fields specific for a function class", + ), + sa.PrimaryKeyConstraint("uuid", name="function_jobs_pk"), +) + +function_job_collections = sa.Table( + "function_job_collections", + metadata, + sa.Column( + "uuid", + UUID(as_uuid=True), + default=uuid.uuid4, + primary_key=True, + index=True, + doc="Unique id of the function job collection", + ), + sa.Column( + "name", + sa.String, + doc="Name of the function job collection", + ), + sa.PrimaryKeyConstraint("uuid", name="function_job_collections_pk"), +) + +function_job_collections_to_function_jobs = sa.Table( + "function_job_collections_to_function_jobs", + metadata, + sa.Column( + "function_job_collection_uuid", + sa.ForeignKey( + function_job_collections.c.uuid, + onupdate=RefActions.CASCADE, + ondelete=RefActions.CASCADE, + name="fk_func_job_coll_to_func_jobs_to_func_job_coll_uuid", + ), + doc="Unique identifier of the function job collection", + ), + sa.Column( + "function_job_uuid", + sa.ForeignKey( + function_jobs.c.uuid, + onupdate=RefActions.CASCADE, + ondelete=RefActions.CASCADE, + name="fk_func_job_coll_to_func_jobs_to_func_job_uuid", + ), + doc="Unique identifier of the function job", + ), +) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions.py deleted file mode 100644 index d53adfafa66..00000000000 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions.py +++ /dev/null @@ -1,22 +0,0 @@ -import logging - -from models_library.api_schemas_webserver import WEBSERVER_RPC_NAMESPACE -from models_library.rabbitmq_basic_types import RPCMethodName -from pydantic import TypeAdapter - -from .....logging_utils import log_decorator -from .....rabbitmq import RabbitMQRPCClient - -_logger = logging.getLogger(__name__) - - -@log_decorator(_logger, level=logging.DEBUG) -async def ping( - rabbitmq_rpc_client: RabbitMQRPCClient, -) -> str: - result = await rabbitmq_rpc_client.request( - WEBSERVER_RPC_NAMESPACE, - TypeAdapter(RPCMethodName).validate_python("ping"), - ) - assert isinstance(result, str) # nosec - return result diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py new file mode 100644 index 00000000000..7e06bef912c --- /dev/null +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py @@ -0,0 +1,187 @@ +import logging + +from models_library.api_schemas_webserver import ( + WEBSERVER_RPC_NAMESPACE, +) +from models_library.api_schemas_webserver.functions_wb_schema import ( + Function, + FunctionID, + FunctionInputs, + FunctionInputSchema, + FunctionJob, + FunctionJobID, + FunctionOutputSchema, +) +from models_library.rabbitmq_basic_types import RPCMethodName +from pydantic import TypeAdapter + +from .....logging_utils import log_decorator +from .... import RabbitMQRPCClient + +_logger = logging.getLogger(__name__) + + +@log_decorator(_logger, level=logging.DEBUG) +async def ping( + rabbitmq_rpc_client: RabbitMQRPCClient, +) -> str: + result = await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("ping"), + ) + assert isinstance(result, str) # nosec + return result + + +@log_decorator(_logger, level=logging.DEBUG) +async def register_function( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + function: Function, +) -> Function: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("register_function"), + function=function, + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def get_function( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + function_id: FunctionID, +) -> Function: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("get_function"), + function_id=function_id, + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def get_function_input_schema( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + function_id: FunctionID, +) -> FunctionInputSchema: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("get_function_input_schema"), + function_id=function_id, + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def get_function_output_schema( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + function_id: FunctionID, +) -> FunctionOutputSchema: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("get_function_output_schema"), + function_id=function_id, + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def delete_function( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + function_id: FunctionID, +) -> None: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("delete_function"), + function_id=function_id, + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def list_functions( + rabbitmq_rpc_client: RabbitMQRPCClient, +) -> list[Function]: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("list_functions"), + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def run_function( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + function_id: FunctionID, + inputs: FunctionInputs, +) -> FunctionJob: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("run_function"), + function_id=function_id, + inputs=inputs, + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def register_function_job( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + function_job: FunctionJob, +) -> FunctionJob: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("register_function_job"), + function_job=function_job, + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def get_function_job( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + function_job_id: FunctionJobID, +) -> FunctionJob: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("get_function_job"), + function_job_id=function_job_id, + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def list_function_jobs( + rabbitmq_rpc_client: RabbitMQRPCClient, +) -> list[FunctionJob]: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("list_function_jobs"), + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def delete_function_job( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + function_job_id: FunctionJobID, +) -> None: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("delete_function_job"), + function_job_id=function_job_id, + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def find_cached_function_job( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + function_id: FunctionID, + inputs: FunctionInputs, +) -> FunctionJob | None: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("find_cached_function_job"), + function_id=function_id, + inputs=inputs, + ) diff --git a/services/api-server/openapi.json b/services/api-server/openapi.json index 78309bd37eb..cd319f32373 100644 --- a/services/api-server/openapi.json +++ b/services/api-server/openapi.json @@ -5267,6 +5267,924 @@ } } }, + "/v0/functions/ping": { + "post": { + "tags": [ + "functions" + ], + "summary": "Ping", + "operationId": "ping", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + } + } + } + }, + "/v0/functions": { + "get": { + "tags": [ + "functions" + ], + "summary": "List Functions", + "description": "List functions", + "operationId": "list_functions", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "items": { + "oneOf": [ + { + "$ref": "#/components/schemas/ProjectFunction" + }, + { + "$ref": "#/components/schemas/PythonCodeFunction" + } + ], + "discriminator": { + "propertyName": "function_class", + "mapping": { + "project": "#/components/schemas/ProjectFunction", + "python_code": "#/components/schemas/PythonCodeFunction" + } + } + }, + "type": "array", + "title": "Response List Functions V0 Functions Get" + } + } + } + } + } + }, + "post": { + "tags": [ + "functions" + ], + "summary": "Register Function", + "description": "Create function", + "operationId": "register_function", + "requestBody": { + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/ProjectFunction" + }, + { + "$ref": "#/components/schemas/PythonCodeFunction" + } + ], + "title": "Function", + "discriminator": { + "propertyName": "function_class", + "mapping": { + "project": "#/components/schemas/ProjectFunction", + "python_code": "#/components/schemas/PythonCodeFunction" + } + } + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/ProjectFunction" + }, + { + "$ref": "#/components/schemas/PythonCodeFunction" + } + ], + "title": "Response Register Function V0 Functions Post", + "discriminator": { + "propertyName": "function_class", + "mapping": { + "project": "#/components/schemas/ProjectFunction", + "python_code": "#/components/schemas/PythonCodeFunction" + } + } + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/functions/{function_id}": { + "get": { + "tags": [ + "functions" + ], + "summary": "Get Function", + "description": "Get function", + "operationId": "get_function", + "parameters": [ + { + "name": "function_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/ProjectFunction" + }, + { + "$ref": "#/components/schemas/PythonCodeFunction" + } + ], + "discriminator": { + "propertyName": "function_class", + "mapping": { + "project": "#/components/schemas/ProjectFunction", + "python_code": "#/components/schemas/PythonCodeFunction" + } + }, + "title": "Response Get Function V0 Functions Function Id Get" + } + } + } + }, + "404": { + "description": "Function not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "delete": { + "tags": [ + "functions" + ], + "summary": "Delete Function", + "description": "Delete function", + "operationId": "delete_function", + "parameters": [ + { + "name": "function_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/ProjectFunction" + }, + { + "$ref": "#/components/schemas/PythonCodeFunction" + } + ], + "discriminator": { + "propertyName": "function_class", + "mapping": { + "project": "#/components/schemas/ProjectFunction", + "python_code": "#/components/schemas/PythonCodeFunction" + } + }, + "title": "Response Delete Function V0 Functions Function Id Delete" + } + } + } + }, + "404": { + "description": "Function not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/functions/{function_id}:run": { + "post": { + "tags": [ + "functions" + ], + "summary": "Run Function", + "description": "Run function", + "operationId": "run_function", + "security": [ + { + "HTTPBasic": [] + } + ], + "parameters": [ + { + "name": "function_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Id" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Function Inputs" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/ProjectFunctionJob" + }, + { + "$ref": "#/components/schemas/PythonCodeFunctionJob" + } + ], + "discriminator": { + "propertyName": "function_class", + "mapping": { + "project": "#/components/schemas/ProjectFunctionJob", + "python_code": "#/components/schemas/PythonCodeFunctionJob" + } + }, + "title": "Response Run Function V0 Functions Function Id Run Post" + } + } + } + }, + "404": { + "description": "Function not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/functions/{function_id}/input_schema": { + "get": { + "tags": [ + "functions" + ], + "summary": "Get Function Input Schema", + "description": "Get function", + "operationId": "get_function_input_schema", + "parameters": [ + { + "name": "function_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FunctionInputSchema" + } + } + } + }, + "404": { + "description": "Function not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/functions/{function_id}/output_schema": { + "get": { + "tags": [ + "functions" + ], + "summary": "Get Function Output Schema", + "description": "Get function", + "operationId": "get_function_output_schema", + "parameters": [ + { + "name": "function_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FunctionOutputSchema" + } + } + } + }, + "404": { + "description": "Function not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/functions/{function_id}:map": { + "post": { + "tags": [ + "functions" + ], + "summary": "Map Function", + "description": "Map function over input parameters", + "operationId": "map_function", + "security": [ + { + "HTTPBasic": [] + } + ], + "parameters": [ + { + "name": "function_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Id" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ] + }, + "title": "Function Inputs List" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "oneOf": [ + { + "$ref": "#/components/schemas/ProjectFunctionJob" + }, + { + "$ref": "#/components/schemas/PythonCodeFunctionJob" + } + ], + "discriminator": { + "propertyName": "function_class", + "mapping": { + "project": "#/components/schemas/ProjectFunctionJob", + "python_code": "#/components/schemas/PythonCodeFunctionJob" + } + } + }, + "title": "Response Map Function V0 Functions Function Id Map Post" + } + } + } + }, + "404": { + "description": "Function not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/function_jobs": { + "get": { + "tags": [ + "function_jobs" + ], + "summary": "List Function Jobs", + "description": "List function jobs", + "operationId": "list_function_jobs", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "items": { + "oneOf": [ + { + "$ref": "#/components/schemas/ProjectFunctionJob" + }, + { + "$ref": "#/components/schemas/PythonCodeFunctionJob" + } + ], + "discriminator": { + "propertyName": "function_class", + "mapping": { + "project": "#/components/schemas/ProjectFunctionJob", + "python_code": "#/components/schemas/PythonCodeFunctionJob" + } + } + }, + "type": "array", + "title": "Response List Function Jobs V0 Function Jobs Get" + } + } + } + } + } + }, + "post": { + "tags": [ + "function_jobs" + ], + "summary": "Register Function Job", + "description": "Create function job", + "operationId": "register_function_job", + "requestBody": { + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/ProjectFunctionJob" + }, + { + "$ref": "#/components/schemas/PythonCodeFunctionJob" + } + ], + "title": "Function Job", + "discriminator": { + "propertyName": "function_class", + "mapping": { + "project": "#/components/schemas/ProjectFunctionJob", + "python_code": "#/components/schemas/PythonCodeFunctionJob" + } + } + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/ProjectFunctionJob" + }, + { + "$ref": "#/components/schemas/PythonCodeFunctionJob" + } + ], + "title": "Response Register Function Job V0 Function Jobs Post", + "discriminator": { + "propertyName": "function_class", + "mapping": { + "project": "#/components/schemas/ProjectFunctionJob", + "python_code": "#/components/schemas/PythonCodeFunctionJob" + } + } + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/function_jobs/{function_job_id}": { + "get": { + "tags": [ + "function_jobs" + ], + "summary": "Get Function Job", + "description": "Get function job", + "operationId": "get_function_job", + "parameters": [ + { + "name": "function_job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Job Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/ProjectFunctionJob" + }, + { + "$ref": "#/components/schemas/PythonCodeFunctionJob" + } + ], + "discriminator": { + "propertyName": "function_class", + "mapping": { + "project": "#/components/schemas/ProjectFunctionJob", + "python_code": "#/components/schemas/PythonCodeFunctionJob" + } + }, + "title": "Response Get Function Job V0 Function Jobs Function Job Id Get" + } + } + } + }, + "404": { + "description": "Function job not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "delete": { + "tags": [ + "function_jobs" + ], + "summary": "Delete Function Job", + "description": "Delete function job", + "operationId": "delete_function_job", + "parameters": [ + { + "name": "function_job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Job Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "404": { + "description": "Function job not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/function_jobs/{function_job_id}/status": { + "get": { + "tags": [ + "function_jobs" + ], + "summary": "Function Job Status", + "description": "Get function job status", + "operationId": "function_job_status", + "security": [ + { + "HTTPBasic": [] + } + ], + "parameters": [ + { + "name": "function_job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Job Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FunctionJobStatus" + } + } + } + }, + "404": { + "description": "Function job not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/function_jobs/{function_job_id}/outputs": { + "get": { + "tags": [ + "function_jobs" + ], + "summary": "Function Job Outputs", + "description": "Get function job outputs", + "operationId": "function_job_outputs", + "security": [ + { + "HTTPBasic": [] + } + ], + "parameters": [ + { + "name": "function_job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Job Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Response Function Job Outputs V0 Function Jobs Function Job Id Outputs Get" + } + } + } + }, + "404": { + "description": "Function job not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, "/v0/wallets/default": { "get": { "tags": [ @@ -6207,11 +7125,64 @@ }, "type": "object", "required": [ - "chunk_size", - "urls", - "links" + "chunk_size", + "urls", + "links" + ], + "title": "FileUploadData" + }, + "FunctionInputSchema": { + "properties": { + "schema_dict": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Schema Dict" + } + }, + "type": "object", + "required": [ + "schema_dict" + ], + "title": "FunctionInputSchema" + }, + "FunctionJobStatus": { + "properties": { + "status": { + "type": "string", + "title": "Status" + } + }, + "type": "object", + "required": [ + "status" + ], + "title": "FunctionJobStatus" + }, + "FunctionOutputSchema": { + "properties": { + "schema_dict": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Schema Dict" + } + }, + "type": "object", + "required": [ + "schema_dict" ], - "title": "FileUploadData" + "title": "FunctionOutputSchema" }, "GetCreditPriceLegacy": { "properties": { @@ -7735,6 +8706,316 @@ "version": "8.0.0" } }, + "ProjectFunction": { + "properties": { + "uid": { + "anyOf": [ + { + "type": "string", + "format": "uuid" + }, + { + "type": "null" + } + ], + "title": "Uid" + }, + "title": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Title" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "function_class": { + "type": "string", + "const": "project", + "title": "Function Class", + "default": "project" + }, + "input_schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/FunctionInputSchema" + }, + { + "type": "null" + } + ] + }, + "output_schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/FunctionOutputSchema" + }, + { + "type": "null" + } + ] + }, + "project_id": { + "type": "string", + "format": "uuid", + "title": "Project Id" + } + }, + "type": "object", + "required": [ + "project_id" + ], + "title": "ProjectFunction" + }, + "ProjectFunctionJob": { + "properties": { + "uid": { + "anyOf": [ + { + "type": "string", + "format": "uuid" + }, + { + "type": "null" + } + ], + "title": "Uid" + }, + "title": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Title" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "function_uid": { + "type": "string", + "format": "uuid", + "title": "Function Uid" + }, + "inputs": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Inputs" + }, + "outputs": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Outputs" + }, + "function_class": { + "type": "string", + "const": "project", + "title": "Function Class", + "default": "project" + }, + "project_job_id": { + "type": "string", + "format": "uuid", + "title": "Project Job Id" + } + }, + "type": "object", + "required": [ + "function_uid", + "project_job_id" + ], + "title": "ProjectFunctionJob" + }, + "PythonCodeFunction": { + "properties": { + "uid": { + "anyOf": [ + { + "type": "string", + "format": "uuid" + }, + { + "type": "null" + } + ], + "title": "Uid" + }, + "title": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Title" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "function_class": { + "type": "string", + "const": "python_code", + "title": "Function Class", + "default": "python_code" + }, + "input_schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/FunctionInputSchema" + }, + { + "type": "null" + } + ] + }, + "output_schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/FunctionOutputSchema" + }, + { + "type": "null" + } + ] + }, + "code_url": { + "type": "string", + "title": "Code Url" + } + }, + "type": "object", + "required": [ + "code_url" + ], + "title": "PythonCodeFunction" + }, + "PythonCodeFunctionJob": { + "properties": { + "uid": { + "anyOf": [ + { + "type": "string", + "format": "uuid" + }, + { + "type": "null" + } + ], + "title": "Uid" + }, + "title": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Title" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "function_uid": { + "type": "string", + "format": "uuid", + "title": "Function Uid" + }, + "inputs": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Inputs" + }, + "outputs": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Outputs" + }, + "function_class": { + "type": "string", + "const": "python_code", + "title": "Function Class", + "default": "python_code" + }, + "code_url": { + "type": "string", + "title": "Code Url" + } + }, + "type": "object", + "required": [ + "function_uid", + "code_url" + ], + "title": "PythonCodeFunctionJob" + }, "RunningState": { "type": "string", "enum": [ diff --git a/services/api-server/src/simcore_service_api_server/api/root.py b/services/api-server/src/simcore_service_api_server/api/root.py index 5654601d403..5a1de4a711c 100644 --- a/services/api-server/src/simcore_service_api_server/api/root.py +++ b/services/api-server/src/simcore_service_api_server/api/root.py @@ -6,7 +6,7 @@ from .routes import credits as _credits from .routes import ( files, - functions, + functions_routes, health, licensed_items, meta, @@ -42,12 +42,27 @@ def create_router(settings: ApplicationSettings): ) router.include_router(studies.router, tags=["studies"], prefix="/studies") router.include_router(studies_jobs.router, tags=["studies"], prefix="/studies") + router.include_router( + functions_routes.function_router, tags=["functions"], prefix="/functions" + ) + router.include_router( + functions_routes.function_job_router, + tags=["function_jobs"], + prefix="/function_jobs", + ) + router.include_router( + functions_routes.function_job_collections_router, + tags=["function_job_collections"], + prefix="/function_job_collections", + ) router.include_router(wallets.router, tags=["wallets"], prefix="/wallets") router.include_router(_credits.router, tags=["credits"], prefix="/credits") router.include_router( licensed_items.router, tags=["licensed-items"], prefix="/licensed-items" ) - router.include_router(functions.router, tags=["functions"], prefix="/functions") + router.include_router( + functions_routes.function_router, tags=["functions"], prefix="/functions" + ) # NOTE: multiple-files upload is currently disabled # Web form to upload files at http://localhost:8000/v0/upload-form-view diff --git a/services/api-server/src/simcore_service_api_server/api/routes/functions.py b/services/api-server/src/simcore_service_api_server/api/routes/functions.py deleted file mode 100644 index 6d5c277451d..00000000000 --- a/services/api-server/src/simcore_service_api_server/api/routes/functions.py +++ /dev/null @@ -1,17 +0,0 @@ -from typing import Annotated - -from fastapi import APIRouter, Depends - -from ...services_rpc.wb_api_server import WbApiRpcClient -from ..dependencies.webserver_rpc import ( - get_wb_api_rpc_client, -) - -router = APIRouter() - - -@router.post("/ping", include_in_schema=False) -async def ping( - wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], -): - return await wb_api_rpc.ping() diff --git a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py new file mode 100644 index 00000000000..dc2a80abcf5 --- /dev/null +++ b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py @@ -0,0 +1,453 @@ +from collections.abc import Callable +from typing import Annotated, Final + +from fastapi import APIRouter, Depends, Request, status +from models_library.api_schemas_webserver.functions_wb_schema import ( + Function, + FunctionClass, + FunctionID, + FunctionInputs, + FunctionInputSchema, + FunctionInputsList, + FunctionJob, + FunctionJobID, + FunctionJobStatus, + FunctionOutputs, + FunctionOutputSchema, + ProjectFunctionJob, +) +from pydantic import PositiveInt +from servicelib.fastapi.dependencies import get_reverse_url_mapper + +from ...models.schemas.errors import ErrorGet +from ...models.schemas.jobs import ( + JobInputs, +) +from ...services_http.director_v2 import DirectorV2Api +from ...services_http.storage import StorageApi +from ...services_http.webserver import AuthSession +from ...services_rpc.wb_api_server import WbApiRpcClient +from ..dependencies.authentication import get_current_user_id, get_product_name +from ..dependencies.services import get_api_client +from ..dependencies.webserver_http import get_webserver_session +from ..dependencies.webserver_rpc import ( + get_wb_api_rpc_client, +) +from . import studies_jobs + +function_router = APIRouter() +function_job_router = APIRouter() +function_job_collections_router = APIRouter() + +_COMMON_FUNCTION_ERROR_RESPONSES: Final[dict] = { + status.HTTP_404_NOT_FOUND: { + "description": "Function not found", + "model": ErrorGet, + }, +} + + +@function_router.post("/ping") +async def ping( + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + return await wb_api_rpc.ping() + + +@function_router.get("", response_model=list[Function], description="List functions") +async def list_functions( + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + return await wb_api_rpc.list_functions() + + +@function_router.post("", response_model=Function, description="Create function") +async def register_function( + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], + function: Function, +): + return await wb_api_rpc.register_function(function=function) + + +@function_router.get( + "/{function_id:uuid}", + response_model=Function, + responses={**_COMMON_FUNCTION_ERROR_RESPONSES}, + description="Get function", +) +async def get_function( + function_id: FunctionID, + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + return await wb_api_rpc.get_function(function_id=function_id) + + +@function_router.post( + "/{function_id:uuid}:run", + response_model=FunctionJob, + responses={**_COMMON_FUNCTION_ERROR_RESPONSES}, + description="Run function", +) +async def run_function( + request: Request, + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], + webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], + url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], + director2_api: Annotated[DirectorV2Api, Depends(get_api_client(DirectorV2Api))], + function_id: FunctionID, + function_inputs: FunctionInputs, + user_id: Annotated[PositiveInt, Depends(get_current_user_id)], + product_name: Annotated[str, Depends(get_product_name)], +): + + to_run_function = await wb_api_rpc.get_function(function_id=function_id) + + assert to_run_function.uid is not None + + if cached_function_job := await wb_api_rpc.find_cached_function_job( + function_id=to_run_function.uid, + inputs=function_inputs, + ): + return cached_function_job + + if to_run_function.function_class == FunctionClass.project: + study_job = await studies_jobs.create_study_job( + study_id=to_run_function.project_id, + job_inputs=JobInputs(values=function_inputs or {}), + webserver_api=webserver_api, + wb_api_rpc=wb_api_rpc, + url_for=url_for, + x_simcore_parent_project_uuid=None, + x_simcore_parent_node_id=None, + user_id=user_id, + product_name=product_name, + ) + await studies_jobs.start_study_job( + request=request, + study_id=to_run_function.project_id, + job_id=study_job.id, + user_id=user_id, + webserver_api=webserver_api, + director2_api=director2_api, + ) + return await register_function_job( + wb_api_rpc=wb_api_rpc, + function_job=ProjectFunctionJob( + function_uid=to_run_function.uid, + title=f"Function job of function {to_run_function.uid}", + description=to_run_function.description, + inputs=function_inputs, + outputs=None, + project_job_id=study_job.id, + ), + ) + else: # noqa: RET505 + msg = f"Function type {type(to_run_function)} not supported" + raise TypeError(msg) + + +@function_router.delete( + "/{function_id:uuid}", + response_model=Function, + responses={**_COMMON_FUNCTION_ERROR_RESPONSES}, + description="Delete function", +) +async def delete_function( + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], + function_id: FunctionID, +): + return await wb_api_rpc.delete_function(function_id=function_id) + + +@function_router.get( + "/{function_id:uuid}/input_schema", + response_model=FunctionInputSchema, + responses={**_COMMON_FUNCTION_ERROR_RESPONSES}, + description="Get function", +) +async def get_function_input_schema( + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], + function_id: FunctionID, +): + return await wb_api_rpc.get_function_input_schema(function_id=function_id) + + +@function_router.get( + "/{function_id:uuid}/output_schema", + response_model=FunctionOutputSchema, + responses={**_COMMON_FUNCTION_ERROR_RESPONSES}, + description="Get function", +) +async def get_function_output_schema( + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], + function_id: FunctionID, +): + return await wb_api_rpc.get_function_output_schema(function_id=function_id) + + +_COMMON_FUNCTION_JOB_ERROR_RESPONSES: Final[dict] = { + status.HTTP_404_NOT_FOUND: { + "description": "Function job not found", + "model": ErrorGet, + }, +} + + +@function_job_router.post( + "", response_model=FunctionJob, description="Create function job" +) +async def register_function_job( + function_job: FunctionJob, + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + return await wb_api_rpc.register_function_job(function_job=function_job) + + +@function_job_router.get( + "/{function_job_id:uuid}", + response_model=FunctionJob, + responses={**_COMMON_FUNCTION_JOB_ERROR_RESPONSES}, + description="Get function job", +) +async def get_function_job( + function_job_id: FunctionJobID, + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + return await wb_api_rpc.get_function_job(function_job_id=function_job_id) + + +@function_job_router.get( + "", response_model=list[FunctionJob], description="List function jobs" +) +async def list_function_jobs( + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + return await wb_api_rpc.list_function_jobs() + + +@function_job_router.delete( + "/{function_job_id:uuid}", + response_model=None, + responses={**_COMMON_FUNCTION_JOB_ERROR_RESPONSES}, + description="Delete function job", +) +async def delete_function_job( + function_job_id: FunctionJobID, + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + return await wb_api_rpc.delete_function_job(function_job_id=function_job_id) + + +async def get_function_from_functionjobid( + wb_api_rpc: WbApiRpcClient, + function_job_id: FunctionJobID, +) -> tuple[Function, FunctionJob]: + function_job = await get_function_job( + wb_api_rpc=wb_api_rpc, function_job_id=function_job_id + ) + + return ( + await get_function( + wb_api_rpc=wb_api_rpc, function_id=function_job.function_uid + ), + function_job, + ) + + +@function_job_router.get( + "/{function_job_id:uuid}/status", + response_model=FunctionJobStatus, + responses={**_COMMON_FUNCTION_JOB_ERROR_RESPONSES}, + description="Get function job status", +) +async def function_job_status( + function_job_id: FunctionJobID, + director2_api: Annotated[DirectorV2Api, Depends(get_api_client(DirectorV2Api))], + user_id: Annotated[PositiveInt, Depends(get_current_user_id)], + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + function, function_job = await get_function_from_functionjobid( + wb_api_rpc=wb_api_rpc, function_job_id=function_job_id + ) + + if ( + function.function_class == FunctionClass.project + and function_job.function_class == FunctionClass.project + ): + job_status = await studies_jobs.inspect_study_job( + study_id=function.project_id, + job_id=function_job.project_job_id, + user_id=user_id, + director2_api=director2_api, + ) + return FunctionJobStatus(status=job_status.state) + else: # noqa: RET505 + msg = f"Function type {function.function_class} / Function job type {function_job.function_class} not supported" + raise TypeError(msg) + + +@function_job_router.get( + "/{function_job_id:uuid}/outputs", + response_model=FunctionOutputs, + responses={**_COMMON_FUNCTION_JOB_ERROR_RESPONSES}, + description="Get function job outputs", +) +async def function_job_outputs( + function_job_id: FunctionJobID, + webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], + user_id: Annotated[PositiveInt, Depends(get_current_user_id)], + storage_client: Annotated[StorageApi, Depends(get_api_client(StorageApi))], + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + function, function_job = await get_function_from_functionjobid( + wb_api_rpc=wb_api_rpc, function_job_id=function_job_id + ) + + if ( + function.function_class != FunctionClass.project + or function_job.function_class != FunctionClass.project + ): + msg = f"Function type {function.function_class} not supported" + raise TypeError(msg) + else: # noqa: RET506 + job_outputs = await studies_jobs.get_study_job_outputs( + study_id=function.project_id, + job_id=function_job.project_job_id, + user_id=user_id, + webserver_api=webserver_api, + storage_client=storage_client, + ) + + return job_outputs.results + + +@function_router.post( + "/{function_id:uuid}:map", + response_model=list[FunctionJob], + responses={**_COMMON_FUNCTION_ERROR_RESPONSES}, + description="Map function over input parameters", +) +async def map_function( + function_id: FunctionID, + function_inputs_list: FunctionInputsList, + request: Request, + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], + webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], + url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], + director2_api: Annotated[DirectorV2Api, Depends(get_api_client(DirectorV2Api))], + user_id: Annotated[PositiveInt, Depends(get_current_user_id)], + product_name: Annotated[str, Depends(get_product_name)], +): + function_jobs = [] + for function_inputs in function_inputs_list: + function_jobs = [ + await run_function( + wb_api_rpc=wb_api_rpc, + function_id=function_id, + function_inputs=function_inputs, + product_name=product_name, + user_id=user_id, + webserver_api=webserver_api, + url_for=url_for, + director2_api=director2_api, + request=request, + ) + for function_inputs in function_inputs_list + ] + # TODO poor system can't handle doing this in parallel, get this fixed # noqa: FIX002 + # function_jobs = await asyncio.gather(*function_jobs_tasks) + + return function_jobs + + +# ruff: noqa: ERA001 + + +# _logger = logging.getLogger(__name__) + +# _COMMON_FUNCTION_JOB_COLLECTION_ERROR_RESPONSES: Final[dict] = { +# status.HTTP_404_NOT_FOUND: { +# "description": "Function job collection not found", +# "model": ErrorGet, +# }, +# } + + +# @function_job_collections_router.get( +# "", +# response_model=FunctionJobCollection, +# description="List function job collections", +# ) +# async def list_function_job_collections( +# page_params: Annotated[PaginationParams, Depends()], +# webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], +# ): +# msg = "list function jobs collection not implemented yet" +# raise NotImplementedError(msg) + + +# @function_job_collections_router.post( +# "", response_model=FunctionJobCollection, description="Create function job" +# ) +# async def create_function_job_collection( +# webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], +# job_ids: Annotated[list[FunctionJob], Depends()], +# ): +# msg = "create function job collection not implemented yet" +# raise NotImplementedError(msg) + + +# @function_job_collections_router.get( +# "/{function_job_collection_id:uuid}", +# response_model=FunctionJobCollection, +# responses={**_COMMON_FUNCTION_JOB_COLLECTION_ERROR_RESPONSES}, +# description="Get function job", +# ) +# async def get_function_job_collection( +# function_job_collection_id: FunctionJobCollectionID, +# webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], +# ): +# msg = "get function job collection not implemented yet" +# raise NotImplementedError(msg) + + +# @function_job_collections_router.delete( +# "/{function_job_collection_id:uuid}", +# response_model=FunctionJob, +# responses={**_COMMON_FUNCTION_JOB_COLLECTION_ERROR_RESPONSES}, +# description="Delete function job collection", +# ) +# async def delete_function_job_collection( +# function_job_collection_id: FunctionJobCollectionID, +# webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], +# ): +# msg = "delete function job collection not implemented yet" +# raise NotImplementedError(msg) + + +# @function_job_collections_router.get( +# "/{function_job_collection_id:uuid}/function_jobs", +# response_model=list[FunctionJob], +# responses={**_COMMON_FUNCTION_JOB_COLLECTION_ERROR_RESPONSES}, +# description="Get the function jobs in function job collection", +# ) +# async def function_job_collection_list_function_jobs( +# function_job_collection_id: FunctionJobCollectionID, +# webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], +# ): +# msg = "function job collection listing not implemented yet" +# raise NotImplementedError(msg) + + +# @function_job_collections_router.get( +# "/{function_job_collection_id:uuid}/status", +# response_model=FunctionJobCollectionStatus, +# responses={**_COMMON_FUNCTION_JOB_COLLECTION_ERROR_RESPONSES}, +# description="Get function job collection status", +# ) +# async def function_job_collection_status( +# function_job_collection_id: FunctionJobCollectionID, +# webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], +# ): +# msg = "function job collection status not implemented yet" +# raise NotImplementedError(msg) diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers.py index 89b220d2d5f..941af0c8803 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers.py @@ -265,7 +265,7 @@ async def list_solver_ports( product_name=product_name, ) - return OnePage[SolverPort].model_validate(dict(items=ports)) + return OnePage[SolverPort].model_validate({"items": ports}) @router.get( diff --git a/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py b/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py index 0043b5daa70..b128541f8b6 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py @@ -94,7 +94,7 @@ async def create_study_job( url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], user_id: Annotated[PositiveInt, Depends(get_current_user_id)], product_name: Annotated[str, Depends(get_product_name)], - hidden: Annotated[bool, Query()] = True, + hidden: Annotated[bool, Query()] = True, # noqa: FBT002 x_simcore_parent_project_uuid: ProjectID | None = Header(default=None), x_simcore_parent_node_id: NodeID | None = Header(default=None), ) -> Job: @@ -257,13 +257,12 @@ async def start_study_job( return JSONResponse( content=jsonable_encoder(job_status), status_code=status.HTTP_200_OK ) - job_status = await inspect_study_job( + return await inspect_study_job( study_id=study_id, job_id=job_id, user_id=user_id, director2_api=director2_api, ) - return job_status @router.post( @@ -340,10 +339,9 @@ async def get_study_job_output_logfile( level=logging.DEBUG, msg=f"get study job output logfile study_id={study_id!r} job_id={job_id!r}.", ): - log_link_map = await director2_api.get_computation_logs( + return await director2_api.get_computation_logs( user_id=user_id, project_id=job_id ) - return log_link_map @router.get( diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/functions_api_schema.py b/services/api-server/src/simcore_service_api_server/models/schemas/functions_api_schema.py new file mode 100644 index 00000000000..ba316c0e88d --- /dev/null +++ b/services/api-server/src/simcore_service_api_server/models/schemas/functions_api_schema.py @@ -0,0 +1,85 @@ +from enum import Enum +from typing import Annotated, Any, Literal, TypeAlias + +from models_library import projects +from pydantic import BaseModel, Field + +FunctionID: TypeAlias = projects.ProjectID + + +class FunctionSchema(BaseModel): + schema_dict: dict[str, Any] | None # JSON Schema + + +class FunctionInputSchema(FunctionSchema): ... + + +class FunctionOutputSchema(FunctionSchema): ... + + +class FunctionClass(str, Enum): + project = "project" + python_code = "python_code" + + +class FunctionInputs(BaseModel): + inputs_dict: dict[str, Any] | None # JSON Schema + + +class FunctionOutputs(BaseModel): + outputs_dict: dict[str, Any] | None # JSON Schema + + +class Function(BaseModel): + uid: FunctionID | None = None + title: str | None = None + description: str | None = None + input_schema: FunctionInputSchema | None = None + output_schema: FunctionOutputSchema | None = None + + +class StudyFunction(Function): + function_type: Literal["study"] = "study" + study_url: str + + +class PythonCodeFunction(Function): + function_type: Literal["python_code"] = "python_code" + code_url: str + + +FunctionUnion: TypeAlias = Annotated[ + StudyFunction | PythonCodeFunction, + Field(discriminator="function_type"), +] + +FunctionJobID: TypeAlias = projects.ProjectID +FunctionJobCollectionID: TypeAlias = projects.ProjectID + + +class FunctionJob(BaseModel): + uid: FunctionJobID + title: str | None + description: str | None + status: str + function_uid: FunctionID + inputs: FunctionInputs | None + outputs: FunctionOutputs | None + + +class FunctionJobStatus(BaseModel): + status: str + + +class FunctionJobCollection(BaseModel): + """Model for a collection of function jobs""" + + id: FunctionJobCollectionID + title: str | None + description: str | None + job_ids: list[FunctionJobID] + status: str + + +class FunctionJobCollectionStatus(BaseModel): + status: list[str] diff --git a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py index ca771d913b1..8766198f68c 100644 --- a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py +++ b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py @@ -4,6 +4,15 @@ from fastapi import FastAPI from fastapi_pagination import create_page +from models_library.api_schemas_webserver.functions_wb_schema import ( + Function, + FunctionID, + FunctionInputs, + FunctionInputSchema, + FunctionJob, + FunctionJobID, + FunctionOutputSchema, +) from models_library.api_schemas_webserver.licensed_items import LicensedItemRpcGetPage from models_library.licenses import LicensedItemID from models_library.products import ProductName @@ -29,9 +38,45 @@ NotEnoughAvailableSeatsError, ) from servicelib.rabbitmq.rpc_interfaces.webserver import projects as projects_rpc -from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions import ( +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + delete_function as _delete_function, +) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + delete_function_job as _delete_function_job, +) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + find_cached_function_job as _find_cached_function_job, +) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + get_function as _get_function, +) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + get_function_input_schema as _get_function_input_schema, +) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + get_function_job as _get_function_job, +) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + get_function_output_schema as _get_function_output_schema, +) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + list_function_jobs as _list_function_jobs, +) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + list_functions as _list_functions, +) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( ping as _ping, ) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + register_function as _register_function, +) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + register_function_job as _register_function_job, +) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + run_function as _run_function, +) from servicelib.rabbitmq.rpc_interfaces.webserver.licenses.licensed_items import ( checkout_licensed_item_for_wallet as _checkout_licensed_item_for_wallet, ) @@ -219,6 +264,65 @@ async def mark_project_as_job( job_parent_resource_name=job_parent_resource_name, ) + async def register_function(self, *, function: Function) -> Function: + function.input_schema = ( + FunctionInputSchema(**function.input_schema.model_dump()) + if function.input_schema is not None + else None + ) + function.output_schema = ( + FunctionOutputSchema(**function.output_schema.model_dump()) + if function.output_schema is not None + else None + ) + return await _register_function( + self._client, + function=function, + ) + + async def get_function(self, *, function_id: FunctionID) -> Function: + return await _get_function(self._client, function_id=function_id) + + async def delete_function(self, *, function_id: FunctionID) -> None: + return await _delete_function(self._client, function_id=function_id) + + async def list_functions(self) -> list[Function]: + return await _list_functions(self._client) + + async def run_function( + self, *, function_id: FunctionID, inputs: FunctionInputs + ) -> FunctionJob: + return await _run_function(self._client, function_id=function_id, inputs=inputs) + + async def get_function_job(self, *, function_job_id: FunctionJobID) -> FunctionJob: + return await _get_function_job(self._client, function_job_id=function_job_id) + + async def delete_function_job(self, *, function_job_id: FunctionJobID) -> None: + return await _delete_function_job(self._client, function_job_id=function_job_id) + + async def register_function_job(self, *, function_job: FunctionJob) -> FunctionJob: + return await _register_function_job(self._client, function_job=function_job) + + async def get_function_input_schema( + self, *, function_id: FunctionID + ) -> FunctionInputSchema: + return await _get_function_input_schema(self._client, function_id=function_id) + + async def get_function_output_schema( + self, *, function_id: FunctionID + ) -> FunctionOutputSchema: + return await _get_function_output_schema(self._client, function_id=function_id) + + async def find_cached_function_job( + self, *, function_id: FunctionID, inputs: FunctionInputs + ) -> FunctionJob | None: + return await _find_cached_function_job( + self._client, function_id=function_id, inputs=inputs + ) + + async def list_function_jobs(self) -> list[FunctionJob]: + return await _list_function_jobs(self._client) + def setup(app: FastAPI, rabbitmq_rmp_client: RabbitMQRPCClient): wb_api_rpc_client = WbApiRpcClient(_client=rabbitmq_rmp_client) diff --git a/services/web/server/src/simcore_service_webserver/functions/_controller_rpc.py b/services/web/server/src/simcore_service_webserver/functions/_controller_rpc.py deleted file mode 100644 index 483fcdc26e3..00000000000 --- a/services/web/server/src/simcore_service_webserver/functions/_controller_rpc.py +++ /dev/null @@ -1,21 +0,0 @@ -from aiohttp import web -from models_library.api_schemas_webserver import WEBSERVER_RPC_NAMESPACE -from servicelib.rabbitmq import RPCRouter - -from ..rabbitmq import get_rabbitmq_rpc_server - -# this is the rpc interface exposed to the api-server -# this interface should call the service layer - -router = RPCRouter() - - -@router.expose() -async def ping(app: web.Application) -> str: - assert app - return "pong from webserver" - - -async def register_rpc_routes_on_startup(app: web.Application): - rpc_server = get_rabbitmq_rpc_server(app) - await rpc_server.register_router(router, WEBSERVER_RPC_NAMESPACE, app) diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py new file mode 100644 index 00000000000..3a689d37ab9 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py @@ -0,0 +1,262 @@ +from aiohttp import web +from models_library.api_schemas_webserver import WEBSERVER_RPC_NAMESPACE +from models_library.api_schemas_webserver.functions_wb_schema import ( + Function, + FunctionClass, + FunctionClassSpecificData, + FunctionDB, + FunctionID, + FunctionInputs, + FunctionInputSchema, + FunctionJob, + FunctionJobClassSpecificData, + FunctionJobDB, + FunctionJobID, + FunctionOutputSchema, + ProjectFunction, + ProjectFunctionJob, +) +from servicelib.rabbitmq import RPCRouter + +from ..rabbitmq import get_rabbitmq_rpc_server +from . import _functions_repository + +router = RPCRouter() + + +@router.expose() +async def ping(app: web.Application) -> str: + assert app + return "pong from webserver" + + +@router.expose() +async def register_function(app: web.Application, *, function: Function) -> Function: + assert app + if function.function_class == FunctionClass.project: + saved_function = await _functions_repository.create_function( + app=app, + function=FunctionDB( + title=function.title, + description=function.description, + input_schema=function.input_schema, + output_schema=function.output_schema, + function_class=function.function_class, + class_specific_data=FunctionClassSpecificData( + { + "project_id": str(function.project_id), + } + ), + ), + ) + return ProjectFunction( + uid=saved_function.uuid, + title=saved_function.title, + description=saved_function.description, + input_schema=saved_function.input_schema, + output_schema=saved_function.output_schema, + project_id=saved_function.class_specific_data["project_id"], + ) + else: # noqa: RET505 + msg = f"Unsupported function class: {function.function_class}" + raise TypeError(msg) + + +def _decode_function( + function: FunctionDB, +) -> Function: + if function.function_class == "project": + return ProjectFunction( + uid=function.uuid, + title=function.title, + description=function.description, + input_schema=function.input_schema, + output_schema=function.output_schema, + project_id=function.class_specific_data["project_id"], + ) + else: # noqa: RET505 + msg = f"Unsupported function class: [{function.function_class}]" + raise TypeError(msg) + + +@router.expose() +async def get_function(app: web.Application, *, function_id: FunctionID) -> Function: + assert app + returned_function = await _functions_repository.get_function( + app=app, + function_id=function_id, + ) + return _decode_function( + returned_function, + ) + + +@router.expose() +async def get_function_job( + app: web.Application, *, function_job_id: FunctionJobID +) -> FunctionJob: + assert app + returned_function_job = await _functions_repository.get_function_job( + app=app, + function_job_id=function_job_id, + ) + assert returned_function_job is not None + + if returned_function_job.function_class == FunctionClass.project: + return ProjectFunctionJob( + uid=returned_function_job.uuid, + title=returned_function_job.title, + description="", + function_uid=returned_function_job.function_uuid, + inputs=returned_function_job.inputs, + outputs=None, + project_job_id=returned_function_job.class_specific_data["project_job_id"], + ) + else: # noqa: RET505 + msg = f"Unsupported function class: [{returned_function_job.function_class}]" + raise TypeError(msg) + + +@router.expose() +async def list_function_jobs(app: web.Application) -> list[FunctionJob]: + assert app + returned_function_jobs = await _functions_repository.list_function_jobs( + app=app, + ) + return [ + ProjectFunctionJob( + uid=returned_function_job.uuid, + title=returned_function_job.title, + description="", + function_uid=returned_function_job.function_uuid, + inputs=returned_function_job.inputs, + outputs=None, + project_job_id=returned_function_job.class_specific_data["project_job_id"], + ) + for returned_function_job in returned_function_jobs + ] + + +@router.expose() +async def get_function_input_schema( + app: web.Application, *, function_id: FunctionID +) -> FunctionInputSchema: + assert app + returned_function = await _functions_repository.get_function( + app=app, + function_id=function_id, + ) + return FunctionInputSchema( + schema_dict=( + returned_function.input_schema.schema_dict + if returned_function.input_schema + else None + ) + ) + + +@router.expose() +async def get_function_output_schema( + app: web.Application, *, function_id: FunctionID +) -> FunctionOutputSchema: + assert app + returned_function = await _functions_repository.get_function( + app=app, + function_id=function_id, + ) + return FunctionOutputSchema( + schema_dict=( + returned_function.output_schema.schema_dict + if returned_function.output_schema + else None + ) + ) + + +@router.expose() +async def list_functions(app: web.Application) -> list[Function]: + assert app + returned_functions = await _functions_repository.list_functions( + app=app, + ) + return [ + _decode_function(returned_function) for returned_function in returned_functions + ] + + +@router.expose() +async def delete_function(app: web.Application, *, function_id: FunctionID) -> None: + assert app + await _functions_repository.delete_function( + app=app, + function_id=function_id, + ) + + +@router.expose() +async def register_function_job( + app: web.Application, *, function_job: FunctionJob +) -> FunctionJob: + assert app + if function_job.function_class == FunctionClass.project: + created_function_job_db = await _functions_repository.register_function_job( + app=app, + function_job=FunctionJobDB( + title=function_job.title, + function_uuid=function_job.function_uid, + inputs=function_job.inputs, + outputs=None, + class_specific_data=FunctionJobClassSpecificData( + { + "project_job_id": str(function_job.project_job_id), + } + ), + function_class=function_job.function_class, + ), + ) + + return ProjectFunctionJob( + uid=created_function_job_db.uuid, + title=created_function_job_db.title, + description="", + function_uid=created_function_job_db.function_uuid, + inputs=created_function_job_db.inputs, + outputs=None, + project_job_id=created_function_job_db.class_specific_data[ + "project_job_id" + ], + ) + else: # noqa: RET505 + msg = f"Unsupported function class: [{function_job.function_class}]" + raise TypeError(msg) + + +@router.expose() +async def find_cached_function_job( + app: web.Application, *, function_id: FunctionID, inputs: FunctionInputs +) -> FunctionJob | None: + assert app + returned_function_job = await _functions_repository.find_cached_function_job( + app=app, function_id=function_id, inputs=inputs + ) + if returned_function_job is None: + return None + + if returned_function_job.function_class == FunctionClass.project: + return ProjectFunctionJob( + uid=returned_function_job.uuid, + title=returned_function_job.title, + description="", + function_uid=returned_function_job.function_uuid, + inputs=returned_function_job.inputs, + outputs=None, + project_job_id=returned_function_job.class_specific_data["project_job_id"], + ) + else: # noqa: RET505 + msg = f"Unsupported function class: [{returned_function_job.function_class}]" + raise TypeError(msg) + + +async def register_rpc_routes_on_startup(app: web.Application): + rpc_server = get_rabbitmq_rpc_server(app) + await rpc_server.register_router(router, WEBSERVER_RPC_NAMESPACE, app) diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py new file mode 100644 index 00000000000..495f93c0c25 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py @@ -0,0 +1,206 @@ +import json + +from aiohttp import web +from models_library.api_schemas_webserver.functions_wb_schema import ( + FunctionDB, + FunctionID, + FunctionInputs, + FunctionJobDB, +) +from simcore_postgres_database.models.functions_models_db import ( + function_jobs as function_jobs_table, +) +from simcore_postgres_database.models.functions_models_db import ( + functions as functions_table, +) +from simcore_postgres_database.utils_repos import ( + get_columns_from_db_model, + transaction_context, +) +from sqlalchemy import Text, cast +from sqlalchemy.ext.asyncio import AsyncConnection + +from ..db.plugin import get_asyncpg_engine + +_FUNCTIONS_TABLE_COLS = get_columns_from_db_model(functions_table, FunctionDB) +_FUNCTION_JOBS_TABLE_COLS = get_columns_from_db_model( + function_jobs_table, FunctionJobDB +) + + +async def create_function( + app: web.Application, + connection: AsyncConnection | None = None, + *, + function: FunctionDB, +) -> FunctionDB: + + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream( + functions_table.insert() + .values( + title=function.title, + description=function.description, + input_schema=( + function.input_schema.model_dump() + if function.input_schema is not None + else None + ), + output_schema=( + function.output_schema.model_dump() + if function.output_schema is not None + else None + ), + function_class=function.function_class, + class_specific_data=function.class_specific_data, + ) + .returning(*_FUNCTIONS_TABLE_COLS) + ) + row = await result.first() + + if row is None: + msg = "No row was returned from the database after creating function." + raise ValueError(msg) + + return FunctionDB.model_validate(dict(row)) + + +async def get_function( + app: web.Application, + connection: AsyncConnection | None = None, + *, + function_id: FunctionID, +) -> FunctionDB: + + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream( + functions_table.select().where(functions_table.c.uuid == function_id) + ) + row = await result.first() + + if row is None: + msg = f"No function found with id {function_id}." + raise web.HTTPNotFound(reason=msg) + + return FunctionDB.model_validate(dict(row)) + + +async def list_functions( + app: web.Application, + connection: AsyncConnection | None = None, +) -> list[FunctionDB]: + + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream(functions_table.select().where()) + rows = await result.all() + if rows is None: + return [] + + return [FunctionDB.model_validate(dict(row)) for row in rows] + + +async def delete_function( + app: web.Application, + connection: AsyncConnection | None = None, + *, + function_id: FunctionID, +) -> None: + + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + await conn.execute( + functions_table.delete().where(functions_table.c.uuid == int(function_id)) + ) + + +async def register_function_job( + app: web.Application, + connection: AsyncConnection | None = None, + *, + function_job: FunctionJobDB, +) -> FunctionJobDB: + + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream( + function_jobs_table.insert() + .values( + function_uuid=function_job.function_uuid, + inputs=function_job.inputs, + function_class=function_job.function_class, + class_specific_data=function_job.class_specific_data, + title=function_job.title, + status="created", + ) + .returning(*_FUNCTION_JOBS_TABLE_COLS) + ) + row = await result.first() + + if row is None: + msg = "No row was returned from the database after creating function job." + raise ValueError(msg) + + return FunctionJobDB.model_validate(dict(row)) + + +async def get_function_job( + app: web.Application, + connection: AsyncConnection | None = None, + *, + function_job_id: FunctionID, +) -> FunctionJobDB: + + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream( + function_jobs_table.select().where( + function_jobs_table.c.uuid == function_job_id + ) + ) + row = await result.first() + + if row is None: + msg = f"No function job found with id {function_job_id}." + raise web.HTTPNotFound(reason=msg) + + return FunctionJobDB.model_validate(dict(row)) + + +async def list_function_jobs( + app: web.Application, + connection: AsyncConnection | None = None, +) -> list[FunctionJobDB]: + + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream(function_jobs_table.select().where()) + rows = await result.all() + if rows is None: + return [] + + return [FunctionJobDB.model_validate(dict(row)) for row in rows] + + +async def find_cached_function_job( + app: web.Application, + connection: AsyncConnection | None = None, + *, + function_id: FunctionID, + inputs: FunctionInputs, +) -> FunctionJobDB | None: + + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream( + function_jobs_table.select().where( + function_jobs_table.c.function_uuid == function_id, + cast(function_jobs_table.c.inputs, Text) == json.dumps(inputs), + ), + ) + + rows = await result.all() + + if rows is None: + return None + + for row in rows: + job = FunctionJobDB.model_validate(dict(row)) + if job.inputs == inputs: + return job + + return None diff --git a/services/web/server/src/simcore_service_webserver/functions/_service.py b/services/web/server/src/simcore_service_webserver/functions/_service.py index 2c967d7c841..899b0b1c681 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_service.py +++ b/services/web/server/src/simcore_service_webserver/functions/_service.py @@ -5,7 +5,7 @@ from aiohttp import web from models_library.users import UserID -from ..projects import _projects_service +from ..projects import projects_service from ..projects.models import ProjectDict @@ -16,6 +16,6 @@ async def get_project_from_function( user_id: UserID, ) -> ProjectDict: - return await _projects_service.get_project_for_user( + return await projects_service.get_project_for_user( app=app, project_uuid=function_uuid, user_id=user_id ) diff --git a/services/web/server/src/simcore_service_webserver/functions/plugin.py b/services/web/server/src/simcore_service_webserver/functions/plugin.py index 364436f4898..8d18e55a034 100644 --- a/services/web/server/src/simcore_service_webserver/functions/plugin.py +++ b/services/web/server/src/simcore_service_webserver/functions/plugin.py @@ -3,7 +3,7 @@ from aiohttp import web from servicelib.aiohttp.application_setup import ModuleCategory, app_module_setup -from . import _controller_rpc +from . import _functions_controller_rpc _logger = logging.getLogger(__name__) @@ -15,4 +15,4 @@ logger=_logger, ) def setup_functions(app: web.Application): - app.on_startup.append(_controller_rpc.register_rpc_routes_on_startup) + app.on_startup.append(_functions_controller_rpc.register_rpc_routes_on_startup) From 080e4fb965bce51fefd0809a3ccebdd08d9b2b0a Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Wed, 16 Apr 2025 11:33:23 +0200 Subject: [PATCH 02/69] Add db migration script for functions api --- .../93dbd49553ae_add_function_tables.py | 133 ++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/93dbd49553ae_add_function_tables.py diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/93dbd49553ae_add_function_tables.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/93dbd49553ae_add_function_tables.py new file mode 100644 index 00000000000..d44b8e271e2 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/93dbd49553ae_add_function_tables.py @@ -0,0 +1,133 @@ +"""Add function tables + +Revision ID: 93dbd49553ae +Revises: cf8f743fd0b7 +Create Date: 2025-04-16 09:32:48.976846+00:00 + +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "93dbd49553ae" +down_revision = "cf8f743fd0b7" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "function_job_collections", + sa.Column("uuid", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("name", sa.String(), nullable=True), + sa.PrimaryKeyConstraint("uuid", name="function_job_collections_pk"), + ) + op.create_index( + op.f("ix_function_job_collections_uuid"), + "function_job_collections", + ["uuid"], + unique=False, + ) + op.create_table( + "functions", + sa.Column("uuid", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("title", sa.String(), nullable=True), + sa.Column("function_class", sa.String(), nullable=True), + sa.Column("description", sa.String(), nullable=True), + sa.Column("input_schema", sa.JSON(), nullable=True), + sa.Column("output_schema", sa.JSON(), nullable=True), + sa.Column("system_tags", sa.JSON(), nullable=True), + sa.Column("user_tags", sa.JSON(), nullable=True), + sa.Column("class_specific_data", sa.JSON(), nullable=True), + sa.PrimaryKeyConstraint("uuid", name="functions_pk"), + ) + op.create_index(op.f("ix_functions_uuid"), "functions", ["uuid"], unique=False) + op.create_table( + "function_jobs", + sa.Column("uuid", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("title", sa.String(), nullable=True), + sa.Column("function_uuid", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("function_class", sa.String(), nullable=True), + sa.Column("status", sa.String(), nullable=True), + sa.Column("inputs", sa.JSON(), nullable=True), + sa.Column("outputs", sa.JSON(), nullable=True), + sa.Column("class_specific_data", sa.JSON(), nullable=True), + sa.ForeignKeyConstraint( + ["function_uuid"], + ["functions.uuid"], + name="fk_functions_to_function_jobs_to_function_uuid", + onupdate="CASCADE", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("uuid", name="function_jobs_pk"), + ) + op.create_index( + op.f("ix_function_jobs_function_uuid"), + "function_jobs", + ["function_uuid"], + unique=False, + ) + op.create_index( + op.f("ix_function_jobs_uuid"), "function_jobs", ["uuid"], unique=False + ) + op.create_table( + "function_job_collections_to_function_jobs", + sa.Column( + "function_job_collection_uuid", postgresql.UUID(as_uuid=True), nullable=True + ), + sa.Column("function_job_uuid", postgresql.UUID(as_uuid=True), nullable=True), + sa.ForeignKeyConstraint( + ["function_job_collection_uuid"], + ["function_job_collections.uuid"], + name="fk_func_job_coll_to_func_jobs_to_func_job_coll_uuid", + onupdate="CASCADE", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["function_job_uuid"], + ["function_jobs.uuid"], + name="fk_func_job_coll_to_func_jobs_to_func_job_uuid", + onupdate="CASCADE", + ondelete="CASCADE", + ), + ) + op.drop_index("idx_projects_last_change_date_desc", table_name="projects") + op.create_index( + "idx_projects_last_change_date_desc", + "projects", + ["last_change_date"], + unique=False, + postgresql_using="btree", + postgresql_ops={"last_change_date": "DESC"}, + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index( + "idx_projects_last_change_date_desc", + table_name="projects", + postgresql_using="btree", + postgresql_ops={"last_change_date": "DESC"}, + ) + op.create_index( + "idx_projects_last_change_date_desc", + "projects", + [sa.text("last_change_date DESC")], + unique=False, + ) + op.drop_table("function_job_collections_to_function_jobs") + op.drop_index(op.f("ix_function_jobs_uuid"), table_name="function_jobs") + op.drop_index(op.f("ix_function_jobs_function_uuid"), table_name="function_jobs") + op.drop_table("function_jobs") + op.drop_index(op.f("ix_functions_uuid"), table_name="functions") + op.drop_table("functions") + op.drop_index( + op.f("ix_function_job_collections_uuid"), table_name="function_job_collections" + ) + op.drop_table("function_job_collections") + # ### end Alembic commands ### From db68658430babe9676b9d844195cc92535f966f8 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Tue, 6 May 2025 11:13:42 +0200 Subject: [PATCH 03/69] Add functions api. New commit to clean up db migration --- .../api_schemas_api_server/functions.py | 44 + .../functions_wb_schema.py | 132 ++ .../models/functions_models_db.py | 170 +++ .../webserver/functions/functions.py | 22 - .../functions/functions_rpc_interface.py | 187 +++ services/api-server/openapi.json | 1289 ++++++++++++++++- .../simcore_service_api_server/api/root.py | 19 +- .../api/routes/functions.py | 17 - .../api/routes/functions_routes.py | 453 ++++++ .../api/routes/studies_jobs.py | 8 +- .../models/schemas/functions_api_schema.py | 85 ++ .../services_rpc/wb_api_server.py | 106 +- .../functions/_controller_rpc.py | 21 - .../functions/_functions_controller_rpc.py | 262 ++++ .../functions/_functions_repository.py | 206 +++ .../functions/_service.py | 4 +- .../functions/plugin.py | 4 +- 17 files changed, 2953 insertions(+), 76 deletions(-) create mode 100644 packages/models-library/src/models_library/api_schemas_api_server/functions.py create mode 100644 packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py create mode 100644 packages/postgres-database/src/simcore_postgres_database/models/functions_models_db.py delete mode 100644 packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions.py create mode 100644 packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py delete mode 100644 services/api-server/src/simcore_service_api_server/api/routes/functions.py create mode 100644 services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py create mode 100644 services/api-server/src/simcore_service_api_server/models/schemas/functions_api_schema.py delete mode 100644 services/web/server/src/simcore_service_webserver/functions/_controller_rpc.py create mode 100644 services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py create mode 100644 services/web/server/src/simcore_service_webserver/functions/_functions_repository.py diff --git a/packages/models-library/src/models_library/api_schemas_api_server/functions.py b/packages/models-library/src/models_library/api_schemas_api_server/functions.py new file mode 100644 index 00000000000..44678efd539 --- /dev/null +++ b/packages/models-library/src/models_library/api_schemas_api_server/functions.py @@ -0,0 +1,44 @@ +from typing import Annotated, Any, Literal, TypeAlias + +from models_library import projects +from pydantic import BaseModel, Field + +FunctionID: TypeAlias = projects.ProjectID + + +class FunctionSchema(BaseModel): + schema_dict: dict[str, Any] | None # JSON Schema + + +class FunctionInputSchema(FunctionSchema): ... + + +class FunctionOutputSchema(FunctionSchema): ... + + +class Function(BaseModel): + uid: FunctionID | None = None + title: str | None = None + description: str | None = None + input_schema: FunctionInputSchema | None = None + output_schema: FunctionOutputSchema | None = None + + # @classmethod + # def compose_resource_name(cls, function_key) -> api_resources.RelativeResourceName: + # return api_resources.compose_resource_name("functions", function_key) + + +class StudyFunction(Function): + function_type: Literal["study"] = "study" + study_url: str + + +class PythonCodeFunction(Function): + function_type: Literal["python_code"] = "python_code" + code_url: str + + +FunctionUnion: TypeAlias = Annotated[ + StudyFunction | PythonCodeFunction, + Field(discriminator="function_type"), +] diff --git a/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py b/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py new file mode 100644 index 00000000000..4391a77a658 --- /dev/null +++ b/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py @@ -0,0 +1,132 @@ +from enum import Enum +from typing import Annotated, Any, Literal, TypeAlias +from uuid import UUID + +from models_library import projects +from pydantic import BaseModel, Field + +from ..projects import ProjectID + +FunctionID: TypeAlias = projects.ProjectID +FunctionJobID: TypeAlias = projects.ProjectID +FileID: TypeAlias = UUID + +InputTypes: TypeAlias = FileID | float | int | bool | str | list | None + + +class FunctionSchema(BaseModel): + schema_dict: dict[str, Any] | None # JSON Schema + + +class FunctionInputSchema(FunctionSchema): ... + + +class FunctionOutputSchema(FunctionSchema): ... + + +class FunctionClass(str, Enum): + project = "project" + python_code = "python_code" + + +FunctionClassSpecificData: TypeAlias = dict[str, Any] +FunctionJobClassSpecificData: TypeAlias = FunctionClassSpecificData + + +# TODO, use InputTypes here, but api is throwing weird errors and asking for dict for elements # noqa: FIX002 +FunctionInputs: TypeAlias = dict[str, Any] | None + +FunctionInputsList: TypeAlias = list[FunctionInputs] + +FunctionOutputs: TypeAlias = dict[str, Any] | None + + +class FunctionBase(BaseModel): + uid: FunctionID | None = None + title: str | None = None + description: str | None = None + function_class: FunctionClass + input_schema: FunctionInputSchema | None = None + output_schema: FunctionOutputSchema | None = None + + +class FunctionDB(BaseModel): + uuid: FunctionJobID | None = None + title: str | None = None + description: str | None = None + function_class: FunctionClass + input_schema: FunctionInputSchema | None = None + output_schema: FunctionOutputSchema | None = None + class_specific_data: FunctionClassSpecificData + + +class FunctionJobDB(BaseModel): + uuid: FunctionJobID | None = None + function_uuid: FunctionID + title: str | None = None + inputs: FunctionInputs | None = None + outputs: FunctionOutputs | None = None + class_specific_data: FunctionJobClassSpecificData + function_class: FunctionClass + + +class ProjectFunction(FunctionBase): + function_class: Literal[FunctionClass.project] = FunctionClass.project + project_id: ProjectID + + +class PythonCodeFunction(FunctionBase): + function_class: Literal[FunctionClass.python_code] = FunctionClass.python_code + code_url: str + + +Function: TypeAlias = Annotated[ + ProjectFunction | PythonCodeFunction, + Field(discriminator="function_class"), +] + +FunctionJobCollectionID: TypeAlias = projects.ProjectID + + +class FunctionJobBase(BaseModel): + uid: FunctionJobID | None = None + title: str | None = None + description: str | None = None + function_uid: FunctionID + inputs: FunctionInputs | None = None + outputs: FunctionOutputs | None = None + function_class: FunctionClass + + +class ProjectFunctionJob(FunctionJobBase): + function_class: Literal[FunctionClass.project] = FunctionClass.project + project_job_id: ProjectID + + +class PythonCodeFunctionJob(FunctionJobBase): + function_class: Literal[FunctionClass.python_code] = FunctionClass.python_code + code_url: str + + +FunctionJob: TypeAlias = Annotated[ + ProjectFunctionJob | PythonCodeFunctionJob, + Field(discriminator="function_class"), +] + + +class FunctionJobStatus(BaseModel): + status: str + + +class FunctionJobCollection(BaseModel): + """Model for a collection of function jobs""" + + id: FunctionJobCollectionID + title: str | None + description: str | None + job_ids: list[FunctionJobID] + status: str + + +class FunctionJobCollectionStatus(BaseModel): + status: list[str] diff --git a/packages/postgres-database/src/simcore_postgres_database/models/functions_models_db.py b/packages/postgres-database/src/simcore_postgres_database/models/functions_models_db.py new file mode 100644 index 00000000000..e8a8ba6f2ec --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/models/functions_models_db.py @@ -0,0 +1,170 @@ +"""Functions table + +- List of functions served by the simcore platform +""" + +import uuid + +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import UUID + +from ._common import RefActions +from .base import metadata + +functions = sa.Table( + "functions", + metadata, + sa.Column( + "uuid", + UUID(as_uuid=True), + primary_key=True, + index=True, + default=uuid.uuid4, + doc="Unique id of the function", + ), + sa.Column( + "title", + sa.String, + doc="Name of the function", + ), + sa.Column( + "function_class", + sa.String, + doc="Class of the function", + ), + sa.Column( + "description", + sa.String, + doc="Description of the function", + ), + sa.Column( + "input_schema", + sa.JSON, + doc="Input schema of the function", + ), + sa.Column( + "output_schema", + sa.JSON, + doc="Output schema of the function", + ), + sa.Column( + "system_tags", + sa.JSON, + nullable=True, + doc="System-level tags of the function", + ), + sa.Column( + "user_tags", + sa.JSON, + nullable=True, + doc="User-level tags of the function", + ), + sa.Column( + "class_specific_data", + sa.JSON, + nullable=True, + doc="Fields specific for a function class", + ), + sa.PrimaryKeyConstraint("uuid", name="functions_pk"), +) + +function_jobs = sa.Table( + "function_jobs", + metadata, + sa.Column( + "uuid", + UUID(as_uuid=True), + primary_key=True, + index=True, + default=uuid.uuid4, + doc="Unique id of the function job", + ), + sa.Column( + "title", + sa.String, + doc="Name of the function job", + ), + sa.Column( + "function_uuid", + sa.ForeignKey( + functions.c.uuid, + onupdate=RefActions.CASCADE, + ondelete=RefActions.CASCADE, + name="fk_functions_to_function_jobs_to_function_uuid", + ), + nullable=False, + index=True, + doc="Unique identifier of the function", + ), + sa.Column( + "function_class", + sa.String, + doc="Class of the function", + ), + sa.Column( + "status", + sa.String, + doc="Status of the function job", + ), + sa.Column( + "inputs", + sa.JSON, + doc="Inputs of the function job", + ), + sa.Column( + "outputs", + sa.JSON, + doc="Outputs of the function job", + ), + sa.Column( + "class_specific_data", + sa.JSON, + nullable=True, + doc="Fields specific for a function class", + ), + sa.PrimaryKeyConstraint("uuid", name="function_jobs_pk"), +) + +function_job_collections = sa.Table( + "function_job_collections", + metadata, + sa.Column( + "uuid", + UUID(as_uuid=True), + default=uuid.uuid4, + primary_key=True, + index=True, + doc="Unique id of the function job collection", + ), + sa.Column( + "name", + sa.String, + doc="Name of the function job collection", + ), + sa.PrimaryKeyConstraint("uuid", name="function_job_collections_pk"), +) + +function_job_collections_to_function_jobs = sa.Table( + "function_job_collections_to_function_jobs", + metadata, + sa.Column( + "function_job_collection_uuid", + sa.ForeignKey( + function_job_collections.c.uuid, + onupdate=RefActions.CASCADE, + ondelete=RefActions.CASCADE, + name="fk_func_job_coll_to_func_jobs_to_func_job_coll_uuid", + ), + doc="Unique identifier of the function job collection", + ), + sa.Column( + "function_job_uuid", + sa.ForeignKey( + function_jobs.c.uuid, + onupdate=RefActions.CASCADE, + ondelete=RefActions.CASCADE, + name="fk_func_job_coll_to_func_jobs_to_func_job_uuid", + ), + doc="Unique identifier of the function job", + ), +) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions.py deleted file mode 100644 index d53adfafa66..00000000000 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions.py +++ /dev/null @@ -1,22 +0,0 @@ -import logging - -from models_library.api_schemas_webserver import WEBSERVER_RPC_NAMESPACE -from models_library.rabbitmq_basic_types import RPCMethodName -from pydantic import TypeAdapter - -from .....logging_utils import log_decorator -from .....rabbitmq import RabbitMQRPCClient - -_logger = logging.getLogger(__name__) - - -@log_decorator(_logger, level=logging.DEBUG) -async def ping( - rabbitmq_rpc_client: RabbitMQRPCClient, -) -> str: - result = await rabbitmq_rpc_client.request( - WEBSERVER_RPC_NAMESPACE, - TypeAdapter(RPCMethodName).validate_python("ping"), - ) - assert isinstance(result, str) # nosec - return result diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py new file mode 100644 index 00000000000..7e06bef912c --- /dev/null +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py @@ -0,0 +1,187 @@ +import logging + +from models_library.api_schemas_webserver import ( + WEBSERVER_RPC_NAMESPACE, +) +from models_library.api_schemas_webserver.functions_wb_schema import ( + Function, + FunctionID, + FunctionInputs, + FunctionInputSchema, + FunctionJob, + FunctionJobID, + FunctionOutputSchema, +) +from models_library.rabbitmq_basic_types import RPCMethodName +from pydantic import TypeAdapter + +from .....logging_utils import log_decorator +from .... import RabbitMQRPCClient + +_logger = logging.getLogger(__name__) + + +@log_decorator(_logger, level=logging.DEBUG) +async def ping( + rabbitmq_rpc_client: RabbitMQRPCClient, +) -> str: + result = await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("ping"), + ) + assert isinstance(result, str) # nosec + return result + + +@log_decorator(_logger, level=logging.DEBUG) +async def register_function( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + function: Function, +) -> Function: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("register_function"), + function=function, + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def get_function( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + function_id: FunctionID, +) -> Function: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("get_function"), + function_id=function_id, + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def get_function_input_schema( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + function_id: FunctionID, +) -> FunctionInputSchema: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("get_function_input_schema"), + function_id=function_id, + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def get_function_output_schema( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + function_id: FunctionID, +) -> FunctionOutputSchema: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("get_function_output_schema"), + function_id=function_id, + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def delete_function( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + function_id: FunctionID, +) -> None: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("delete_function"), + function_id=function_id, + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def list_functions( + rabbitmq_rpc_client: RabbitMQRPCClient, +) -> list[Function]: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("list_functions"), + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def run_function( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + function_id: FunctionID, + inputs: FunctionInputs, +) -> FunctionJob: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("run_function"), + function_id=function_id, + inputs=inputs, + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def register_function_job( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + function_job: FunctionJob, +) -> FunctionJob: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("register_function_job"), + function_job=function_job, + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def get_function_job( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + function_job_id: FunctionJobID, +) -> FunctionJob: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("get_function_job"), + function_job_id=function_job_id, + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def list_function_jobs( + rabbitmq_rpc_client: RabbitMQRPCClient, +) -> list[FunctionJob]: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("list_function_jobs"), + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def delete_function_job( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + function_job_id: FunctionJobID, +) -> None: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("delete_function_job"), + function_job_id=function_job_id, + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def find_cached_function_job( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + function_id: FunctionID, + inputs: FunctionInputs, +) -> FunctionJob | None: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("find_cached_function_job"), + function_id=function_id, + inputs=inputs, + ) diff --git a/services/api-server/openapi.json b/services/api-server/openapi.json index e7ff27a22a1..3196a2cee37 100644 --- a/services/api-server/openapi.json +++ b/services/api-server/openapi.json @@ -5276,6 +5276,924 @@ } } }, + "/v0/functions/ping": { + "post": { + "tags": [ + "functions" + ], + "summary": "Ping", + "operationId": "ping", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + } + } + } + }, + "/v0/functions": { + "get": { + "tags": [ + "functions" + ], + "summary": "List Functions", + "description": "List functions", + "operationId": "list_functions", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "items": { + "oneOf": [ + { + "$ref": "#/components/schemas/ProjectFunction" + }, + { + "$ref": "#/components/schemas/PythonCodeFunction" + } + ], + "discriminator": { + "propertyName": "function_class", + "mapping": { + "project": "#/components/schemas/ProjectFunction", + "python_code": "#/components/schemas/PythonCodeFunction" + } + } + }, + "type": "array", + "title": "Response List Functions V0 Functions Get" + } + } + } + } + } + }, + "post": { + "tags": [ + "functions" + ], + "summary": "Register Function", + "description": "Create function", + "operationId": "register_function", + "requestBody": { + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/ProjectFunction" + }, + { + "$ref": "#/components/schemas/PythonCodeFunction" + } + ], + "title": "Function", + "discriminator": { + "propertyName": "function_class", + "mapping": { + "project": "#/components/schemas/ProjectFunction", + "python_code": "#/components/schemas/PythonCodeFunction" + } + } + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/ProjectFunction" + }, + { + "$ref": "#/components/schemas/PythonCodeFunction" + } + ], + "title": "Response Register Function V0 Functions Post", + "discriminator": { + "propertyName": "function_class", + "mapping": { + "project": "#/components/schemas/ProjectFunction", + "python_code": "#/components/schemas/PythonCodeFunction" + } + } + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/functions/{function_id}": { + "get": { + "tags": [ + "functions" + ], + "summary": "Get Function", + "description": "Get function", + "operationId": "get_function", + "parameters": [ + { + "name": "function_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/ProjectFunction" + }, + { + "$ref": "#/components/schemas/PythonCodeFunction" + } + ], + "discriminator": { + "propertyName": "function_class", + "mapping": { + "project": "#/components/schemas/ProjectFunction", + "python_code": "#/components/schemas/PythonCodeFunction" + } + }, + "title": "Response Get Function V0 Functions Function Id Get" + } + } + } + }, + "404": { + "description": "Function not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "delete": { + "tags": [ + "functions" + ], + "summary": "Delete Function", + "description": "Delete function", + "operationId": "delete_function", + "parameters": [ + { + "name": "function_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/ProjectFunction" + }, + { + "$ref": "#/components/schemas/PythonCodeFunction" + } + ], + "discriminator": { + "propertyName": "function_class", + "mapping": { + "project": "#/components/schemas/ProjectFunction", + "python_code": "#/components/schemas/PythonCodeFunction" + } + }, + "title": "Response Delete Function V0 Functions Function Id Delete" + } + } + } + }, + "404": { + "description": "Function not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/functions/{function_id}:run": { + "post": { + "tags": [ + "functions" + ], + "summary": "Run Function", + "description": "Run function", + "operationId": "run_function", + "security": [ + { + "HTTPBasic": [] + } + ], + "parameters": [ + { + "name": "function_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Id" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Function Inputs" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/ProjectFunctionJob" + }, + { + "$ref": "#/components/schemas/PythonCodeFunctionJob" + } + ], + "discriminator": { + "propertyName": "function_class", + "mapping": { + "project": "#/components/schemas/ProjectFunctionJob", + "python_code": "#/components/schemas/PythonCodeFunctionJob" + } + }, + "title": "Response Run Function V0 Functions Function Id Run Post" + } + } + } + }, + "404": { + "description": "Function not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/functions/{function_id}/input_schema": { + "get": { + "tags": [ + "functions" + ], + "summary": "Get Function Input Schema", + "description": "Get function", + "operationId": "get_function_input_schema", + "parameters": [ + { + "name": "function_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FunctionInputSchema" + } + } + } + }, + "404": { + "description": "Function not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/functions/{function_id}/output_schema": { + "get": { + "tags": [ + "functions" + ], + "summary": "Get Function Output Schema", + "description": "Get function", + "operationId": "get_function_output_schema", + "parameters": [ + { + "name": "function_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FunctionOutputSchema" + } + } + } + }, + "404": { + "description": "Function not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/functions/{function_id}:map": { + "post": { + "tags": [ + "functions" + ], + "summary": "Map Function", + "description": "Map function over input parameters", + "operationId": "map_function", + "security": [ + { + "HTTPBasic": [] + } + ], + "parameters": [ + { + "name": "function_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Id" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ] + }, + "title": "Function Inputs List" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "oneOf": [ + { + "$ref": "#/components/schemas/ProjectFunctionJob" + }, + { + "$ref": "#/components/schemas/PythonCodeFunctionJob" + } + ], + "discriminator": { + "propertyName": "function_class", + "mapping": { + "project": "#/components/schemas/ProjectFunctionJob", + "python_code": "#/components/schemas/PythonCodeFunctionJob" + } + } + }, + "title": "Response Map Function V0 Functions Function Id Map Post" + } + } + } + }, + "404": { + "description": "Function not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/function_jobs": { + "get": { + "tags": [ + "function_jobs" + ], + "summary": "List Function Jobs", + "description": "List function jobs", + "operationId": "list_function_jobs", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "items": { + "oneOf": [ + { + "$ref": "#/components/schemas/ProjectFunctionJob" + }, + { + "$ref": "#/components/schemas/PythonCodeFunctionJob" + } + ], + "discriminator": { + "propertyName": "function_class", + "mapping": { + "project": "#/components/schemas/ProjectFunctionJob", + "python_code": "#/components/schemas/PythonCodeFunctionJob" + } + } + }, + "type": "array", + "title": "Response List Function Jobs V0 Function Jobs Get" + } + } + } + } + } + }, + "post": { + "tags": [ + "function_jobs" + ], + "summary": "Register Function Job", + "description": "Create function job", + "operationId": "register_function_job", + "requestBody": { + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/ProjectFunctionJob" + }, + { + "$ref": "#/components/schemas/PythonCodeFunctionJob" + } + ], + "title": "Function Job", + "discriminator": { + "propertyName": "function_class", + "mapping": { + "project": "#/components/schemas/ProjectFunctionJob", + "python_code": "#/components/schemas/PythonCodeFunctionJob" + } + } + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/ProjectFunctionJob" + }, + { + "$ref": "#/components/schemas/PythonCodeFunctionJob" + } + ], + "title": "Response Register Function Job V0 Function Jobs Post", + "discriminator": { + "propertyName": "function_class", + "mapping": { + "project": "#/components/schemas/ProjectFunctionJob", + "python_code": "#/components/schemas/PythonCodeFunctionJob" + } + } + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/function_jobs/{function_job_id}": { + "get": { + "tags": [ + "function_jobs" + ], + "summary": "Get Function Job", + "description": "Get function job", + "operationId": "get_function_job", + "parameters": [ + { + "name": "function_job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Job Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/ProjectFunctionJob" + }, + { + "$ref": "#/components/schemas/PythonCodeFunctionJob" + } + ], + "discriminator": { + "propertyName": "function_class", + "mapping": { + "project": "#/components/schemas/ProjectFunctionJob", + "python_code": "#/components/schemas/PythonCodeFunctionJob" + } + }, + "title": "Response Get Function Job V0 Function Jobs Function Job Id Get" + } + } + } + }, + "404": { + "description": "Function job not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "delete": { + "tags": [ + "function_jobs" + ], + "summary": "Delete Function Job", + "description": "Delete function job", + "operationId": "delete_function_job", + "parameters": [ + { + "name": "function_job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Job Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "404": { + "description": "Function job not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/function_jobs/{function_job_id}/status": { + "get": { + "tags": [ + "function_jobs" + ], + "summary": "Function Job Status", + "description": "Get function job status", + "operationId": "function_job_status", + "security": [ + { + "HTTPBasic": [] + } + ], + "parameters": [ + { + "name": "function_job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Job Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FunctionJobStatus" + } + } + } + }, + "404": { + "description": "Function job not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/function_jobs/{function_job_id}/outputs": { + "get": { + "tags": [ + "function_jobs" + ], + "summary": "Function Job Outputs", + "description": "Get function job outputs", + "operationId": "function_job_outputs", + "security": [ + { + "HTTPBasic": [] + } + ], + "parameters": [ + { + "name": "function_job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Job Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Response Function Job Outputs V0 Function Jobs Function Job Id Outputs Get" + } + } + } + }, + "404": { + "description": "Function job not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, "/v0/wallets/default": { "get": { "tags": [ @@ -6246,11 +7164,64 @@ }, "type": "object", "required": [ - "chunk_size", - "urls", - "links" + "chunk_size", + "urls", + "links" + ], + "title": "FileUploadData" + }, + "FunctionInputSchema": { + "properties": { + "schema_dict": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Schema Dict" + } + }, + "type": "object", + "required": [ + "schema_dict" + ], + "title": "FunctionInputSchema" + }, + "FunctionJobStatus": { + "properties": { + "status": { + "type": "string", + "title": "Status" + } + }, + "type": "object", + "required": [ + "status" + ], + "title": "FunctionJobStatus" + }, + "FunctionOutputSchema": { + "properties": { + "schema_dict": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Schema Dict" + } + }, + "type": "object", + "required": [ + "schema_dict" ], - "title": "FileUploadData" + "title": "FunctionOutputSchema" }, "GetCreditPriceLegacy": { "properties": { @@ -7787,6 +8758,316 @@ "version_display": "8.0.0" } }, + "ProjectFunction": { + "properties": { + "uid": { + "anyOf": [ + { + "type": "string", + "format": "uuid" + }, + { + "type": "null" + } + ], + "title": "Uid" + }, + "title": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Title" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "function_class": { + "type": "string", + "const": "project", + "title": "Function Class", + "default": "project" + }, + "input_schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/FunctionInputSchema" + }, + { + "type": "null" + } + ] + }, + "output_schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/FunctionOutputSchema" + }, + { + "type": "null" + } + ] + }, + "project_id": { + "type": "string", + "format": "uuid", + "title": "Project Id" + } + }, + "type": "object", + "required": [ + "project_id" + ], + "title": "ProjectFunction" + }, + "ProjectFunctionJob": { + "properties": { + "uid": { + "anyOf": [ + { + "type": "string", + "format": "uuid" + }, + { + "type": "null" + } + ], + "title": "Uid" + }, + "title": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Title" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "function_uid": { + "type": "string", + "format": "uuid", + "title": "Function Uid" + }, + "inputs": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Inputs" + }, + "outputs": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Outputs" + }, + "function_class": { + "type": "string", + "const": "project", + "title": "Function Class", + "default": "project" + }, + "project_job_id": { + "type": "string", + "format": "uuid", + "title": "Project Job Id" + } + }, + "type": "object", + "required": [ + "function_uid", + "project_job_id" + ], + "title": "ProjectFunctionJob" + }, + "PythonCodeFunction": { + "properties": { + "uid": { + "anyOf": [ + { + "type": "string", + "format": "uuid" + }, + { + "type": "null" + } + ], + "title": "Uid" + }, + "title": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Title" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "function_class": { + "type": "string", + "const": "python_code", + "title": "Function Class", + "default": "python_code" + }, + "input_schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/FunctionInputSchema" + }, + { + "type": "null" + } + ] + }, + "output_schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/FunctionOutputSchema" + }, + { + "type": "null" + } + ] + }, + "code_url": { + "type": "string", + "title": "Code Url" + } + }, + "type": "object", + "required": [ + "code_url" + ], + "title": "PythonCodeFunction" + }, + "PythonCodeFunctionJob": { + "properties": { + "uid": { + "anyOf": [ + { + "type": "string", + "format": "uuid" + }, + { + "type": "null" + } + ], + "title": "Uid" + }, + "title": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Title" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "function_uid": { + "type": "string", + "format": "uuid", + "title": "Function Uid" + }, + "inputs": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Inputs" + }, + "outputs": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Outputs" + }, + "function_class": { + "type": "string", + "const": "python_code", + "title": "Function Class", + "default": "python_code" + }, + "code_url": { + "type": "string", + "title": "Code Url" + } + }, + "type": "object", + "required": [ + "function_uid", + "code_url" + ], + "title": "PythonCodeFunctionJob" + }, "RunningState": { "type": "string", "enum": [ diff --git a/services/api-server/src/simcore_service_api_server/api/root.py b/services/api-server/src/simcore_service_api_server/api/root.py index 5654601d403..5a1de4a711c 100644 --- a/services/api-server/src/simcore_service_api_server/api/root.py +++ b/services/api-server/src/simcore_service_api_server/api/root.py @@ -6,7 +6,7 @@ from .routes import credits as _credits from .routes import ( files, - functions, + functions_routes, health, licensed_items, meta, @@ -42,12 +42,27 @@ def create_router(settings: ApplicationSettings): ) router.include_router(studies.router, tags=["studies"], prefix="/studies") router.include_router(studies_jobs.router, tags=["studies"], prefix="/studies") + router.include_router( + functions_routes.function_router, tags=["functions"], prefix="/functions" + ) + router.include_router( + functions_routes.function_job_router, + tags=["function_jobs"], + prefix="/function_jobs", + ) + router.include_router( + functions_routes.function_job_collections_router, + tags=["function_job_collections"], + prefix="/function_job_collections", + ) router.include_router(wallets.router, tags=["wallets"], prefix="/wallets") router.include_router(_credits.router, tags=["credits"], prefix="/credits") router.include_router( licensed_items.router, tags=["licensed-items"], prefix="/licensed-items" ) - router.include_router(functions.router, tags=["functions"], prefix="/functions") + router.include_router( + functions_routes.function_router, tags=["functions"], prefix="/functions" + ) # NOTE: multiple-files upload is currently disabled # Web form to upload files at http://localhost:8000/v0/upload-form-view diff --git a/services/api-server/src/simcore_service_api_server/api/routes/functions.py b/services/api-server/src/simcore_service_api_server/api/routes/functions.py deleted file mode 100644 index 6d5c277451d..00000000000 --- a/services/api-server/src/simcore_service_api_server/api/routes/functions.py +++ /dev/null @@ -1,17 +0,0 @@ -from typing import Annotated - -from fastapi import APIRouter, Depends - -from ...services_rpc.wb_api_server import WbApiRpcClient -from ..dependencies.webserver_rpc import ( - get_wb_api_rpc_client, -) - -router = APIRouter() - - -@router.post("/ping", include_in_schema=False) -async def ping( - wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], -): - return await wb_api_rpc.ping() diff --git a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py new file mode 100644 index 00000000000..dc2a80abcf5 --- /dev/null +++ b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py @@ -0,0 +1,453 @@ +from collections.abc import Callable +from typing import Annotated, Final + +from fastapi import APIRouter, Depends, Request, status +from models_library.api_schemas_webserver.functions_wb_schema import ( + Function, + FunctionClass, + FunctionID, + FunctionInputs, + FunctionInputSchema, + FunctionInputsList, + FunctionJob, + FunctionJobID, + FunctionJobStatus, + FunctionOutputs, + FunctionOutputSchema, + ProjectFunctionJob, +) +from pydantic import PositiveInt +from servicelib.fastapi.dependencies import get_reverse_url_mapper + +from ...models.schemas.errors import ErrorGet +from ...models.schemas.jobs import ( + JobInputs, +) +from ...services_http.director_v2 import DirectorV2Api +from ...services_http.storage import StorageApi +from ...services_http.webserver import AuthSession +from ...services_rpc.wb_api_server import WbApiRpcClient +from ..dependencies.authentication import get_current_user_id, get_product_name +from ..dependencies.services import get_api_client +from ..dependencies.webserver_http import get_webserver_session +from ..dependencies.webserver_rpc import ( + get_wb_api_rpc_client, +) +from . import studies_jobs + +function_router = APIRouter() +function_job_router = APIRouter() +function_job_collections_router = APIRouter() + +_COMMON_FUNCTION_ERROR_RESPONSES: Final[dict] = { + status.HTTP_404_NOT_FOUND: { + "description": "Function not found", + "model": ErrorGet, + }, +} + + +@function_router.post("/ping") +async def ping( + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + return await wb_api_rpc.ping() + + +@function_router.get("", response_model=list[Function], description="List functions") +async def list_functions( + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + return await wb_api_rpc.list_functions() + + +@function_router.post("", response_model=Function, description="Create function") +async def register_function( + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], + function: Function, +): + return await wb_api_rpc.register_function(function=function) + + +@function_router.get( + "/{function_id:uuid}", + response_model=Function, + responses={**_COMMON_FUNCTION_ERROR_RESPONSES}, + description="Get function", +) +async def get_function( + function_id: FunctionID, + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + return await wb_api_rpc.get_function(function_id=function_id) + + +@function_router.post( + "/{function_id:uuid}:run", + response_model=FunctionJob, + responses={**_COMMON_FUNCTION_ERROR_RESPONSES}, + description="Run function", +) +async def run_function( + request: Request, + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], + webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], + url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], + director2_api: Annotated[DirectorV2Api, Depends(get_api_client(DirectorV2Api))], + function_id: FunctionID, + function_inputs: FunctionInputs, + user_id: Annotated[PositiveInt, Depends(get_current_user_id)], + product_name: Annotated[str, Depends(get_product_name)], +): + + to_run_function = await wb_api_rpc.get_function(function_id=function_id) + + assert to_run_function.uid is not None + + if cached_function_job := await wb_api_rpc.find_cached_function_job( + function_id=to_run_function.uid, + inputs=function_inputs, + ): + return cached_function_job + + if to_run_function.function_class == FunctionClass.project: + study_job = await studies_jobs.create_study_job( + study_id=to_run_function.project_id, + job_inputs=JobInputs(values=function_inputs or {}), + webserver_api=webserver_api, + wb_api_rpc=wb_api_rpc, + url_for=url_for, + x_simcore_parent_project_uuid=None, + x_simcore_parent_node_id=None, + user_id=user_id, + product_name=product_name, + ) + await studies_jobs.start_study_job( + request=request, + study_id=to_run_function.project_id, + job_id=study_job.id, + user_id=user_id, + webserver_api=webserver_api, + director2_api=director2_api, + ) + return await register_function_job( + wb_api_rpc=wb_api_rpc, + function_job=ProjectFunctionJob( + function_uid=to_run_function.uid, + title=f"Function job of function {to_run_function.uid}", + description=to_run_function.description, + inputs=function_inputs, + outputs=None, + project_job_id=study_job.id, + ), + ) + else: # noqa: RET505 + msg = f"Function type {type(to_run_function)} not supported" + raise TypeError(msg) + + +@function_router.delete( + "/{function_id:uuid}", + response_model=Function, + responses={**_COMMON_FUNCTION_ERROR_RESPONSES}, + description="Delete function", +) +async def delete_function( + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], + function_id: FunctionID, +): + return await wb_api_rpc.delete_function(function_id=function_id) + + +@function_router.get( + "/{function_id:uuid}/input_schema", + response_model=FunctionInputSchema, + responses={**_COMMON_FUNCTION_ERROR_RESPONSES}, + description="Get function", +) +async def get_function_input_schema( + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], + function_id: FunctionID, +): + return await wb_api_rpc.get_function_input_schema(function_id=function_id) + + +@function_router.get( + "/{function_id:uuid}/output_schema", + response_model=FunctionOutputSchema, + responses={**_COMMON_FUNCTION_ERROR_RESPONSES}, + description="Get function", +) +async def get_function_output_schema( + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], + function_id: FunctionID, +): + return await wb_api_rpc.get_function_output_schema(function_id=function_id) + + +_COMMON_FUNCTION_JOB_ERROR_RESPONSES: Final[dict] = { + status.HTTP_404_NOT_FOUND: { + "description": "Function job not found", + "model": ErrorGet, + }, +} + + +@function_job_router.post( + "", response_model=FunctionJob, description="Create function job" +) +async def register_function_job( + function_job: FunctionJob, + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + return await wb_api_rpc.register_function_job(function_job=function_job) + + +@function_job_router.get( + "/{function_job_id:uuid}", + response_model=FunctionJob, + responses={**_COMMON_FUNCTION_JOB_ERROR_RESPONSES}, + description="Get function job", +) +async def get_function_job( + function_job_id: FunctionJobID, + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + return await wb_api_rpc.get_function_job(function_job_id=function_job_id) + + +@function_job_router.get( + "", response_model=list[FunctionJob], description="List function jobs" +) +async def list_function_jobs( + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + return await wb_api_rpc.list_function_jobs() + + +@function_job_router.delete( + "/{function_job_id:uuid}", + response_model=None, + responses={**_COMMON_FUNCTION_JOB_ERROR_RESPONSES}, + description="Delete function job", +) +async def delete_function_job( + function_job_id: FunctionJobID, + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + return await wb_api_rpc.delete_function_job(function_job_id=function_job_id) + + +async def get_function_from_functionjobid( + wb_api_rpc: WbApiRpcClient, + function_job_id: FunctionJobID, +) -> tuple[Function, FunctionJob]: + function_job = await get_function_job( + wb_api_rpc=wb_api_rpc, function_job_id=function_job_id + ) + + return ( + await get_function( + wb_api_rpc=wb_api_rpc, function_id=function_job.function_uid + ), + function_job, + ) + + +@function_job_router.get( + "/{function_job_id:uuid}/status", + response_model=FunctionJobStatus, + responses={**_COMMON_FUNCTION_JOB_ERROR_RESPONSES}, + description="Get function job status", +) +async def function_job_status( + function_job_id: FunctionJobID, + director2_api: Annotated[DirectorV2Api, Depends(get_api_client(DirectorV2Api))], + user_id: Annotated[PositiveInt, Depends(get_current_user_id)], + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + function, function_job = await get_function_from_functionjobid( + wb_api_rpc=wb_api_rpc, function_job_id=function_job_id + ) + + if ( + function.function_class == FunctionClass.project + and function_job.function_class == FunctionClass.project + ): + job_status = await studies_jobs.inspect_study_job( + study_id=function.project_id, + job_id=function_job.project_job_id, + user_id=user_id, + director2_api=director2_api, + ) + return FunctionJobStatus(status=job_status.state) + else: # noqa: RET505 + msg = f"Function type {function.function_class} / Function job type {function_job.function_class} not supported" + raise TypeError(msg) + + +@function_job_router.get( + "/{function_job_id:uuid}/outputs", + response_model=FunctionOutputs, + responses={**_COMMON_FUNCTION_JOB_ERROR_RESPONSES}, + description="Get function job outputs", +) +async def function_job_outputs( + function_job_id: FunctionJobID, + webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], + user_id: Annotated[PositiveInt, Depends(get_current_user_id)], + storage_client: Annotated[StorageApi, Depends(get_api_client(StorageApi))], + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + function, function_job = await get_function_from_functionjobid( + wb_api_rpc=wb_api_rpc, function_job_id=function_job_id + ) + + if ( + function.function_class != FunctionClass.project + or function_job.function_class != FunctionClass.project + ): + msg = f"Function type {function.function_class} not supported" + raise TypeError(msg) + else: # noqa: RET506 + job_outputs = await studies_jobs.get_study_job_outputs( + study_id=function.project_id, + job_id=function_job.project_job_id, + user_id=user_id, + webserver_api=webserver_api, + storage_client=storage_client, + ) + + return job_outputs.results + + +@function_router.post( + "/{function_id:uuid}:map", + response_model=list[FunctionJob], + responses={**_COMMON_FUNCTION_ERROR_RESPONSES}, + description="Map function over input parameters", +) +async def map_function( + function_id: FunctionID, + function_inputs_list: FunctionInputsList, + request: Request, + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], + webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], + url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], + director2_api: Annotated[DirectorV2Api, Depends(get_api_client(DirectorV2Api))], + user_id: Annotated[PositiveInt, Depends(get_current_user_id)], + product_name: Annotated[str, Depends(get_product_name)], +): + function_jobs = [] + for function_inputs in function_inputs_list: + function_jobs = [ + await run_function( + wb_api_rpc=wb_api_rpc, + function_id=function_id, + function_inputs=function_inputs, + product_name=product_name, + user_id=user_id, + webserver_api=webserver_api, + url_for=url_for, + director2_api=director2_api, + request=request, + ) + for function_inputs in function_inputs_list + ] + # TODO poor system can't handle doing this in parallel, get this fixed # noqa: FIX002 + # function_jobs = await asyncio.gather(*function_jobs_tasks) + + return function_jobs + + +# ruff: noqa: ERA001 + + +# _logger = logging.getLogger(__name__) + +# _COMMON_FUNCTION_JOB_COLLECTION_ERROR_RESPONSES: Final[dict] = { +# status.HTTP_404_NOT_FOUND: { +# "description": "Function job collection not found", +# "model": ErrorGet, +# }, +# } + + +# @function_job_collections_router.get( +# "", +# response_model=FunctionJobCollection, +# description="List function job collections", +# ) +# async def list_function_job_collections( +# page_params: Annotated[PaginationParams, Depends()], +# webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], +# ): +# msg = "list function jobs collection not implemented yet" +# raise NotImplementedError(msg) + + +# @function_job_collections_router.post( +# "", response_model=FunctionJobCollection, description="Create function job" +# ) +# async def create_function_job_collection( +# webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], +# job_ids: Annotated[list[FunctionJob], Depends()], +# ): +# msg = "create function job collection not implemented yet" +# raise NotImplementedError(msg) + + +# @function_job_collections_router.get( +# "/{function_job_collection_id:uuid}", +# response_model=FunctionJobCollection, +# responses={**_COMMON_FUNCTION_JOB_COLLECTION_ERROR_RESPONSES}, +# description="Get function job", +# ) +# async def get_function_job_collection( +# function_job_collection_id: FunctionJobCollectionID, +# webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], +# ): +# msg = "get function job collection not implemented yet" +# raise NotImplementedError(msg) + + +# @function_job_collections_router.delete( +# "/{function_job_collection_id:uuid}", +# response_model=FunctionJob, +# responses={**_COMMON_FUNCTION_JOB_COLLECTION_ERROR_RESPONSES}, +# description="Delete function job collection", +# ) +# async def delete_function_job_collection( +# function_job_collection_id: FunctionJobCollectionID, +# webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], +# ): +# msg = "delete function job collection not implemented yet" +# raise NotImplementedError(msg) + + +# @function_job_collections_router.get( +# "/{function_job_collection_id:uuid}/function_jobs", +# response_model=list[FunctionJob], +# responses={**_COMMON_FUNCTION_JOB_COLLECTION_ERROR_RESPONSES}, +# description="Get the function jobs in function job collection", +# ) +# async def function_job_collection_list_function_jobs( +# function_job_collection_id: FunctionJobCollectionID, +# webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], +# ): +# msg = "function job collection listing not implemented yet" +# raise NotImplementedError(msg) + + +# @function_job_collections_router.get( +# "/{function_job_collection_id:uuid}/status", +# response_model=FunctionJobCollectionStatus, +# responses={**_COMMON_FUNCTION_JOB_COLLECTION_ERROR_RESPONSES}, +# description="Get function job collection status", +# ) +# async def function_job_collection_status( +# function_job_collection_id: FunctionJobCollectionID, +# webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], +# ): +# msg = "function job collection status not implemented yet" +# raise NotImplementedError(msg) diff --git a/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py b/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py index 436633b1c18..0da6c51574c 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py @@ -133,7 +133,7 @@ async def create_study_job( url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], user_id: Annotated[PositiveInt, Depends(get_current_user_id)], product_name: Annotated[str, Depends(get_product_name)], - hidden: Annotated[bool, Query()] = True, + hidden: Annotated[bool, Query()] = True, # noqa: FBT002 x_simcore_parent_project_uuid: ProjectID | None = Header(default=None), x_simcore_parent_node_id: NodeID | None = Header(default=None), ) -> Job: @@ -304,13 +304,12 @@ async def start_study_job( return JSONResponse( content=jsonable_encoder(job_status), status_code=status.HTTP_200_OK ) - job_status = await inspect_study_job( + return await inspect_study_job( study_id=study_id, job_id=job_id, user_id=user_id, director2_api=director2_api, ) - return job_status @router.post( @@ -387,10 +386,9 @@ async def get_study_job_output_logfile( level=logging.DEBUG, msg=f"get study job output logfile study_id={study_id!r} job_id={job_id!r}.", ): - log_link_map = await director2_api.get_computation_logs( + return await director2_api.get_computation_logs( user_id=user_id, project_id=job_id ) - return log_link_map @router.get( diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/functions_api_schema.py b/services/api-server/src/simcore_service_api_server/models/schemas/functions_api_schema.py new file mode 100644 index 00000000000..ba316c0e88d --- /dev/null +++ b/services/api-server/src/simcore_service_api_server/models/schemas/functions_api_schema.py @@ -0,0 +1,85 @@ +from enum import Enum +from typing import Annotated, Any, Literal, TypeAlias + +from models_library import projects +from pydantic import BaseModel, Field + +FunctionID: TypeAlias = projects.ProjectID + + +class FunctionSchema(BaseModel): + schema_dict: dict[str, Any] | None # JSON Schema + + +class FunctionInputSchema(FunctionSchema): ... + + +class FunctionOutputSchema(FunctionSchema): ... + + +class FunctionClass(str, Enum): + project = "project" + python_code = "python_code" + + +class FunctionInputs(BaseModel): + inputs_dict: dict[str, Any] | None # JSON Schema + + +class FunctionOutputs(BaseModel): + outputs_dict: dict[str, Any] | None # JSON Schema + + +class Function(BaseModel): + uid: FunctionID | None = None + title: str | None = None + description: str | None = None + input_schema: FunctionInputSchema | None = None + output_schema: FunctionOutputSchema | None = None + + +class StudyFunction(Function): + function_type: Literal["study"] = "study" + study_url: str + + +class PythonCodeFunction(Function): + function_type: Literal["python_code"] = "python_code" + code_url: str + + +FunctionUnion: TypeAlias = Annotated[ + StudyFunction | PythonCodeFunction, + Field(discriminator="function_type"), +] + +FunctionJobID: TypeAlias = projects.ProjectID +FunctionJobCollectionID: TypeAlias = projects.ProjectID + + +class FunctionJob(BaseModel): + uid: FunctionJobID + title: str | None + description: str | None + status: str + function_uid: FunctionID + inputs: FunctionInputs | None + outputs: FunctionOutputs | None + + +class FunctionJobStatus(BaseModel): + status: str + + +class FunctionJobCollection(BaseModel): + """Model for a collection of function jobs""" + + id: FunctionJobCollectionID + title: str | None + description: str | None + job_ids: list[FunctionJobID] + status: str + + +class FunctionJobCollectionStatus(BaseModel): + status: list[str] diff --git a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py index fa9da284649..80139346f25 100644 --- a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py +++ b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py @@ -4,6 +4,15 @@ from fastapi import FastAPI from fastapi_pagination import create_page +from models_library.api_schemas_webserver.functions_wb_schema import ( + Function, + FunctionID, + FunctionInputs, + FunctionInputSchema, + FunctionJob, + FunctionJobID, + FunctionOutputSchema, +) from models_library.api_schemas_webserver.licensed_items import LicensedItemRpcGetPage from models_library.licenses import LicensedItemID from models_library.products import ProductName @@ -29,9 +38,45 @@ NotEnoughAvailableSeatsError, ) from servicelib.rabbitmq.rpc_interfaces.webserver import projects as projects_rpc -from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions import ( +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + delete_function as _delete_function, +) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + delete_function_job as _delete_function_job, +) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + find_cached_function_job as _find_cached_function_job, +) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + get_function as _get_function, +) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + get_function_input_schema as _get_function_input_schema, +) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + get_function_job as _get_function_job, +) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + get_function_output_schema as _get_function_output_schema, +) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + list_function_jobs as _list_function_jobs, +) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + list_functions as _list_functions, +) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( ping as _ping, ) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + register_function as _register_function, +) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + register_function_job as _register_function_job, +) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + run_function as _run_function, +) from servicelib.rabbitmq.rpc_interfaces.webserver.licenses.licensed_items import ( checkout_licensed_item_for_wallet as _checkout_licensed_item_for_wallet, ) @@ -237,6 +282,65 @@ async def list_projects_marked_as_jobs( job_parent_resource_name_prefix=job_parent_resource_name_prefix, ) + async def register_function(self, *, function: Function) -> Function: + function.input_schema = ( + FunctionInputSchema(**function.input_schema.model_dump()) + if function.input_schema is not None + else None + ) + function.output_schema = ( + FunctionOutputSchema(**function.output_schema.model_dump()) + if function.output_schema is not None + else None + ) + return await _register_function( + self._client, + function=function, + ) + + async def get_function(self, *, function_id: FunctionID) -> Function: + return await _get_function(self._client, function_id=function_id) + + async def delete_function(self, *, function_id: FunctionID) -> None: + return await _delete_function(self._client, function_id=function_id) + + async def list_functions(self) -> list[Function]: + return await _list_functions(self._client) + + async def run_function( + self, *, function_id: FunctionID, inputs: FunctionInputs + ) -> FunctionJob: + return await _run_function(self._client, function_id=function_id, inputs=inputs) + + async def get_function_job(self, *, function_job_id: FunctionJobID) -> FunctionJob: + return await _get_function_job(self._client, function_job_id=function_job_id) + + async def delete_function_job(self, *, function_job_id: FunctionJobID) -> None: + return await _delete_function_job(self._client, function_job_id=function_job_id) + + async def register_function_job(self, *, function_job: FunctionJob) -> FunctionJob: + return await _register_function_job(self._client, function_job=function_job) + + async def get_function_input_schema( + self, *, function_id: FunctionID + ) -> FunctionInputSchema: + return await _get_function_input_schema(self._client, function_id=function_id) + + async def get_function_output_schema( + self, *, function_id: FunctionID + ) -> FunctionOutputSchema: + return await _get_function_output_schema(self._client, function_id=function_id) + + async def find_cached_function_job( + self, *, function_id: FunctionID, inputs: FunctionInputs + ) -> FunctionJob | None: + return await _find_cached_function_job( + self._client, function_id=function_id, inputs=inputs + ) + + async def list_function_jobs(self) -> list[FunctionJob]: + return await _list_function_jobs(self._client) + def setup(app: FastAPI, rabbitmq_rmp_client: RabbitMQRPCClient): wb_api_rpc_client = WbApiRpcClient(_client=rabbitmq_rmp_client) diff --git a/services/web/server/src/simcore_service_webserver/functions/_controller_rpc.py b/services/web/server/src/simcore_service_webserver/functions/_controller_rpc.py deleted file mode 100644 index 483fcdc26e3..00000000000 --- a/services/web/server/src/simcore_service_webserver/functions/_controller_rpc.py +++ /dev/null @@ -1,21 +0,0 @@ -from aiohttp import web -from models_library.api_schemas_webserver import WEBSERVER_RPC_NAMESPACE -from servicelib.rabbitmq import RPCRouter - -from ..rabbitmq import get_rabbitmq_rpc_server - -# this is the rpc interface exposed to the api-server -# this interface should call the service layer - -router = RPCRouter() - - -@router.expose() -async def ping(app: web.Application) -> str: - assert app - return "pong from webserver" - - -async def register_rpc_routes_on_startup(app: web.Application): - rpc_server = get_rabbitmq_rpc_server(app) - await rpc_server.register_router(router, WEBSERVER_RPC_NAMESPACE, app) diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py new file mode 100644 index 00000000000..3a689d37ab9 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py @@ -0,0 +1,262 @@ +from aiohttp import web +from models_library.api_schemas_webserver import WEBSERVER_RPC_NAMESPACE +from models_library.api_schemas_webserver.functions_wb_schema import ( + Function, + FunctionClass, + FunctionClassSpecificData, + FunctionDB, + FunctionID, + FunctionInputs, + FunctionInputSchema, + FunctionJob, + FunctionJobClassSpecificData, + FunctionJobDB, + FunctionJobID, + FunctionOutputSchema, + ProjectFunction, + ProjectFunctionJob, +) +from servicelib.rabbitmq import RPCRouter + +from ..rabbitmq import get_rabbitmq_rpc_server +from . import _functions_repository + +router = RPCRouter() + + +@router.expose() +async def ping(app: web.Application) -> str: + assert app + return "pong from webserver" + + +@router.expose() +async def register_function(app: web.Application, *, function: Function) -> Function: + assert app + if function.function_class == FunctionClass.project: + saved_function = await _functions_repository.create_function( + app=app, + function=FunctionDB( + title=function.title, + description=function.description, + input_schema=function.input_schema, + output_schema=function.output_schema, + function_class=function.function_class, + class_specific_data=FunctionClassSpecificData( + { + "project_id": str(function.project_id), + } + ), + ), + ) + return ProjectFunction( + uid=saved_function.uuid, + title=saved_function.title, + description=saved_function.description, + input_schema=saved_function.input_schema, + output_schema=saved_function.output_schema, + project_id=saved_function.class_specific_data["project_id"], + ) + else: # noqa: RET505 + msg = f"Unsupported function class: {function.function_class}" + raise TypeError(msg) + + +def _decode_function( + function: FunctionDB, +) -> Function: + if function.function_class == "project": + return ProjectFunction( + uid=function.uuid, + title=function.title, + description=function.description, + input_schema=function.input_schema, + output_schema=function.output_schema, + project_id=function.class_specific_data["project_id"], + ) + else: # noqa: RET505 + msg = f"Unsupported function class: [{function.function_class}]" + raise TypeError(msg) + + +@router.expose() +async def get_function(app: web.Application, *, function_id: FunctionID) -> Function: + assert app + returned_function = await _functions_repository.get_function( + app=app, + function_id=function_id, + ) + return _decode_function( + returned_function, + ) + + +@router.expose() +async def get_function_job( + app: web.Application, *, function_job_id: FunctionJobID +) -> FunctionJob: + assert app + returned_function_job = await _functions_repository.get_function_job( + app=app, + function_job_id=function_job_id, + ) + assert returned_function_job is not None + + if returned_function_job.function_class == FunctionClass.project: + return ProjectFunctionJob( + uid=returned_function_job.uuid, + title=returned_function_job.title, + description="", + function_uid=returned_function_job.function_uuid, + inputs=returned_function_job.inputs, + outputs=None, + project_job_id=returned_function_job.class_specific_data["project_job_id"], + ) + else: # noqa: RET505 + msg = f"Unsupported function class: [{returned_function_job.function_class}]" + raise TypeError(msg) + + +@router.expose() +async def list_function_jobs(app: web.Application) -> list[FunctionJob]: + assert app + returned_function_jobs = await _functions_repository.list_function_jobs( + app=app, + ) + return [ + ProjectFunctionJob( + uid=returned_function_job.uuid, + title=returned_function_job.title, + description="", + function_uid=returned_function_job.function_uuid, + inputs=returned_function_job.inputs, + outputs=None, + project_job_id=returned_function_job.class_specific_data["project_job_id"], + ) + for returned_function_job in returned_function_jobs + ] + + +@router.expose() +async def get_function_input_schema( + app: web.Application, *, function_id: FunctionID +) -> FunctionInputSchema: + assert app + returned_function = await _functions_repository.get_function( + app=app, + function_id=function_id, + ) + return FunctionInputSchema( + schema_dict=( + returned_function.input_schema.schema_dict + if returned_function.input_schema + else None + ) + ) + + +@router.expose() +async def get_function_output_schema( + app: web.Application, *, function_id: FunctionID +) -> FunctionOutputSchema: + assert app + returned_function = await _functions_repository.get_function( + app=app, + function_id=function_id, + ) + return FunctionOutputSchema( + schema_dict=( + returned_function.output_schema.schema_dict + if returned_function.output_schema + else None + ) + ) + + +@router.expose() +async def list_functions(app: web.Application) -> list[Function]: + assert app + returned_functions = await _functions_repository.list_functions( + app=app, + ) + return [ + _decode_function(returned_function) for returned_function in returned_functions + ] + + +@router.expose() +async def delete_function(app: web.Application, *, function_id: FunctionID) -> None: + assert app + await _functions_repository.delete_function( + app=app, + function_id=function_id, + ) + + +@router.expose() +async def register_function_job( + app: web.Application, *, function_job: FunctionJob +) -> FunctionJob: + assert app + if function_job.function_class == FunctionClass.project: + created_function_job_db = await _functions_repository.register_function_job( + app=app, + function_job=FunctionJobDB( + title=function_job.title, + function_uuid=function_job.function_uid, + inputs=function_job.inputs, + outputs=None, + class_specific_data=FunctionJobClassSpecificData( + { + "project_job_id": str(function_job.project_job_id), + } + ), + function_class=function_job.function_class, + ), + ) + + return ProjectFunctionJob( + uid=created_function_job_db.uuid, + title=created_function_job_db.title, + description="", + function_uid=created_function_job_db.function_uuid, + inputs=created_function_job_db.inputs, + outputs=None, + project_job_id=created_function_job_db.class_specific_data[ + "project_job_id" + ], + ) + else: # noqa: RET505 + msg = f"Unsupported function class: [{function_job.function_class}]" + raise TypeError(msg) + + +@router.expose() +async def find_cached_function_job( + app: web.Application, *, function_id: FunctionID, inputs: FunctionInputs +) -> FunctionJob | None: + assert app + returned_function_job = await _functions_repository.find_cached_function_job( + app=app, function_id=function_id, inputs=inputs + ) + if returned_function_job is None: + return None + + if returned_function_job.function_class == FunctionClass.project: + return ProjectFunctionJob( + uid=returned_function_job.uuid, + title=returned_function_job.title, + description="", + function_uid=returned_function_job.function_uuid, + inputs=returned_function_job.inputs, + outputs=None, + project_job_id=returned_function_job.class_specific_data["project_job_id"], + ) + else: # noqa: RET505 + msg = f"Unsupported function class: [{returned_function_job.function_class}]" + raise TypeError(msg) + + +async def register_rpc_routes_on_startup(app: web.Application): + rpc_server = get_rabbitmq_rpc_server(app) + await rpc_server.register_router(router, WEBSERVER_RPC_NAMESPACE, app) diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py new file mode 100644 index 00000000000..495f93c0c25 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py @@ -0,0 +1,206 @@ +import json + +from aiohttp import web +from models_library.api_schemas_webserver.functions_wb_schema import ( + FunctionDB, + FunctionID, + FunctionInputs, + FunctionJobDB, +) +from simcore_postgres_database.models.functions_models_db import ( + function_jobs as function_jobs_table, +) +from simcore_postgres_database.models.functions_models_db import ( + functions as functions_table, +) +from simcore_postgres_database.utils_repos import ( + get_columns_from_db_model, + transaction_context, +) +from sqlalchemy import Text, cast +from sqlalchemy.ext.asyncio import AsyncConnection + +from ..db.plugin import get_asyncpg_engine + +_FUNCTIONS_TABLE_COLS = get_columns_from_db_model(functions_table, FunctionDB) +_FUNCTION_JOBS_TABLE_COLS = get_columns_from_db_model( + function_jobs_table, FunctionJobDB +) + + +async def create_function( + app: web.Application, + connection: AsyncConnection | None = None, + *, + function: FunctionDB, +) -> FunctionDB: + + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream( + functions_table.insert() + .values( + title=function.title, + description=function.description, + input_schema=( + function.input_schema.model_dump() + if function.input_schema is not None + else None + ), + output_schema=( + function.output_schema.model_dump() + if function.output_schema is not None + else None + ), + function_class=function.function_class, + class_specific_data=function.class_specific_data, + ) + .returning(*_FUNCTIONS_TABLE_COLS) + ) + row = await result.first() + + if row is None: + msg = "No row was returned from the database after creating function." + raise ValueError(msg) + + return FunctionDB.model_validate(dict(row)) + + +async def get_function( + app: web.Application, + connection: AsyncConnection | None = None, + *, + function_id: FunctionID, +) -> FunctionDB: + + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream( + functions_table.select().where(functions_table.c.uuid == function_id) + ) + row = await result.first() + + if row is None: + msg = f"No function found with id {function_id}." + raise web.HTTPNotFound(reason=msg) + + return FunctionDB.model_validate(dict(row)) + + +async def list_functions( + app: web.Application, + connection: AsyncConnection | None = None, +) -> list[FunctionDB]: + + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream(functions_table.select().where()) + rows = await result.all() + if rows is None: + return [] + + return [FunctionDB.model_validate(dict(row)) for row in rows] + + +async def delete_function( + app: web.Application, + connection: AsyncConnection | None = None, + *, + function_id: FunctionID, +) -> None: + + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + await conn.execute( + functions_table.delete().where(functions_table.c.uuid == int(function_id)) + ) + + +async def register_function_job( + app: web.Application, + connection: AsyncConnection | None = None, + *, + function_job: FunctionJobDB, +) -> FunctionJobDB: + + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream( + function_jobs_table.insert() + .values( + function_uuid=function_job.function_uuid, + inputs=function_job.inputs, + function_class=function_job.function_class, + class_specific_data=function_job.class_specific_data, + title=function_job.title, + status="created", + ) + .returning(*_FUNCTION_JOBS_TABLE_COLS) + ) + row = await result.first() + + if row is None: + msg = "No row was returned from the database after creating function job." + raise ValueError(msg) + + return FunctionJobDB.model_validate(dict(row)) + + +async def get_function_job( + app: web.Application, + connection: AsyncConnection | None = None, + *, + function_job_id: FunctionID, +) -> FunctionJobDB: + + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream( + function_jobs_table.select().where( + function_jobs_table.c.uuid == function_job_id + ) + ) + row = await result.first() + + if row is None: + msg = f"No function job found with id {function_job_id}." + raise web.HTTPNotFound(reason=msg) + + return FunctionJobDB.model_validate(dict(row)) + + +async def list_function_jobs( + app: web.Application, + connection: AsyncConnection | None = None, +) -> list[FunctionJobDB]: + + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream(function_jobs_table.select().where()) + rows = await result.all() + if rows is None: + return [] + + return [FunctionJobDB.model_validate(dict(row)) for row in rows] + + +async def find_cached_function_job( + app: web.Application, + connection: AsyncConnection | None = None, + *, + function_id: FunctionID, + inputs: FunctionInputs, +) -> FunctionJobDB | None: + + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream( + function_jobs_table.select().where( + function_jobs_table.c.function_uuid == function_id, + cast(function_jobs_table.c.inputs, Text) == json.dumps(inputs), + ), + ) + + rows = await result.all() + + if rows is None: + return None + + for row in rows: + job = FunctionJobDB.model_validate(dict(row)) + if job.inputs == inputs: + return job + + return None diff --git a/services/web/server/src/simcore_service_webserver/functions/_service.py b/services/web/server/src/simcore_service_webserver/functions/_service.py index 2c967d7c841..899b0b1c681 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_service.py +++ b/services/web/server/src/simcore_service_webserver/functions/_service.py @@ -5,7 +5,7 @@ from aiohttp import web from models_library.users import UserID -from ..projects import _projects_service +from ..projects import projects_service from ..projects.models import ProjectDict @@ -16,6 +16,6 @@ async def get_project_from_function( user_id: UserID, ) -> ProjectDict: - return await _projects_service.get_project_for_user( + return await projects_service.get_project_for_user( app=app, project_uuid=function_uuid, user_id=user_id ) diff --git a/services/web/server/src/simcore_service_webserver/functions/plugin.py b/services/web/server/src/simcore_service_webserver/functions/plugin.py index 364436f4898..8d18e55a034 100644 --- a/services/web/server/src/simcore_service_webserver/functions/plugin.py +++ b/services/web/server/src/simcore_service_webserver/functions/plugin.py @@ -3,7 +3,7 @@ from aiohttp import web from servicelib.aiohttp.application_setup import ModuleCategory, app_module_setup -from . import _controller_rpc +from . import _functions_controller_rpc _logger = logging.getLogger(__name__) @@ -15,4 +15,4 @@ logger=_logger, ) def setup_functions(app: web.Application): - app.on_startup.append(_controller_rpc.register_rpc_routes_on_startup) + app.on_startup.append(_functions_controller_rpc.register_rpc_routes_on_startup) From 01709827652450060e7026690b8e12cf099fc635 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Wed, 16 Apr 2025 11:33:23 +0200 Subject: [PATCH 04/69] Add db migration script for functions api --- .../93dbd49553ae_add_function_tables.py | 133 ++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/93dbd49553ae_add_function_tables.py diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/93dbd49553ae_add_function_tables.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/93dbd49553ae_add_function_tables.py new file mode 100644 index 00000000000..d44b8e271e2 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/93dbd49553ae_add_function_tables.py @@ -0,0 +1,133 @@ +"""Add function tables + +Revision ID: 93dbd49553ae +Revises: cf8f743fd0b7 +Create Date: 2025-04-16 09:32:48.976846+00:00 + +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "93dbd49553ae" +down_revision = "cf8f743fd0b7" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "function_job_collections", + sa.Column("uuid", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("name", sa.String(), nullable=True), + sa.PrimaryKeyConstraint("uuid", name="function_job_collections_pk"), + ) + op.create_index( + op.f("ix_function_job_collections_uuid"), + "function_job_collections", + ["uuid"], + unique=False, + ) + op.create_table( + "functions", + sa.Column("uuid", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("title", sa.String(), nullable=True), + sa.Column("function_class", sa.String(), nullable=True), + sa.Column("description", sa.String(), nullable=True), + sa.Column("input_schema", sa.JSON(), nullable=True), + sa.Column("output_schema", sa.JSON(), nullable=True), + sa.Column("system_tags", sa.JSON(), nullable=True), + sa.Column("user_tags", sa.JSON(), nullable=True), + sa.Column("class_specific_data", sa.JSON(), nullable=True), + sa.PrimaryKeyConstraint("uuid", name="functions_pk"), + ) + op.create_index(op.f("ix_functions_uuid"), "functions", ["uuid"], unique=False) + op.create_table( + "function_jobs", + sa.Column("uuid", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("title", sa.String(), nullable=True), + sa.Column("function_uuid", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("function_class", sa.String(), nullable=True), + sa.Column("status", sa.String(), nullable=True), + sa.Column("inputs", sa.JSON(), nullable=True), + sa.Column("outputs", sa.JSON(), nullable=True), + sa.Column("class_specific_data", sa.JSON(), nullable=True), + sa.ForeignKeyConstraint( + ["function_uuid"], + ["functions.uuid"], + name="fk_functions_to_function_jobs_to_function_uuid", + onupdate="CASCADE", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("uuid", name="function_jobs_pk"), + ) + op.create_index( + op.f("ix_function_jobs_function_uuid"), + "function_jobs", + ["function_uuid"], + unique=False, + ) + op.create_index( + op.f("ix_function_jobs_uuid"), "function_jobs", ["uuid"], unique=False + ) + op.create_table( + "function_job_collections_to_function_jobs", + sa.Column( + "function_job_collection_uuid", postgresql.UUID(as_uuid=True), nullable=True + ), + sa.Column("function_job_uuid", postgresql.UUID(as_uuid=True), nullable=True), + sa.ForeignKeyConstraint( + ["function_job_collection_uuid"], + ["function_job_collections.uuid"], + name="fk_func_job_coll_to_func_jobs_to_func_job_coll_uuid", + onupdate="CASCADE", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["function_job_uuid"], + ["function_jobs.uuid"], + name="fk_func_job_coll_to_func_jobs_to_func_job_uuid", + onupdate="CASCADE", + ondelete="CASCADE", + ), + ) + op.drop_index("idx_projects_last_change_date_desc", table_name="projects") + op.create_index( + "idx_projects_last_change_date_desc", + "projects", + ["last_change_date"], + unique=False, + postgresql_using="btree", + postgresql_ops={"last_change_date": "DESC"}, + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index( + "idx_projects_last_change_date_desc", + table_name="projects", + postgresql_using="btree", + postgresql_ops={"last_change_date": "DESC"}, + ) + op.create_index( + "idx_projects_last_change_date_desc", + "projects", + [sa.text("last_change_date DESC")], + unique=False, + ) + op.drop_table("function_job_collections_to_function_jobs") + op.drop_index(op.f("ix_function_jobs_uuid"), table_name="function_jobs") + op.drop_index(op.f("ix_function_jobs_function_uuid"), table_name="function_jobs") + op.drop_table("function_jobs") + op.drop_index(op.f("ix_functions_uuid"), table_name="functions") + op.drop_table("functions") + op.drop_index( + op.f("ix_function_job_collections_uuid"), table_name="function_job_collections" + ) + op.drop_table("function_job_collections") + # ### end Alembic commands ### From acdcfd40b1636924c27295aea988b318af8c86e4 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Wed, 16 Apr 2025 17:57:43 +0200 Subject: [PATCH 05/69] Add solver functions --- .../api_schemas_api_server/functions.py | 44 ---- .../functions_wb_schema.py | 30 ++- services/api-server/openapi.json | 234 ++++++++++++++++-- .../api/routes/functions_routes.py | 85 ++++++- .../api/routes/solvers_jobs.py | 2 +- .../models/schemas/functions_api_schema.py | 85 ------- .../functions/_functions_controller_rpc.py | 124 ++++++++-- 7 files changed, 416 insertions(+), 188 deletions(-) delete mode 100644 packages/models-library/src/models_library/api_schemas_api_server/functions.py delete mode 100644 services/api-server/src/simcore_service_api_server/models/schemas/functions_api_schema.py diff --git a/packages/models-library/src/models_library/api_schemas_api_server/functions.py b/packages/models-library/src/models_library/api_schemas_api_server/functions.py deleted file mode 100644 index 44678efd539..00000000000 --- a/packages/models-library/src/models_library/api_schemas_api_server/functions.py +++ /dev/null @@ -1,44 +0,0 @@ -from typing import Annotated, Any, Literal, TypeAlias - -from models_library import projects -from pydantic import BaseModel, Field - -FunctionID: TypeAlias = projects.ProjectID - - -class FunctionSchema(BaseModel): - schema_dict: dict[str, Any] | None # JSON Schema - - -class FunctionInputSchema(FunctionSchema): ... - - -class FunctionOutputSchema(FunctionSchema): ... - - -class Function(BaseModel): - uid: FunctionID | None = None - title: str | None = None - description: str | None = None - input_schema: FunctionInputSchema | None = None - output_schema: FunctionOutputSchema | None = None - - # @classmethod - # def compose_resource_name(cls, function_key) -> api_resources.RelativeResourceName: - # return api_resources.compose_resource_name("functions", function_key) - - -class StudyFunction(Function): - function_type: Literal["study"] = "study" - study_url: str - - -class PythonCodeFunction(Function): - function_type: Literal["python_code"] = "python_code" - code_url: str - - -FunctionUnion: TypeAlias = Annotated[ - StudyFunction | PythonCodeFunction, - Field(discriminator="function_type"), -] diff --git a/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py b/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py index 4391a77a658..a17a31ee113 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py @@ -3,7 +3,9 @@ from uuid import UUID from models_library import projects -from pydantic import BaseModel, Field +from models_library.basic_regex import SIMPLE_VERSION_RE +from models_library.services_regex import COMPUTATIONAL_SERVICE_KEY_RE +from pydantic import BaseModel, Field, StringConstraints from ..projects import ProjectID @@ -26,6 +28,7 @@ class FunctionOutputSchema(FunctionSchema): ... class FunctionClass(str, Enum): project = "project" + solver = "solver" python_code = "python_code" @@ -75,13 +78,28 @@ class ProjectFunction(FunctionBase): project_id: ProjectID +SolverKeyId = Annotated[ + str, StringConstraints(strip_whitespace=True, pattern=COMPUTATIONAL_SERVICE_KEY_RE) +] +VersionStr: TypeAlias = Annotated[ + str, StringConstraints(strip_whitespace=True, pattern=SIMPLE_VERSION_RE) +] +SolverJobID: TypeAlias = UUID + + +class SolverFunction(FunctionBase): + function_class: Literal[FunctionClass.solver] = FunctionClass.solver + solver_key: SolverKeyId + solver_version: str + + class PythonCodeFunction(FunctionBase): function_class: Literal[FunctionClass.python_code] = FunctionClass.python_code code_url: str Function: TypeAlias = Annotated[ - ProjectFunction | PythonCodeFunction, + ProjectFunction | PythonCodeFunction | SolverFunction, Field(discriminator="function_class"), ] @@ -103,13 +121,17 @@ class ProjectFunctionJob(FunctionJobBase): project_job_id: ProjectID +class SolverFunctionJob(FunctionJobBase): + function_class: Literal[FunctionClass.solver] = FunctionClass.solver + solver_job_id: ProjectID + + class PythonCodeFunctionJob(FunctionJobBase): function_class: Literal[FunctionClass.python_code] = FunctionClass.python_code - code_url: str FunctionJob: TypeAlias = Annotated[ - ProjectFunctionJob | PythonCodeFunctionJob, + ProjectFunctionJob | PythonCodeFunctionJob | SolverFunctionJob, Field(discriminator="function_class"), ] diff --git a/services/api-server/openapi.json b/services/api-server/openapi.json index 3196a2cee37..2d22489c953 100644 --- a/services/api-server/openapi.json +++ b/services/api-server/openapi.json @@ -5316,13 +5316,17 @@ }, { "$ref": "#/components/schemas/PythonCodeFunction" + }, + { + "$ref": "#/components/schemas/SolverFunction" } ], "discriminator": { "propertyName": "function_class", "mapping": { "project": "#/components/schemas/ProjectFunction", - "python_code": "#/components/schemas/PythonCodeFunction" + "python_code": "#/components/schemas/PythonCodeFunction", + "solver": "#/components/schemas/SolverFunction" } } }, @@ -5351,6 +5355,9 @@ }, { "$ref": "#/components/schemas/PythonCodeFunction" + }, + { + "$ref": "#/components/schemas/SolverFunction" } ], "title": "Function", @@ -5358,7 +5365,8 @@ "propertyName": "function_class", "mapping": { "project": "#/components/schemas/ProjectFunction", - "python_code": "#/components/schemas/PythonCodeFunction" + "python_code": "#/components/schemas/PythonCodeFunction", + "solver": "#/components/schemas/SolverFunction" } } } @@ -5378,6 +5386,9 @@ }, { "$ref": "#/components/schemas/PythonCodeFunction" + }, + { + "$ref": "#/components/schemas/SolverFunction" } ], "title": "Response Register Function V0 Functions Post", @@ -5385,7 +5396,8 @@ "propertyName": "function_class", "mapping": { "project": "#/components/schemas/ProjectFunction", - "python_code": "#/components/schemas/PythonCodeFunction" + "python_code": "#/components/schemas/PythonCodeFunction", + "solver": "#/components/schemas/SolverFunction" } } } @@ -5437,13 +5449,17 @@ }, { "$ref": "#/components/schemas/PythonCodeFunction" + }, + { + "$ref": "#/components/schemas/SolverFunction" } ], "discriminator": { "propertyName": "function_class", "mapping": { "project": "#/components/schemas/ProjectFunction", - "python_code": "#/components/schemas/PythonCodeFunction" + "python_code": "#/components/schemas/PythonCodeFunction", + "solver": "#/components/schemas/SolverFunction" } }, "title": "Response Get Function V0 Functions Function Id Get" @@ -5504,13 +5520,17 @@ }, { "$ref": "#/components/schemas/PythonCodeFunction" + }, + { + "$ref": "#/components/schemas/SolverFunction" } ], "discriminator": { "propertyName": "function_class", "mapping": { "project": "#/components/schemas/ProjectFunction", - "python_code": "#/components/schemas/PythonCodeFunction" + "python_code": "#/components/schemas/PythonCodeFunction", + "solver": "#/components/schemas/SolverFunction" } }, "title": "Response Delete Function V0 Functions Function Id Delete" @@ -5596,13 +5616,17 @@ }, { "$ref": "#/components/schemas/PythonCodeFunctionJob" + }, + { + "$ref": "#/components/schemas/SolverFunctionJob" } ], "discriminator": { "propertyName": "function_class", "mapping": { "project": "#/components/schemas/ProjectFunctionJob", - "python_code": "#/components/schemas/PythonCodeFunctionJob" + "python_code": "#/components/schemas/PythonCodeFunctionJob", + "solver": "#/components/schemas/SolverFunctionJob" } }, "title": "Response Run Function V0 Functions Function Id Run Post" @@ -5801,13 +5825,17 @@ }, { "$ref": "#/components/schemas/PythonCodeFunctionJob" + }, + { + "$ref": "#/components/schemas/SolverFunctionJob" } ], "discriminator": { "propertyName": "function_class", "mapping": { "project": "#/components/schemas/ProjectFunctionJob", - "python_code": "#/components/schemas/PythonCodeFunctionJob" + "python_code": "#/components/schemas/PythonCodeFunctionJob", + "solver": "#/components/schemas/SolverFunctionJob" } } }, @@ -5860,13 +5888,17 @@ }, { "$ref": "#/components/schemas/PythonCodeFunctionJob" + }, + { + "$ref": "#/components/schemas/SolverFunctionJob" } ], "discriminator": { "propertyName": "function_class", "mapping": { "project": "#/components/schemas/ProjectFunctionJob", - "python_code": "#/components/schemas/PythonCodeFunctionJob" + "python_code": "#/components/schemas/PythonCodeFunctionJob", + "solver": "#/components/schemas/SolverFunctionJob" } } }, @@ -5895,6 +5927,9 @@ }, { "$ref": "#/components/schemas/PythonCodeFunctionJob" + }, + { + "$ref": "#/components/schemas/SolverFunctionJob" } ], "title": "Function Job", @@ -5902,7 +5937,8 @@ "propertyName": "function_class", "mapping": { "project": "#/components/schemas/ProjectFunctionJob", - "python_code": "#/components/schemas/PythonCodeFunctionJob" + "python_code": "#/components/schemas/PythonCodeFunctionJob", + "solver": "#/components/schemas/SolverFunctionJob" } } } @@ -5922,6 +5958,9 @@ }, { "$ref": "#/components/schemas/PythonCodeFunctionJob" + }, + { + "$ref": "#/components/schemas/SolverFunctionJob" } ], "title": "Response Register Function Job V0 Function Jobs Post", @@ -5929,7 +5968,8 @@ "propertyName": "function_class", "mapping": { "project": "#/components/schemas/ProjectFunctionJob", - "python_code": "#/components/schemas/PythonCodeFunctionJob" + "python_code": "#/components/schemas/PythonCodeFunctionJob", + "solver": "#/components/schemas/SolverFunctionJob" } } } @@ -5981,13 +6021,17 @@ }, { "$ref": "#/components/schemas/PythonCodeFunctionJob" + }, + { + "$ref": "#/components/schemas/SolverFunctionJob" } ], "discriminator": { "propertyName": "function_class", "mapping": { "project": "#/components/schemas/ProjectFunctionJob", - "python_code": "#/components/schemas/PythonCodeFunctionJob" + "python_code": "#/components/schemas/PythonCodeFunctionJob", + "solver": "#/components/schemas/SolverFunctionJob" } }, "title": "Response Get Function Job V0 Function Jobs Function Job Id Get" @@ -9055,16 +9099,11 @@ "const": "python_code", "title": "Function Class", "default": "python_code" - }, - "code_url": { - "type": "string", - "title": "Code Url" } }, "type": "object", "required": [ - "function_uid", - "code_url" + "function_uid" ], "title": "PythonCodeFunctionJob" }, @@ -9205,6 +9244,167 @@ "version": "2.1.1" } }, + "SolverFunction": { + "properties": { + "uid": { + "anyOf": [ + { + "type": "string", + "format": "uuid" + }, + { + "type": "null" + } + ], + "title": "Uid" + }, + "title": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Title" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "function_class": { + "type": "string", + "const": "solver", + "title": "Function Class", + "default": "solver" + }, + "input_schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/FunctionInputSchema" + }, + { + "type": "null" + } + ] + }, + "output_schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/FunctionOutputSchema" + }, + { + "type": "null" + } + ] + }, + "solver_key": { + "type": "string", + "pattern": "^simcore/services/comp/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", + "title": "Solver Key" + }, + "solver_version": { + "type": "string", + "title": "Solver Version" + } + }, + "type": "object", + "required": [ + "solver_key", + "solver_version" + ], + "title": "SolverFunction" + }, + "SolverFunctionJob": { + "properties": { + "uid": { + "anyOf": [ + { + "type": "string", + "format": "uuid" + }, + { + "type": "null" + } + ], + "title": "Uid" + }, + "title": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Title" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "function_uid": { + "type": "string", + "format": "uuid", + "title": "Function Uid" + }, + "inputs": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Inputs" + }, + "outputs": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Outputs" + }, + "function_class": { + "type": "string", + "const": "solver", + "title": "Function Class", + "default": "solver" + }, + "solver_job_id": { + "type": "string", + "format": "uuid", + "title": "Solver Job Id" + } + }, + "type": "object", + "required": [ + "function_uid", + "solver_job_id" + ], + "title": "SolverFunctionJob" + }, "SolverPort": { "properties": { "key": { diff --git a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py index dc2a80abcf5..e94bf7fd450 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py @@ -15,6 +15,7 @@ FunctionOutputs, FunctionOutputSchema, ProjectFunctionJob, + SolverFunctionJob, ) from pydantic import PositiveInt from servicelib.fastapi.dependencies import get_reverse_url_mapper @@ -23,17 +24,19 @@ from ...models.schemas.jobs import ( JobInputs, ) +from ...services_http.catalog import CatalogApi from ...services_http.director_v2 import DirectorV2Api from ...services_http.storage import StorageApi from ...services_http.webserver import AuthSession from ...services_rpc.wb_api_server import WbApiRpcClient from ..dependencies.authentication import get_current_user_id, get_product_name +from ..dependencies.database import Engine, get_db_engine from ..dependencies.services import get_api_client from ..dependencies.webserver_http import get_webserver_session from ..dependencies.webserver_rpc import ( get_wb_api_rpc_client, ) -from . import studies_jobs +from . import solvers_jobs, solvers_jobs_getters, studies_jobs function_router = APIRouter() function_job_router = APIRouter() @@ -98,6 +101,7 @@ async def run_function( function_inputs: FunctionInputs, user_id: Annotated[PositiveInt, Depends(get_current_user_id)], product_name: Annotated[str, Depends(get_product_name)], + catalog_client: Annotated[CatalogApi, Depends(get_api_client(CatalogApi))], ): to_run_function = await wb_api_rpc.get_function(function_id=function_id) @@ -141,7 +145,41 @@ async def run_function( project_job_id=study_job.id, ), ) - else: # noqa: RET505 + elif to_run_function.function_class == FunctionClass.solver: # noqa: RET505 + solver_job = await solvers_jobs.create_solver_job( + solver_key=to_run_function.solver_key, + version=to_run_function.solver_version, + inputs=JobInputs(values=function_inputs or {}), + webserver_api=webserver_api, + wb_api_rpc=wb_api_rpc, + url_for=url_for, + x_simcore_parent_project_uuid=None, + x_simcore_parent_node_id=None, + user_id=user_id, + product_name=product_name, + catalog_client=catalog_client, + ) + await solvers_jobs.start_job( + request=request, + solver_key=to_run_function.solver_key, + version=to_run_function.solver_version, + job_id=solver_job.id, + user_id=user_id, + webserver_api=webserver_api, + director2_api=director2_api, + ) + return await register_function_job( + wb_api_rpc=wb_api_rpc, + function_job=SolverFunctionJob( + function_uid=to_run_function.uid, + title=f"Function job of function {to_run_function.uid}", + description=to_run_function.description, + inputs=function_inputs, + outputs=None, + solver_job_id=solver_job.id, + ), + ) + else: msg = f"Function type {type(to_run_function)} not supported" raise TypeError(msg) @@ -276,12 +314,23 @@ async def function_job_status( ): job_status = await studies_jobs.inspect_study_job( study_id=function.project_id, - job_id=function_job.project_job_id, + job_id=function_job.project_job_id, # type: ignore + user_id=user_id, + director2_api=director2_api, + ) + return FunctionJobStatus(status=job_status.state) + elif (function.function_class == FunctionClass.solver) and ( # noqa: RET505 + function_job.function_class == FunctionClass.solver + ): + job_status = await solvers_jobs.inspect_job( + solver_key=function.solver_key, + version=function.solver_version, + job_id=function_job.solver_job_id, user_id=user_id, director2_api=director2_api, ) return FunctionJobStatus(status=job_status.state) - else: # noqa: RET505 + else: msg = f"Function type {function.function_class} / Function job type {function_job.function_class} not supported" raise TypeError(msg) @@ -298,27 +347,41 @@ async def function_job_outputs( user_id: Annotated[PositiveInt, Depends(get_current_user_id)], storage_client: Annotated[StorageApi, Depends(get_api_client(StorageApi))], wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], + db_engine: Annotated[Engine, Depends(get_db_engine)], ): function, function_job = await get_function_from_functionjobid( wb_api_rpc=wb_api_rpc, function_job_id=function_job_id ) if ( - function.function_class != FunctionClass.project - or function_job.function_class != FunctionClass.project + function.function_class == FunctionClass.project + and function_job.function_class == FunctionClass.project ): - msg = f"Function type {function.function_class} not supported" - raise TypeError(msg) - else: # noqa: RET506 job_outputs = await studies_jobs.get_study_job_outputs( study_id=function.project_id, - job_id=function_job.project_job_id, + job_id=function_job.project_job_id, # type: ignore user_id=user_id, webserver_api=webserver_api, storage_client=storage_client, ) return job_outputs.results + elif (function.function_class == FunctionClass.solver) and ( # noqa: RET505 + function_job.function_class == FunctionClass.solver + ): + job_outputs = await solvers_jobs_getters.get_job_outputs( + solver_key=function.solver_key, + version=function.solver_version, + job_id=function_job.solver_job_id, + user_id=user_id, + webserver_api=webserver_api, + storage_client=storage_client, + db_engine=db_engine, + ) + return job_outputs.results + else: + msg = f"Function type {function.function_class} not supported" + raise TypeError(msg) @function_router.post( @@ -337,6 +400,7 @@ async def map_function( director2_api: Annotated[DirectorV2Api, Depends(get_api_client(DirectorV2Api))], user_id: Annotated[PositiveInt, Depends(get_current_user_id)], product_name: Annotated[str, Depends(get_product_name)], + catalog_client: Annotated[CatalogApi, Depends(get_api_client(CatalogApi))], ): function_jobs = [] for function_inputs in function_inputs_list: @@ -351,6 +415,7 @@ async def map_function( url_for=url_for, director2_api=director2_api, request=request, + catalog_client=catalog_client, ) for function_inputs in function_inputs_list ] diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py index 7b724de24dc..4490db6e628 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py @@ -86,7 +86,7 @@ def compose_job_resource_name(solver_key, solver_version, job_id) -> str: status_code=status.HTTP_201_CREATED, responses=JOBS_STATUS_CODES, ) -async def create_solver_job( +async def create_solver_job( # noqa: PLR0913 solver_key: SolverKeyId, version: VersionStr, inputs: JobInputs, diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/functions_api_schema.py b/services/api-server/src/simcore_service_api_server/models/schemas/functions_api_schema.py deleted file mode 100644 index ba316c0e88d..00000000000 --- a/services/api-server/src/simcore_service_api_server/models/schemas/functions_api_schema.py +++ /dev/null @@ -1,85 +0,0 @@ -from enum import Enum -from typing import Annotated, Any, Literal, TypeAlias - -from models_library import projects -from pydantic import BaseModel, Field - -FunctionID: TypeAlias = projects.ProjectID - - -class FunctionSchema(BaseModel): - schema_dict: dict[str, Any] | None # JSON Schema - - -class FunctionInputSchema(FunctionSchema): ... - - -class FunctionOutputSchema(FunctionSchema): ... - - -class FunctionClass(str, Enum): - project = "project" - python_code = "python_code" - - -class FunctionInputs(BaseModel): - inputs_dict: dict[str, Any] | None # JSON Schema - - -class FunctionOutputs(BaseModel): - outputs_dict: dict[str, Any] | None # JSON Schema - - -class Function(BaseModel): - uid: FunctionID | None = None - title: str | None = None - description: str | None = None - input_schema: FunctionInputSchema | None = None - output_schema: FunctionOutputSchema | None = None - - -class StudyFunction(Function): - function_type: Literal["study"] = "study" - study_url: str - - -class PythonCodeFunction(Function): - function_type: Literal["python_code"] = "python_code" - code_url: str - - -FunctionUnion: TypeAlias = Annotated[ - StudyFunction | PythonCodeFunction, - Field(discriminator="function_type"), -] - -FunctionJobID: TypeAlias = projects.ProjectID -FunctionJobCollectionID: TypeAlias = projects.ProjectID - - -class FunctionJob(BaseModel): - uid: FunctionJobID - title: str | None - description: str | None - status: str - function_uid: FunctionID - inputs: FunctionInputs | None - outputs: FunctionOutputs | None - - -class FunctionJobStatus(BaseModel): - status: str - - -class FunctionJobCollection(BaseModel): - """Model for a collection of function jobs""" - - id: FunctionJobCollectionID - title: str | None - description: str | None - job_ids: list[FunctionJobID] - status: str - - -class FunctionJobCollectionStatus(BaseModel): - status: list[str] diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py index 3a689d37ab9..eb826058ec2 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py @@ -15,6 +15,8 @@ FunctionOutputSchema, ProjectFunction, ProjectFunctionJob, + SolverFunction, + SolverFunctionJob, ) from servicelib.rabbitmq import RPCRouter @@ -57,7 +59,33 @@ async def register_function(app: web.Application, *, function: Function) -> Func output_schema=saved_function.output_schema, project_id=saved_function.class_specific_data["project_id"], ) - else: # noqa: RET505 + elif function.function_class == FunctionClass.solver: # noqa: RET505 + saved_function = await _functions_repository.create_function( + app=app, + function=FunctionDB( + title=function.title, + description=function.description, + input_schema=function.input_schema, + output_schema=function.output_schema, + function_class=function.function_class, + class_specific_data=FunctionClassSpecificData( + { + "solver_key": str(function.solver_key), + "solver_version": str(function.solver_version), + } + ), + ), + ) + return SolverFunction( + uid=saved_function.uuid, + title=saved_function.title, + description=saved_function.description, + input_schema=saved_function.input_schema, + output_schema=saved_function.output_schema, + solver_key=saved_function.class_specific_data["solver_key"], + solver_version=saved_function.class_specific_data["solver_version"], + ) + else: msg = f"Unsupported function class: {function.function_class}" raise TypeError(msg) @@ -74,7 +102,17 @@ def _decode_function( output_schema=function.output_schema, project_id=function.class_specific_data["project_id"], ) - else: # noqa: RET505 + elif function.function_class == "solver": # noqa: RET505 + return SolverFunction( + uid=function.uuid, + title=function.title, + description=function.description, + input_schema=function.input_schema, + output_schema=function.output_schema, + solver_key=function.class_specific_data["solver_key"], + solver_version=function.class_specific_data["solver_version"], + ) + else: msg = f"Unsupported function class: [{function.function_class}]" raise TypeError(msg) @@ -91,6 +129,34 @@ async def get_function(app: web.Application, *, function_id: FunctionID) -> Func ) +def convert_functionjobdb_to_functionjob( + functionjob_db: FunctionJobDB, +) -> FunctionJob: + if functionjob_db.function_class == FunctionClass.project: + return ProjectFunctionJob( + uid=functionjob_db.uuid, + title=functionjob_db.title, + description="", + function_uid=functionjob_db.function_uuid, + inputs=functionjob_db.inputs, + outputs=None, + project_job_id=functionjob_db.class_specific_data["project_job_id"], + ) + elif functionjob_db.function_class == FunctionClass.solver: # noqa: RET505 + return SolverFunctionJob( + uid=functionjob_db.uuid, + title=functionjob_db.title, + description="", + function_uid=functionjob_db.function_uuid, + inputs=functionjob_db.inputs, + outputs=None, + solver_job_id=functionjob_db.class_specific_data["solver_job_id"], + ) + else: + msg = f"Unsupported function class: [{functionjob_db.function_class}]" + raise TypeError(msg) + + @router.expose() async def get_function_job( app: web.Application, *, function_job_id: FunctionJobID @@ -102,19 +168,7 @@ async def get_function_job( ) assert returned_function_job is not None - if returned_function_job.function_class == FunctionClass.project: - return ProjectFunctionJob( - uid=returned_function_job.uuid, - title=returned_function_job.title, - description="", - function_uid=returned_function_job.function_uuid, - inputs=returned_function_job.inputs, - outputs=None, - project_job_id=returned_function_job.class_specific_data["project_job_id"], - ) - else: # noqa: RET505 - msg = f"Unsupported function class: [{returned_function_job.function_class}]" - raise TypeError(msg) + return convert_functionjobdb_to_functionjob(returned_function_job) @router.expose() @@ -124,15 +178,7 @@ async def list_function_jobs(app: web.Application) -> list[FunctionJob]: app=app, ) return [ - ProjectFunctionJob( - uid=returned_function_job.uuid, - title=returned_function_job.title, - description="", - function_uid=returned_function_job.function_uuid, - inputs=returned_function_job.inputs, - outputs=None, - project_job_id=returned_function_job.class_specific_data["project_job_id"], - ) + convert_functionjobdb_to_functionjob(returned_function_job) for returned_function_job in returned_function_jobs ] @@ -208,13 +254,12 @@ async def register_function_job( outputs=None, class_specific_data=FunctionJobClassSpecificData( { - "project_job_id": str(function_job.project_job_id), + "project_job_id": str(function_job.project_job_id), # type: ignore } ), function_class=function_job.function_class, ), ) - return ProjectFunctionJob( uid=created_function_job_db.uuid, title=created_function_job_db.title, @@ -226,7 +271,32 @@ async def register_function_job( "project_job_id" ], ) - else: # noqa: RET505 + elif function_job.function_class == FunctionClass.solver: # noqa: RET505 + created_function_job_db = await _functions_repository.register_function_job( + app=app, + function_job=FunctionJobDB( + title=function_job.title, + function_uuid=function_job.function_uid, + inputs=function_job.inputs, + outputs=None, + class_specific_data=FunctionJobClassSpecificData( + { + "solver_job_id": str(function_job.solver_job_id), + } + ), + function_class=function_job.function_class, + ), + ) + return SolverFunctionJob( + uid=created_function_job_db.uuid, + title=created_function_job_db.title, + description="", + function_uid=created_function_job_db.function_uuid, + inputs=created_function_job_db.inputs, + outputs=None, + solver_job_id=created_function_job_db.class_specific_data["solver_job_id"], + ) + else: msg = f"Unsupported function class: [{function_job.function_class}]" raise TypeError(msg) From 8fbd814ca0215f554034f7b80b3c207008c8b8bf Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Wed, 16 Apr 2025 18:21:16 +0200 Subject: [PATCH 06/69] Add default inputs to functions --- .../functions_wb_schema.py | 6 +- .../models/functions_models_db.py | 6 + services/api-server/openapi.json | 69 +++++-- .../functions/_functions_controller_rpc.py | 192 ++++++++---------- 4 files changed, 142 insertions(+), 131 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py b/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py index a17a31ee113..5e0f30f6666 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py @@ -45,21 +45,23 @@ class FunctionClass(str, Enum): class FunctionBase(BaseModel): + function_class: FunctionClass uid: FunctionID | None = None title: str | None = None description: str | None = None - function_class: FunctionClass input_schema: FunctionInputSchema | None = None output_schema: FunctionOutputSchema | None = None + default_inputs: FunctionInputs | None = None class FunctionDB(BaseModel): + function_class: FunctionClass uuid: FunctionJobID | None = None title: str | None = None description: str | None = None - function_class: FunctionClass input_schema: FunctionInputSchema | None = None output_schema: FunctionOutputSchema | None = None + default_inputs: FunctionInputs | None = None class_specific_data: FunctionClassSpecificData diff --git a/packages/postgres-database/src/simcore_postgres_database/models/functions_models_db.py b/packages/postgres-database/src/simcore_postgres_database/models/functions_models_db.py index e8a8ba6f2ec..6d9b80b0509 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/functions_models_db.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/functions_models_db.py @@ -65,6 +65,12 @@ nullable=True, doc="Fields specific for a function class", ), + sa.Column( + "default_inputs", + sa.JSON, + nullable=True, + doc="Default inputs of the function", + ), sa.PrimaryKeyConstraint("uuid", name="functions_pk"), ) diff --git a/services/api-server/openapi.json b/services/api-server/openapi.json index 2d22489c953..e28cd250482 100644 --- a/services/api-server/openapi.json +++ b/services/api-server/openapi.json @@ -8804,6 +8804,12 @@ }, "ProjectFunction": { "properties": { + "function_class": { + "type": "string", + "const": "project", + "title": "Function Class", + "default": "project" + }, "uid": { "anyOf": [ { @@ -8838,12 +8844,6 @@ ], "title": "Description" }, - "function_class": { - "type": "string", - "const": "project", - "title": "Function Class", - "default": "project" - }, "input_schema": { "anyOf": [ { @@ -8864,6 +8864,17 @@ } ] }, + "default_inputs": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Default Inputs" + }, "project_id": { "type": "string", "format": "uuid", @@ -8960,6 +8971,12 @@ }, "PythonCodeFunction": { "properties": { + "function_class": { + "type": "string", + "const": "python_code", + "title": "Function Class", + "default": "python_code" + }, "uid": { "anyOf": [ { @@ -8994,12 +9011,6 @@ ], "title": "Description" }, - "function_class": { - "type": "string", - "const": "python_code", - "title": "Function Class", - "default": "python_code" - }, "input_schema": { "anyOf": [ { @@ -9020,6 +9031,17 @@ } ] }, + "default_inputs": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Default Inputs" + }, "code_url": { "type": "string", "title": "Code Url" @@ -9246,6 +9268,12 @@ }, "SolverFunction": { "properties": { + "function_class": { + "type": "string", + "const": "solver", + "title": "Function Class", + "default": "solver" + }, "uid": { "anyOf": [ { @@ -9280,12 +9308,6 @@ ], "title": "Description" }, - "function_class": { - "type": "string", - "const": "solver", - "title": "Function Class", - "default": "solver" - }, "input_schema": { "anyOf": [ { @@ -9306,6 +9328,17 @@ } ] }, + "default_inputs": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Default Inputs" + }, "solver_key": { "type": "string", "pattern": "^simcore/services/comp/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py index eb826058ec2..64bf15a364a 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py @@ -35,59 +35,10 @@ async def ping(app: web.Application) -> str: @router.expose() async def register_function(app: web.Application, *, function: Function) -> Function: assert app - if function.function_class == FunctionClass.project: - saved_function = await _functions_repository.create_function( - app=app, - function=FunctionDB( - title=function.title, - description=function.description, - input_schema=function.input_schema, - output_schema=function.output_schema, - function_class=function.function_class, - class_specific_data=FunctionClassSpecificData( - { - "project_id": str(function.project_id), - } - ), - ), - ) - return ProjectFunction( - uid=saved_function.uuid, - title=saved_function.title, - description=saved_function.description, - input_schema=saved_function.input_schema, - output_schema=saved_function.output_schema, - project_id=saved_function.class_specific_data["project_id"], - ) - elif function.function_class == FunctionClass.solver: # noqa: RET505 - saved_function = await _functions_repository.create_function( - app=app, - function=FunctionDB( - title=function.title, - description=function.description, - input_schema=function.input_schema, - output_schema=function.output_schema, - function_class=function.function_class, - class_specific_data=FunctionClassSpecificData( - { - "solver_key": str(function.solver_key), - "solver_version": str(function.solver_version), - } - ), - ), - ) - return SolverFunction( - uid=saved_function.uuid, - title=saved_function.title, - description=saved_function.description, - input_schema=saved_function.input_schema, - output_schema=saved_function.output_schema, - solver_key=saved_function.class_specific_data["solver_key"], - solver_version=saved_function.class_specific_data["solver_version"], - ) - else: - msg = f"Unsupported function class: {function.function_class}" - raise TypeError(msg) + saved_function = await _functions_repository.create_function( + app=app, function=_encode_function(function) + ) + return _decode_function(saved_function) def _decode_function( @@ -117,6 +68,42 @@ def _decode_function( raise TypeError(msg) +def _encode_function( + function: Function, +) -> FunctionDB: + if function.function_class == FunctionClass.project: + return FunctionDB( + title=function.title, + description=function.description, + input_schema=function.input_schema, + output_schema=function.output_schema, + function_class=function.function_class, + default_inputs=function.default_inputs, + class_specific_data=FunctionClassSpecificData( + { + "project_id": str(function.project_id), + } + ), + ) + elif function.function_class == FunctionClass.solver: # noqa: RET505 + return FunctionDB( + title=function.title, + description=function.description, + input_schema=function.input_schema, + output_schema=function.output_schema, + function_class=function.function_class, + class_specific_data=FunctionClassSpecificData( + { + "solver_key": str(function.solver_key), + "solver_version": str(function.solver_version), + } + ), + ) + else: + msg = f"Unsupported function class: {function.function_class}" + raise TypeError(msg) + + @router.expose() async def get_function(app: web.Application, *, function_id: FunctionID) -> Function: assert app @@ -129,7 +116,7 @@ async def get_function(app: web.Application, *, function_id: FunctionID) -> Func ) -def convert_functionjobdb_to_functionjob( +def _decode_functionjob( functionjob_db: FunctionJobDB, ) -> FunctionJob: if functionjob_db.function_class == FunctionClass.project: @@ -157,6 +144,40 @@ def convert_functionjobdb_to_functionjob( raise TypeError(msg) +def _encode_functionjob( + functionjob: FunctionJob, +) -> FunctionJobDB: + if functionjob.function_class == FunctionClass.project: + return FunctionJobDB( + title=functionjob.title, + function_uuid=functionjob.function_uid, + inputs=functionjob.inputs, + outputs=None, + class_specific_data=FunctionJobClassSpecificData( + { + "project_job_id": str(functionjob.project_job_id), + } + ), + function_class=functionjob.function_class, + ) + elif functionjob.function_class == FunctionClass.solver: # noqa: RET505 + return FunctionJobDB( + title=functionjob.title, + function_uuid=functionjob.function_uid, + inputs=functionjob.inputs, + outputs=None, + class_specific_data=FunctionJobClassSpecificData( + { + "solver_job_id": str(functionjob.solver_job_id), + } + ), + function_class=functionjob.function_class, + ) + else: + msg = f"Unsupported function class: [{functionjob.function_class}]" + raise TypeError(msg) + + @router.expose() async def get_function_job( app: web.Application, *, function_job_id: FunctionJobID @@ -168,7 +189,7 @@ async def get_function_job( ) assert returned_function_job is not None - return convert_functionjobdb_to_functionjob(returned_function_job) + return _decode_functionjob(returned_function_job) @router.expose() @@ -178,7 +199,7 @@ async def list_function_jobs(app: web.Application) -> list[FunctionJob]: app=app, ) return [ - convert_functionjobdb_to_functionjob(returned_function_job) + _decode_functionjob(returned_function_job) for returned_function_job in returned_function_jobs ] @@ -244,61 +265,10 @@ async def register_function_job( app: web.Application, *, function_job: FunctionJob ) -> FunctionJob: assert app - if function_job.function_class == FunctionClass.project: - created_function_job_db = await _functions_repository.register_function_job( - app=app, - function_job=FunctionJobDB( - title=function_job.title, - function_uuid=function_job.function_uid, - inputs=function_job.inputs, - outputs=None, - class_specific_data=FunctionJobClassSpecificData( - { - "project_job_id": str(function_job.project_job_id), # type: ignore - } - ), - function_class=function_job.function_class, - ), - ) - return ProjectFunctionJob( - uid=created_function_job_db.uuid, - title=created_function_job_db.title, - description="", - function_uid=created_function_job_db.function_uuid, - inputs=created_function_job_db.inputs, - outputs=None, - project_job_id=created_function_job_db.class_specific_data[ - "project_job_id" - ], - ) - elif function_job.function_class == FunctionClass.solver: # noqa: RET505 - created_function_job_db = await _functions_repository.register_function_job( - app=app, - function_job=FunctionJobDB( - title=function_job.title, - function_uuid=function_job.function_uid, - inputs=function_job.inputs, - outputs=None, - class_specific_data=FunctionJobClassSpecificData( - { - "solver_job_id": str(function_job.solver_job_id), - } - ), - function_class=function_job.function_class, - ), - ) - return SolverFunctionJob( - uid=created_function_job_db.uuid, - title=created_function_job_db.title, - description="", - function_uid=created_function_job_db.function_uuid, - inputs=created_function_job_db.inputs, - outputs=None, - solver_job_id=created_function_job_db.class_specific_data["solver_job_id"], - ) - else: - msg = f"Unsupported function class: [{function_job.function_class}]" - raise TypeError(msg) + created_function_job_db = await _functions_repository.register_function_job( + app=app, function_job=_encode_functionjob(function_job) + ) + return _decode_functionjob(created_function_job_db) @router.expose() From b5cf30a859640ef1bee8c2bc44c2e99479653dca Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Wed, 16 Apr 2025 19:36:21 +0200 Subject: [PATCH 07/69] Add default inputs to functions --- ...94af8f28b25_add_function_default_inputs.py | 49 +++++++++++++++++++ .../api/routes/functions_routes.py | 29 +++++++++-- .../functions/_functions_controller_rpc.py | 45 ++++++++--------- .../functions/_functions_repository.py | 1 + 4 files changed, 94 insertions(+), 30 deletions(-) create mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/d94af8f28b25_add_function_default_inputs.py diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/d94af8f28b25_add_function_default_inputs.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/d94af8f28b25_add_function_default_inputs.py new file mode 100644 index 00000000000..621916c8233 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/d94af8f28b25_add_function_default_inputs.py @@ -0,0 +1,49 @@ +"""Add function default inputs + +Revision ID: d94af8f28b25 +Revises: 93dbd49553ae +Create Date: 2025-04-16 16:23:12.224948+00:00 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "d94af8f28b25" +down_revision = "93dbd49553ae" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column("functions", sa.Column("default_inputs", sa.JSON(), nullable=True)) + op.drop_index("idx_projects_last_change_date_desc", table_name="projects") + op.create_index( + "idx_projects_last_change_date_desc", + "projects", + ["last_change_date"], + unique=False, + postgresql_using="btree", + postgresql_ops={"last_change_date": "DESC"}, + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index( + "idx_projects_last_change_date_desc", + table_name="projects", + postgresql_using="btree", + postgresql_ops={"last_change_date": "DESC"}, + ) + op.create_index( + "idx_projects_last_change_date_desc", + "projects", + [sa.text("last_change_date DESC")], + unique=False, + ) + op.drop_column("functions", "default_inputs") + # ### end Alembic commands ### diff --git a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py index e94bf7fd450..ec0059d3d88 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py @@ -85,6 +85,20 @@ async def get_function( return await wb_api_rpc.get_function(function_id=function_id) +def join_inputs( + default_inputs: FunctionInputs | None, + function_inputs: FunctionInputs | None, +) -> FunctionInputs: + if default_inputs is None: + return function_inputs + + if function_inputs is None: + return default_inputs + + # last dict will override defaults + return {**default_inputs, **function_inputs} + + @function_router.post( "/{function_id:uuid}:run", response_model=FunctionJob, @@ -108,16 +122,21 @@ async def run_function( assert to_run_function.uid is not None + joined_inputs = join_inputs( + to_run_function.default_inputs, + function_inputs, + ) + if cached_function_job := await wb_api_rpc.find_cached_function_job( function_id=to_run_function.uid, - inputs=function_inputs, + inputs=joined_inputs, ): return cached_function_job if to_run_function.function_class == FunctionClass.project: study_job = await studies_jobs.create_study_job( study_id=to_run_function.project_id, - job_inputs=JobInputs(values=function_inputs or {}), + job_inputs=JobInputs(values=joined_inputs or {}), webserver_api=webserver_api, wb_api_rpc=wb_api_rpc, url_for=url_for, @@ -140,7 +159,7 @@ async def run_function( function_uid=to_run_function.uid, title=f"Function job of function {to_run_function.uid}", description=to_run_function.description, - inputs=function_inputs, + inputs=joined_inputs, outputs=None, project_job_id=study_job.id, ), @@ -149,7 +168,7 @@ async def run_function( solver_job = await solvers_jobs.create_solver_job( solver_key=to_run_function.solver_key, version=to_run_function.solver_version, - inputs=JobInputs(values=function_inputs or {}), + inputs=JobInputs(values=joined_inputs or {}), webserver_api=webserver_api, wb_api_rpc=wb_api_rpc, url_for=url_for, @@ -174,7 +193,7 @@ async def run_function( function_uid=to_run_function.uid, title=f"Function job of function {to_run_function.uid}", description=to_run_function.description, - inputs=function_inputs, + inputs=joined_inputs, outputs=None, solver_job_id=solver_job.id, ), diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py index 64bf15a364a..53b3aad7f2b 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py @@ -52,6 +52,7 @@ def _decode_function( input_schema=function.input_schema, output_schema=function.output_schema, project_id=function.class_specific_data["project_id"], + default_inputs=function.default_inputs, ) elif function.function_class == "solver": # noqa: RET505 return SolverFunction( @@ -62,6 +63,7 @@ def _decode_function( output_schema=function.output_schema, solver_key=function.class_specific_data["solver_key"], solver_version=function.class_specific_data["solver_version"], + default_inputs=function.default_inputs, ) else: msg = f"Unsupported function class: [{function.function_class}]" @@ -72,37 +74,30 @@ def _encode_function( function: Function, ) -> FunctionDB: if function.function_class == FunctionClass.project: - return FunctionDB( - title=function.title, - description=function.description, - input_schema=function.input_schema, - output_schema=function.output_schema, - function_class=function.function_class, - default_inputs=function.default_inputs, - class_specific_data=FunctionClassSpecificData( - { - "project_id": str(function.project_id), - } - ), + class_specific_data = FunctionClassSpecificData( + {"project_id": str(function.project_id)} ) - elif function.function_class == FunctionClass.solver: # noqa: RET505 - return FunctionDB( - title=function.title, - description=function.description, - input_schema=function.input_schema, - output_schema=function.output_schema, - function_class=function.function_class, - class_specific_data=FunctionClassSpecificData( - { - "solver_key": str(function.solver_key), - "solver_version": str(function.solver_version), - } - ), + elif function.function_class == FunctionClass.solver: + class_specific_data = FunctionClassSpecificData( + { + "solver_key": str(function.solver_key), + "solver_version": str(function.solver_version), + } ) else: msg = f"Unsupported function class: {function.function_class}" raise TypeError(msg) + return FunctionDB( + title=function.title, + description=function.description, + input_schema=function.input_schema, + output_schema=function.output_schema, + function_class=function.function_class, + default_inputs=function.default_inputs, + class_specific_data=class_specific_data, + ) + @router.expose() async def get_function(app: web.Application, *, function_id: FunctionID) -> Function: diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py index 495f93c0c25..ada1980f4f1 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py @@ -53,6 +53,7 @@ async def create_function( ), function_class=function.function_class, class_specific_data=function.class_specific_data, + default_inputs=function.default_inputs, ) .returning(*_FUNCTIONS_TABLE_COLS) ) From 4e1ac9a9d48034f636e8fbff9131a5ca5140eb8c Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Tue, 29 Apr 2025 13:11:24 +0200 Subject: [PATCH 08/69] Add function collections --- .../functions_wb_schema.py | 11 +- .../models/functions_models_db.py | 9 +- .../functions/functions_rpc_interface.py | 51 +++ services/api-server/openapi.json | 400 +++++++++++++++++- .../api/routes/functions_routes.py | 259 +++++++----- .../services_rpc/wb_api_server.py | 38 ++ .../functions/_functions_controller_rpc.py | 71 ++++ .../functions/_functions_repository.py | 131 ++++++ 8 files changed, 833 insertions(+), 137 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py b/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py index 5e0f30f6666..53362663569 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py @@ -145,11 +145,18 @@ class FunctionJobStatus(BaseModel): class FunctionJobCollection(BaseModel): """Model for a collection of function jobs""" - id: FunctionJobCollectionID + uid: FunctionJobCollectionID | None title: str | None description: str | None job_ids: list[FunctionJobID] - status: str + + +class FunctionJobCollectionDB(BaseModel): + """Model for a collection of function jobs""" + + uuid: FunctionJobCollectionID | None + title: str | None + description: str | None class FunctionJobCollectionStatus(BaseModel): diff --git a/packages/postgres-database/src/simcore_postgres_database/models/functions_models_db.py b/packages/postgres-database/src/simcore_postgres_database/models/functions_models_db.py index 6d9b80b0509..bfaabb39372 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/functions_models_db.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/functions_models_db.py @@ -143,9 +143,14 @@ doc="Unique id of the function job collection", ), sa.Column( - "name", + "title", + sa.String, + doc="Title of the function job collection", + ), + sa.Column( + "description", sa.String, - doc="Name of the function job collection", + doc="Description of the function job collection", ), sa.PrimaryKeyConstraint("uuid", name="function_job_collections_pk"), ) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py index 7e06bef912c..7bee95d0601 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py @@ -9,6 +9,8 @@ FunctionInputs, FunctionInputSchema, FunctionJob, + FunctionJobCollection, + FunctionJobCollectionID, FunctionJobID, FunctionOutputSchema, ) @@ -185,3 +187,52 @@ async def find_cached_function_job( function_id=function_id, inputs=inputs, ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def list_function_job_collections( + rabbitmq_rpc_client: RabbitMQRPCClient, +) -> list[FunctionJobCollection]: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("list_function_job_collections"), + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def register_function_job_collection( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + function_job_collection: FunctionJobCollection, +) -> FunctionJobCollection: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("register_function_job_collection"), + function_job_collection=function_job_collection, + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def get_function_job_collection( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + function_job_collection_id: FunctionJobCollectionID, +) -> FunctionJobCollection: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("get_function_job_collection"), + function_job_collection_id=function_job_collection_id, + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def delete_function_job_collection( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + function_job_collection_id: FunctionJobCollectionID, +) -> None: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("delete_function_job_collection"), + function_job_collection_id=function_job_collection_id, + ) diff --git a/services/api-server/openapi.json b/services/api-server/openapi.json index e28cd250482..8d3abb3289a 100644 --- a/services/api-server/openapi.json +++ b/services/api-server/openapi.json @@ -5817,29 +5817,7 @@ "content": { "application/json": { "schema": { - "type": "array", - "items": { - "oneOf": [ - { - "$ref": "#/components/schemas/ProjectFunctionJob" - }, - { - "$ref": "#/components/schemas/PythonCodeFunctionJob" - }, - { - "$ref": "#/components/schemas/SolverFunctionJob" - } - ], - "discriminator": { - "propertyName": "function_class", - "mapping": { - "project": "#/components/schemas/ProjectFunctionJob", - "python_code": "#/components/schemas/PythonCodeFunctionJob", - "solver": "#/components/schemas/SolverFunctionJob" - } - } - }, - "title": "Response Map Function V0 Functions Function Id Map Post" + "$ref": "#/components/schemas/FunctionJobCollection" } } } @@ -6238,6 +6216,311 @@ } } }, + "/v0/function_job_collections": { + "get": { + "tags": [ + "function_job_collections" + ], + "summary": "List Function Job Collections", + "description": "List function job collections", + "operationId": "list_function_job_collections", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/FunctionJobCollection" + }, + "type": "array", + "title": "Response List Function Job Collections V0 Function Job Collections Get" + } + } + } + } + } + }, + "post": { + "tags": [ + "function_job_collections" + ], + "summary": "Register Function Job Collection", + "description": "Register function job collection", + "operationId": "register_function_job_collection", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FunctionJobCollection" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FunctionJobCollection" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/function_job_collections/{function_job_collection_id}": { + "get": { + "tags": [ + "function_job_collections" + ], + "summary": "Get Function Job Collection", + "description": "Get function job collection", + "operationId": "get_function_job_collection", + "parameters": [ + { + "name": "function_job_collection_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Job Collection Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FunctionJobCollection" + } + } + } + }, + "404": { + "description": "Function job collection not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "delete": { + "tags": [ + "function_job_collections" + ], + "summary": "Delete Function Job Collection", + "description": "Delete function job collection", + "operationId": "delete_function_job_collection", + "parameters": [ + { + "name": "function_job_collection_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Job Collection Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "404": { + "description": "Function job collection not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/function_job_collections/{function_job_collection_id}/function_jobs": { + "get": { + "tags": [ + "function_job_collections" + ], + "summary": "Function Job Collection List Function Jobs", + "description": "Get the function jobs in function job collection", + "operationId": "function_job_collection_list_function_jobs", + "parameters": [ + { + "name": "function_job_collection_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Job Collection Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "oneOf": [ + { + "$ref": "#/components/schemas/ProjectFunctionJob" + }, + { + "$ref": "#/components/schemas/PythonCodeFunctionJob" + }, + { + "$ref": "#/components/schemas/SolverFunctionJob" + } + ], + "discriminator": { + "propertyName": "function_class", + "mapping": { + "project": "#/components/schemas/ProjectFunctionJob", + "python_code": "#/components/schemas/PythonCodeFunctionJob", + "solver": "#/components/schemas/SolverFunctionJob" + } + } + }, + "title": "Response Function Job Collection List Function Jobs V0 Function Job Collections Function Job Collection Id Function Jobs Get" + } + } + } + }, + "404": { + "description": "Function job collection not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/function_job_collections/{function_job_collection_id}/status": { + "get": { + "tags": [ + "function_job_collections" + ], + "summary": "Function Job Collection Status", + "description": "Get function job collection status", + "operationId": "function_job_collection_status", + "security": [ + { + "HTTPBasic": [] + } + ], + "parameters": [ + { + "name": "function_job_collection_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Job Collection Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FunctionJobCollectionStatus" + } + } + } + }, + "404": { + "description": "Function job collection not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, "/v0/wallets/default": { "get": { "tags": [ @@ -7234,6 +7517,77 @@ ], "title": "FunctionInputSchema" }, + "FunctionJobCollection": { + "properties": { + "uid": { + "anyOf": [ + { + "type": "string", + "format": "uuid" + }, + { + "type": "null" + } + ], + "title": "Uid" + }, + "title": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Title" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "job_ids": { + "items": { + "type": "string", + "format": "uuid" + }, + "type": "array", + "title": "Job Ids" + } + }, + "type": "object", + "required": [ + "uid", + "title", + "description", + "job_ids" + ], + "title": "FunctionJobCollection", + "description": "Model for a collection of function jobs" + }, + "FunctionJobCollectionStatus": { + "properties": { + "status": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Status" + } + }, + "type": "object", + "required": [ + "status" + ], + "title": "FunctionJobCollectionStatus" + }, "FunctionJobStatus": { "properties": { "status": { diff --git a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py index ec0059d3d88..b8aeacc7c75 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py @@ -1,3 +1,4 @@ +import asyncio from collections.abc import Callable from typing import Annotated, Final @@ -10,6 +11,9 @@ FunctionInputSchema, FunctionInputsList, FunctionJob, + FunctionJobCollection, + FunctionJobCollectionID, + FunctionJobCollectionStatus, FunctionJobID, FunctionJobStatus, FunctionOutputs, @@ -405,7 +409,7 @@ async def function_job_outputs( @function_router.post( "/{function_id:uuid}:map", - response_model=list[FunctionJob], + response_model=FunctionJobCollection, responses={**_COMMON_FUNCTION_ERROR_RESPONSES}, description="Map function over input parameters", ) @@ -422,116 +426,151 @@ async def map_function( catalog_client: Annotated[CatalogApi, Depends(get_api_client(CatalogApi))], ): function_jobs = [] - for function_inputs in function_inputs_list: - function_jobs = [ - await run_function( + function_jobs = [ + await run_function( + wb_api_rpc=wb_api_rpc, + function_id=function_id, + function_inputs=function_inputs, + product_name=product_name, + user_id=user_id, + webserver_api=webserver_api, + url_for=url_for, + director2_api=director2_api, + request=request, + catalog_client=catalog_client, + ) + for function_inputs in function_inputs_list + ] + + assert all( + function_job.uid is not None for function_job in function_jobs + ), "Function job uid should not be None" + + return await register_function_job_collection( + wb_api_rpc=wb_api_rpc, + function_job_collection=FunctionJobCollection( + id=None, + title=f"Function job collection of function map {[function_job.uid for function_job in function_jobs]}", + description="", + job_ids=[function_job.uid for function_job in function_jobs], # type: ignore + ), + ) + + +_COMMON_FUNCTION_JOB_COLLECTION_ERROR_RESPONSES: Final[dict] = { + status.HTTP_404_NOT_FOUND: { + "description": "Function job collection not found", + "model": ErrorGet, + }, +} + + +@function_job_collections_router.get( + "", + response_model=list[FunctionJobCollection], + description="List function job collections", +) +async def list_function_job_collections( + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + return await wb_api_rpc.list_function_job_collections() + + +@function_job_collections_router.get( + "/{function_job_collection_id:uuid}", + response_model=FunctionJobCollection, + responses={**_COMMON_FUNCTION_JOB_COLLECTION_ERROR_RESPONSES}, + description="Get function job collection", +) +async def get_function_job_collection( + function_job_collection_id: FunctionJobCollectionID, + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + return await wb_api_rpc.get_function_job_collection( + function_job_collection_id=function_job_collection_id + ) + + +@function_job_collections_router.post( + "", + response_model=FunctionJobCollection, + description="Register function job collection", +) +async def register_function_job_collection( + function_job_collection: FunctionJobCollection, + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + return await wb_api_rpc.register_function_job_collection( + function_job_collection=function_job_collection + ) + + +@function_job_collections_router.delete( + "/{function_job_collection_id:uuid}", + response_model=None, + responses={**_COMMON_FUNCTION_JOB_COLLECTION_ERROR_RESPONSES}, + description="Delete function job collection", +) +async def delete_function_job_collection( + function_job_collection_id: FunctionJobCollectionID, + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + return await wb_api_rpc.delete_function_job_collection( + function_job_collection_id=function_job_collection_id + ) + + +@function_job_collections_router.get( + "/{function_job_collection_id:uuid}/function_jobs", + response_model=list[FunctionJob], + responses={**_COMMON_FUNCTION_JOB_COLLECTION_ERROR_RESPONSES}, + description="Get the function jobs in function job collection", +) +async def function_job_collection_list_function_jobs( + function_job_collection_id: FunctionJobCollectionID, + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + function_job_collection = await get_function_job_collection( + function_job_collection_id=function_job_collection_id, + wb_api_rpc=wb_api_rpc, + ) + return [ + await get_function_job( + job_id, + wb_api_rpc=wb_api_rpc, + ) + for job_id in function_job_collection.job_ids + ] + + +@function_job_collections_router.get( + "/{function_job_collection_id:uuid}/status", + response_model=FunctionJobCollectionStatus, + responses={**_COMMON_FUNCTION_JOB_COLLECTION_ERROR_RESPONSES}, + description="Get function job collection status", +) +async def function_job_collection_status( + function_job_collection_id: FunctionJobCollectionID, + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], + director2_api: Annotated[DirectorV2Api, Depends(get_api_client(DirectorV2Api))], + user_id: Annotated[PositiveInt, Depends(get_current_user_id)], +): + function_job_collection = await get_function_job_collection( + function_job_collection_id=function_job_collection_id, + wb_api_rpc=wb_api_rpc, + ) + + job_statuses = await asyncio.gather( + *[ + function_job_status( + job_id, wb_api_rpc=wb_api_rpc, - function_id=function_id, - function_inputs=function_inputs, - product_name=product_name, - user_id=user_id, - webserver_api=webserver_api, - url_for=url_for, director2_api=director2_api, - request=request, - catalog_client=catalog_client, + user_id=user_id, ) - for function_inputs in function_inputs_list + for job_id in function_job_collection.job_ids ] - # TODO poor system can't handle doing this in parallel, get this fixed # noqa: FIX002 - # function_jobs = await asyncio.gather(*function_jobs_tasks) - - return function_jobs - - -# ruff: noqa: ERA001 - - -# _logger = logging.getLogger(__name__) - -# _COMMON_FUNCTION_JOB_COLLECTION_ERROR_RESPONSES: Final[dict] = { -# status.HTTP_404_NOT_FOUND: { -# "description": "Function job collection not found", -# "model": ErrorGet, -# }, -# } - - -# @function_job_collections_router.get( -# "", -# response_model=FunctionJobCollection, -# description="List function job collections", -# ) -# async def list_function_job_collections( -# page_params: Annotated[PaginationParams, Depends()], -# webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], -# ): -# msg = "list function jobs collection not implemented yet" -# raise NotImplementedError(msg) - - -# @function_job_collections_router.post( -# "", response_model=FunctionJobCollection, description="Create function job" -# ) -# async def create_function_job_collection( -# webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], -# job_ids: Annotated[list[FunctionJob], Depends()], -# ): -# msg = "create function job collection not implemented yet" -# raise NotImplementedError(msg) - - -# @function_job_collections_router.get( -# "/{function_job_collection_id:uuid}", -# response_model=FunctionJobCollection, -# responses={**_COMMON_FUNCTION_JOB_COLLECTION_ERROR_RESPONSES}, -# description="Get function job", -# ) -# async def get_function_job_collection( -# function_job_collection_id: FunctionJobCollectionID, -# webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], -# ): -# msg = "get function job collection not implemented yet" -# raise NotImplementedError(msg) - - -# @function_job_collections_router.delete( -# "/{function_job_collection_id:uuid}", -# response_model=FunctionJob, -# responses={**_COMMON_FUNCTION_JOB_COLLECTION_ERROR_RESPONSES}, -# description="Delete function job collection", -# ) -# async def delete_function_job_collection( -# function_job_collection_id: FunctionJobCollectionID, -# webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], -# ): -# msg = "delete function job collection not implemented yet" -# raise NotImplementedError(msg) - - -# @function_job_collections_router.get( -# "/{function_job_collection_id:uuid}/function_jobs", -# response_model=list[FunctionJob], -# responses={**_COMMON_FUNCTION_JOB_COLLECTION_ERROR_RESPONSES}, -# description="Get the function jobs in function job collection", -# ) -# async def function_job_collection_list_function_jobs( -# function_job_collection_id: FunctionJobCollectionID, -# webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], -# ): -# msg = "function job collection listing not implemented yet" -# raise NotImplementedError(msg) - - -# @function_job_collections_router.get( -# "/{function_job_collection_id:uuid}/status", -# response_model=FunctionJobCollectionStatus, -# responses={**_COMMON_FUNCTION_JOB_COLLECTION_ERROR_RESPONSES}, -# description="Get function job collection status", -# ) -# async def function_job_collection_status( -# function_job_collection_id: FunctionJobCollectionID, -# webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], -# ): -# msg = "function job collection status not implemented yet" -# raise NotImplementedError(msg) + ) + return FunctionJobCollectionStatus( + status=[job_status.status for job_status in job_statuses] + ) diff --git a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py index 80139346f25..c5a35499ef2 100644 --- a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py +++ b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py @@ -10,6 +10,8 @@ FunctionInputs, FunctionInputSchema, FunctionJob, + FunctionJobCollection, + FunctionJobCollectionID, FunctionJobID, FunctionOutputSchema, ) @@ -44,6 +46,9 @@ from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( delete_function_job as _delete_function_job, ) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + delete_function_job_collection as _delete_function_job_collection, +) from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( find_cached_function_job as _find_cached_function_job, ) @@ -56,9 +61,15 @@ from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( get_function_job as _get_function_job, ) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + get_function_job_collection as _get_function_job_collection, +) from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( get_function_output_schema as _get_function_output_schema, ) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + list_function_job_collections as _list_function_job_collections, +) from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( list_function_jobs as _list_function_jobs, ) @@ -74,6 +85,9 @@ from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( register_function_job as _register_function_job, ) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + register_function_job_collection as _register_function_job_collection, +) from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( run_function as _run_function, ) @@ -341,6 +355,30 @@ async def find_cached_function_job( async def list_function_jobs(self) -> list[FunctionJob]: return await _list_function_jobs(self._client) + async def list_function_job_collections(self) -> list[FunctionJobCollection]: + return await _list_function_job_collections(self._client) + + async def get_function_job_collection( + self, *, function_job_collection_id: FunctionJobCollectionID + ) -> FunctionJobCollection: + return await _get_function_job_collection( + self._client, function_job_collection_id=function_job_collection_id + ) + + async def register_function_job_collection( + self, *, function_job_collection: FunctionJobCollection + ) -> FunctionJobCollection: + return await _register_function_job_collection( + self._client, function_job_collection=function_job_collection + ) + + async def delete_function_job_collection( + self, *, function_job_collection_id: FunctionJobCollectionID + ) -> None: + return await _delete_function_job_collection( + self._client, function_job_collection_id=function_job_collection_id + ) + def setup(app: FastAPI, rabbitmq_rmp_client: RabbitMQRPCClient): wb_api_rpc_client = WbApiRpcClient(_client=rabbitmq_rmp_client) diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py index 53b3aad7f2b..c59c444d384 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py @@ -10,6 +10,7 @@ FunctionInputSchema, FunctionJob, FunctionJobClassSpecificData, + FunctionJobCollection, FunctionJobDB, FunctionJobID, FunctionOutputSchema, @@ -292,6 +293,76 @@ async def find_cached_function_job( raise TypeError(msg) +@router.expose() +async def list_function_job_collections( + app: web.Application, +) -> list[FunctionJobCollection]: + assert app + returned_function_job_collections = ( + await _functions_repository.list_function_job_collections( + app=app, + ) + ) + return [ + FunctionJobCollection( + uid=function_job_collection.uuid, + title=function_job_collection.title, + description=function_job_collection.description, + job_ids=job_ids, + ) + for function_job_collection, job_ids in returned_function_job_collections + ] + + +@router.expose() +async def register_function_job_collection( + app: web.Application, *, function_job_collection: FunctionJobCollection +) -> FunctionJobCollection: + assert app + registered_function_job_collection, registered_job_ids = ( + await _functions_repository.register_function_job_collection( + app=app, + function_job_collection=function_job_collection, + ) + ) + return FunctionJobCollection( + uid=registered_function_job_collection.uuid, + title=registered_function_job_collection.title, + description=registered_function_job_collection.description, + job_ids=registered_job_ids, + ) + + +@router.expose() +async def get_function_job_collection( + app: web.Application, *, function_job_collection_id: FunctionJobID +) -> FunctionJobCollection: + assert app + returned_function_job_collection, job_ids = ( + await _functions_repository.get_function_job_collection( + app=app, + function_job_collection_id=function_job_collection_id, + ) + ) + return FunctionJobCollection( + uid=returned_function_job_collection.uuid, + title=returned_function_job_collection.title, + description=returned_function_job_collection.description, + job_ids=job_ids, + ) + + +@router.expose() +async def delete_function_job_collection( + app: web.Application, *, function_job_collection_id: FunctionJobID +) -> None: + assert app + await _functions_repository.delete_function_job_collection( + app=app, + function_job_collection_id=function_job_collection_id, + ) + + async def register_rpc_routes_on_startup(app: web.Application): rpc_server = get_rabbitmq_rpc_server(app) await rpc_server.register_router(router, WEBSERVER_RPC_NAMESPACE, app) diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py index ada1980f4f1..d2397851233 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py @@ -5,7 +5,16 @@ FunctionDB, FunctionID, FunctionInputs, + FunctionJobCollection, + FunctionJobCollectionDB, FunctionJobDB, + FunctionJobID, +) +from simcore_postgres_database.models.functions_models_db import ( + function_job_collections as function_job_collections_table, +) +from simcore_postgres_database.models.functions_models_db import ( + function_job_collections_to_function_jobs as function_job_collections_to_function_jobs_table, ) from simcore_postgres_database.models.functions_models_db import ( function_jobs as function_jobs_table, @@ -26,6 +35,9 @@ _FUNCTION_JOBS_TABLE_COLS = get_columns_from_db_model( function_jobs_table, FunctionJobDB ) +_FUNCTION_JOB_COLLECTIONS_TABLE_COLS = get_columns_from_db_model( + function_jobs_table, FunctionJobCollectionDB +) async def create_function( @@ -205,3 +217,122 @@ async def find_cached_function_job( return job return None + + +async def list_function_job_collections( + app: web.Application, + connection: AsyncConnection | None = None, +) -> list[tuple[FunctionJobCollectionDB, list[FunctionJobID]]]: + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream(function_job_collections_table.select().where()) + rows = await result.all() + if rows is None: + return [] + + collections = [] + for row in rows: + collection = FunctionJobCollection.model_validate(dict(row)) + job_result = await conn.stream( + function_job_collections_to_function_jobs_table.select().where( + function_job_collections_to_function_jobs_table.c.function_job_collection_uuid + == row["uuid"] + ) + ) + job_rows = await job_result.all() + job_ids = ( + [job_row["function_job_uuid"] for job_row in job_rows] + if job_rows + else [] + ) + collections.append((collection, job_ids)) + return collections + + +async def get_function_job_collection( + app: web.Application, + connection: AsyncConnection | None = None, + *, + function_job_collection_id: FunctionID, +) -> tuple[FunctionJobCollectionDB, list[FunctionJobID]]: + + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream( + function_job_collections_table.select().where( + function_job_collections_table.c.uuid == function_job_collection_id + ) + ) + row = await result.first() + + if row is None: + msg = f"No function job collection found with id {function_job_collection_id}." + raise web.HTTPNotFound(reason=msg) + + # Retrieve associated job ids from the join table + job_result = await conn.stream( + function_job_collections_to_function_jobs_table.select().where( + function_job_collections_to_function_jobs_table.c.function_job_collection_uuid + == row["uuid"] + ) + ) + job_rows = await job_result.all() + + job_ids = ( + [job_row["function_job_uuid"] for job_row in job_rows] if job_rows else [] + ) + + job_collection = FunctionJobCollectionDB.model_validate(dict(row)) + return job_collection, job_ids + + +async def register_function_job_collection( + app: web.Application, + connection: AsyncConnection | None = None, + *, + function_job_collection: FunctionJobCollection, +) -> tuple[FunctionJobCollectionDB, list[FunctionJobID]]: + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream( + function_job_collections_table.insert() + .values( + title=function_job_collection.title, + description=function_job_collection.description, + ) + .returning(*_FUNCTION_JOB_COLLECTIONS_TABLE_COLS) + ) + row = await result.first() + + if row is None: + msg = "No row was returned from the database after creating function job collection." + raise ValueError(msg) + + for job_id in function_job_collection.job_ids: + await conn.execute( + function_job_collections_to_function_jobs_table.insert().values( + function_job_collection_uuid=row["uuid"], + function_job_uuid=job_id, + ) + ) + + job_collection = FunctionJobCollectionDB.model_validate(dict(row)) + return job_collection, function_job_collection.job_ids + + +async def delete_function_job_collection( + app: web.Application, + connection: AsyncConnection | None = None, + *, + function_job_collection_id: FunctionID, +) -> None: + + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + await conn.execute( + function_job_collections_table.delete().where( + function_job_collections_table.c.uuid == function_job_collection_id + ) + ) + await conn.execute( + function_job_collections_to_function_jobs_table.delete().where( + function_job_collections_to_function_jobs_table.c.function_job_collection_uuid + == function_job_collection_id + ) + ) From 9fabebfd72649e423a13805e871c4c9bfb79e001 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Tue, 29 Apr 2025 14:45:13 +0200 Subject: [PATCH 09/69] Working project function job collection --- .../functions_wb_schema.py | 2 + services/api-server/openapi.json | 59 +++++++++++++++++++ .../api/routes/functions_routes.py | 51 +++++++++++++++- .../functions/_functions_repository.py | 2 +- 4 files changed, 110 insertions(+), 4 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py b/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py index 53362663569..94b47183b37 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py @@ -43,6 +43,8 @@ class FunctionClass(str, Enum): FunctionOutputs: TypeAlias = dict[str, Any] | None +FunctionOutputsLogfile: TypeAlias = Any + class FunctionBase(BaseModel): function_class: FunctionClass diff --git a/services/api-server/openapi.json b/services/api-server/openapi.json index 8d3abb3289a..f85b6502196 100644 --- a/services/api-server/openapi.json +++ b/services/api-server/openapi.json @@ -6216,6 +6216,65 @@ } } }, + "/v0/function_jobs/{function_job_id}/outputs/logfile": { + "get": { + "tags": [ + "function_jobs" + ], + "summary": "Function Job Logfile", + "description": "Get function job outputs", + "operationId": "function_job_logfile", + "security": [ + { + "HTTPBasic": [] + } + ], + "parameters": [ + { + "name": "function_job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Job Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "title": "Response Function Job Logfile V0 Function Jobs Function Job Id Outputs Logfile Get" + } + } + } + }, + "404": { + "description": "Function job not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, "/v0/function_job_collections": { "get": { "tags": [ diff --git a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py index b8aeacc7c75..f8335552cfc 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py @@ -446,12 +446,13 @@ async def map_function( function_job.uid is not None for function_job in function_jobs ), "Function job uid should not be None" + function_job_collection_description = f"Function job collection of map of function {function_id} with {len(function_inputs_list)} inputs" return await register_function_job_collection( wb_api_rpc=wb_api_rpc, function_job_collection=FunctionJobCollection( - id=None, - title=f"Function job collection of function map {[function_job.uid for function_job in function_jobs]}", - description="", + uid=None, + title="Function job collection of function map", + description=function_job_collection_description, job_ids=[function_job.uid for function_job in function_jobs], # type: ignore ), ) @@ -574,3 +575,47 @@ async def function_job_collection_status( return FunctionJobCollectionStatus( status=[job_status.status for job_status in job_statuses] ) + + +# @function_job_router.get( +# "/{function_job_id:uuid}/outputs/logfile", +# response_model=FunctionOutputsLogfile, +# responses={**_COMMON_FUNCTION_JOB_ERROR_RESPONSES}, +# description="Get function job outputs", +# ) +# async def function_job_logfile( +# function_job_id: FunctionJobID, +# user_id: Annotated[PositiveInt, Depends(get_current_user_id)], +# wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +# director2_api: Annotated[DirectorV2Api, Depends(get_api_client(DirectorV2Api))], +# ): +# function, function_job = await get_function_from_functionjobid( +# wb_api_rpc=wb_api_rpc, function_job_id=function_job_id +# ) + +# if ( +# function.function_class == FunctionClass.project +# and function_job.function_class == FunctionClass.project +# ): +# job_outputs = await studies_jobs.get_study_job_output_logfile( +# study_id=function.project_id, +# job_id=function_job.project_job_id, # type: ignore +# user_id=user_id, +# director2_api=director2_api, +# ) + +# return job_outputs +# elif (function.function_class == FunctionClass.solver) and ( # noqa: RET505 +# function_job.function_class == FunctionClass.solver +# ): +# job_outputs_logfile = await solvers_jobs_getters.get_job_output_logfile( +# director2_api=director2_api, +# solver_key=function.solver_key, +# version=function.solver_version, +# job_id=function_job.solver_job_id, +# user_id=user_id, +# ) +# return job_outputs_logfile +# else: +# msg = f"Function type {function.function_class} not supported" +# raise TypeError(msg) diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py index d2397851233..f9d0cce1c71 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py @@ -36,7 +36,7 @@ function_jobs_table, FunctionJobDB ) _FUNCTION_JOB_COLLECTIONS_TABLE_COLS = get_columns_from_db_model( - function_jobs_table, FunctionJobCollectionDB + function_job_collections_table, FunctionJobCollectionDB ) From 3c9f819cc05470f03a9b8941ab3d89550a01d897 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Tue, 29 Apr 2025 15:20:29 +0200 Subject: [PATCH 10/69] Add db migration for job collections --- ...b7f433b_fix_function_job_collections_db.py | 60 +++++++++++++++++++ .../functions/_functions_controller_rpc.py | 14 ++++- 2 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/0b64fb7f433b_fix_function_job_collections_db.py diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/0b64fb7f433b_fix_function_job_collections_db.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/0b64fb7f433b_fix_function_job_collections_db.py new file mode 100644 index 00000000000..2fb484396f8 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/0b64fb7f433b_fix_function_job_collections_db.py @@ -0,0 +1,60 @@ +"""Fix function job collections db + +Revision ID: 0b64fb7f433b +Revises: d94af8f28b25 +Create Date: 2025-04-29 11:12:23.529262+00:00 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "0b64fb7f433b" +down_revision = "d94af8f28b25" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "function_job_collections", sa.Column("title", sa.String(), nullable=True) + ) + op.add_column( + "function_job_collections", sa.Column("description", sa.String(), nullable=True) + ) + op.drop_column("function_job_collections", "name") + op.drop_index("idx_projects_last_change_date_desc", table_name="projects") + op.create_index( + "idx_projects_last_change_date_desc", + "projects", + ["last_change_date"], + unique=False, + postgresql_using="btree", + postgresql_ops={"last_change_date": "DESC"}, + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index( + "idx_projects_last_change_date_desc", + table_name="projects", + postgresql_using="btree", + postgresql_ops={"last_change_date": "DESC"}, + ) + op.create_index( + "idx_projects_last_change_date_desc", + "projects", + [sa.text("last_change_date DESC")], + unique=False, + ) + op.add_column( + "function_job_collections", + sa.Column("name", sa.VARCHAR(), autoincrement=False, nullable=True), + ) + op.drop_column("function_job_collections", "description") + op.drop_column("function_job_collections", "title") + # ### end Alembic commands ### diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py index c59c444d384..da4737553d5 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py @@ -288,6 +288,16 @@ async def find_cached_function_job( outputs=None, project_job_id=returned_function_job.class_specific_data["project_job_id"], ) + elif returned_function_job.function_class == FunctionClass.solver: # noqa: RET505 + return SolverFunctionJob( + uid=returned_function_job.uuid, + title=returned_function_job.title, + description="", + function_uid=returned_function_job.function_uuid, + inputs=returned_function_job.inputs, + outputs=None, + solver_job_id=returned_function_job.class_specific_data["solver_job_id"], + ) else: # noqa: RET505 msg = f"Unsupported function class: [{returned_function_job.function_class}]" raise TypeError(msg) @@ -338,7 +348,7 @@ async def get_function_job_collection( app: web.Application, *, function_job_collection_id: FunctionJobID ) -> FunctionJobCollection: assert app - returned_function_job_collection, job_ids = ( + returned_function_job_collection, returned_job_ids = ( await _functions_repository.get_function_job_collection( app=app, function_job_collection_id=function_job_collection_id, @@ -348,7 +358,7 @@ async def get_function_job_collection( uid=returned_function_job_collection.uuid, title=returned_function_job_collection.title, description=returned_function_job_collection.description, - job_ids=job_ids, + job_ids=returned_job_ids, ) From 77e0a00b7c9c9e3b6e0a687aec875c78b5fb457f Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Tue, 29 Apr 2025 15:40:07 +0200 Subject: [PATCH 11/69] Adapt for changes in solver job api --- services/api-server/openapi.json | 59 ------------------- .../api/routes/functions_routes.py | 24 ++++---- 2 files changed, 14 insertions(+), 69 deletions(-) diff --git a/services/api-server/openapi.json b/services/api-server/openapi.json index f85b6502196..8d3abb3289a 100644 --- a/services/api-server/openapi.json +++ b/services/api-server/openapi.json @@ -6216,65 +6216,6 @@ } } }, - "/v0/function_jobs/{function_job_id}/outputs/logfile": { - "get": { - "tags": [ - "function_jobs" - ], - "summary": "Function Job Logfile", - "description": "Get function job outputs", - "operationId": "function_job_logfile", - "security": [ - { - "HTTPBasic": [] - } - ], - "parameters": [ - { - "name": "function_job_id", - "in": "path", - "required": true, - "schema": { - "type": "string", - "format": "uuid", - "title": "Function Job Id" - } - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "title": "Response Function Job Logfile V0 Function Jobs Function Job Id Outputs Logfile Get" - } - } - } - }, - "404": { - "description": "Function job not found", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorGet" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, "/v0/function_job_collections": { "get": { "tags": [ diff --git a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py index f8335552cfc..82bbf6ce9ff 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py @@ -23,18 +23,20 @@ ) from pydantic import PositiveInt from servicelib.fastapi.dependencies import get_reverse_url_mapper +from sqlalchemy.ext.asyncio import AsyncEngine +from ..._service_job import JobService +from ..._service_solvers import SolverService from ...models.schemas.errors import ErrorGet from ...models.schemas.jobs import ( JobInputs, ) -from ...services_http.catalog import CatalogApi from ...services_http.director_v2 import DirectorV2Api from ...services_http.storage import StorageApi from ...services_http.webserver import AuthSession from ...services_rpc.wb_api_server import WbApiRpcClient from ..dependencies.authentication import get_current_user_id, get_product_name -from ..dependencies.database import Engine, get_db_engine +from ..dependencies.database import get_db_asyncpg_engine from ..dependencies.services import get_api_client from ..dependencies.webserver_http import get_webserver_session from ..dependencies.webserver_rpc import ( @@ -119,7 +121,8 @@ async def run_function( function_inputs: FunctionInputs, user_id: Annotated[PositiveInt, Depends(get_current_user_id)], product_name: Annotated[str, Depends(get_product_name)], - catalog_client: Annotated[CatalogApi, Depends(get_api_client(CatalogApi))], + solver_service: Annotated[SolverService, Depends()], + job_service: Annotated[JobService, Depends()], ): to_run_function = await wb_api_rpc.get_function(function_id=function_id) @@ -173,14 +176,13 @@ async def run_function( solver_key=to_run_function.solver_key, version=to_run_function.solver_version, inputs=JobInputs(values=joined_inputs or {}), - webserver_api=webserver_api, - wb_api_rpc=wb_api_rpc, + solver_service=solver_service, + job_service=job_service, url_for=url_for, x_simcore_parent_project_uuid=None, x_simcore_parent_node_id=None, user_id=user_id, product_name=product_name, - catalog_client=catalog_client, ) await solvers_jobs.start_job( request=request, @@ -370,7 +372,7 @@ async def function_job_outputs( user_id: Annotated[PositiveInt, Depends(get_current_user_id)], storage_client: Annotated[StorageApi, Depends(get_api_client(StorageApi))], wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], - db_engine: Annotated[Engine, Depends(get_db_engine)], + async_pg_engine: Annotated[AsyncEngine, Depends(get_db_asyncpg_engine)], ): function, function_job = await get_function_from_functionjobid( wb_api_rpc=wb_api_rpc, function_job_id=function_job_id @@ -399,7 +401,7 @@ async def function_job_outputs( user_id=user_id, webserver_api=webserver_api, storage_client=storage_client, - db_engine=db_engine, + async_pg_engine=async_pg_engine, ) return job_outputs.results else: @@ -423,7 +425,8 @@ async def map_function( director2_api: Annotated[DirectorV2Api, Depends(get_api_client(DirectorV2Api))], user_id: Annotated[PositiveInt, Depends(get_current_user_id)], product_name: Annotated[str, Depends(get_product_name)], - catalog_client: Annotated[CatalogApi, Depends(get_api_client(CatalogApi))], + solver_service: Annotated[SolverService, Depends()], + job_service: Annotated[JobService, Depends()], ): function_jobs = [] function_jobs = [ @@ -437,7 +440,8 @@ async def map_function( url_for=url_for, director2_api=director2_api, request=request, - catalog_client=catalog_client, + solver_service=solver_service, + job_service=job_service, ) for function_inputs in function_inputs_list ] From b8722cc2fe48c818c62d2815fd35051052c45221 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Tue, 29 Apr 2025 16:02:29 +0200 Subject: [PATCH 12/69] Db merge heads functions draft and master rebase --- ...3b85134_merge_742123f0933a_0b64fb7f433b.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/ecd7a3b85134_merge_742123f0933a_0b64fb7f433b.py diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/ecd7a3b85134_merge_742123f0933a_0b64fb7f433b.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/ecd7a3b85134_merge_742123f0933a_0b64fb7f433b.py new file mode 100644 index 00000000000..0118aaa4a77 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/ecd7a3b85134_merge_742123f0933a_0b64fb7f433b.py @@ -0,0 +1,21 @@ +"""merge 742123f0933a 0b64fb7f433b + +Revision ID: ecd7a3b85134 +Revises: 742123f0933a, 0b64fb7f433b +Create Date: 2025-04-29 13:40:28.311099+00:00 + +""" + +# revision identifiers, used by Alembic. +revision = "ecd7a3b85134" +down_revision = ("742123f0933a", "0b64fb7f433b") +branch_labels = None +depends_on = None + + +def upgrade(): + pass + + +def downgrade(): + pass From efdd470bbe09b42833e29f22f2f2765274ab0b07 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Wed, 30 Apr 2025 15:11:03 +0200 Subject: [PATCH 13/69] Add tests for functions api server --- .../functions_wb_schema.py | 2 +- services/api-server/openapi.json | 248 ++++--- .../api/routes/functions_routes.py | 132 ++-- .../test_api_routers_functions.py | 660 ++++++++++++++++++ 4 files changed, 895 insertions(+), 147 deletions(-) create mode 100644 services/api-server/tests/unit/api_functions/test_api_routers_functions.py diff --git a/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py b/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py index 94b47183b37..0961b75765b 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py @@ -147,7 +147,7 @@ class FunctionJobStatus(BaseModel): class FunctionJobCollection(BaseModel): """Model for a collection of function jobs""" - uid: FunctionJobCollectionID | None + uid: FunctionJobCollectionID | None = None title: str | None description: str | None job_ids: list[FunctionJobID] diff --git a/services/api-server/openapi.json b/services/api-server/openapi.json index 8d3abb3289a..a2a22f8d5e4 100644 --- a/services/api-server/openapi.json +++ b/services/api-server/openapi.json @@ -5276,25 +5276,6 @@ } } }, - "/v0/functions/ping": { - "post": { - "tags": [ - "functions" - ], - "summary": "Ping", - "operationId": "ping", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": {} - } - } - } - } - } - }, "/v0/functions": { "get": { "tags": [ @@ -5404,6 +5385,16 @@ } } }, + "404": { + "description": "Function not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, "422": { "description": "Validation Error", "content": { @@ -5513,28 +5504,7 @@ "description": "Successful Response", "content": { "application/json": { - "schema": { - "oneOf": [ - { - "$ref": "#/components/schemas/ProjectFunction" - }, - { - "$ref": "#/components/schemas/PythonCodeFunction" - }, - { - "$ref": "#/components/schemas/SolverFunction" - } - ], - "discriminator": { - "propertyName": "function_class", - "mapping": { - "project": "#/components/schemas/ProjectFunction", - "python_code": "#/components/schemas/PythonCodeFunction", - "solver": "#/components/schemas/SolverFunction" - } - }, - "title": "Response Delete Function V0 Functions Function Id Delete" - } + "schema": {} } } }, @@ -5561,19 +5531,14 @@ } } }, - "/v0/functions/{function_id}:run": { - "post": { + "/v0/functions/{function_id}/input_schema": { + "get": { "tags": [ "functions" ], - "summary": "Run Function", - "description": "Run function", - "operationId": "run_function", - "security": [ - { - "HTTPBasic": [] - } - ], + "summary": "Get Function Inputschema", + "description": "Get function input schema", + "operationId": "get_function_inputschema", "parameters": [ { "name": "function_id", @@ -5586,50 +5551,13 @@ } } ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "anyOf": [ - { - "type": "object" - }, - { - "type": "null" - } - ], - "title": "Function Inputs" - } - } - } - }, "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { - "oneOf": [ - { - "$ref": "#/components/schemas/ProjectFunctionJob" - }, - { - "$ref": "#/components/schemas/PythonCodeFunctionJob" - }, - { - "$ref": "#/components/schemas/SolverFunctionJob" - } - ], - "discriminator": { - "propertyName": "function_class", - "mapping": { - "project": "#/components/schemas/ProjectFunctionJob", - "python_code": "#/components/schemas/PythonCodeFunctionJob", - "solver": "#/components/schemas/SolverFunctionJob" - } - }, - "title": "Response Run Function V0 Functions Function Id Run Post" + "$ref": "#/components/schemas/FunctionInputSchema" } } } @@ -5657,14 +5585,14 @@ } } }, - "/v0/functions/{function_id}/input_schema": { + "/v0/functions/{function_id}/output_schema": { "get": { "tags": [ "functions" ], - "summary": "Get Function Input Schema", - "description": "Get function", - "operationId": "get_function_input_schema", + "summary": "Get Function Outputschema", + "description": "Get function input schema", + "operationId": "get_function_outputschema", "parameters": [ { "name": "function_id", @@ -5711,14 +5639,98 @@ } } }, - "/v0/functions/{function_id}/output_schema": { - "get": { + "/v0/functions/{function_id}:validate_inputs": { + "post": { "tags": [ "functions" ], - "summary": "Get Function Output Schema", - "description": "Get function", - "operationId": "get_function_output_schema", + "summary": "Validate Function Inputs", + "description": "Validate inputs against the function's input schema", + "operationId": "validate_function_inputs", + "parameters": [ + { + "name": "function_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Id" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Inputs" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "array", + "prefixItems": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "minItems": 2, + "maxItems": 2, + "title": "Response Validate Function Inputs V0 Functions Function Id Validate Inputs Post" + } + } + } + }, + "400": { + "description": "Invalid inputs" + }, + "404": { + "description": "Function not found" + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/functions/{function_id}:run": { + "post": { + "tags": [ + "functions" + ], + "summary": "Run Function", + "description": "Run function", + "operationId": "run_function", + "security": [ + { + "HTTPBasic": [] + } + ], "parameters": [ { "name": "function_id", @@ -5731,13 +5743,50 @@ } } ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Function Inputs" + } + } + } + }, "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/FunctionOutputSchema" + "oneOf": [ + { + "$ref": "#/components/schemas/ProjectFunctionJob" + }, + { + "$ref": "#/components/schemas/PythonCodeFunctionJob" + }, + { + "$ref": "#/components/schemas/SolverFunctionJob" + } + ], + "discriminator": { + "propertyName": "function_class", + "mapping": { + "project": "#/components/schemas/ProjectFunctionJob", + "python_code": "#/components/schemas/PythonCodeFunctionJob", + "solver": "#/components/schemas/SolverFunctionJob" + } + }, + "title": "Response Run Function V0 Functions Function Id Run Post" } } } @@ -7564,7 +7613,6 @@ }, "type": "object", "required": [ - "uid", "title", "description", "job_ids" diff --git a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py index 82bbf6ce9ff..d14739cdd9d 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py @@ -2,7 +2,9 @@ from collections.abc import Callable from typing import Annotated, Final +import jsonschema from fastapi import APIRouter, Depends, Request, status +from jsonschema import ValidationError from models_library.api_schemas_webserver.functions_wb_schema import ( Function, FunctionClass, @@ -17,7 +19,6 @@ FunctionJobID, FunctionJobStatus, FunctionOutputs, - FunctionOutputSchema, ProjectFunctionJob, SolverFunctionJob, ) @@ -56,21 +57,12 @@ } -@function_router.post("/ping") -async def ping( - wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], -): - return await wb_api_rpc.ping() - - -@function_router.get("", response_model=list[Function], description="List functions") -async def list_functions( - wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], -): - return await wb_api_rpc.list_functions() - - -@function_router.post("", response_model=Function, description="Create function") +@function_router.post( + "", + response_model=Function, + responses={**_COMMON_FUNCTION_ERROR_RESPONSES}, + description="Create function", +) async def register_function( wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], function: Function, @@ -91,6 +83,13 @@ async def get_function( return await wb_api_rpc.get_function(function_id=function_id) +@function_router.get("", response_model=list[Function], description="List functions") +async def list_functions( + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + return await wb_api_rpc.list_functions() + + def join_inputs( default_inputs: FunctionInputs | None, function_inputs: FunctionInputs | None, @@ -105,13 +104,66 @@ def join_inputs( return {**default_inputs, **function_inputs} +@function_router.get( + "/{function_id:uuid}/input_schema", + response_model=FunctionInputSchema, + responses={**_COMMON_FUNCTION_ERROR_RESPONSES}, + description="Get function input schema", +) +async def get_function_inputschema( + function_id: FunctionID, + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + function = await wb_api_rpc.get_function(function_id=function_id) + return function.input_schema + + +@function_router.get( + "/{function_id:uuid}/output_schema", + response_model=FunctionInputSchema, + responses={**_COMMON_FUNCTION_ERROR_RESPONSES}, + description="Get function input schema", +) +async def get_function_outputschema( + function_id: FunctionID, + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + function = await wb_api_rpc.get_function(function_id=function_id) + return function.output_schema + + +@function_router.post( + "/{function_id:uuid}:validate_inputs", + response_model=tuple[bool, str], + responses={ + status.HTTP_400_BAD_REQUEST: {"description": "Invalid inputs"}, + status.HTTP_404_NOT_FOUND: {"description": "Function not found"}, + }, + description="Validate inputs against the function's input schema", +) +async def validate_function_inputs( + function_id: FunctionID, + inputs: FunctionInputs, + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + function = await wb_api_rpc.get_function(function_id=function_id) + + if function.input_schema is None: + return True, "No input schema defined for this function" + try: + jsonschema.validate(instance=inputs, schema=function.input_schema.schema_dict) # type: ignore + except ValidationError as err: + return False, str(err) + return True, "Inputs are valid" + + @function_router.post( "/{function_id:uuid}:run", response_model=FunctionJob, responses={**_COMMON_FUNCTION_ERROR_RESPONSES}, description="Run function", ) -async def run_function( +async def run_function( # noqa: PLR0913 request: Request, wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], @@ -134,6 +186,18 @@ async def run_function( function_inputs, ) + if to_run_function.input_schema is not None: + is_valid, validation_str = await validate_function_inputs( + function_id=to_run_function.uid, + inputs=joined_inputs, + wb_api_rpc=wb_api_rpc, + ) + if not is_valid: + msg = ( + f"Function {to_run_function.uid} inputs are not valid: {validation_str}" + ) + raise ValidationError(msg) + if cached_function_job := await wb_api_rpc.find_cached_function_job( function_id=to_run_function.uid, inputs=joined_inputs, @@ -211,7 +275,7 @@ async def run_function( @function_router.delete( "/{function_id:uuid}", - response_model=Function, + response_model=None, responses={**_COMMON_FUNCTION_ERROR_RESPONSES}, description="Delete function", ) @@ -222,32 +286,6 @@ async def delete_function( return await wb_api_rpc.delete_function(function_id=function_id) -@function_router.get( - "/{function_id:uuid}/input_schema", - response_model=FunctionInputSchema, - responses={**_COMMON_FUNCTION_ERROR_RESPONSES}, - description="Get function", -) -async def get_function_input_schema( - wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], - function_id: FunctionID, -): - return await wb_api_rpc.get_function_input_schema(function_id=function_id) - - -@function_router.get( - "/{function_id:uuid}/output_schema", - response_model=FunctionOutputSchema, - responses={**_COMMON_FUNCTION_ERROR_RESPONSES}, - description="Get function", -) -async def get_function_output_schema( - wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], - function_id: FunctionID, -): - return await wb_api_rpc.get_function_output_schema(function_id=function_id) - - _COMMON_FUNCTION_JOB_ERROR_RESPONSES: Final[dict] = { status.HTTP_404_NOT_FOUND: { "description": "Function job not found", @@ -415,7 +453,7 @@ async def function_job_outputs( responses={**_COMMON_FUNCTION_ERROR_RESPONSES}, description="Map function over input parameters", ) -async def map_function( +async def map_function( # noqa: PLR0913 function_id: FunctionID, function_inputs_list: FunctionInputsList, request: Request, @@ -581,6 +619,8 @@ async def function_job_collection_status( ) +# ruff: noqa: ERA001 + # @function_job_router.get( # "/{function_job_id:uuid}/outputs/logfile", # response_model=FunctionOutputsLogfile, @@ -609,7 +649,7 @@ async def function_job_collection_status( # ) # return job_outputs -# elif (function.function_class == FunctionClass.solver) and ( # noqa: RET505 +# elif (function.function_class == FunctionClass.solver) and ( # function_job.function_class == FunctionClass.solver # ): # job_outputs_logfile = await solvers_jobs_getters.get_job_output_logfile( diff --git a/services/api-server/tests/unit/api_functions/test_api_routers_functions.py b/services/api-server/tests/unit/api_functions/test_api_routers_functions.py new file mode 100644 index 00000000000..8223e7e5813 --- /dev/null +++ b/services/api-server/tests/unit/api_functions/test_api_routers_functions.py @@ -0,0 +1,660 @@ +from unittest.mock import MagicMock +from uuid import uuid4 + +import pytest +from fastapi import FastAPI, HTTPException +from fastapi.testclient import TestClient +from models_library.api_schemas_webserver.functions_wb_schema import ( + Function, + FunctionJob, + FunctionJobCollection, +) +from pydantic import TypeAdapter +from simcore_service_api_server.api.routes.functions_routes import ( + function_job_collections_router, + function_job_router, + function_router, + get_current_user_id, + get_wb_api_rpc_client, +) +from sqlalchemy.ext.asyncio import AsyncEngine + + +@pytest.fixture(name="api_app") +def _api_app() -> FastAPI: + fastapi_app = FastAPI() + fastapi_app.include_router(function_router, prefix="/functions") + fastapi_app.include_router(function_job_router, prefix="/function_jobs") + fastapi_app.include_router( + function_job_collections_router, prefix="/function_job_collections" + ) + + # Mock authentication dependency + async def mock_auth_dependency() -> int: + # Mock a valid user ID + return 100 + + fastapi_app.dependency_overrides[get_current_user_id] = mock_auth_dependency + + fake_wb_api_rpc = FakeWbApiRpc() + + async def fake_get_wb_api_rpc_client() -> FakeWbApiRpc: + return fake_wb_api_rpc + + fastapi_app.dependency_overrides[get_wb_api_rpc_client] = fake_get_wb_api_rpc_client + + mock_engine = MagicMock(spec=AsyncEngine) + mock_engine.pool = MagicMock() + mock_engine.pool.checkedin = MagicMock(return_value=[]) + fastapi_app.state.engine = mock_engine + + return fastapi_app + + +class FakeWbApiRpc: + def __init__(self) -> None: + self._functions = {} + self._function_jobs = {} + self._function_job_collections = {} + + async def register_function(self, function: Function) -> Function: + # Mimic returning the same function that was passed and store it for later retrieval + function.uid = uuid4() + self._functions[function.uid] = TypeAdapter(Function).validate_python( + { + "uid": str(function.uid), + "title": function.title, + "function_class": function.function_class, + "project_id": getattr(function, "project_id", None), + "description": function.description, + "input_schema": function.input_schema, + "output_schema": function.output_schema, + "default_inputs": None, + } + ) + return self._functions[function.uid] + + async def get_function(self, function_id: str) -> dict: + # Mimic retrieval of a function based on function_id and raise 404 if not found + if function_id not in self._functions: + raise HTTPException(status_code=404, detail="Function not found") + return self._functions[function_id] + + async def run_function(self, function_id: str, inputs: dict) -> dict: + # Mimic running a function and returning a success status + if function_id not in self._functions: + raise HTTPException( + status_code=404, + detail=f"Function {function_id} not found in {self._functions}", + ) + return {"status": "success", "function_id": function_id, "inputs": inputs} + + async def list_functions(self) -> list: + # Mimic listing all functions + return list(self._functions.values()) + + async def delete_function(self, function_id: str) -> None: + # Mimic deleting a function + if function_id in self._functions: + del self._functions[function_id] + else: + raise HTTPException(status_code=404, detail="Function not found") + + async def register_function_job(self, function_job: FunctionJob) -> FunctionJob: + # Mimic registering a function job + function_job.uid = uuid4() + self._function_jobs[function_job.uid] = TypeAdapter( + FunctionJob + ).validate_python( + { + "uid": str(function_job.uid), + "function_uid": function_job.function_uid, + "title": function_job.title, + "description": function_job.description, + "project_job_id": getattr(function_job, "project_job_id", None), + "inputs": function_job.inputs, + "outputs": function_job.outputs, + "function_class": function_job.function_class, + } + ) + return self._function_jobs[function_job.uid] + + async def get_function_job(self, function_job_id: str) -> dict: + # Mimic retrieval of a function job based on function_job_id and raise 404 if not found + if function_job_id not in self._function_jobs: + raise HTTPException(status_code=404, detail="Function job not found") + return self._function_jobs[function_job_id] + + async def list_function_jobs(self) -> list: + # Mimic listing all function jobs + return list(self._function_jobs.values()) + + async def delete_function_job(self, function_job_id: str) -> None: + # Mimic deleting a function job + if function_job_id in self._function_jobs: + del self._function_jobs[function_job_id] + else: + raise HTTPException(status_code=404, detail="Function job not found") + + async def register_function_job_collection( + self, function_job_collection: FunctionJobCollection + ) -> FunctionJobCollection: + # Mimic registering a function job collection + function_job_collection.uid = uuid4() + self._function_job_collections[function_job_collection.uid] = TypeAdapter( + FunctionJobCollection + ).validate_python( + { + "uid": str(function_job_collection.uid), + "title": function_job_collection.title, + "description": function_job_collection.description, + "job_ids": function_job_collection.job_ids, + } + ) + return self._function_job_collections[function_job_collection.uid] + + async def get_function_job_collection( + self, function_job_collection_id: str + ) -> dict: + # Mimic retrieval of a function job collection based on collection_id and raise 404 if not found + if function_job_collection_id not in self._function_job_collections: + raise HTTPException( + status_code=404, detail="Function job collection not found" + ) + return self._function_job_collections[function_job_collection_id] + + async def list_function_job_collections(self) -> list: + # Mimic listing all function job collections + return list(self._function_job_collections.values()) + + async def delete_function_job_collection( + self, function_job_collection_id: str + ) -> None: + # Mimic deleting a function job collection + if function_job_collection_id in self._function_job_collections: + del self._function_job_collections[function_job_collection_id] + else: + raise HTTPException( + status_code=404, detail="Function job collection not found" + ) + + +def test_register_function(api_app) -> None: + client = TestClient(api_app) + sample_function = { + "title": "test_function", + "function_class": "project", + "project_id": str(uuid4()), + "description": "A test function", + "input_schema": {"schema_dict": {}}, + "output_schema": {"schema_dict": {}}, + } + response = client.post("/functions", json=sample_function) + assert response.status_code == 200 + data = response.json() + assert data["uid"] is not None + assert data["function_class"] == sample_function["function_class"] + assert data["project_id"] == sample_function["project_id"] + assert data["input_schema"] == sample_function["input_schema"] + assert data["output_schema"] == sample_function["output_schema"] + assert data["title"] == sample_function["title"] + assert data["description"] == sample_function["description"] + + +def test_register_function_invalid(api_app: FastAPI) -> None: + client = TestClient(api_app) + invalid_function = { + "title": "test_function", + "function_class": "invalid_class", # Invalid class + "project_id": str(uuid4()), + } + response = client.post("/functions", json=invalid_function) + assert response.status_code == 422 # Unprocessable Entity + assert ( + "Input tag 'invalid_class' found using 'function_class' does no" + in response.json()["detail"][0]["msg"] + ) + + +def test_get_function(api_app: FastAPI) -> None: + client = TestClient(api_app) + project_id = str(uuid4()) + # First, register a sample function so that it exists + sample_function = { + "title": "example_function", + "function_class": "project", + "project_id": project_id, + "description": "An example function", + "input_schema": {"schema_dict": {}}, + "output_schema": {"schema_dict": {}}, + } + post_response = client.post("/functions", json=sample_function) + assert post_response.status_code == 200 + data = post_response.json() + function_id = data["uid"] + + expected_function = { + "uid": function_id, + "title": "example_function", + "description": "An example function", + "function_class": "project", + "project_id": project_id, + "input_schema": {"schema_dict": {}}, + "output_schema": {"schema_dict": {}}, + "default_inputs": None, + } + response = client.get(f"/functions/{function_id}") + assert response.status_code == 200 + data = response.json() + # Exclude the 'project_id' field from both expected and actual results before comparing + assert data == expected_function + + +def test_get_function_not_found(api_app: FastAPI) -> None: + client = TestClient(api_app) + non_existent_function_id = str(uuid4()) + response = client.get(f"/functions/{non_existent_function_id}") + assert response.status_code == 404 + assert response.json() == {"detail": "Function not found"} + + +def test_list_functions(api_app: FastAPI) -> None: + client = TestClient(api_app) + # Register a sample function + sample_function = { + "title": "example_function", + "function_class": "project", + "project_id": str(uuid4()), + "description": "An example function", + "input_schema": {"schema_dict": {}}, + "output_schema": {"schema_dict": {}}, + } + post_response = client.post("/functions", json=sample_function) + assert post_response.status_code == 200 + + # List functions + response = client.get("/functions") + assert response.status_code == 200 + data = response.json() + assert len(data) > 0 + assert data[0]["title"] == sample_function["title"] + + +def test_get_function_input_schema(api_app: FastAPI) -> None: + client = TestClient(api_app) + project_id = str(uuid4()) + # Register a sample function + sample_function = { + "title": "example_function", + "function_class": "project", + "project_id": project_id, + "description": "An example function", + "input_schema": { + "schema_dict": { + "type": "object", + "properties": {"input1": {"type": "integer"}}, + } + }, + "output_schema": {"schema_dict": {}}, + } + post_response = client.post("/functions", json=sample_function) + assert post_response.status_code == 200 + data = post_response.json() + function_id = data["uid"] + + # Get the input schema + # assert f"/functions/{function_id}/input-schema" is None + response = client.get(f"/functions/{function_id}/input_schema") + assert response.status_code == 200 + data = response.json() + assert data["schema_dict"] == sample_function["input_schema"]["schema_dict"] + + +def test_get_function_output_schema(api_app: FastAPI) -> None: + client = TestClient(api_app) + project_id = str(uuid4()) + # Register a sample function + sample_function = { + "title": "example_function", + "function_class": "project", + "project_id": project_id, + "description": "An example function", + "input_schema": {"schema_dict": {}}, + "output_schema": { + "schema_dict": { + "type": "object", + "properties": {"output1": {"type": "string"}}, + } + }, + } + post_response = client.post("/functions", json=sample_function) + assert post_response.status_code == 200 + data = post_response.json() + function_id = data["uid"] + + # Get the output schema + response = client.get(f"/functions/{function_id}/output_schema") + assert response.status_code == 200 + data = response.json() + assert data["schema_dict"] == sample_function["output_schema"]["schema_dict"] + + +def test_validate_function_inputs(api_app: FastAPI) -> None: + client = TestClient(api_app) + project_id = str(uuid4()) + # Register a sample function + sample_function = { + "title": "example_function", + "function_class": "project", + "project_id": project_id, + "description": "An example function", + "input_schema": { + "schema_dict": { + "type": "object", + "properties": {"input1": {"type": "integer"}}, + } + }, + "output_schema": {"schema_dict": {}}, + } + post_response = client.post("/functions", json=sample_function) + assert post_response.status_code == 200 + data = post_response.json() + function_id = data["uid"] + + # Validate inputs + validate_payload = {"input1": 10} + response = client.post( + f"/functions/{function_id}:validate_inputs", json=validate_payload + ) + assert response.status_code == 200 + data = response.json() + assert data == [True, "Inputs are valid"] + + +def test_delete_function(api_app: FastAPI) -> None: + client = TestClient(api_app) + project_id = str(uuid4()) + # Register a sample function + sample_function = { + "title": "example_function", + "function_class": "project", + "project_id": project_id, + "description": "An example function", + "input_schema": {"schema_dict": {}}, + "output_schema": {"schema_dict": {}}, + } + post_response = client.post("/functions", json=sample_function) + assert post_response.status_code == 200 + data = post_response.json() + function_id = data["uid"] + + # Delete the function + response = client.delete(f"/functions/{function_id}") + assert response.status_code == 200 + + +def test_register_function_job(api_app: FastAPI) -> None: + """Test the register_function_job endpoint.""" + + client = TestClient(api_app) + mock_function_job = { + "function_uid": str(uuid4()), + "title": "Test Function Job", + "description": "A test function job", + "inputs": {"key": "value"}, + "outputs": None, + "project_job_id": str(uuid4()), + "function_class": "project", + } + + # Act + response = client.post("/function_jobs", json=mock_function_job) + + # Assert + assert response.status_code == 200 + response_data = response.json() + assert response_data["uid"] is not None + response_data.pop("uid", None) # Remove the uid field + assert response_data == mock_function_job + + +def test_get_function_job(api_app: FastAPI) -> None: + """Test the get_function_job endpoint.""" + + client = TestClient(api_app) + mock_function_job = { + "function_uid": str(uuid4()), + "title": "Test Function Job", + "description": "A test function job", + "inputs": {"key": "value"}, + "outputs": None, + "project_job_id": str(uuid4()), + "function_class": "project", + } + + # First, register a function job + post_response = client.post("/function_jobs", json=mock_function_job) + assert post_response.status_code == 200 + data = post_response.json() + function_job_id = data["uid"] + + # Now, get the function job + response = client.get(f"/function_jobs/{function_job_id}") + assert response.status_code == 200 + data = response.json() + assert data["uid"] == function_job_id + assert data["title"] == mock_function_job["title"] + assert data["description"] == mock_function_job["description"] + assert data["inputs"] == mock_function_job["inputs"] + assert data["outputs"] == mock_function_job["outputs"] + + +def test_list_function_jobs(api_app: FastAPI) -> None: + """Test the list_function_jobs endpoint.""" + + client = TestClient(api_app) + mock_function_job = { + "function_uid": str(uuid4()), + "title": "Test Function Job", + "description": "A test function job", + "inputs": {"key": "value"}, + "outputs": None, + "project_job_id": str(uuid4()), + "function_class": "project", + } + + # First, register a function job + post_response = client.post("/function_jobs", json=mock_function_job) + assert post_response.status_code == 200 + + # Now, list function jobs + response = client.get("/function_jobs") + assert response.status_code == 200 + data = response.json() + assert len(data) > 0 + assert data[0]["title"] == mock_function_job["title"] + + +def test_delete_function_job(api_app: FastAPI) -> None: + """Test the delete_function_job endpoint.""" + + client = TestClient(api_app) + mock_function_job = { + "function_uid": str(uuid4()), + "title": "Test Function Job", + "description": "A test function job", + "inputs": {"key": "value"}, + "outputs": None, + "project_job_id": str(uuid4()), + "function_class": "project", + } + + # First, register a function job + post_response = client.post("/function_jobs", json=mock_function_job) + assert post_response.status_code == 200 + data = post_response.json() + function_job_id = data["uid"] + + # Now, delete the function job + response = client.delete(f"/function_jobs/{function_job_id}") + assert response.status_code == 200 + + +def test_register_function_job_collection(api_app: FastAPI) -> None: + # Arrange + client = TestClient(api_app) + + mock_function_job_collection = { + "title": "Test Collection", + "description": "A test function job collection", + "job_ids": [str(uuid4()), str(uuid4())], + } + + # Act + response = client.post( + "/function_job_collections", json=mock_function_job_collection + ) + + # Assert + assert response.status_code == 200 + response_data = response.json() + assert response_data["uid"] is not None + response_data.pop("uid", None) # Remove the uid field + assert response_data == mock_function_job_collection + + +def test_get_function_job_collection(api_app: FastAPI) -> None: + # Arrange + client = TestClient(api_app) + mock_function_job_collection = { + "title": "Test Collection", + "description": "A test function job collection", + "job_ids": [str(uuid4()), str(uuid4())], + } + + # First, register a function job collection + post_response = client.post( + "/function_job_collections", json=mock_function_job_collection + ) + assert post_response.status_code == 200 + data = post_response.json() + collection_id = data["uid"] + + # Act + response = client.get(f"/function_job_collections/{collection_id}") + + # Assert + assert response.status_code == 200 + data = response.json() + assert data["uid"] == collection_id + assert data["title"] == mock_function_job_collection["title"] + assert data["description"] == mock_function_job_collection["description"] + assert data["job_ids"] == mock_function_job_collection["job_ids"] + + +# def test_run_function_project_class(api_app: FastAPI) -> None: +# client = TestClient(api_app) +# project_id = str(uuid4()) +# # Register a sample function with "project" class +# sample_function = { +# "title": "example_function", +# "function_class": "project", +# "project_id": project_id, +# "description": "An example function", +# "input_schema": {"schema_dict": {"type": "object", "properties": {"input1": {"type": "integer"}}}}, +# "output_schema": {"schema_dict": {}}, +# "default_inputs": {"input1": 5}, +# } +# post_response = client.post("/functions", json=sample_function) +# assert post_response.status_code == 200 +# data = post_response.json() +# function_id = data["uid"] + +# # Run the function +# run_payload = {"input1": 10} +# response = client.post(f"/functions/{function_id}:run", json=run_payload) +# assert response.status_code == 200 +# data = response.json() +# assert data["function_uid"] == function_id +# assert data["inputs"] == {"input1": 10} + +# def test_run_function_solver_class(api_app: FastAPI) -> None: +# client = TestClient(api_app) +# # Register a sample function with "solver" class +# sample_function = { +# "title": "solver_function", +# "function_class": "solver", +# "solver_key": "example_solver", +# "solver_version": "1.0.0", +# "description": "A solver function", +# "input_schema": {"schema_dict": {"type": "object", "properties": {"input1": {"type": "integer"}}}}, +# "output_schema": {"schema_dict": {}}, +# "default_inputs": {"input1": 5}, +# } +# post_response = client.post("/functions", json=sample_function) +# assert post_response.status_code == 200 +# data = post_response.json() +# function_id = data["uid"] + +# # Run the function +# run_payload = {"input1": 15} +# response = client.post(f"/functions/{function_id}:run", json=run_payload) +# assert response.status_code == 200 +# data = response.json() +# assert data["function_uid"] == function_id +# assert data["inputs"] == {"input1": 15} + +# def test_run_function_invalid_inputs(api_app: FastAPI) -> None: +# client = TestClient(api_app) +# # Register a sample function with input schema +# sample_function = { +# "title": "example_function", +# "function_class": "project", +# "project_id": str(uuid4()), +# "description": "An example function", +# "input_schema": {"schema_dict": {"type": "object", "properties": {"input1": {"type": "integer"}}}}, +# "output_schema": {"schema_dict": {}}, +# } +# post_response = client.post("/functions", json=sample_function) +# assert post_response.status_code == 200 +# data = post_response.json() +# function_id = data["uid"] + +# # Run the function with invalid inputs +# run_payload = {"input1": "invalid_value"} +# response = client.post(f"/functions/{function_id}:run", json=run_payload) +# assert response.status_code == 400 +# assert "inputs are not valid" in response.json()["detail"] + +# def test_run_function_not_found(api_app: FastAPI) -> None: +# client = TestClient(api_app) +# non_existent_function_id = str(uuid4()) +# run_payload = {"input1": 10} +# response = client.post(f"/functions/{non_existent_function_id}:run", json=run_payload) +# assert response.status_code == 404 +# assert response.json() == {"detail": "Function not found"} + +# def test_run_function_cached_job(api_app: FastAPI) -> None: +# client = TestClient(api_app) +# # Register a sample function +# sample_function = { +# "title": "example_function", +# "function_class": "project", +# "project_id": str(uuid4()), +# "description": "An example function", +# "input_schema": {"schema_dict": {"type": "object", "properties": {"input1": {"type": "integer"}}}}, +# "output_schema": {"schema_dict": {}}, +# "default_inputs": {"input1": 5}, +# } +# post_response = client.post("/functions", json=sample_function) +# assert post_response.status_code == 200 +# data = post_response.json() +# function_id = data["uid"] + +# # Mimic a cached job +# run_payload = {"input1": 10} +# response = client.post(f"/functions/{function_id}:run", json=run_payload) +# assert response.status_code == 200 +# data = response.json() +# assert data["function_uid"] == function_id +# assert data["inputs"] == {"input1": 10} From be9ac7ccc8645bc0d339637a9ffac67a29c64da0 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Wed, 30 Apr 2025 20:06:41 +0200 Subject: [PATCH 14/69] Add function rpc tests --- .../functions/_functions_controller_rpc.py | 21 +- .../functions/_functions_repository.py | 17 +- .../functions/_repo.py | 2 - .../functions/_service.py | 21 - .../05/test_functions_controller_rpc.py | 449 ++++++++++++++++++ 5 files changed, 481 insertions(+), 29 deletions(-) delete mode 100644 services/web/server/src/simcore_service_webserver/functions/_repo.py delete mode 100644 services/web/server/src/simcore_service_webserver/functions/_service.py create mode 100644 services/web/server/tests/unit/with_dbs/05/test_functions_controller_rpc.py diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py index da4737553d5..f8e39796249 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py @@ -122,7 +122,7 @@ def _decode_functionjob( description="", function_uid=functionjob_db.function_uuid, inputs=functionjob_db.inputs, - outputs=None, + outputs=functionjob_db.outputs, project_job_id=functionjob_db.class_specific_data["project_job_id"], ) elif functionjob_db.function_class == FunctionClass.solver: # noqa: RET505 @@ -132,7 +132,7 @@ def _decode_functionjob( description="", function_uid=functionjob_db.function_uuid, inputs=functionjob_db.inputs, - outputs=None, + outputs=functionjob_db.outputs, solver_job_id=functionjob_db.class_specific_data["solver_job_id"], ) else: @@ -148,7 +148,7 @@ def _encode_functionjob( title=functionjob.title, function_uuid=functionjob.function_uid, inputs=functionjob.inputs, - outputs=None, + outputs=functionjob.outputs, class_specific_data=FunctionJobClassSpecificData( { "project_job_id": str(functionjob.project_job_id), @@ -161,7 +161,7 @@ def _encode_functionjob( title=functionjob.title, function_uuid=functionjob.function_uid, inputs=functionjob.inputs, - outputs=None, + outputs=functionjob.outputs, class_specific_data=FunctionJobClassSpecificData( { "solver_job_id": str(functionjob.solver_job_id), @@ -267,6 +267,17 @@ async def register_function_job( return _decode_functionjob(created_function_job_db) +@router.expose() +async def delete_function_job( + app: web.Application, *, function_job_id: FunctionJobID +) -> None: + assert app + await _functions_repository.delete_function_job( + app=app, + function_job_id=function_job_id, + ) + + @router.expose() async def find_cached_function_job( app: web.Application, *, function_id: FunctionID, inputs: FunctionInputs @@ -298,7 +309,7 @@ async def find_cached_function_job( outputs=None, solver_job_id=returned_function_job.class_specific_data["solver_job_id"], ) - else: # noqa: RET505 + else: msg = f"Unsupported function class: [{returned_function_job.function_class}]" raise TypeError(msg) diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py index f9d0cce1c71..6e3fa98b94d 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py @@ -121,7 +121,7 @@ async def delete_function( async with transaction_context(get_asyncpg_engine(app), connection) as conn: await conn.execute( - functions_table.delete().where(functions_table.c.uuid == int(function_id)) + functions_table.delete().where(functions_table.c.uuid == function_id) ) @@ -138,6 +138,7 @@ async def register_function_job( .values( function_uuid=function_job.function_uuid, inputs=function_job.inputs, + outputs=function_job.outputs, function_class=function_job.function_class, class_specific_data=function_job.class_specific_data, title=function_job.title, @@ -190,6 +191,20 @@ async def list_function_jobs( return [FunctionJobDB.model_validate(dict(row)) for row in rows] +async def delete_function_job( + app: web.Application, + connection: AsyncConnection | None = None, + *, + function_job_id: FunctionID, +) -> None: + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + await conn.execute( + function_jobs_table.delete().where( + function_jobs_table.c.uuid == function_job_id + ) + ) + + async def find_cached_function_job( app: web.Application, connection: AsyncConnection | None = None, diff --git a/services/web/server/src/simcore_service_webserver/functions/_repo.py b/services/web/server/src/simcore_service_webserver/functions/_repo.py deleted file mode 100644 index baf76a9ee8d..00000000000 --- a/services/web/server/src/simcore_service_webserver/functions/_repo.py +++ /dev/null @@ -1,2 +0,0 @@ -# the repository layer. Calls directly to the db (function table) should be done here. -# see e.g. licenses/_licensed_resources_repository.py file for an example diff --git a/services/web/server/src/simcore_service_webserver/functions/_service.py b/services/web/server/src/simcore_service_webserver/functions/_service.py deleted file mode 100644 index 899b0b1c681..00000000000 --- a/services/web/server/src/simcore_service_webserver/functions/_service.py +++ /dev/null @@ -1,21 +0,0 @@ -# This is where calls to the business logic is done. -# calls to the `projects` interface should be done here. -# calls to _repo.py should also be done here - -from aiohttp import web -from models_library.users import UserID - -from ..projects import projects_service -from ..projects.models import ProjectDict - - -# example function -async def get_project_from_function( - app: web.Application, - function_uuid: str, - user_id: UserID, -) -> ProjectDict: - - return await projects_service.get_project_for_user( - app=app, project_uuid=function_uuid, user_id=user_id - ) diff --git a/services/web/server/tests/unit/with_dbs/05/test_functions_controller_rpc.py b/services/web/server/tests/unit/with_dbs/05/test_functions_controller_rpc.py new file mode 100644 index 00000000000..400813e43e9 --- /dev/null +++ b/services/web/server/tests/unit/with_dbs/05/test_functions_controller_rpc.py @@ -0,0 +1,449 @@ +# pylint: disable=redefined-outer-name +from uuid import uuid4 + +import pytest +import simcore_service_webserver.functions._functions_controller_rpc as functions_rpc +from aiohttp import web +from models_library.api_schemas_webserver.functions_wb_schema import ( + Function, + FunctionInputSchema, + FunctionJobCollection, + FunctionOutputSchema, + ProjectFunction, + ProjectFunctionJob, +) + + +@pytest.fixture +def mock_function() -> Function: + return ProjectFunction( + title="Test Function", + description="A test function", + input_schema=FunctionInputSchema( + schema_dict={"type": "object", "properties": {"input1": {"type": "string"}}} + ), + output_schema=FunctionOutputSchema( + schema_dict={ + "type": "object", + "properties": {"output1": {"type": "string"}}, + } + ), + project_id=uuid4(), + default_inputs=None, + ) + + +@pytest.mark.asyncio +async def test_register_function(client, mock_function): + # Register the function + registered_function = await functions_rpc.register_function( + app=client.app, function=mock_function + ) + assert registered_function.uid is not None + # Retrieve the function from the repository to verify it was saved + saved_function = await functions_rpc.get_function( + app=client.app, function_id=registered_function.uid + ) + + # Assert the saved function matches the input function + assert saved_function.uid is not None + assert saved_function.title == mock_function.title + assert saved_function.description == mock_function.description + + # Ensure saved_function is of type ProjectFunction before accessing project_id + assert isinstance(saved_function, ProjectFunction) + assert saved_function.project_id == mock_function.project_id + + # Assert the returned function matches the expected result + assert registered_function.title == mock_function.title + assert registered_function.description == mock_function.description + assert isinstance(registered_function, ProjectFunction) + assert registered_function.project_id == mock_function.project_id + + +@pytest.mark.asyncio +async def test_get_function(client, mock_function): + # Register the function first + registered_function = await functions_rpc.register_function( + app=client.app, function=mock_function + ) + assert registered_function.uid is not None + + # Retrieve the function using its ID + retrieved_function = await functions_rpc.get_function( + app=client.app, function_id=registered_function.uid + ) + + # Assert the retrieved function matches the registered function + assert retrieved_function.uid == registered_function.uid + assert retrieved_function.title == registered_function.title + assert retrieved_function.description == registered_function.description + + # Ensure retrieved_function is of type ProjectFunction before accessing project_id + assert isinstance(retrieved_function, ProjectFunction) + assert isinstance(registered_function, ProjectFunction) + assert retrieved_function.project_id == registered_function.project_id + + +@pytest.mark.asyncio +async def test_get_function_not_found(client): + # Attempt to retrieve a function that does not exist + with pytest.raises(web.HTTPNotFound): + await functions_rpc.get_function(app=client.app, function_id=uuid4()) + + +@pytest.mark.asyncio +async def test_list_functions(client): + # Register a function first + mock_function = ProjectFunction( + title="Test Function", + description="A test function", + input_schema=None, + output_schema=None, + project_id=uuid4(), + default_inputs=None, + ) + registered_function = await functions_rpc.register_function( + app=client.app, function=mock_function + ) + assert registered_function.uid is not None + + # List functions + functions = await functions_rpc.list_functions(app=client.app) + + # Assert the list contains the registered function + assert len(functions) > 0 + assert any(f.uid == registered_function.uid for f in functions) + + +@pytest.mark.asyncio +async def test_get_function_input_schema(client, mock_function): + # Register the function first + registered_function = await functions_rpc.register_function( + app=client.app, function=mock_function + ) + assert registered_function.uid is not None + + # Retrieve the input schema using its ID + input_schema = await functions_rpc.get_function_input_schema( + app=client.app, function_id=registered_function.uid + ) + + # Assert the input schema matches the registered function's input schema + assert input_schema == registered_function.input_schema + + +@pytest.mark.asyncio +async def test_get_function_output_schema(client, mock_function): + # Register the function first + registered_function = await functions_rpc.register_function( + app=client.app, function=mock_function + ) + assert registered_function.uid is not None + + # Retrieve the output schema using its ID + output_schema = await functions_rpc.get_function_output_schema( + app=client.app, function_id=registered_function.uid + ) + + # Assert the output schema matches the registered function's output schema + assert output_schema == registered_function.output_schema + + +@pytest.mark.asyncio +async def test_delete_function(client, mock_function): + # Register the function first + registered_function = await functions_rpc.register_function( + app=client.app, function=mock_function + ) + assert registered_function.uid is not None + + # Delete the function using its ID + await functions_rpc.delete_function( + app=client.app, function_id=registered_function.uid + ) + + # Attempt to retrieve the deleted function + with pytest.raises(web.HTTPNotFound): + await functions_rpc.get_function( + app=client.app, function_id=registered_function.uid + ) + + +@pytest.mark.asyncio +async def test_register_function_job(client, mock_function): + # Register the function first + registered_function = await functions_rpc.register_function( + app=client.app, function=mock_function + ) + assert registered_function.uid is not None + + function_job = ProjectFunctionJob( + function_uid=registered_function.uid, + title="Test Function Job", + description="A test function job", + project_job_id=uuid4(), + inputs={"input1": "value1"}, + outputs={"output1": "result1"}, + ) + + # Register the function job + registered_job = await functions_rpc.register_function_job( + app=client.app, function_job=function_job + ) + + # Assert the registered job matches the input job + assert registered_job.function_uid == function_job.function_uid + assert registered_job.inputs == function_job.inputs + assert registered_job.outputs == function_job.outputs + + +@pytest.mark.asyncio +async def test_get_function_job(client, mock_function): + # Register the function first + registered_function = await functions_rpc.register_function( + app=client.app, function=mock_function + ) + assert registered_function.uid is not None + + function_job = ProjectFunctionJob( + function_uid=registered_function.uid, + title="Test Function Job", + description="A test function job", + project_job_id=uuid4(), + inputs={"input1": "value1"}, + outputs={"output1": "result1"}, + ) + + # Register the function job + registered_job = await functions_rpc.register_function_job( + app=client.app, function_job=function_job + ) + assert registered_job.uid is not None + + # Retrieve the function job using its ID + retrieved_job = await functions_rpc.get_function_job( + app=client.app, function_job_id=registered_job.uid + ) + + # Assert the retrieved job matches the registered job + assert retrieved_job.function_uid == registered_job.function_uid + assert retrieved_job.inputs == registered_job.inputs + assert retrieved_job.outputs == registered_job.outputs + + +@pytest.mark.asyncio +async def test_get_function_job_not_found(client): + # Attempt to retrieve a function job that does not exist + with pytest.raises(web.HTTPNotFound): + await functions_rpc.get_function_job(app=client.app, function_job_id=uuid4()) + + +@pytest.mark.asyncio +async def test_list_function_jobs(client, mock_function): + # Register the function first + registered_function = await functions_rpc.register_function( + app=client.app, function=mock_function + ) + assert registered_function.uid is not None + + function_job = ProjectFunctionJob( + function_uid=registered_function.uid, + title="Test Function Job", + description="A test function job", + project_job_id=uuid4(), + inputs={"input1": "value1"}, + outputs={"output1": "result1"}, + ) + + # Register the function job + registered_job = await functions_rpc.register_function_job( + app=client.app, function_job=function_job + ) + + # List function jobs + jobs = await functions_rpc.list_function_jobs(app=client.app) + + # Assert the list contains the registered job + assert len(jobs) > 0 + assert any(j.uid == registered_job.uid for j in jobs) + + +@pytest.mark.asyncio +async def test_delete_function_job(client, mock_function): + # Register the function first + registered_function = await functions_rpc.register_function( + app=client.app, function=mock_function + ) + assert registered_function.uid is not None + + function_job = ProjectFunctionJob( + function_uid=registered_function.uid, + title="Test Function Job", + description="A test function job", + project_job_id=uuid4(), + inputs={"input1": "value1"}, + outputs={"output1": "result1"}, + ) + + # Register the function job + registered_job = await functions_rpc.register_function_job( + app=client.app, function_job=function_job + ) + assert registered_job.uid is not None + + # Delete the function job using its ID + await functions_rpc.delete_function_job( + app=client.app, function_job_id=registered_job.uid + ) + + # Attempt to retrieve the deleted job + with pytest.raises(web.HTTPNotFound): + await functions_rpc.get_function_job( + app=client.app, function_job_id=registered_job.uid + ) + + +@pytest.mark.asyncio +async def test_function_job_collection(client, mock_function): + # Register the function first + registered_function = await functions_rpc.register_function( + app=client.app, function=mock_function + ) + assert registered_function.uid is not None + + registered_function_job = ProjectFunctionJob( + function_uid=registered_function.uid, + title="Test Function Job", + description="A test function job", + project_job_id=uuid4(), + inputs={"input1": "value1"}, + outputs={"output1": "result1"}, + ) + # Register the function job + function_job_ids = [] + for _ in range(3): + registered_function_job = ProjectFunctionJob( + function_uid=registered_function.uid, + title="Test Function Job", + description="A test function job", + project_job_id=uuid4(), + inputs={"input1": "value1"}, + outputs={"output1": "result1"}, + ) + # Register the function job + registered_job = await functions_rpc.register_function_job( + app=client.app, function_job=registered_function_job + ) + assert registered_job.uid is not None + function_job_ids.append(registered_job.uid) + + function_job_collection = FunctionJobCollection( + title="Test Function Job Collection", + description="A test function job collection", + job_ids=function_job_ids, + ) + + # Register the function job collection + registered_collection = await functions_rpc.register_function_job_collection( + app=client.app, function_job_collection=function_job_collection + ) + assert registered_collection.uid is not None + + # Assert the registered collection matches the input collection + assert registered_collection.job_ids == function_job_ids + + await functions_rpc.delete_function_job_collection( + app=client.app, function_job_collection_id=registered_collection.uid + ) + # Attempt to retrieve the deleted collection + with pytest.raises(web.HTTPNotFound): + await functions_rpc.get_function_job( + app=client.app, function_job_id=registered_collection.uid + ) + + +# @pytest.mark.asyncio +# async def test_find_cached_function_job_project_class( +# mock_app, mock_function_id, mock_function_inputs +# ): +# mock_function_job = AsyncMock() +# mock_function_job.function_class = FunctionClass.project +# mock_function_job.uuid = "mock-uuid" +# mock_function_job.title = "mock-title" +# mock_function_job.function_uuid = "mock-function-uuid" +# mock_function_job.inputs = {"key": "value"} +# mock_function_job.class_specific_data = {"project_job_id": "mock-project-job-id"} + +# with patch( +# "simcore_service_webserver.functions._functions_repository.find_cached_function_job", +# return_value=mock_function_job, +# ): +# result = await find_cached_function_job( +# app=mock_app, function_id=mock_function_id, inputs=mock_function_inputs +# ) + +# assert isinstance(result, ProjectFunctionJob) +# assert result.uid == "mock-uuid" +# assert result.title == "mock-title" +# assert result.function_uid == "mock-function-uuid" +# assert result.inputs == {"key": "value"} +# assert result.project_job_id == "mock-project-job-id" + + +# @pytest.mark.asyncio +# async def test_find_cached_function_job_solver_class( +# mock_app, mock_function_id, mock_function_inputs +# ): +# mock_function_job = AsyncMock() +# mock_function_job.function_class = FunctionClass.solver +# mock_function_job.uuid = "mock-uuid" +# mock_function_job.title = "mock-title" +# mock_function_job.function_uuid = "mock-function-uuid" +# mock_function_job.inputs = {"key": "value"} +# mock_function_job.class_specific_data = {"solver_job_id": "mock-solver-job-id"} + +# with patch( +# "simcore_service_webserver.functions._functions_repository.find_cached_function_job", +# return_value=mock_function_job, +# ): +# result = await find_cached_function_job( +# app=mock_app, function_id=mock_function_id, inputs=mock_function_inputs +# ) + +# assert isinstance(result, SolverFunctionJob) +# assert result.uid == "mock-uuid" +# assert result.title == "mock-title" +# assert result.function_uid == "mock-function-uuid" +# assert result.inputs == {"key": "value"} +# assert result.solver_job_id == "mock-solver-job-id" + + +# @pytest.mark.asyncio +# async def test_find_cached_function_job_none(mock_app, mock_function_id, mock_function_inputs): +# with patch( +# "simcore_service_webserver.functions._functions_repository.find_cached_function_job", +# return_value=None, +# ): +# result = await find_cached_function_job( +# app=mock_app, function_id=mock_function_id, inputs=mock_function_inputs +# ) + +# assert result is None + + +# @pytest.mark.asyncio +# async def test_find_cached_function_job_unsupported_class( +# mock_app, mock_function_id, mock_function_inputs +# ): +# mock_function_job = AsyncMock() +# mock_function_job.function_class = "unsupported_class" + +# with patch( +# "simcore_service_webserver.functions._functions_repository.find_cached_function_job", +# return_value=mock_function_job, +# ): +# with pytest.raises(TypeError, match="Unsupported function class:"): +# await find_cached_function_job( +# app=mock_app, function_id=mock_function_id, inputs=mock_function_inputs +# ) From 6a7ba23287af2e5c1c2a3ee71a950a33b1fb13c1 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Wed, 30 Apr 2025 20:17:39 +0200 Subject: [PATCH 15/69] Move function rpc test dir --- .../{05 => functions_rpc}/test_functions_controller_rpc.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename services/web/server/tests/unit/with_dbs/{05 => functions_rpc}/test_functions_controller_rpc.py (100%) diff --git a/services/web/server/tests/unit/with_dbs/05/test_functions_controller_rpc.py b/services/web/server/tests/unit/with_dbs/functions_rpc/test_functions_controller_rpc.py similarity index 100% rename from services/web/server/tests/unit/with_dbs/05/test_functions_controller_rpc.py rename to services/web/server/tests/unit/with_dbs/functions_rpc/test_functions_controller_rpc.py From 23b36e904c1b052a91591711e72c42bf5ed060c3 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Wed, 7 May 2025 10:45:27 +0200 Subject: [PATCH 16/69] Remove Nullable fields --- .../functions_wb_schema.py | 58 ++--- services/api-server/openapi.json | 200 ++++++------------ .../api/routes/functions_routes.py | 16 +- .../services_rpc/wb_api_server.py | 6 - 4 files changed, 105 insertions(+), 175 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py b/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py index 0961b75765b..ad005fd8bc2 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py @@ -13,7 +13,7 @@ FunctionJobID: TypeAlias = projects.ProjectID FileID: TypeAlias = UUID -InputTypes: TypeAlias = FileID | float | int | bool | str | list | None +InputTypes: TypeAlias = FileID | float | int | bool | str | list class FunctionSchema(BaseModel): @@ -48,31 +48,31 @@ class FunctionClass(str, Enum): class FunctionBase(BaseModel): function_class: FunctionClass - uid: FunctionID | None = None - title: str | None = None - description: str | None = None - input_schema: FunctionInputSchema | None = None - output_schema: FunctionOutputSchema | None = None - default_inputs: FunctionInputs | None = None + uid: FunctionID | None + title: str = "" + description: str = "" + input_schema: FunctionInputSchema | None + output_schema: FunctionOutputSchema | None + default_inputs: FunctionInputs class FunctionDB(BaseModel): function_class: FunctionClass - uuid: FunctionJobID | None = None - title: str | None = None - description: str | None = None - input_schema: FunctionInputSchema | None = None - output_schema: FunctionOutputSchema | None = None - default_inputs: FunctionInputs | None = None + uuid: FunctionJobID | None + title: str = "" + description: str = "" + input_schema: FunctionInputSchema | None + output_schema: FunctionOutputSchema | None + default_inputs: FunctionInputs class_specific_data: FunctionClassSpecificData class FunctionJobDB(BaseModel): - uuid: FunctionJobID | None = None + uuid: FunctionJobID | None function_uuid: FunctionID - title: str | None = None - inputs: FunctionInputs | None = None - outputs: FunctionOutputs | None = None + title: str = "" + inputs: FunctionInputs + outputs: FunctionOutputs class_specific_data: FunctionJobClassSpecificData function_class: FunctionClass @@ -94,7 +94,7 @@ class ProjectFunction(FunctionBase): class SolverFunction(FunctionBase): function_class: Literal[FunctionClass.solver] = FunctionClass.solver solver_key: SolverKeyId - solver_version: str + solver_version: str = "" class PythonCodeFunction(FunctionBase): @@ -111,12 +111,12 @@ class PythonCodeFunction(FunctionBase): class FunctionJobBase(BaseModel): - uid: FunctionJobID | None = None - title: str | None = None - description: str | None = None + uid: FunctionJobID | None + title: str = "" + description: str = "" function_uid: FunctionID - inputs: FunctionInputs | None = None - outputs: FunctionOutputs | None = None + inputs: FunctionInputs + outputs: FunctionOutputs function_class: FunctionClass @@ -147,18 +147,18 @@ class FunctionJobStatus(BaseModel): class FunctionJobCollection(BaseModel): """Model for a collection of function jobs""" - uid: FunctionJobCollectionID | None = None - title: str | None - description: str | None + uid: FunctionJobCollectionID | None + title: str = "" + description: str = "" job_ids: list[FunctionJobID] class FunctionJobCollectionDB(BaseModel): """Model for a collection of function jobs""" - uuid: FunctionJobCollectionID | None - title: str | None - description: str | None + uuid: FunctionJobCollectionID + title: str = "" + description: str = "" class FunctionJobCollectionStatus(BaseModel): diff --git a/services/api-server/openapi.json b/services/api-server/openapi.json index a2a22f8d5e4..d38dafb42bb 100644 --- a/services/api-server/openapi.json +++ b/services/api-server/openapi.json @@ -7581,26 +7581,14 @@ "title": "Uid" }, "title": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Title" + "type": "string", + "title": "Title", + "default": "" }, "description": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Description" + "type": "string", + "title": "Description", + "default": "" }, "job_ids": { "items": { @@ -7613,8 +7601,7 @@ }, "type": "object", "required": [ - "title", - "description", + "uid", "job_ids" ], "title": "FunctionJobCollection", @@ -9225,26 +9212,14 @@ "title": "Uid" }, "title": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Title" + "type": "string", + "title": "Title", + "default": "" }, "description": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Description" + "type": "string", + "title": "Description", + "default": "" }, "input_schema": { "anyOf": [ @@ -9285,6 +9260,10 @@ }, "type": "object", "required": [ + "uid", + "input_schema", + "output_schema", + "default_inputs", "project_id" ], "title": "ProjectFunction" @@ -9304,26 +9283,14 @@ "title": "Uid" }, "title": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Title" + "type": "string", + "title": "Title", + "default": "" }, "description": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Description" + "type": "string", + "title": "Description", + "default": "" }, "function_uid": { "type": "string", @@ -9366,7 +9333,10 @@ }, "type": "object", "required": [ + "uid", "function_uid", + "inputs", + "outputs", "project_job_id" ], "title": "ProjectFunctionJob" @@ -9392,26 +9362,14 @@ "title": "Uid" }, "title": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Title" + "type": "string", + "title": "Title", + "default": "" }, "description": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Description" + "type": "string", + "title": "Description", + "default": "" }, "input_schema": { "anyOf": [ @@ -9451,6 +9409,10 @@ }, "type": "object", "required": [ + "uid", + "input_schema", + "output_schema", + "default_inputs", "code_url" ], "title": "PythonCodeFunction" @@ -9470,26 +9432,14 @@ "title": "Uid" }, "title": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Title" + "type": "string", + "title": "Title", + "default": "" }, "description": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Description" + "type": "string", + "title": "Description", + "default": "" }, "function_uid": { "type": "string", @@ -9527,7 +9477,10 @@ }, "type": "object", "required": [ - "function_uid" + "uid", + "function_uid", + "inputs", + "outputs" ], "title": "PythonCodeFunctionJob" }, @@ -9689,26 +9642,14 @@ "title": "Uid" }, "title": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Title" + "type": "string", + "title": "Title", + "default": "" }, "description": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Description" + "type": "string", + "title": "Description", + "default": "" }, "input_schema": { "anyOf": [ @@ -9748,13 +9689,17 @@ }, "solver_version": { "type": "string", - "title": "Solver Version" + "title": "Solver Version", + "default": "" } }, "type": "object", "required": [ - "solver_key", - "solver_version" + "uid", + "input_schema", + "output_schema", + "default_inputs", + "solver_key" ], "title": "SolverFunction" }, @@ -9773,26 +9718,14 @@ "title": "Uid" }, "title": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Title" + "type": "string", + "title": "Title", + "default": "" }, "description": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Description" + "type": "string", + "title": "Description", + "default": "" }, "function_uid": { "type": "string", @@ -9835,7 +9768,10 @@ }, "type": "object", "required": [ + "uid", "function_uid", + "inputs", + "outputs", "solver_job_id" ], "title": "SolverFunctionJob" diff --git a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py index d14739cdd9d..a5d31066c8d 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py @@ -24,9 +24,9 @@ ) from pydantic import PositiveInt from servicelib.fastapi.dependencies import get_reverse_url_mapper +from simcore_service_api_server._service_jobs import JobService from sqlalchemy.ext.asyncio import AsyncEngine -from ..._service_job import JobService from ..._service_solvers import SolverService from ...models.schemas.errors import ErrorGet from ...models.schemas.jobs import ( @@ -38,7 +38,7 @@ from ...services_rpc.wb_api_server import WbApiRpcClient from ..dependencies.authentication import get_current_user_id, get_product_name from ..dependencies.database import get_db_asyncpg_engine -from ..dependencies.services import get_api_client +from ..dependencies.services import get_api_client, get_job_service, get_solver_service from ..dependencies.webserver_http import get_webserver_session from ..dependencies.webserver_rpc import ( get_wb_api_rpc_client, @@ -173,8 +173,8 @@ async def run_function( # noqa: PLR0913 function_inputs: FunctionInputs, user_id: Annotated[PositiveInt, Depends(get_current_user_id)], product_name: Annotated[str, Depends(get_product_name)], - solver_service: Annotated[SolverService, Depends()], - job_service: Annotated[JobService, Depends()], + solver_service: Annotated[SolverService, Depends(get_solver_service)], + job_service: Annotated[JobService, Depends(get_job_service)], ): to_run_function = await wb_api_rpc.get_function(function_id=function_id) @@ -227,6 +227,7 @@ async def run_function( # noqa: PLR0913 return await register_function_job( wb_api_rpc=wb_api_rpc, function_job=ProjectFunctionJob( + uid=None, function_uid=to_run_function.uid, title=f"Function job of function {to_run_function.uid}", description=to_run_function.description, @@ -245,8 +246,6 @@ async def run_function( # noqa: PLR0913 url_for=url_for, x_simcore_parent_project_uuid=None, x_simcore_parent_node_id=None, - user_id=user_id, - product_name=product_name, ) await solvers_jobs.start_job( request=request, @@ -260,6 +259,7 @@ async def run_function( # noqa: PLR0913 return await register_function_job( wb_api_rpc=wb_api_rpc, function_job=SolverFunctionJob( + uid=None, function_uid=to_run_function.uid, title=f"Function job of function {to_run_function.uid}", description=to_run_function.description, @@ -463,8 +463,8 @@ async def map_function( # noqa: PLR0913 director2_api: Annotated[DirectorV2Api, Depends(get_api_client(DirectorV2Api))], user_id: Annotated[PositiveInt, Depends(get_current_user_id)], product_name: Annotated[str, Depends(get_product_name)], - solver_service: Annotated[SolverService, Depends()], - job_service: Annotated[JobService, Depends()], + solver_service: Annotated[SolverService, Depends(get_solver_service)], + job_service: Annotated[JobService, Depends(get_job_service)], ): function_jobs = [] function_jobs = [ diff --git a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py index c5a35499ef2..9c8787acc5a 100644 --- a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py +++ b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py @@ -76,9 +76,6 @@ from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( list_functions as _list_functions, ) -from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( - ping as _ping, -) from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( register_function as _register_function, ) @@ -260,9 +257,6 @@ async def release_licensed_item_for_wallet( num_of_seats=licensed_item_checkout_get.num_of_seats, ) - async def ping(self) -> str: - return await _ping(self._client) - async def mark_project_as_job( self, product_name: ProductName, From f956360e0a1e61f89f3c9e046e9fe99786d16a4d Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Wed, 7 May 2025 10:46:18 +0200 Subject: [PATCH 17/69] Merge alembic heads after rebase --- ...8debc3e_merge_0d52976dc616_ecd7a3b85134.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/1b5c88debc3e_merge_0d52976dc616_ecd7a3b85134.py diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/1b5c88debc3e_merge_0d52976dc616_ecd7a3b85134.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/1b5c88debc3e_merge_0d52976dc616_ecd7a3b85134.py new file mode 100644 index 00000000000..be5f96cccd6 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/1b5c88debc3e_merge_0d52976dc616_ecd7a3b85134.py @@ -0,0 +1,21 @@ +"""merge 0d52976dc616 ecd7a3b85134 + +Revision ID: 1b5c88debc3e +Revises: 0d52976dc616, ecd7a3b85134 +Create Date: 2025-05-07 08:45:47.779512+00:00 + +""" + +# revision identifiers, used by Alembic. +revision = "1b5c88debc3e" +down_revision = ("0d52976dc616", "ecd7a3b85134") +branch_labels = None +depends_on = None + + +def upgrade(): + pass + + +def downgrade(): + pass From 94c44c8e43c162f0876626e6e15d24bd6ccb4ce7 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Wed, 7 May 2025 11:54:13 +0200 Subject: [PATCH 18/69] Fix tests after rebase --- .../test_api_routers_functions.py | 22 +++++++++++++++++++ .../functions/_functions_controller_rpc.py | 5 ++++- .../functions/_functions_repository.py | 6 ++++- .../test_functions_controller_rpc.py | 9 ++++++++ 4 files changed, 40 insertions(+), 2 deletions(-) diff --git a/services/api-server/tests/unit/api_functions/test_api_routers_functions.py b/services/api-server/tests/unit/api_functions/test_api_routers_functions.py index 8223e7e5813..21ac2be2a90 100644 --- a/services/api-server/tests/unit/api_functions/test_api_routers_functions.py +++ b/services/api-server/tests/unit/api_functions/test_api_routers_functions.py @@ -182,12 +182,14 @@ async def delete_function_job_collection( def test_register_function(api_app) -> None: client = TestClient(api_app) sample_function = { + "uid": None, "title": "test_function", "function_class": "project", "project_id": str(uuid4()), "description": "A test function", "input_schema": {"schema_dict": {}}, "output_schema": {"schema_dict": {}}, + "default_inputs": None, } response = client.post("/functions", json=sample_function) assert response.status_code == 200 @@ -221,12 +223,14 @@ def test_get_function(api_app: FastAPI) -> None: project_id = str(uuid4()) # First, register a sample function so that it exists sample_function = { + "uid": None, "title": "example_function", "function_class": "project", "project_id": project_id, "description": "An example function", "input_schema": {"schema_dict": {}}, "output_schema": {"schema_dict": {}}, + "default_inputs": None, } post_response = client.post("/functions", json=sample_function) assert post_response.status_code == 200 @@ -262,12 +266,14 @@ def test_list_functions(api_app: FastAPI) -> None: client = TestClient(api_app) # Register a sample function sample_function = { + "uid": None, "title": "example_function", "function_class": "project", "project_id": str(uuid4()), "description": "An example function", "input_schema": {"schema_dict": {}}, "output_schema": {"schema_dict": {}}, + "default_inputs": None, } post_response = client.post("/functions", json=sample_function) assert post_response.status_code == 200 @@ -285,6 +291,7 @@ def test_get_function_input_schema(api_app: FastAPI) -> None: project_id = str(uuid4()) # Register a sample function sample_function = { + "uid": None, "title": "example_function", "function_class": "project", "project_id": project_id, @@ -296,6 +303,7 @@ def test_get_function_input_schema(api_app: FastAPI) -> None: } }, "output_schema": {"schema_dict": {}}, + "default_inputs": None, } post_response = client.post("/functions", json=sample_function) assert post_response.status_code == 200 @@ -315,6 +323,7 @@ def test_get_function_output_schema(api_app: FastAPI) -> None: project_id = str(uuid4()) # Register a sample function sample_function = { + "uid": None, "title": "example_function", "function_class": "project", "project_id": project_id, @@ -326,6 +335,7 @@ def test_get_function_output_schema(api_app: FastAPI) -> None: "properties": {"output1": {"type": "string"}}, } }, + "default_inputs": None, } post_response = client.post("/functions", json=sample_function) assert post_response.status_code == 200 @@ -344,6 +354,7 @@ def test_validate_function_inputs(api_app: FastAPI) -> None: project_id = str(uuid4()) # Register a sample function sample_function = { + "uid": None, "title": "example_function", "function_class": "project", "project_id": project_id, @@ -355,6 +366,7 @@ def test_validate_function_inputs(api_app: FastAPI) -> None: } }, "output_schema": {"schema_dict": {}}, + "default_inputs": None, } post_response = client.post("/functions", json=sample_function) assert post_response.status_code == 200 @@ -376,12 +388,14 @@ def test_delete_function(api_app: FastAPI) -> None: project_id = str(uuid4()) # Register a sample function sample_function = { + "uid": None, "title": "example_function", "function_class": "project", "project_id": project_id, "description": "An example function", "input_schema": {"schema_dict": {}}, "output_schema": {"schema_dict": {}}, + "default_inputs": None, } post_response = client.post("/functions", json=sample_function) assert post_response.status_code == 200 @@ -398,6 +412,7 @@ def test_register_function_job(api_app: FastAPI) -> None: client = TestClient(api_app) mock_function_job = { + "uid": None, "function_uid": str(uuid4()), "title": "Test Function Job", "description": "A test function job", @@ -415,6 +430,7 @@ def test_register_function_job(api_app: FastAPI) -> None: response_data = response.json() assert response_data["uid"] is not None response_data.pop("uid", None) # Remove the uid field + mock_function_job.pop("uid", None) # Remove the uid field assert response_data == mock_function_job @@ -423,6 +439,7 @@ def test_get_function_job(api_app: FastAPI) -> None: client = TestClient(api_app) mock_function_job = { + "uid": None, "function_uid": str(uuid4()), "title": "Test Function Job", "description": "A test function job", @@ -454,6 +471,7 @@ def test_list_function_jobs(api_app: FastAPI) -> None: client = TestClient(api_app) mock_function_job = { + "uid": None, "function_uid": str(uuid4()), "title": "Test Function Job", "description": "A test function job", @@ -480,6 +498,7 @@ def test_delete_function_job(api_app: FastAPI) -> None: client = TestClient(api_app) mock_function_job = { + "uid": None, "function_uid": str(uuid4()), "title": "Test Function Job", "description": "A test function job", @@ -505,6 +524,7 @@ def test_register_function_job_collection(api_app: FastAPI) -> None: client = TestClient(api_app) mock_function_job_collection = { + "uid": None, "title": "Test Collection", "description": "A test function job collection", "job_ids": [str(uuid4()), str(uuid4())], @@ -520,6 +540,7 @@ def test_register_function_job_collection(api_app: FastAPI) -> None: response_data = response.json() assert response_data["uid"] is not None response_data.pop("uid", None) # Remove the uid field + mock_function_job_collection.pop("uid", None) # Remove the uid field assert response_data == mock_function_job_collection @@ -527,6 +548,7 @@ def test_get_function_job_collection(api_app: FastAPI) -> None: # Arrange client = TestClient(api_app) mock_function_job_collection = { + "uid": None, "title": "Test Collection", "description": "A test function job collection", "job_ids": [str(uuid4()), str(uuid4())], diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py index f8e39796249..90cf67bf71e 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py @@ -36,7 +36,7 @@ async def ping(app: web.Application) -> str: @router.expose() async def register_function(app: web.Application, *, function: Function) -> Function: assert app - saved_function = await _functions_repository.create_function( + saved_function = await _functions_repository.register_function( app=app, function=_encode_function(function) ) return _decode_function(saved_function) @@ -90,6 +90,7 @@ def _encode_function( raise TypeError(msg) return FunctionDB( + uuid=function.uid, title=function.title, description=function.description, input_schema=function.input_schema, @@ -145,6 +146,7 @@ def _encode_functionjob( ) -> FunctionJobDB: if functionjob.function_class == FunctionClass.project: return FunctionJobDB( + uuid=functionjob.uid, title=functionjob.title, function_uuid=functionjob.function_uid, inputs=functionjob.inputs, @@ -158,6 +160,7 @@ def _encode_functionjob( ) elif functionjob.function_class == FunctionClass.solver: # noqa: RET505 return FunctionJobDB( + uuid=functionjob.uid, title=functionjob.title, function_uuid=functionjob.function_uid, inputs=functionjob.inputs, diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py index 6e3fa98b94d..fbcc4ad6c2b 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py @@ -40,13 +40,17 @@ ) -async def create_function( +async def register_function( app: web.Application, connection: AsyncConnection | None = None, *, function: FunctionDB, ) -> FunctionDB: + if function.uuid is not None: + msg = "Function uid is not None. Cannot register function." + raise ValueError(msg) + async with transaction_context(get_asyncpg_engine(app), connection) as conn: result = await conn.stream( functions_table.insert() diff --git a/services/web/server/tests/unit/with_dbs/functions_rpc/test_functions_controller_rpc.py b/services/web/server/tests/unit/with_dbs/functions_rpc/test_functions_controller_rpc.py index 400813e43e9..556885f4b56 100644 --- a/services/web/server/tests/unit/with_dbs/functions_rpc/test_functions_controller_rpc.py +++ b/services/web/server/tests/unit/with_dbs/functions_rpc/test_functions_controller_rpc.py @@ -17,6 +17,7 @@ @pytest.fixture def mock_function() -> Function: return ProjectFunction( + uid=None, title="Test Function", description="A test function", input_schema=FunctionInputSchema( @@ -96,6 +97,7 @@ async def test_get_function_not_found(client): async def test_list_functions(client): # Register a function first mock_function = ProjectFunction( + uid=None, title="Test Function", description="A test function", input_schema=None, @@ -179,6 +181,7 @@ async def test_register_function_job(client, mock_function): assert registered_function.uid is not None function_job = ProjectFunctionJob( + uid=None, function_uid=registered_function.uid, title="Test Function Job", description="A test function job", @@ -207,6 +210,7 @@ async def test_get_function_job(client, mock_function): assert registered_function.uid is not None function_job = ProjectFunctionJob( + uid=None, function_uid=registered_function.uid, title="Test Function Job", description="A test function job", @@ -248,6 +252,7 @@ async def test_list_function_jobs(client, mock_function): assert registered_function.uid is not None function_job = ProjectFunctionJob( + uid=None, function_uid=registered_function.uid, title="Test Function Job", description="A test function job", @@ -278,6 +283,7 @@ async def test_delete_function_job(client, mock_function): assert registered_function.uid is not None function_job = ProjectFunctionJob( + uid=None, function_uid=registered_function.uid, title="Test Function Job", description="A test function job", @@ -313,6 +319,7 @@ async def test_function_job_collection(client, mock_function): assert registered_function.uid is not None registered_function_job = ProjectFunctionJob( + uid=None, function_uid=registered_function.uid, title="Test Function Job", description="A test function job", @@ -324,6 +331,7 @@ async def test_function_job_collection(client, mock_function): function_job_ids = [] for _ in range(3): registered_function_job = ProjectFunctionJob( + uid=None, function_uid=registered_function.uid, title="Test Function Job", description="A test function job", @@ -339,6 +347,7 @@ async def test_function_job_collection(client, mock_function): function_job_ids.append(registered_job.uid) function_job_collection = FunctionJobCollection( + uid=None, title="Test Function Job Collection", description="A test function job collection", job_ids=function_job_ids, From 61c3bbf723a6d51ed4bb250297a063ba8e86c988 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Wed, 7 May 2025 15:19:27 +0200 Subject: [PATCH 19/69] Add pagenation to function listing --- .../models/functions_models_db.py | 2 +- .../functions/functions_rpc_interface.py | 71 ++++---- .../api/routes/functions_routes.py | 20 ++- .../services_rpc/wb_api_server.py | 50 +++++- .../functions/_functions_controller_rpc.py | 91 ++++++---- .../functions/_functions_repository.py | 157 ++++++++++++------ .../test_functions_controller_rpc.py | 121 +++++++++++++- 7 files changed, 382 insertions(+), 130 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/functions_models_db.py b/packages/postgres-database/src/simcore_postgres_database/models/functions_models_db.py index bfaabb39372..ee8dd1474c0 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/functions_models_db.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/functions_models_db.py @@ -96,7 +96,7 @@ functions.c.uuid, onupdate=RefActions.CASCADE, ondelete=RefActions.CASCADE, - name="fk_functions_to_function_jobs_to_function_uuid", + name="fk_function_jobs_to_function_uuid", ), nullable=False, index=True, diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py index 7bee95d0601..56987fc8693 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py @@ -15,6 +15,9 @@ FunctionOutputSchema, ) from models_library.rabbitmq_basic_types import RPCMethodName +from models_library.rest_pagination import ( + PageMetaInfoLimitOffset, +) from pydantic import TypeAdapter from .....logging_utils import log_decorator @@ -23,18 +26,6 @@ _logger = logging.getLogger(__name__) -@log_decorator(_logger, level=logging.DEBUG) -async def ping( - rabbitmq_rpc_client: RabbitMQRPCClient, -) -> str: - result = await rabbitmq_rpc_client.request( - WEBSERVER_RPC_NAMESPACE, - TypeAdapter(RPCMethodName).validate_python("ping"), - ) - assert isinstance(result, str) # nosec - return result - - @log_decorator(_logger, level=logging.DEBUG) async def register_function( rabbitmq_rpc_client: RabbitMQRPCClient, @@ -103,10 +94,44 @@ async def delete_function( @log_decorator(_logger, level=logging.DEBUG) async def list_functions( rabbitmq_rpc_client: RabbitMQRPCClient, -) -> list[Function]: + *pagination_limit: int, + pagination_offset: int, +) -> tuple[list[Function], PageMetaInfoLimitOffset]: return await rabbitmq_rpc_client.request( WEBSERVER_RPC_NAMESPACE, TypeAdapter(RPCMethodName).validate_python("list_functions"), + pagination_offset=pagination_offset, + pagination_limit=pagination_limit, + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def list_function_jobs( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + pagination_limit: int, + pagination_offset: int, +) -> tuple[list[FunctionJob], PageMetaInfoLimitOffset]: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("list_function_jobs"), + pagination_offset=pagination_offset, + pagination_limit=pagination_limit, + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def list_function_job_collections( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + pagination_limit: int, + pagination_offset: int, +) -> tuple[list[FunctionJobCollection], PageMetaInfoLimitOffset]: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("list_function_job_collections"), + pagination_offset=pagination_offset, + pagination_limit=pagination_limit, ) @@ -151,16 +176,6 @@ async def get_function_job( ) -@log_decorator(_logger, level=logging.DEBUG) -async def list_function_jobs( - rabbitmq_rpc_client: RabbitMQRPCClient, -) -> list[FunctionJob]: - return await rabbitmq_rpc_client.request( - WEBSERVER_RPC_NAMESPACE, - TypeAdapter(RPCMethodName).validate_python("list_function_jobs"), - ) - - @log_decorator(_logger, level=logging.DEBUG) async def delete_function_job( rabbitmq_rpc_client: RabbitMQRPCClient, @@ -189,16 +204,6 @@ async def find_cached_function_job( ) -@log_decorator(_logger, level=logging.DEBUG) -async def list_function_job_collections( - rabbitmq_rpc_client: RabbitMQRPCClient, -) -> list[FunctionJobCollection]: - return await rabbitmq_rpc_client.request( - WEBSERVER_RPC_NAMESPACE, - TypeAdapter(RPCMethodName).validate_python("list_function_job_collections"), - ) - - @log_decorator(_logger, level=logging.DEBUG) async def register_function_job_collection( rabbitmq_rpc_client: RabbitMQRPCClient, diff --git a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py index a5d31066c8d..b25b541058d 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py @@ -4,6 +4,7 @@ import jsonschema from fastapi import APIRouter, Depends, Request, status +from fastapi_pagination.api import create_page from jsonschema import ValidationError from models_library.api_schemas_webserver.functions_wb_schema import ( Function, @@ -28,6 +29,7 @@ from sqlalchemy.ext.asyncio import AsyncEngine from ..._service_solvers import SolverService +from ...models.pagination import PaginationParams from ...models.schemas.errors import ErrorGet from ...models.schemas.jobs import ( JobInputs, @@ -86,8 +88,18 @@ async def get_function( @function_router.get("", response_model=list[Function], description="List functions") async def list_functions( wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], + page_params: Annotated[PaginationParams, Depends()], ): - return await wb_api_rpc.list_functions() + functions_list, meta = await wb_api_rpc.list_functions( + pagination_offset=page_params.offset, + pagination_limit=page_params.limit, + ) + + return create_page( + functions_list, + total=meta.total, + params=page_params, + ) def join_inputs( @@ -322,8 +334,9 @@ async def get_function_job( ) async def list_function_jobs( wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], + page_params: Annotated[PaginationParams, Depends()], ): - return await wb_api_rpc.list_function_jobs() + return await wb_api_rpc.list_function_jobs(page_params=page_params) @function_job_router.delete( @@ -515,8 +528,9 @@ async def map_function( # noqa: PLR0913 ) async def list_function_job_collections( wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], + page_params: Annotated[PaginationParams, Depends()], ): - return await wb_api_rpc.list_function_job_collections() + return await wb_api_rpc.list_function_job_collections(page_params=page_params) @function_job_collections_router.get( diff --git a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py index 9c8787acc5a..91adebd2606 100644 --- a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py +++ b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py @@ -22,6 +22,12 @@ from models_library.resource_tracker_licensed_items_checkouts import ( LicensedItemCheckoutID, ) +from models_library.rest_pagination import ( + DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, + PageLimitInt, + PageMetaInfoLimitOffset, + PageOffsetInt, +) from models_library.services_types import ServiceRunID from models_library.users import UserID from models_library.wallets import WalletID @@ -312,8 +318,42 @@ async def get_function(self, *, function_id: FunctionID) -> Function: async def delete_function(self, *, function_id: FunctionID) -> None: return await _delete_function(self._client, function_id=function_id) - async def list_functions(self) -> list[Function]: - return await _list_functions(self._client) + async def list_functions( + self, + *, + pagination_offset: PageOffsetInt = 0, + pagination_limit: PageLimitInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, + ) -> tuple[list[Function], PageMetaInfoLimitOffset]: + + return await _list_functions( + self._client, + pagination_offset=pagination_offset, + pagination_limit=pagination_limit, + ) + + async def list_function_jobs( + self, + *, + pagination_offset: PageOffsetInt = 0, + pagination_limit: PageLimitInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, + ) -> tuple[list[FunctionJob], PageMetaInfoLimitOffset]: + return await _list_function_jobs( + self._client, + pagination_offset=pagination_offset, + pagination_limit=pagination_limit, + ) + + async def list_function_job_collections( + self, + *, + pagination_offset: PageOffsetInt = 0, + pagination_limit: PageLimitInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, + ) -> tuple[list[FunctionJobCollection], PageMetaInfoLimitOffset]: + return await _list_function_job_collections( + self._client, + pagination_offset=pagination_offset, + pagination_limit=pagination_limit, + ) async def run_function( self, *, function_id: FunctionID, inputs: FunctionInputs @@ -346,12 +386,6 @@ async def find_cached_function_job( self._client, function_id=function_id, inputs=inputs ) - async def list_function_jobs(self) -> list[FunctionJob]: - return await _list_function_jobs(self._client) - - async def list_function_job_collections(self) -> list[FunctionJobCollection]: - return await _list_function_job_collections(self._client) - async def get_function_job_collection( self, *, function_job_collection_id: FunctionJobCollectionID ) -> FunctionJobCollection: diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py index 90cf67bf71e..fa14c220d1f 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py @@ -19,6 +19,9 @@ SolverFunction, SolverFunctionJob, ) +from models_library.rest_pagination import ( + PageMetaInfoLimitOffset, +) from servicelib.rabbitmq import RPCRouter from ..rabbitmq import get_rabbitmq_rpc_server @@ -191,18 +194,6 @@ async def get_function_job( return _decode_functionjob(returned_function_job) -@router.expose() -async def list_function_jobs(app: web.Application) -> list[FunctionJob]: - assert app - returned_function_jobs = await _functions_repository.list_function_jobs( - app=app, - ) - return [ - _decode_functionjob(returned_function_job) - for returned_function_job in returned_function_jobs - ] - - @router.expose() async def get_function_input_schema( app: web.Application, *, function_id: FunctionID @@ -240,14 +231,63 @@ async def get_function_output_schema( @router.expose() -async def list_functions(app: web.Application) -> list[Function]: +async def list_functions( + app: web.Application, + pagination_limit: int, + pagination_offset: int, +) -> tuple[list[Function], PageMetaInfoLimitOffset]: assert app - returned_functions = await _functions_repository.list_functions( + returned_functions, page = await _functions_repository.list_functions( app=app, + pagination_limit=pagination_limit, + pagination_offset=pagination_offset, ) return [ _decode_function(returned_function) for returned_function in returned_functions - ] + ], page + + +@router.expose() +async def list_function_jobs( + app: web.Application, + pagination_limit: int, + pagination_offset: int, +) -> tuple[list[FunctionJob], PageMetaInfoLimitOffset]: + assert app + returned_function_jobs, page = await _functions_repository.list_function_jobs( + app=app, + pagination_limit=pagination_limit, + pagination_offset=pagination_offset, + ) + return [ + _decode_functionjob(returned_function_job) + for returned_function_job in returned_function_jobs + ], page + + +@router.expose() +async def list_function_job_collections( + app: web.Application, + pagination_limit: int, + pagination_offset: int, +) -> tuple[list[FunctionJobCollection], PageMetaInfoLimitOffset]: + assert app + returned_function_job_collections, page = ( + await _functions_repository.list_function_job_collections( + app=app, + pagination_limit=pagination_limit, + pagination_offset=pagination_offset, + ) + ) + return [ + FunctionJobCollection( + uid=function_job_collection.uuid, + title=function_job_collection.title, + description=function_job_collection.description, + job_ids=job_ids, + ) + for function_job_collection, job_ids in returned_function_job_collections + ], page @router.expose() @@ -317,27 +357,6 @@ async def find_cached_function_job( raise TypeError(msg) -@router.expose() -async def list_function_job_collections( - app: web.Application, -) -> list[FunctionJobCollection]: - assert app - returned_function_job_collections = ( - await _functions_repository.list_function_job_collections( - app=app, - ) - ) - return [ - FunctionJobCollection( - uid=function_job_collection.uuid, - title=function_job_collection.title, - description=function_job_collection.description, - job_ids=job_ids, - ) - for function_job_collection, job_ids in returned_function_job_collections - ] - - @router.expose() async def register_function_job_collection( app: web.Application, *, function_job_collection: FunctionJobCollection diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py index fbcc4ad6c2b..5d8e7252064 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py @@ -10,6 +10,9 @@ FunctionJobDB, FunctionJobID, ) +from models_library.rest_pagination import ( + PageMetaInfoLimitOffset, +) from simcore_postgres_database.models.functions_models_db import ( function_job_collections as function_job_collections_table, ) @@ -28,6 +31,7 @@ ) from sqlalchemy import Text, cast from sqlalchemy.ext.asyncio import AsyncConnection +from sqlalchemy.sql import func from ..db.plugin import get_asyncpg_engine @@ -105,15 +109,117 @@ async def get_function( async def list_functions( app: web.Application, connection: AsyncConnection | None = None, -) -> list[FunctionDB]: + *, + pagination_limit: int, + pagination_offset: int, +) -> tuple[list[FunctionDB], PageMetaInfoLimitOffset]: + + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + total_count_result = await conn.scalar( + func.count().select().select_from(functions_table) + ) + result = await conn.stream( + functions_table.select().offset(pagination_offset).limit(pagination_limit) + ) + rows = await result.all() + if rows is None: + return [], PageMetaInfoLimitOffset( + total=0, offset=pagination_offset, limit=pagination_limit, count=0 + ) + + return [ + FunctionDB.model_validate(dict(row)) for row in rows + ], PageMetaInfoLimitOffset( + total=total_count_result, + offset=pagination_offset, + limit=pagination_limit, + count=len(rows), + ) + + +async def list_function_jobs( + app: web.Application, + connection: AsyncConnection | None = None, + *, + pagination_limit: int, + pagination_offset: int, +) -> tuple[list[FunctionJobDB], PageMetaInfoLimitOffset]: async with transaction_context(get_asyncpg_engine(app), connection) as conn: - result = await conn.stream(functions_table.select().where()) + total_count_result = await conn.scalar( + func.count().select().select_from(function_jobs_table) + ) + result = await conn.stream( + function_jobs_table.select() + .offset(pagination_offset) + .limit(pagination_limit) + ) rows = await result.all() if rows is None: - return [] + return [], PageMetaInfoLimitOffset( + total=0, offset=pagination_offset, limit=pagination_limit, count=0 + ) + + return [ + FunctionJobDB.model_validate(dict(row)) for row in rows + ], PageMetaInfoLimitOffset( + total=total_count_result, + offset=pagination_offset, + limit=pagination_limit, + count=len(rows), + ) - return [FunctionDB.model_validate(dict(row)) for row in rows] + +async def list_function_job_collections( + app: web.Application, + connection: AsyncConnection | None = None, + *, + pagination_limit: int, + pagination_offset: int, +) -> tuple[ + list[tuple[FunctionJobCollectionDB, list[FunctionJobID]]], + PageMetaInfoLimitOffset, +]: + """ + Returns a list of function job collections and their associated job ids. + """ + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + total_count_result = await conn.scalar( + func.count().select().select_from(function_job_collections_table) + ) + result = await conn.stream( + function_job_collections_table.select() + .offset(pagination_offset) + .limit(pagination_limit) + ) + rows = await result.all() + if rows is None: + return [], PageMetaInfoLimitOffset( + total=0, offset=pagination_offset, limit=pagination_limit, count=0 + ) + + collections = [] + for row in rows: + collection = FunctionJobCollectionDB.model_validate(dict(row)) + job_result = await conn.stream( + function_job_collections_to_function_jobs_table.select().where( + function_job_collections_to_function_jobs_table.c.function_job_collection_uuid + == row["uuid"] + ) + ) + job_rows = await job_result.all() + job_ids = ( + [job_row["function_job_uuid"] for job_row in job_rows] + if job_rows + else [] + ) + collections.append((collection, job_ids)) + return collections, PageMetaInfoLimitOffset( + total=total_count_result, + offset=pagination_offset, + limit=pagination_limit, + count=len(rows), + ) async def delete_function( @@ -181,20 +287,6 @@ async def get_function_job( return FunctionJobDB.model_validate(dict(row)) -async def list_function_jobs( - app: web.Application, - connection: AsyncConnection | None = None, -) -> list[FunctionJobDB]: - - async with transaction_context(get_asyncpg_engine(app), connection) as conn: - result = await conn.stream(function_jobs_table.select().where()) - rows = await result.all() - if rows is None: - return [] - - return [FunctionJobDB.model_validate(dict(row)) for row in rows] - - async def delete_function_job( app: web.Application, connection: AsyncConnection | None = None, @@ -238,35 +330,6 @@ async def find_cached_function_job( return None -async def list_function_job_collections( - app: web.Application, - connection: AsyncConnection | None = None, -) -> list[tuple[FunctionJobCollectionDB, list[FunctionJobID]]]: - async with transaction_context(get_asyncpg_engine(app), connection) as conn: - result = await conn.stream(function_job_collections_table.select().where()) - rows = await result.all() - if rows is None: - return [] - - collections = [] - for row in rows: - collection = FunctionJobCollection.model_validate(dict(row)) - job_result = await conn.stream( - function_job_collections_to_function_jobs_table.select().where( - function_job_collections_to_function_jobs_table.c.function_job_collection_uuid - == row["uuid"] - ) - ) - job_rows = await job_result.all() - job_ids = ( - [job_row["function_job_uuid"] for job_row in job_rows] - if job_rows - else [] - ) - collections.append((collection, job_ids)) - return collections - - async def get_function_job_collection( app: web.Application, connection: AsyncConnection | None = None, diff --git a/services/web/server/tests/unit/with_dbs/functions_rpc/test_functions_controller_rpc.py b/services/web/server/tests/unit/with_dbs/functions_rpc/test_functions_controller_rpc.py index 556885f4b56..403fe959374 100644 --- a/services/web/server/tests/unit/with_dbs/functions_rpc/test_functions_controller_rpc.py +++ b/services/web/server/tests/unit/with_dbs/functions_rpc/test_functions_controller_rpc.py @@ -111,13 +111,72 @@ async def test_list_functions(client): assert registered_function.uid is not None # List functions - functions = await functions_rpc.list_functions(app=client.app) + functions, _ = await functions_rpc.list_functions( + app=client.app, pagination_limit=10, pagination_offset=0 + ) # Assert the list contains the registered function assert len(functions) > 0 assert any(f.uid == registered_function.uid for f in functions) +async def delete_all_registered_functions(client): + # This function is a placeholder for the actual implementation + # that deletes all registered functions from the database. + functions, _ = await functions_rpc.list_functions( + app=client.app, pagination_limit=100, pagination_offset=0 + ) + for function in functions: + assert function.uid is not None + await functions_rpc.delete_function(app=client.app, function_id=function.uid) + + +@pytest.mark.asyncio +async def test_list_functions_empty(client): + await delete_all_registered_functions(client) + # List functions when none are registered + functions, _ = await functions_rpc.list_functions( + app=client.app, pagination_limit=10, pagination_offset=0 + ) + + # Assert the list is empty + assert len(functions) == 0 + + +@pytest.mark.asyncio +async def test_list_functions_with_pagination(client, mock_function): + await delete_all_registered_functions(client) + + # Register multiple functions + TOTAL_FUNCTIONS = 3 + for _ in range(TOTAL_FUNCTIONS): + await functions_rpc.register_function(app=client.app, function=mock_function) + + functions, page_info = await functions_rpc.list_functions( + app=client.app, pagination_limit=2, pagination_offset=0 + ) + + # List functions with pagination + functions, page_info = await functions_rpc.list_functions( + app=client.app, pagination_limit=2, pagination_offset=0 + ) + + # Assert the list contains the correct number of functions + assert len(functions) == 2 + assert page_info.count == 2 + assert page_info.total == TOTAL_FUNCTIONS + + # List the next page of functions + functions, page_info = await functions_rpc.list_functions( + app=client.app, pagination_limit=2, pagination_offset=2 + ) + + # Assert the list contains the correct number of functions + assert len(functions) == 1 + assert page_info.count == 1 + assert page_info.total == TOTAL_FUNCTIONS + + @pytest.mark.asyncio async def test_get_function_input_schema(client, mock_function): # Register the function first @@ -267,7 +326,9 @@ async def test_list_function_jobs(client, mock_function): ) # List function jobs - jobs = await functions_rpc.list_function_jobs(app=client.app) + jobs, _ = await functions_rpc.list_function_jobs( + app=client.app, pagination_limit=10, pagination_offset=0 + ) # Assert the list contains the registered job assert len(jobs) > 0 @@ -372,6 +433,62 @@ async def test_function_job_collection(client, mock_function): ) +@pytest.mark.asyncio +async def test_list_function_job_collections(client, mock_function): + # Register the function first + registered_function = await functions_rpc.register_function( + app=client.app, function=mock_function + ) + assert registered_function.uid is not None + + # Create a function job collection + function_job_ids = [] + for _ in range(3): + registered_function_job = ProjectFunctionJob( + uid=None, + function_uid=registered_function.uid, + title="Test Function Job", + description="A test function job", + project_job_id=uuid4(), + inputs={"input1": "value1"}, + outputs={"output1": "result1"}, + ) + # Register the function job + registered_job = await functions_rpc.register_function_job( + app=client.app, function_job=registered_function_job + ) + assert registered_job.uid is not None + function_job_ids.append(registered_job.uid) + + function_job_collection = FunctionJobCollection( + uid=None, + title="Test Function Job Collection", + description="A test function job collection", + job_ids=function_job_ids, + ) + + # Register the function job collection + registered_collections = [ + await functions_rpc.register_function_job_collection( + app=client.app, function_job_collection=function_job_collection + ) + for _ in range(3) + ] + assert all( + registered_collection.uid is not None + for registered_collection in registered_collections + ) + + # List function job collections + collections, _ = await functions_rpc.list_function_job_collections( + app=client.app, pagination_limit=1, pagination_offset=1 + ) + + # Assert the list contains the registered collection + assert len(collections) == 1 + assert collections[0].uid == registered_collections[1].uid + + # @pytest.mark.asyncio # async def test_find_cached_function_job_project_class( # mock_app, mock_function_id, mock_function_inputs From 9f3d968c38f63e8163e5a5c0ba1ffed11114e352 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Wed, 7 May 2025 15:23:00 +0200 Subject: [PATCH 20/69] Fix function routes --- .../api/routes/functions_routes.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py index b25b541058d..614119542b2 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py @@ -336,7 +336,10 @@ async def list_function_jobs( wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], page_params: Annotated[PaginationParams, Depends()], ): - return await wb_api_rpc.list_function_jobs(page_params=page_params) + return await wb_api_rpc.list_function_jobs( + pagination_offset=page_params.offset, + pagination_limit=page_params.limit, + ) @function_job_router.delete( @@ -530,7 +533,10 @@ async def list_function_job_collections( wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], page_params: Annotated[PaginationParams, Depends()], ): - return await wb_api_rpc.list_function_job_collections(page_params=page_params) + return await wb_api_rpc.list_function_job_collections( + pagination_offset=page_params.offset, + pagination_limit=page_params.limit, + ) @function_job_collections_router.get( From 8619a2c20c6bf1d260084c6c4003275f887c0826 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Wed, 7 May 2025 16:48:48 +0200 Subject: [PATCH 21/69] Fix function pagination api tests --- .github/copilot-instructions.md | 53 --- scripts/common-service.Makefile | 3 +- services/api-server/Makefile | 2 +- services/api-server/openapi.json | 438 ++++++++++++++---- .../api/routes/functions_routes.py | 71 +-- .../test_api_routers_functions.py | 65 ++- 6 files changed, 453 insertions(+), 179 deletions(-) delete mode 100644 .github/copilot-instructions.md diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md deleted file mode 100644 index ebd8c6030a6..00000000000 --- a/.github/copilot-instructions.md +++ /dev/null @@ -1,53 +0,0 @@ -# GitHub Copilot Instructions - -This document provides guidelines and best practices for using GitHub Copilot in the `osparc-simcore` repository and other Python and Node.js projects. - -## General Guidelines - -1. **Use Python 3.11**: Ensure that all Python-related suggestions align with Python 3.11 features and syntax. -2. **Node.js Compatibility**: For Node.js projects, ensure compatibility with the version specified in the project (e.g., Node.js 14 or later). -3. **Follow Coding Conventions**: Adhere to the coding conventions outlined in the `docs/coding-conventions.md` file. -4. **Test-Driven Development**: Write unit tests for all new functions and features. Use `pytest` for Python and appropriate testing frameworks for Node.js. -5. **Environment Variables**: Use environment variables as specified in `docs/env-vars.md` for configuration. Avoid hardcoding sensitive information. -6. **Documentation**: Prefer self-explanatory code; add documentation only if explicitly requested by the developer. - -## Python-Specific Instructions - -- Always use type hints and annotations to improve code clarity and compatibility with tools like `mypy`. - - An exception to that rule is in `test_*` functions return type hint must not be added -- Follow the dependency management practices outlined in `requirements/`. -- Use `ruff` for code formatting and for linting. -- Use `black` for code formatting and `pylint` for linting. -- ensure we use `sqlalchemy` >2 compatible code. -- ensure we use `pydantic` >2 compatible code. -- ensure we use `fastapi` >0.100 compatible code -- use f-string formatting -- Only add comments in function if strictly necessary - - -### Json serialization - -- Generally use `json_dumps`/`json_loads` from `common_library.json_serialization` to built-in `json.dumps` / `json.loads`. -- Prefer Pydantic model methods (e.g., `model.model_dump_json()`) for serialization. - - -## Node.js-Specific Instructions - -- Use ES6+ syntax and features. -- Follow the `package.json` configuration for dependencies and scripts. -- Use `eslint` for linting and `prettier` for code formatting. -- Write modular and reusable code, adhering to the project's structure. - -## Copilot Usage Tips - -1. **Be Specific**: Provide clear and detailed prompts to Copilot for better suggestions. -2. **Iterate**: Review and refine Copilot's suggestions to ensure they meet project standards. -3. **Split Tasks**: Break down complex tasks into smaller, manageable parts for better suggestions. -4. **Test Suggestions**: Always test Copilot-generated code to ensure it works as expected. - -## Additional Resources - -- [Python Coding Conventions](../docs/coding-conventions.md) -- [Environment Variables Guide](../docs/env-vars.md) -- [Steps to Upgrade Python](../docs/steps-to-upgrade-python.md) -- [Node.js Installation Script](../scripts/install_nodejs_14.bash) diff --git a/scripts/common-service.Makefile b/scripts/common-service.Makefile index 57fb6e3b5b4..394f0a861ca 100644 --- a/scripts/common-service.Makefile +++ b/scripts/common-service.Makefile @@ -137,7 +137,7 @@ info: ## displays service info .PHONY: _run-test-dev _run-test-ci -TEST_TARGET := $(if $(target),$(target),$(CURDIR)/tests/unit) +# TEST_TARGET := $(if $(target),$(target),$(CURDIR)/tests/unit) PYTEST_ADDITIONAL_PARAMETERS := $(if $(pytest-parameters),$(pytest-parameters),) _run-test-dev: _check_venv_active # runs tests for development (e.g w/ pdb) @@ -153,7 +153,6 @@ _run-test-dev: _check_venv_active --failed-first \ --junitxml=junit.xml -o junit_family=legacy \ --keep-docker-up \ - --pdb \ -vv \ $(PYTEST_ADDITIONAL_PARAMETERS) \ $(TEST_TARGET) diff --git a/services/api-server/Makefile b/services/api-server/Makefile index e923de11db8..64ff30491ea 100644 --- a/services/api-server/Makefile +++ b/services/api-server/Makefile @@ -89,7 +89,7 @@ APP_URL:=http://$(get_my_ip).nip.io:8006 test-api: ## Runs schemathesis against development server (NOTE: make up-devel first) - @docker run schemathesis/schemathesis:stable run \ + @docker run schemathesis/schemathesis:stable run --experimental=openapi-3.1 \ "$(APP_URL)/api/v0/openapi.json" diff --git a/services/api-server/openapi.json b/services/api-server/openapi.json index d38dafb42bb..65f40210e36 100644 --- a/services/api-server/openapi.json +++ b/services/api-server/openapi.json @@ -5277,48 +5277,6 @@ } }, "/v0/functions": { - "get": { - "tags": [ - "functions" - ], - "summary": "List Functions", - "description": "List functions", - "operationId": "list_functions", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "items": { - "oneOf": [ - { - "$ref": "#/components/schemas/ProjectFunction" - }, - { - "$ref": "#/components/schemas/PythonCodeFunction" - }, - { - "$ref": "#/components/schemas/SolverFunction" - } - ], - "discriminator": { - "propertyName": "function_class", - "mapping": { - "project": "#/components/schemas/ProjectFunction", - "python_code": "#/components/schemas/PythonCodeFunction", - "solver": "#/components/schemas/SolverFunction" - } - } - }, - "type": "array", - "title": "Response List Functions V0 Functions Get" - } - } - } - } - } - }, "post": { "tags": [ "functions" @@ -5327,6 +5285,7 @@ "description": "Create function", "operationId": "register_function", "requestBody": { + "required": true, "content": { "application/json": { "schema": { @@ -5341,7 +5300,6 @@ "$ref": "#/components/schemas/SolverFunction" } ], - "title": "Function", "discriminator": { "propertyName": "function_class", "mapping": { @@ -5349,11 +5307,11 @@ "python_code": "#/components/schemas/PythonCodeFunction", "solver": "#/components/schemas/SolverFunction" } - } + }, + "title": "Function" } } - }, - "required": true + } }, "responses": { "200": { @@ -5372,7 +5330,6 @@ "$ref": "#/components/schemas/SolverFunction" } ], - "title": "Response Register Function V0 Functions Post", "discriminator": { "propertyName": "function_class", "mapping": { @@ -5380,7 +5337,8 @@ "python_code": "#/components/schemas/PythonCodeFunction", "solver": "#/components/schemas/SolverFunction" } - } + }, + "title": "Response Register Function V0 Functions Post" } } } @@ -5406,6 +5364,61 @@ } } } + }, + "get": { + "tags": [ + "functions" + ], + "summary": "List Functions", + "description": "List functions", + "operationId": "list_functions", + "parameters": [ + { + "name": "limit", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "maximum": 50, + "minimum": 1, + "default": 20, + "title": "Limit" + } + }, + { + "name": "offset", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "minimum": 0, + "default": 0, + "title": "Offset" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Page_Annotated_Union_ProjectFunction__PythonCodeFunction__SolverFunction___FieldInfo_annotation_NoneType__required_True__discriminator__function_class____" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } } }, "/v0/functions/{function_id}": { @@ -5902,35 +5915,48 @@ "summary": "List Function Jobs", "description": "List function jobs", "operationId": "list_function_jobs", + "parameters": [ + { + "name": "limit", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "maximum": 50, + "minimum": 1, + "default": 20, + "title": "Limit" + } + }, + { + "name": "offset", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "minimum": 0, + "default": 0, + "title": "Offset" + } + } + ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { - "items": { - "oneOf": [ - { - "$ref": "#/components/schemas/ProjectFunctionJob" - }, - { - "$ref": "#/components/schemas/PythonCodeFunctionJob" - }, - { - "$ref": "#/components/schemas/SolverFunctionJob" - } - ], - "discriminator": { - "propertyName": "function_class", - "mapping": { - "project": "#/components/schemas/ProjectFunctionJob", - "python_code": "#/components/schemas/PythonCodeFunctionJob", - "solver": "#/components/schemas/SolverFunctionJob" - } - } - }, - "type": "array", - "title": "Response List Function Jobs V0 Function Jobs Get" + "$ref": "#/components/schemas/Page_Annotated_Union_ProjectFunctionJob__PythonCodeFunctionJob__SolverFunctionJob___FieldInfo_annotation_NoneType__required_True__discriminator__function_class____" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" } } } @@ -5945,6 +5971,7 @@ "description": "Create function job", "operationId": "register_function_job", "requestBody": { + "required": true, "content": { "application/json": { "schema": { @@ -5959,7 +5986,6 @@ "$ref": "#/components/schemas/SolverFunctionJob" } ], - "title": "Function Job", "discriminator": { "propertyName": "function_class", "mapping": { @@ -5967,11 +5993,11 @@ "python_code": "#/components/schemas/PythonCodeFunctionJob", "solver": "#/components/schemas/SolverFunctionJob" } - } + }, + "title": "Function Job" } } - }, - "required": true + } }, "responses": { "200": { @@ -5990,7 +6016,6 @@ "$ref": "#/components/schemas/SolverFunctionJob" } ], - "title": "Response Register Function Job V0 Function Jobs Post", "discriminator": { "propertyName": "function_class", "mapping": { @@ -5998,7 +6023,8 @@ "python_code": "#/components/schemas/PythonCodeFunctionJob", "solver": "#/components/schemas/SolverFunctionJob" } - } + }, + "title": "Response Register Function Job V0 Function Jobs Post" } } } @@ -6273,17 +6299,48 @@ "summary": "List Function Job Collections", "description": "List function job collections", "operationId": "list_function_job_collections", + "parameters": [ + { + "name": "limit", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "maximum": 50, + "minimum": 1, + "default": 20, + "title": "Limit" + } + }, + { + "name": "offset", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "minimum": 0, + "default": 0, + "title": "Offset" + } + } + ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { - "items": { - "$ref": "#/components/schemas/FunctionJobCollection" - }, - "type": "array", - "title": "Response List Function Job Collections V0 Function Job Collections Get" + "$ref": "#/components/schemas/Page_FunctionJobCollection_" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" } } } @@ -6298,14 +6355,14 @@ "description": "Register function job collection", "operationId": "register_function_job_collection", "requestBody": { + "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/FunctionJobCollection" } } - }, - "required": true + } }, "responses": { "200": { @@ -8710,6 +8767,160 @@ ], "title": "OnePage[StudyPort]" }, + "Page_Annotated_Union_ProjectFunctionJob__PythonCodeFunctionJob__SolverFunctionJob___FieldInfo_annotation_NoneType__required_True__discriminator__function_class____": { + "properties": { + "items": { + "items": { + "oneOf": [ + { + "$ref": "#/components/schemas/ProjectFunctionJob" + }, + { + "$ref": "#/components/schemas/PythonCodeFunctionJob" + }, + { + "$ref": "#/components/schemas/SolverFunctionJob" + } + ], + "discriminator": { + "propertyName": "function_class", + "mapping": { + "project": "#/components/schemas/ProjectFunctionJob", + "python_code": "#/components/schemas/PythonCodeFunctionJob", + "solver": "#/components/schemas/SolverFunctionJob" + } + } + }, + "type": "array", + "title": "Items" + }, + "total": { + "anyOf": [ + { + "type": "integer", + "minimum": 0 + }, + { + "type": "null" + } + ], + "title": "Total" + }, + "limit": { + "anyOf": [ + { + "type": "integer", + "minimum": 1 + }, + { + "type": "null" + } + ], + "title": "Limit" + }, + "offset": { + "anyOf": [ + { + "type": "integer", + "minimum": 0 + }, + { + "type": "null" + } + ], + "title": "Offset" + }, + "links": { + "$ref": "#/components/schemas/Links" + } + }, + "type": "object", + "required": [ + "items", + "total", + "limit", + "offset", + "links" + ], + "title": "Page[Annotated[Union[ProjectFunctionJob, PythonCodeFunctionJob, SolverFunctionJob], FieldInfo(annotation=NoneType, required=True, discriminator='function_class')]]" + }, + "Page_Annotated_Union_ProjectFunction__PythonCodeFunction__SolverFunction___FieldInfo_annotation_NoneType__required_True__discriminator__function_class____": { + "properties": { + "items": { + "items": { + "oneOf": [ + { + "$ref": "#/components/schemas/ProjectFunction" + }, + { + "$ref": "#/components/schemas/PythonCodeFunction" + }, + { + "$ref": "#/components/schemas/SolverFunction" + } + ], + "discriminator": { + "propertyName": "function_class", + "mapping": { + "project": "#/components/schemas/ProjectFunction", + "python_code": "#/components/schemas/PythonCodeFunction", + "solver": "#/components/schemas/SolverFunction" + } + } + }, + "type": "array", + "title": "Items" + }, + "total": { + "anyOf": [ + { + "type": "integer", + "minimum": 0 + }, + { + "type": "null" + } + ], + "title": "Total" + }, + "limit": { + "anyOf": [ + { + "type": "integer", + "minimum": 1 + }, + { + "type": "null" + } + ], + "title": "Limit" + }, + "offset": { + "anyOf": [ + { + "type": "integer", + "minimum": 0 + }, + { + "type": "null" + } + ], + "title": "Offset" + }, + "links": { + "$ref": "#/components/schemas/Links" + } + }, + "type": "object", + "required": [ + "items", + "total", + "limit", + "offset", + "links" + ], + "title": "Page[Annotated[Union[ProjectFunction, PythonCodeFunction, SolverFunction], FieldInfo(annotation=NoneType, required=True, discriminator='function_class')]]" + }, "Page_File_": { "properties": { "items": { @@ -8769,6 +8980,65 @@ ], "title": "Page[File]" }, + "Page_FunctionJobCollection_": { + "properties": { + "items": { + "items": { + "$ref": "#/components/schemas/FunctionJobCollection" + }, + "type": "array", + "title": "Items" + }, + "total": { + "anyOf": [ + { + "type": "integer", + "minimum": 0 + }, + { + "type": "null" + } + ], + "title": "Total" + }, + "limit": { + "anyOf": [ + { + "type": "integer", + "minimum": 1 + }, + { + "type": "null" + } + ], + "title": "Limit" + }, + "offset": { + "anyOf": [ + { + "type": "integer", + "minimum": 0 + }, + { + "type": "null" + } + ], + "title": "Offset" + }, + "links": { + "$ref": "#/components/schemas/Links" + } + }, + "type": "object", + "required": [ + "items", + "total", + "limit", + "offset", + "links" + ], + "title": "Page[FunctionJobCollection]" + }, "Page_Job_": { "properties": { "items": { diff --git a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py index 614119542b2..a03fa39fbdf 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py @@ -29,7 +29,7 @@ from sqlalchemy.ext.asyncio import AsyncEngine from ..._service_solvers import SolverService -from ...models.pagination import PaginationParams +from ...models.pagination import Page, PaginationParams from ...models.schemas.errors import ErrorGet from ...models.schemas.jobs import ( JobInputs, @@ -85,7 +85,7 @@ async def get_function( return await wb_api_rpc.get_function(function_id=function_id) -@function_router.get("", response_model=list[Function], description="List functions") +@function_router.get("", response_model=Page[Function], description="List functions") async def list_functions( wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], page_params: Annotated[PaginationParams, Depends()], @@ -102,6 +102,45 @@ async def list_functions( ) +@function_job_router.get( + "", response_model=Page[FunctionJob], description="List function jobs" +) +async def list_function_jobs( + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], + page_params: Annotated[PaginationParams, Depends()], +): + function_jobs_list, meta = await wb_api_rpc.list_function_jobs( + pagination_offset=page_params.offset, + pagination_limit=page_params.limit, + ) + + return create_page( + function_jobs_list, + total=meta.total, + params=page_params, + ) + + +@function_job_collections_router.get( + "", + response_model=Page[FunctionJobCollection], + description="List function job collections", +) +async def list_function_job_collections( + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], + page_params: Annotated[PaginationParams, Depends()], +): + function_job_collection_list, meta = await wb_api_rpc.list_function_job_collections( + pagination_offset=page_params.offset, + pagination_limit=page_params.limit, + ) + return create_page( + function_job_collection_list, + total=meta.total, + params=page_params, + ) + + def join_inputs( default_inputs: FunctionInputs | None, function_inputs: FunctionInputs | None, @@ -329,19 +368,6 @@ async def get_function_job( return await wb_api_rpc.get_function_job(function_job_id=function_job_id) -@function_job_router.get( - "", response_model=list[FunctionJob], description="List function jobs" -) -async def list_function_jobs( - wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], - page_params: Annotated[PaginationParams, Depends()], -): - return await wb_api_rpc.list_function_jobs( - pagination_offset=page_params.offset, - pagination_limit=page_params.limit, - ) - - @function_job_router.delete( "/{function_job_id:uuid}", response_model=None, @@ -524,21 +550,6 @@ async def map_function( # noqa: PLR0913 } -@function_job_collections_router.get( - "", - response_model=list[FunctionJobCollection], - description="List function job collections", -) -async def list_function_job_collections( - wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], - page_params: Annotated[PaginationParams, Depends()], -): - return await wb_api_rpc.list_function_job_collections( - pagination_offset=page_params.offset, - pagination_limit=page_params.limit, - ) - - @function_job_collections_router.get( "/{function_job_collection_id:uuid}", response_model=FunctionJobCollection, diff --git a/services/api-server/tests/unit/api_functions/test_api_routers_functions.py b/services/api-server/tests/unit/api_functions/test_api_routers_functions.py index 21ac2be2a90..cfb993093b0 100644 --- a/services/api-server/tests/unit/api_functions/test_api_routers_functions.py +++ b/services/api-server/tests/unit/api_functions/test_api_routers_functions.py @@ -4,11 +4,15 @@ import pytest from fastapi import FastAPI, HTTPException from fastapi.testclient import TestClient +from fastapi_pagination import add_pagination from models_library.api_schemas_webserver.functions_wb_schema import ( Function, FunctionJob, FunctionJobCollection, ) +from models_library.rest_pagination import ( + PageMetaInfoLimitOffset, +) from pydantic import TypeAdapter from simcore_service_api_server.api.routes.functions_routes import ( function_job_collections_router, @@ -28,6 +32,7 @@ def _api_app() -> FastAPI: fastapi_app.include_router( function_job_collections_router, prefix="/function_job_collections" ) + add_pagination(fastapi_app) # Mock authentication dependency async def mock_auth_dependency() -> int: @@ -89,9 +94,23 @@ async def run_function(self, function_id: str, inputs: dict) -> dict: ) return {"status": "success", "function_id": function_id, "inputs": inputs} - async def list_functions(self) -> list: + async def list_functions( + self, + pagination_offset: int, + pagination_limit: int, + ) -> tuple[list[Function], PageMetaInfoLimitOffset]: # Mimic listing all functions - return list(self._functions.values()) + functions_list = list(self._functions.values())[ + pagination_offset : pagination_offset + pagination_limit + ] + total_count = len(self._functions) + page_meta_info = PageMetaInfoLimitOffset( + total=total_count, + limit=pagination_limit, + offset=pagination_offset, + count=len(functions_list), + ) + return functions_list, page_meta_info async def delete_function(self, function_id: str) -> None: # Mimic deleting a function @@ -125,9 +144,23 @@ async def get_function_job(self, function_job_id: str) -> dict: raise HTTPException(status_code=404, detail="Function job not found") return self._function_jobs[function_job_id] - async def list_function_jobs(self) -> list: + async def list_function_jobs( + self, + pagination_offset: int, + pagination_limit: int, + ) -> tuple[list[FunctionJob], PageMetaInfoLimitOffset]: # Mimic listing all function jobs - return list(self._function_jobs.values()) + function_jobs_list = list(self._function_jobs.values())[ + pagination_offset : pagination_offset + pagination_limit + ] + total_count = len(self._function_jobs) + page_meta_info = PageMetaInfoLimitOffset( + total=total_count, + limit=pagination_limit, + offset=pagination_offset, + count=len(function_jobs_list), + ) + return function_jobs_list, page_meta_info async def delete_function_job(self, function_job_id: str) -> None: # Mimic deleting a function job @@ -163,9 +196,23 @@ async def get_function_job_collection( ) return self._function_job_collections[function_job_collection_id] - async def list_function_job_collections(self) -> list: + async def list_function_job_collections( + self, + pagination_offset: int, + pagination_limit: int, + ) -> tuple[list[FunctionJobCollection], PageMetaInfoLimitOffset]: # Mimic listing all function job collections - return list(self._function_job_collections.values()) + function_job_collections_list = list(self._function_job_collections.values())[ + pagination_offset : pagination_offset + pagination_limit + ] + total_count = len(self._function_job_collections) + page_meta_info = PageMetaInfoLimitOffset( + total=total_count, + limit=pagination_limit, + offset=pagination_offset, + count=len(function_job_collections_list), + ) + return function_job_collections_list, page_meta_info async def delete_function_job_collection( self, function_job_collection_id: str @@ -279,9 +326,9 @@ def test_list_functions(api_app: FastAPI) -> None: assert post_response.status_code == 200 # List functions - response = client.get("/functions") + response = client.get("/functions", params={"limit": 10, "offset": 0}) assert response.status_code == 200 - data = response.json() + data = response.json()["items"] assert len(data) > 0 assert data[0]["title"] == sample_function["title"] @@ -488,7 +535,7 @@ def test_list_function_jobs(api_app: FastAPI) -> None: # Now, list function jobs response = client.get("/function_jobs") assert response.status_code == 200 - data = response.json() + data = response.json()["items"] assert len(data) > 0 assert data[0]["title"] == mock_function_job["title"] From 8042578984441631e1d5896116f9b847d91f6d2d Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Wed, 7 May 2025 17:03:34 +0200 Subject: [PATCH 22/69] Fix pagination of functions again --- .../webserver/functions/functions_rpc_interface.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py index 56987fc8693..41a020124e2 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py @@ -94,7 +94,8 @@ async def delete_function( @log_decorator(_logger, level=logging.DEBUG) async def list_functions( rabbitmq_rpc_client: RabbitMQRPCClient, - *pagination_limit: int, + *, + pagination_limit: int, pagination_offset: int, ) -> tuple[list[Function], PageMetaInfoLimitOffset]: return await rabbitmq_rpc_client.request( From dc3952348e08233739444bd92cbd44e5b27ef701 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Thu, 8 May 2025 08:46:59 +0200 Subject: [PATCH 23/69] Restore some files from master --- .github/copilot-instructions.md | 53 +++++++++++++++++++++++++++++++++ scripts/common-service.Makefile | 3 +- services/api-server/Makefile | 2 +- 3 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 .github/copilot-instructions.md diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 00000000000..ebd8c6030a6 --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,53 @@ +# GitHub Copilot Instructions + +This document provides guidelines and best practices for using GitHub Copilot in the `osparc-simcore` repository and other Python and Node.js projects. + +## General Guidelines + +1. **Use Python 3.11**: Ensure that all Python-related suggestions align with Python 3.11 features and syntax. +2. **Node.js Compatibility**: For Node.js projects, ensure compatibility with the version specified in the project (e.g., Node.js 14 or later). +3. **Follow Coding Conventions**: Adhere to the coding conventions outlined in the `docs/coding-conventions.md` file. +4. **Test-Driven Development**: Write unit tests for all new functions and features. Use `pytest` for Python and appropriate testing frameworks for Node.js. +5. **Environment Variables**: Use environment variables as specified in `docs/env-vars.md` for configuration. Avoid hardcoding sensitive information. +6. **Documentation**: Prefer self-explanatory code; add documentation only if explicitly requested by the developer. + +## Python-Specific Instructions + +- Always use type hints and annotations to improve code clarity and compatibility with tools like `mypy`. + - An exception to that rule is in `test_*` functions return type hint must not be added +- Follow the dependency management practices outlined in `requirements/`. +- Use `ruff` for code formatting and for linting. +- Use `black` for code formatting and `pylint` for linting. +- ensure we use `sqlalchemy` >2 compatible code. +- ensure we use `pydantic` >2 compatible code. +- ensure we use `fastapi` >0.100 compatible code +- use f-string formatting +- Only add comments in function if strictly necessary + + +### Json serialization + +- Generally use `json_dumps`/`json_loads` from `common_library.json_serialization` to built-in `json.dumps` / `json.loads`. +- Prefer Pydantic model methods (e.g., `model.model_dump_json()`) for serialization. + + +## Node.js-Specific Instructions + +- Use ES6+ syntax and features. +- Follow the `package.json` configuration for dependencies and scripts. +- Use `eslint` for linting and `prettier` for code formatting. +- Write modular and reusable code, adhering to the project's structure. + +## Copilot Usage Tips + +1. **Be Specific**: Provide clear and detailed prompts to Copilot for better suggestions. +2. **Iterate**: Review and refine Copilot's suggestions to ensure they meet project standards. +3. **Split Tasks**: Break down complex tasks into smaller, manageable parts for better suggestions. +4. **Test Suggestions**: Always test Copilot-generated code to ensure it works as expected. + +## Additional Resources + +- [Python Coding Conventions](../docs/coding-conventions.md) +- [Environment Variables Guide](../docs/env-vars.md) +- [Steps to Upgrade Python](../docs/steps-to-upgrade-python.md) +- [Node.js Installation Script](../scripts/install_nodejs_14.bash) diff --git a/scripts/common-service.Makefile b/scripts/common-service.Makefile index 394f0a861ca..57fb6e3b5b4 100644 --- a/scripts/common-service.Makefile +++ b/scripts/common-service.Makefile @@ -137,7 +137,7 @@ info: ## displays service info .PHONY: _run-test-dev _run-test-ci -# TEST_TARGET := $(if $(target),$(target),$(CURDIR)/tests/unit) +TEST_TARGET := $(if $(target),$(target),$(CURDIR)/tests/unit) PYTEST_ADDITIONAL_PARAMETERS := $(if $(pytest-parameters),$(pytest-parameters),) _run-test-dev: _check_venv_active # runs tests for development (e.g w/ pdb) @@ -153,6 +153,7 @@ _run-test-dev: _check_venv_active --failed-first \ --junitxml=junit.xml -o junit_family=legacy \ --keep-docker-up \ + --pdb \ -vv \ $(PYTEST_ADDITIONAL_PARAMETERS) \ $(TEST_TARGET) diff --git a/services/api-server/Makefile b/services/api-server/Makefile index 64ff30491ea..e923de11db8 100644 --- a/services/api-server/Makefile +++ b/services/api-server/Makefile @@ -89,7 +89,7 @@ APP_URL:=http://$(get_my_ip).nip.io:8006 test-api: ## Runs schemathesis against development server (NOTE: make up-devel first) - @docker run schemathesis/schemathesis:stable run --experimental=openapi-3.1 \ + @docker run schemathesis/schemathesis:stable run \ "$(APP_URL)/api/v0/openapi.json" From d246bb455d0f8cab184b7a97fd07ee49bc71680d Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Thu, 8 May 2025 09:33:02 +0200 Subject: [PATCH 24/69] Fix pylint --- .../functions/_functions_controller_rpc.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py index fa14c220d1f..0909f5e68b3 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py @@ -29,6 +29,8 @@ router = RPCRouter() +# pylint: disable=no-else-return + @router.expose() async def ping(app: web.Application) -> str: From 616bf69688520103736631a242e5fc9ee5d16ed2 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Thu, 8 May 2025 10:48:07 +0200 Subject: [PATCH 25/69] Changes based on Mads comments wrt functions api --- .../functions_wb_schema.py | 51 +++++++++++++++++ .../functions/_functions_controller_rpc.py | 29 +++++----- .../functions/_functions_repository.py | 56 +++++++++++-------- .../test_functions_controller_rpc.py | 13 +++-- 4 files changed, 104 insertions(+), 45 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py b/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py index ad005fd8bc2..d237413f8f3 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py @@ -17,6 +17,8 @@ class FunctionSchema(BaseModel): + """Schema for function input/output""" + schema_dict: dict[str, Any] | None # JSON Schema @@ -163,3 +165,52 @@ class FunctionJobCollectionDB(BaseModel): class FunctionJobCollectionStatus(BaseModel): status: list[str] + + +class FunctionNotFoundError(Exception): + """Exception raised when a function is not found""" + + def __init__(self, function_id: FunctionID): + self.function_id = function_id + super().__init__(f"Function {function_id} not found") + + +class FunctionJobNotFoundError(Exception): + """Exception raised when a function job is not found""" + + def __init__(self, function_job_id: FunctionJobID): + self.function_job_id = function_job_id + super().__init__(f"Function job {function_job_id} not found") + + +class FunctionJobCollectionNotFoundError(Exception): + """Exception raised when a function job collection is not found""" + + def __init__(self, function_job_collection_id: FunctionJobCollectionID): + self.function_job_collection_id = function_job_collection_id + super().__init__( + f"Function job collection {function_job_collection_id} not found" + ) + + +class RegisterFunctionWithUIDError(Exception): + """Exception raised when registering a function with a UID""" + + def __init__(self): + super().__init__("Cannot register Function with a UID") + + +class UnsupportedFunctionClassError(Exception): + """Exception raised when a function class is not supported""" + + def __init__(self, function_class: str): + self.function_class = function_class + super().__init__(f"Function class {function_class} is not supported") + + +class UnsupportedFunctionJobClassError(Exception): + """Exception raised when a function job class is not supported""" + + def __init__(self, function_job_class: str): + self.function_job_class = function_job_class + super().__init__(f"Function job class {function_job_class} is not supported") diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py index 0909f5e68b3..6d43c8da66a 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py @@ -18,6 +18,8 @@ ProjectFunctionJob, SolverFunction, SolverFunctionJob, + UnsupportedFunctionClassError, + UnsupportedFunctionJobClassError, ) from models_library.rest_pagination import ( PageMetaInfoLimitOffset, @@ -32,12 +34,6 @@ # pylint: disable=no-else-return -@router.expose() -async def ping(app: web.Application) -> str: - assert app - return "pong from webserver" - - @router.expose() async def register_function(app: web.Application, *, function: Function) -> Function: assert app @@ -72,8 +68,7 @@ def _decode_function( default_inputs=function.default_inputs, ) else: - msg = f"Unsupported function class: [{function.function_class}]" - raise TypeError(msg) + raise UnsupportedFunctionClassError(function_class=function.function_class) def _encode_function( @@ -91,8 +86,7 @@ def _encode_function( } ) else: - msg = f"Unsupported function class: {function.function_class}" - raise TypeError(msg) + raise UnsupportedFunctionClassError(function_class=function.function_class) return FunctionDB( uuid=function.uid, @@ -142,8 +136,9 @@ def _decode_functionjob( solver_job_id=functionjob_db.class_specific_data["solver_job_id"], ) else: - msg = f"Unsupported function class: [{functionjob_db.function_class}]" - raise TypeError(msg) + raise UnsupportedFunctionJobClassError( + function_job_class=functionjob_db.function_class + ) def _encode_functionjob( @@ -178,8 +173,9 @@ def _encode_functionjob( function_class=functionjob.function_class, ) else: - msg = f"Unsupported function class: [{functionjob.function_class}]" - raise TypeError(msg) + raise UnsupportedFunctionJobClassError( + function_job_class=functionjob.function_class + ) @router.expose() @@ -355,8 +351,9 @@ async def find_cached_function_job( solver_job_id=returned_function_job.class_specific_data["solver_job_id"], ) else: - msg = f"Unsupported function class: [{returned_function_job.function_class}]" - raise TypeError(msg) + raise UnsupportedFunctionJobClassError( + function_job_class=returned_function_job.function_class + ) @router.expose() diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py index 5d8e7252064..1de44feb96b 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py @@ -7,8 +7,12 @@ FunctionInputs, FunctionJobCollection, FunctionJobCollectionDB, + FunctionJobCollectionNotFoundError, FunctionJobDB, FunctionJobID, + FunctionJobNotFoundError, + FunctionNotFoundError, + RegisterFunctionWithUIDError, ) from models_library.rest_pagination import ( PageMetaInfoLimitOffset, @@ -52,8 +56,7 @@ async def register_function( ) -> FunctionDB: if function.uuid is not None: - msg = "Function uid is not None. Cannot register function." - raise ValueError(msg) + raise RegisterFunctionWithUIDError async with transaction_context(get_asyncpg_engine(app), connection) as conn: result = await conn.stream( @@ -79,9 +82,10 @@ async def register_function( ) row = await result.first() - if row is None: - msg = "No row was returned from the database after creating function." - raise ValueError(msg) + assert row is not None, ( + "No row was returned from the database after creating function." + f" Function: {function}" + ) # nosec return FunctionDB.model_validate(dict(row)) @@ -100,9 +104,7 @@ async def get_function( row = await result.first() if row is None: - msg = f"No function found with id {function_id}." - raise web.HTTPNotFound(reason=msg) - + raise FunctionNotFoundError(function_id=function_id) return FunctionDB.model_validate(dict(row)) @@ -258,9 +260,10 @@ async def register_function_job( ) row = await result.first() - if row is None: - msg = "No row was returned from the database after creating function job." - raise ValueError(msg) + assert row is not None, ( + "No row was returned from the database after creating function job." + f" Function job: {function_job}" + ) # nosec return FunctionJobDB.model_validate(dict(row)) @@ -281,8 +284,7 @@ async def get_function_job( row = await result.first() if row is None: - msg = f"No function job found with id {function_job_id}." - raise web.HTTPNotFound(reason=msg) + raise FunctionJobNotFoundError(function_job_id=function_job_id) return FunctionJobDB.model_validate(dict(row)) @@ -319,13 +321,19 @@ async def find_cached_function_job( rows = await result.all() - if rows is None: + if rows is None or len(rows) == 0: return None - for row in rows: - job = FunctionJobDB.model_validate(dict(row)) - if job.inputs == inputs: - return job + assert len(rows) == 1, ( + "More than one function job found with the same function id and inputs." + f" Function id: {function_id}, Inputs: {inputs}" + ) # nosec + + row = rows[0] + + job = FunctionJobDB.model_validate(dict(row)) + if job.inputs == inputs: + return job return None @@ -346,8 +354,9 @@ async def get_function_job_collection( row = await result.first() if row is None: - msg = f"No function job collection found with id {function_job_collection_id}." - raise web.HTTPNotFound(reason=msg) + raise FunctionJobCollectionNotFoundError( + function_job_collection_id=function_job_collection_id + ) # Retrieve associated job ids from the join table job_result = await conn.stream( @@ -383,9 +392,10 @@ async def register_function_job_collection( ) row = await result.first() - if row is None: - msg = "No row was returned from the database after creating function job collection." - raise ValueError(msg) + assert row is not None, ( + "No row was returned from the database after creating function job collection." + f" Function job collection: {function_job_collection}" + ) # nosec for job_id in function_job_collection.job_ids: await conn.execute( diff --git a/services/web/server/tests/unit/with_dbs/functions_rpc/test_functions_controller_rpc.py b/services/web/server/tests/unit/with_dbs/functions_rpc/test_functions_controller_rpc.py index 403fe959374..2a5122268f0 100644 --- a/services/web/server/tests/unit/with_dbs/functions_rpc/test_functions_controller_rpc.py +++ b/services/web/server/tests/unit/with_dbs/functions_rpc/test_functions_controller_rpc.py @@ -3,11 +3,12 @@ import pytest import simcore_service_webserver.functions._functions_controller_rpc as functions_rpc -from aiohttp import web from models_library.api_schemas_webserver.functions_wb_schema import ( Function, FunctionInputSchema, FunctionJobCollection, + FunctionJobNotFoundError, + FunctionNotFoundError, FunctionOutputSchema, ProjectFunction, ProjectFunctionJob, @@ -89,7 +90,7 @@ async def test_get_function(client, mock_function): @pytest.mark.asyncio async def test_get_function_not_found(client): # Attempt to retrieve a function that does not exist - with pytest.raises(web.HTTPNotFound): + with pytest.raises(FunctionNotFoundError): await functions_rpc.get_function(app=client.app, function_id=uuid4()) @@ -225,7 +226,7 @@ async def test_delete_function(client, mock_function): ) # Attempt to retrieve the deleted function - with pytest.raises(web.HTTPNotFound): + with pytest.raises(FunctionNotFoundError): await functions_rpc.get_function( app=client.app, function_id=registered_function.uid ) @@ -298,7 +299,7 @@ async def test_get_function_job(client, mock_function): @pytest.mark.asyncio async def test_get_function_job_not_found(client): # Attempt to retrieve a function job that does not exist - with pytest.raises(web.HTTPNotFound): + with pytest.raises(FunctionJobNotFoundError): await functions_rpc.get_function_job(app=client.app, function_job_id=uuid4()) @@ -365,7 +366,7 @@ async def test_delete_function_job(client, mock_function): ) # Attempt to retrieve the deleted job - with pytest.raises(web.HTTPNotFound): + with pytest.raises(FunctionJobNotFoundError): await functions_rpc.get_function_job( app=client.app, function_job_id=registered_job.uid ) @@ -427,7 +428,7 @@ async def test_function_job_collection(client, mock_function): app=client.app, function_job_collection_id=registered_collection.uid ) # Attempt to retrieve the deleted collection - with pytest.raises(web.HTTPNotFound): + with pytest.raises(FunctionJobNotFoundError): await functions_rpc.get_function_job( app=client.app, function_job_id=registered_collection.uid ) From 3da5899cbedc8a4f3531377f736d2aea1cec52dc Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Thu, 8 May 2025 14:55:45 +0200 Subject: [PATCH 26/69] Mention explicit exceptions in functions rpc --- .../functions_wb_schema.py | 22 +- .../functions/_functions_controller_rpc.py | 411 +++++++++--------- .../functions/_functions_repository.py | 49 ++- .../test_functions_controller_rpc.py | 100 +---- 4 files changed, 278 insertions(+), 304 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py b/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py index d237413f8f3..806201cc4de 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py @@ -167,7 +167,7 @@ class FunctionJobCollectionStatus(BaseModel): status: list[str] -class FunctionNotFoundError(Exception): +class FunctionIDNotFoundError(Exception): """Exception raised when a function is not found""" def __init__(self, function_id: FunctionID): @@ -175,7 +175,7 @@ def __init__(self, function_id: FunctionID): super().__init__(f"Function {function_id} not found") -class FunctionJobNotFoundError(Exception): +class FunctionJobIDNotFoundError(Exception): """Exception raised when a function job is not found""" def __init__(self, function_job_id: FunctionJobID): @@ -183,7 +183,7 @@ def __init__(self, function_job_id: FunctionJobID): super().__init__(f"Function job {function_job_id} not found") -class FunctionJobCollectionNotFoundError(Exception): +class FunctionJobCollectionIDNotFoundError(Exception): """Exception raised when a function job collection is not found""" def __init__(self, function_job_collection_id: FunctionJobCollectionID): @@ -193,13 +193,27 @@ def __init__(self, function_job_collection_id: FunctionJobCollectionID): ) -class RegisterFunctionWithUIDError(Exception): +class RegisterFunctionWithIDError(Exception): """Exception raised when registering a function with a UID""" def __init__(self): super().__init__("Cannot register Function with a UID") +class RegisterFunctionJobWithIDError(Exception): + """Exception raised when registering a function job with a UID""" + + def __init__(self): + super().__init__("Cannot register FunctionJob with a UID") + + +class RegisterFunctionJobCollectionWithIDError(Exception): + """Exception raised when registering a function job collection with a UID""" + + def __init__(self): + super().__init__("Cannot register FunctionJobCollection with a UID") + + class UnsupportedFunctionClassError(Exception): """Exception raised when a function class is not supported""" diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py index 6d43c8da66a..5e5cb4d7155 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py @@ -6,16 +6,22 @@ FunctionClassSpecificData, FunctionDB, FunctionID, + FunctionIDNotFoundError, FunctionInputs, FunctionInputSchema, FunctionJob, FunctionJobClassSpecificData, FunctionJobCollection, + FunctionJobCollectionIDNotFoundError, FunctionJobDB, FunctionJobID, + FunctionJobIDNotFoundError, FunctionOutputSchema, ProjectFunction, ProjectFunctionJob, + RegisterFunctionJobCollectionWithIDError, + RegisterFunctionJobWithIDError, + RegisterFunctionWithIDError, SolverFunction, SolverFunctionJob, UnsupportedFunctionClassError, @@ -34,7 +40,9 @@ # pylint: disable=no-else-return -@router.expose() +@router.expose( + reraise_if_error_type=(UnsupportedFunctionClassError, RegisterFunctionWithIDError) +) async def register_function(app: web.Application, *, function: Function) -> Function: assert app saved_function = await _functions_repository.register_function( @@ -43,64 +51,42 @@ async def register_function(app: web.Application, *, function: Function) -> Func return _decode_function(saved_function) -def _decode_function( - function: FunctionDB, -) -> Function: - if function.function_class == "project": - return ProjectFunction( - uid=function.uuid, - title=function.title, - description=function.description, - input_schema=function.input_schema, - output_schema=function.output_schema, - project_id=function.class_specific_data["project_id"], - default_inputs=function.default_inputs, - ) - elif function.function_class == "solver": # noqa: RET505 - return SolverFunction( - uid=function.uuid, - title=function.title, - description=function.description, - input_schema=function.input_schema, - output_schema=function.output_schema, - solver_key=function.class_specific_data["solver_key"], - solver_version=function.class_specific_data["solver_version"], - default_inputs=function.default_inputs, - ) - else: - raise UnsupportedFunctionClassError(function_class=function.function_class) +@router.expose( + reraise_if_error_type=( + UnsupportedFunctionJobClassError, + RegisterFunctionJobWithIDError, + ) +) +async def register_function_job( + app: web.Application, *, function_job: FunctionJob +) -> FunctionJob: + assert app + created_function_job_db = await _functions_repository.register_function_job( + app=app, function_job=_encode_functionjob(function_job) + ) + return _decode_functionjob(created_function_job_db) -def _encode_function( - function: Function, -) -> FunctionDB: - if function.function_class == FunctionClass.project: - class_specific_data = FunctionClassSpecificData( - {"project_id": str(function.project_id)} - ) - elif function.function_class == FunctionClass.solver: - class_specific_data = FunctionClassSpecificData( - { - "solver_key": str(function.solver_key), - "solver_version": str(function.solver_version), - } +@router.expose(reraise_if_error_type=(RegisterFunctionJobCollectionWithIDError,)) +async def register_function_job_collection( + app: web.Application, *, function_job_collection: FunctionJobCollection +) -> FunctionJobCollection: + assert app + registered_function_job_collection, registered_job_ids = ( + await _functions_repository.register_function_job_collection( + app=app, + function_job_collection=function_job_collection, ) - else: - raise UnsupportedFunctionClassError(function_class=function.function_class) - - return FunctionDB( - uuid=function.uid, - title=function.title, - description=function.description, - input_schema=function.input_schema, - output_schema=function.output_schema, - function_class=function.function_class, - default_inputs=function.default_inputs, - class_specific_data=class_specific_data, + ) + return FunctionJobCollection( + uid=registered_function_job_collection.uuid, + title=registered_function_job_collection.title, + description=registered_function_job_collection.description, + job_ids=registered_job_ids, ) -@router.expose() +@router.expose(reraise_if_error_type=(FunctionIDNotFoundError,)) async def get_function(app: web.Application, *, function_id: FunctionID) -> Function: assert app returned_function = await _functions_repository.get_function( @@ -112,73 +98,7 @@ async def get_function(app: web.Application, *, function_id: FunctionID) -> Func ) -def _decode_functionjob( - functionjob_db: FunctionJobDB, -) -> FunctionJob: - if functionjob_db.function_class == FunctionClass.project: - return ProjectFunctionJob( - uid=functionjob_db.uuid, - title=functionjob_db.title, - description="", - function_uid=functionjob_db.function_uuid, - inputs=functionjob_db.inputs, - outputs=functionjob_db.outputs, - project_job_id=functionjob_db.class_specific_data["project_job_id"], - ) - elif functionjob_db.function_class == FunctionClass.solver: # noqa: RET505 - return SolverFunctionJob( - uid=functionjob_db.uuid, - title=functionjob_db.title, - description="", - function_uid=functionjob_db.function_uuid, - inputs=functionjob_db.inputs, - outputs=functionjob_db.outputs, - solver_job_id=functionjob_db.class_specific_data["solver_job_id"], - ) - else: - raise UnsupportedFunctionJobClassError( - function_job_class=functionjob_db.function_class - ) - - -def _encode_functionjob( - functionjob: FunctionJob, -) -> FunctionJobDB: - if functionjob.function_class == FunctionClass.project: - return FunctionJobDB( - uuid=functionjob.uid, - title=functionjob.title, - function_uuid=functionjob.function_uid, - inputs=functionjob.inputs, - outputs=functionjob.outputs, - class_specific_data=FunctionJobClassSpecificData( - { - "project_job_id": str(functionjob.project_job_id), - } - ), - function_class=functionjob.function_class, - ) - elif functionjob.function_class == FunctionClass.solver: # noqa: RET505 - return FunctionJobDB( - uuid=functionjob.uid, - title=functionjob.title, - function_uuid=functionjob.function_uid, - inputs=functionjob.inputs, - outputs=functionjob.outputs, - class_specific_data=FunctionJobClassSpecificData( - { - "solver_job_id": str(functionjob.solver_job_id), - } - ), - function_class=functionjob.function_class, - ) - else: - raise UnsupportedFunctionJobClassError( - function_job_class=functionjob.function_class - ) - - -@router.expose() +@router.expose(reraise_if_error_type=(FunctionJobIDNotFoundError,)) async def get_function_job( app: web.Application, *, function_job_id: FunctionJobID ) -> FunctionJob: @@ -192,39 +112,22 @@ async def get_function_job( return _decode_functionjob(returned_function_job) -@router.expose() -async def get_function_input_schema( - app: web.Application, *, function_id: FunctionID -) -> FunctionInputSchema: +@router.expose(reraise_if_error_type=(FunctionJobCollectionIDNotFoundError,)) +async def get_function_job_collection( + app: web.Application, *, function_job_collection_id: FunctionJobID +) -> FunctionJobCollection: assert app - returned_function = await _functions_repository.get_function( - app=app, - function_id=function_id, - ) - return FunctionInputSchema( - schema_dict=( - returned_function.input_schema.schema_dict - if returned_function.input_schema - else None + returned_function_job_collection, returned_job_ids = ( + await _functions_repository.get_function_job_collection( + app=app, + function_job_collection_id=function_job_collection_id, ) ) - - -@router.expose() -async def get_function_output_schema( - app: web.Application, *, function_id: FunctionID -) -> FunctionOutputSchema: - assert app - returned_function = await _functions_repository.get_function( - app=app, - function_id=function_id, - ) - return FunctionOutputSchema( - schema_dict=( - returned_function.output_schema.schema_dict - if returned_function.output_schema - else None - ) + return FunctionJobCollection( + uid=returned_function_job_collection.uuid, + title=returned_function_job_collection.title, + description=returned_function_job_collection.description, + job_ids=returned_job_ids, ) @@ -288,7 +191,7 @@ async def list_function_job_collections( ], page -@router.expose() +@router.expose(reraise_if_error_type=(FunctionIDNotFoundError,)) async def delete_function(app: web.Application, *, function_id: FunctionID) -> None: assert app await _functions_repository.delete_function( @@ -297,18 +200,7 @@ async def delete_function(app: web.Application, *, function_id: FunctionID) -> N ) -@router.expose() -async def register_function_job( - app: web.Application, *, function_job: FunctionJob -) -> FunctionJob: - assert app - created_function_job_db = await _functions_repository.register_function_job( - app=app, function_job=_encode_functionjob(function_job) - ) - return _decode_functionjob(created_function_job_db) - - -@router.expose() +@router.expose(reraise_if_error_type=(FunctionJobIDNotFoundError,)) async def delete_function_job( app: web.Application, *, function_job_id: FunctionJobID ) -> None: @@ -319,6 +211,17 @@ async def delete_function_job( ) +@router.expose(reraise_if_error_type=(FunctionJobCollectionIDNotFoundError,)) +async def delete_function_job_collection( + app: web.Application, *, function_job_collection_id: FunctionJobID +) -> None: + assert app + await _functions_repository.delete_function_job_collection( + app=app, + function_job_collection_id=function_job_collection_id, + ) + + @router.expose() async def find_cached_function_job( app: web.Application, *, function_id: FunctionID, inputs: FunctionInputs @@ -356,55 +259,165 @@ async def find_cached_function_job( ) -@router.expose() -async def register_function_job_collection( - app: web.Application, *, function_job_collection: FunctionJobCollection -) -> FunctionJobCollection: +@router.expose(reraise_if_error_type=(FunctionIDNotFoundError,)) +async def get_function_input_schema( + app: web.Application, *, function_id: FunctionID +) -> FunctionInputSchema: assert app - registered_function_job_collection, registered_job_ids = ( - await _functions_repository.register_function_job_collection( - app=app, - function_job_collection=function_job_collection, - ) + returned_function = await _functions_repository.get_function( + app=app, + function_id=function_id, ) - return FunctionJobCollection( - uid=registered_function_job_collection.uuid, - title=registered_function_job_collection.title, - description=registered_function_job_collection.description, - job_ids=registered_job_ids, + return FunctionInputSchema( + schema_dict=( + returned_function.input_schema.schema_dict + if returned_function.input_schema + else None + ) ) -@router.expose() -async def get_function_job_collection( - app: web.Application, *, function_job_collection_id: FunctionJobID -) -> FunctionJobCollection: +@router.expose(reraise_if_error_type=(FunctionIDNotFoundError,)) +async def get_function_output_schema( + app: web.Application, *, function_id: FunctionID +) -> FunctionOutputSchema: assert app - returned_function_job_collection, returned_job_ids = ( - await _functions_repository.get_function_job_collection( - app=app, - function_job_collection_id=function_job_collection_id, - ) + returned_function = await _functions_repository.get_function( + app=app, + function_id=function_id, ) - return FunctionJobCollection( - uid=returned_function_job_collection.uuid, - title=returned_function_job_collection.title, - description=returned_function_job_collection.description, - job_ids=returned_job_ids, + return FunctionOutputSchema( + schema_dict=( + returned_function.output_schema.schema_dict + if returned_function.output_schema + else None + ) ) -@router.expose() -async def delete_function_job_collection( - app: web.Application, *, function_job_collection_id: FunctionJobID -) -> None: - assert app - await _functions_repository.delete_function_job_collection( - app=app, - function_job_collection_id=function_job_collection_id, +def _decode_function( + function: FunctionDB, +) -> Function: + if function.function_class == "project": + return ProjectFunction( + uid=function.uuid, + title=function.title, + description=function.description, + input_schema=function.input_schema, + output_schema=function.output_schema, + project_id=function.class_specific_data["project_id"], + default_inputs=function.default_inputs, + ) + elif function.function_class == "solver": # noqa: RET505 + return SolverFunction( + uid=function.uuid, + title=function.title, + description=function.description, + input_schema=function.input_schema, + output_schema=function.output_schema, + solver_key=function.class_specific_data["solver_key"], + solver_version=function.class_specific_data["solver_version"], + default_inputs=function.default_inputs, + ) + else: + raise UnsupportedFunctionClassError(function_class=function.function_class) + + +def _encode_function( + function: Function, +) -> FunctionDB: + if function.function_class == FunctionClass.project: + class_specific_data = FunctionClassSpecificData( + {"project_id": str(function.project_id)} + ) + elif function.function_class == FunctionClass.solver: + class_specific_data = FunctionClassSpecificData( + { + "solver_key": str(function.solver_key), + "solver_version": str(function.solver_version), + } + ) + else: + raise UnsupportedFunctionClassError(function_class=function.function_class) + + return FunctionDB( + uuid=function.uid, + title=function.title, + description=function.description, + input_schema=function.input_schema, + output_schema=function.output_schema, + function_class=function.function_class, + default_inputs=function.default_inputs, + class_specific_data=class_specific_data, ) +def _encode_functionjob( + functionjob: FunctionJob, +) -> FunctionJobDB: + if functionjob.function_class == FunctionClass.project: + return FunctionJobDB( + uuid=functionjob.uid, + title=functionjob.title, + function_uuid=functionjob.function_uid, + inputs=functionjob.inputs, + outputs=functionjob.outputs, + class_specific_data=FunctionJobClassSpecificData( + { + "project_job_id": str(functionjob.project_job_id), + } + ), + function_class=functionjob.function_class, + ) + elif functionjob.function_class == FunctionClass.solver: # noqa: RET505 + return FunctionJobDB( + uuid=functionjob.uid, + title=functionjob.title, + function_uuid=functionjob.function_uid, + inputs=functionjob.inputs, + outputs=functionjob.outputs, + class_specific_data=FunctionJobClassSpecificData( + { + "solver_job_id": str(functionjob.solver_job_id), + } + ), + function_class=functionjob.function_class, + ) + else: + raise UnsupportedFunctionJobClassError( + function_job_class=functionjob.function_class + ) + + +def _decode_functionjob( + functionjob_db: FunctionJobDB, +) -> FunctionJob: + if functionjob_db.function_class == FunctionClass.project: + return ProjectFunctionJob( + uid=functionjob_db.uuid, + title=functionjob_db.title, + description="", + function_uid=functionjob_db.function_uuid, + inputs=functionjob_db.inputs, + outputs=functionjob_db.outputs, + project_job_id=functionjob_db.class_specific_data["project_job_id"], + ) + elif functionjob_db.function_class == FunctionClass.solver: # noqa: RET505 + return SolverFunctionJob( + uid=functionjob_db.uuid, + title=functionjob_db.title, + description="", + function_uid=functionjob_db.function_uuid, + inputs=functionjob_db.inputs, + outputs=functionjob_db.outputs, + solver_job_id=functionjob_db.class_specific_data["solver_job_id"], + ) + else: + raise UnsupportedFunctionJobClassError( + function_job_class=functionjob_db.function_class + ) + + async def register_rpc_routes_on_startup(app: web.Application): rpc_server = get_rabbitmq_rpc_server(app) await rpc_server.register_router(router, WEBSERVER_RPC_NAMESPACE, app) diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py index 1de44feb96b..faa226f7107 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py @@ -4,15 +4,15 @@ from models_library.api_schemas_webserver.functions_wb_schema import ( FunctionDB, FunctionID, + FunctionIDNotFoundError, FunctionInputs, FunctionJobCollection, FunctionJobCollectionDB, - FunctionJobCollectionNotFoundError, + FunctionJobCollectionIDNotFoundError, FunctionJobDB, FunctionJobID, - FunctionJobNotFoundError, - FunctionNotFoundError, - RegisterFunctionWithUIDError, + FunctionJobIDNotFoundError, + RegisterFunctionWithIDError, ) from models_library.rest_pagination import ( PageMetaInfoLimitOffset, @@ -56,7 +56,7 @@ async def register_function( ) -> FunctionDB: if function.uuid is not None: - raise RegisterFunctionWithUIDError + raise RegisterFunctionWithIDError async with transaction_context(get_asyncpg_engine(app), connection) as conn: result = await conn.stream( @@ -104,7 +104,7 @@ async def get_function( row = await result.first() if row is None: - raise FunctionNotFoundError(function_id=function_id) + raise FunctionIDNotFoundError(function_id=function_id) return FunctionDB.model_validate(dict(row)) @@ -232,6 +232,16 @@ async def delete_function( ) -> None: async with transaction_context(get_asyncpg_engine(app), connection) as conn: + # Check if the function exists + result = await conn.stream( + functions_table.select().where(functions_table.c.uuid == function_id) + ) + row = await result.first() + + if row is None: + raise FunctionIDNotFoundError(function_id=function_id) + + # Proceed with deletion await conn.execute( functions_table.delete().where(functions_table.c.uuid == function_id) ) @@ -284,7 +294,7 @@ async def get_function_job( row = await result.first() if row is None: - raise FunctionJobNotFoundError(function_job_id=function_job_id) + raise FunctionJobIDNotFoundError(function_job_id=function_job_id) return FunctionJobDB.model_validate(dict(row)) @@ -296,6 +306,17 @@ async def delete_function_job( function_job_id: FunctionID, ) -> None: async with transaction_context(get_asyncpg_engine(app), connection) as conn: + # Check if the function job exists + result = await conn.stream( + function_jobs_table.select().where( + function_jobs_table.c.uuid == function_job_id + ) + ) + row = await result.first() + if row is None: + raise FunctionJobIDNotFoundError(function_job_id=function_job_id) + + # Proceed with deletion await conn.execute( function_jobs_table.delete().where( function_jobs_table.c.uuid == function_job_id @@ -354,7 +375,7 @@ async def get_function_job_collection( row = await result.first() if row is None: - raise FunctionJobCollectionNotFoundError( + raise FunctionJobCollectionIDNotFoundError( function_job_collection_id=function_job_collection_id ) @@ -417,6 +438,18 @@ async def delete_function_job_collection( ) -> None: async with transaction_context(get_asyncpg_engine(app), connection) as conn: + # Check if the function job collection exists + result = await conn.stream( + function_job_collections_table.select().where( + function_job_collections_table.c.uuid == function_job_collection_id + ) + ) + row = await result.first() + if row is None: + raise FunctionJobCollectionIDNotFoundError( + function_job_collection_id=function_job_collection_id + ) + # Proceed with deletion await conn.execute( function_job_collections_table.delete().where( function_job_collections_table.c.uuid == function_job_collection_id diff --git a/services/web/server/tests/unit/with_dbs/functions_rpc/test_functions_controller_rpc.py b/services/web/server/tests/unit/with_dbs/functions_rpc/test_functions_controller_rpc.py index 2a5122268f0..8e78093115a 100644 --- a/services/web/server/tests/unit/with_dbs/functions_rpc/test_functions_controller_rpc.py +++ b/services/web/server/tests/unit/with_dbs/functions_rpc/test_functions_controller_rpc.py @@ -5,10 +5,10 @@ import simcore_service_webserver.functions._functions_controller_rpc as functions_rpc from models_library.api_schemas_webserver.functions_wb_schema import ( Function, + FunctionIDNotFoundError, FunctionInputSchema, FunctionJobCollection, - FunctionJobNotFoundError, - FunctionNotFoundError, + FunctionJobIDNotFoundError, FunctionOutputSchema, ProjectFunction, ProjectFunctionJob, @@ -90,7 +90,7 @@ async def test_get_function(client, mock_function): @pytest.mark.asyncio async def test_get_function_not_found(client): # Attempt to retrieve a function that does not exist - with pytest.raises(FunctionNotFoundError): + with pytest.raises(FunctionIDNotFoundError): await functions_rpc.get_function(app=client.app, function_id=uuid4()) @@ -226,7 +226,7 @@ async def test_delete_function(client, mock_function): ) # Attempt to retrieve the deleted function - with pytest.raises(FunctionNotFoundError): + with pytest.raises(FunctionIDNotFoundError): await functions_rpc.get_function( app=client.app, function_id=registered_function.uid ) @@ -299,7 +299,7 @@ async def test_get_function_job(client, mock_function): @pytest.mark.asyncio async def test_get_function_job_not_found(client): # Attempt to retrieve a function job that does not exist - with pytest.raises(FunctionJobNotFoundError): + with pytest.raises(FunctionJobIDNotFoundError): await functions_rpc.get_function_job(app=client.app, function_job_id=uuid4()) @@ -366,7 +366,7 @@ async def test_delete_function_job(client, mock_function): ) # Attempt to retrieve the deleted job - with pytest.raises(FunctionJobNotFoundError): + with pytest.raises(FunctionJobIDNotFoundError): await functions_rpc.get_function_job( app=client.app, function_job_id=registered_job.uid ) @@ -428,7 +428,7 @@ async def test_function_job_collection(client, mock_function): app=client.app, function_job_collection_id=registered_collection.uid ) # Attempt to retrieve the deleted collection - with pytest.raises(FunctionJobNotFoundError): + with pytest.raises(FunctionJobIDNotFoundError): await functions_rpc.get_function_job( app=client.app, function_job_id=registered_collection.uid ) @@ -488,89 +488,3 @@ async def test_list_function_job_collections(client, mock_function): # Assert the list contains the registered collection assert len(collections) == 1 assert collections[0].uid == registered_collections[1].uid - - -# @pytest.mark.asyncio -# async def test_find_cached_function_job_project_class( -# mock_app, mock_function_id, mock_function_inputs -# ): -# mock_function_job = AsyncMock() -# mock_function_job.function_class = FunctionClass.project -# mock_function_job.uuid = "mock-uuid" -# mock_function_job.title = "mock-title" -# mock_function_job.function_uuid = "mock-function-uuid" -# mock_function_job.inputs = {"key": "value"} -# mock_function_job.class_specific_data = {"project_job_id": "mock-project-job-id"} - -# with patch( -# "simcore_service_webserver.functions._functions_repository.find_cached_function_job", -# return_value=mock_function_job, -# ): -# result = await find_cached_function_job( -# app=mock_app, function_id=mock_function_id, inputs=mock_function_inputs -# ) - -# assert isinstance(result, ProjectFunctionJob) -# assert result.uid == "mock-uuid" -# assert result.title == "mock-title" -# assert result.function_uid == "mock-function-uuid" -# assert result.inputs == {"key": "value"} -# assert result.project_job_id == "mock-project-job-id" - - -# @pytest.mark.asyncio -# async def test_find_cached_function_job_solver_class( -# mock_app, mock_function_id, mock_function_inputs -# ): -# mock_function_job = AsyncMock() -# mock_function_job.function_class = FunctionClass.solver -# mock_function_job.uuid = "mock-uuid" -# mock_function_job.title = "mock-title" -# mock_function_job.function_uuid = "mock-function-uuid" -# mock_function_job.inputs = {"key": "value"} -# mock_function_job.class_specific_data = {"solver_job_id": "mock-solver-job-id"} - -# with patch( -# "simcore_service_webserver.functions._functions_repository.find_cached_function_job", -# return_value=mock_function_job, -# ): -# result = await find_cached_function_job( -# app=mock_app, function_id=mock_function_id, inputs=mock_function_inputs -# ) - -# assert isinstance(result, SolverFunctionJob) -# assert result.uid == "mock-uuid" -# assert result.title == "mock-title" -# assert result.function_uid == "mock-function-uuid" -# assert result.inputs == {"key": "value"} -# assert result.solver_job_id == "mock-solver-job-id" - - -# @pytest.mark.asyncio -# async def test_find_cached_function_job_none(mock_app, mock_function_id, mock_function_inputs): -# with patch( -# "simcore_service_webserver.functions._functions_repository.find_cached_function_job", -# return_value=None, -# ): -# result = await find_cached_function_job( -# app=mock_app, function_id=mock_function_id, inputs=mock_function_inputs -# ) - -# assert result is None - - -# @pytest.mark.asyncio -# async def test_find_cached_function_job_unsupported_class( -# mock_app, mock_function_id, mock_function_inputs -# ): -# mock_function_job = AsyncMock() -# mock_function_job.function_class = "unsupported_class" - -# with patch( -# "simcore_service_webserver.functions._functions_repository.find_cached_function_job", -# return_value=mock_function_job, -# ): -# with pytest.raises(TypeError, match="Unsupported function class:"): -# await find_cached_function_job( -# app=mock_app, function_id=mock_function_id, inputs=mock_function_inputs -# ) From ced82aa1f05136a3431823d8d341590c8d67278f Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Thu, 8 May 2025 15:10:40 +0200 Subject: [PATCH 27/69] Fix linting --- .../api/routes/functions_routes.py | 48 +------------------ .../services_rpc/wb_api_server.py | 2 + 2 files changed, 4 insertions(+), 46 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py index a03fa39fbdf..de469bb93e6 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py @@ -47,6 +47,8 @@ ) from . import solvers_jobs, solvers_jobs_getters, studies_jobs +# pylint: disable=too-many-arguments,no-else-return + function_router = APIRouter() function_job_router = APIRouter() function_job_collections_router = APIRouter() @@ -648,49 +650,3 @@ async def function_job_collection_status( return FunctionJobCollectionStatus( status=[job_status.status for job_status in job_statuses] ) - - -# ruff: noqa: ERA001 - -# @function_job_router.get( -# "/{function_job_id:uuid}/outputs/logfile", -# response_model=FunctionOutputsLogfile, -# responses={**_COMMON_FUNCTION_JOB_ERROR_RESPONSES}, -# description="Get function job outputs", -# ) -# async def function_job_logfile( -# function_job_id: FunctionJobID, -# user_id: Annotated[PositiveInt, Depends(get_current_user_id)], -# wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], -# director2_api: Annotated[DirectorV2Api, Depends(get_api_client(DirectorV2Api))], -# ): -# function, function_job = await get_function_from_functionjobid( -# wb_api_rpc=wb_api_rpc, function_job_id=function_job_id -# ) - -# if ( -# function.function_class == FunctionClass.project -# and function_job.function_class == FunctionClass.project -# ): -# job_outputs = await studies_jobs.get_study_job_output_logfile( -# study_id=function.project_id, -# job_id=function_job.project_job_id, # type: ignore -# user_id=user_id, -# director2_api=director2_api, -# ) - -# return job_outputs -# elif (function.function_class == FunctionClass.solver) and ( -# function_job.function_class == FunctionClass.solver -# ): -# job_outputs_logfile = await solvers_jobs_getters.get_job_output_logfile( -# director2_api=director2_api, -# solver_key=function.solver_key, -# version=function.solver_version, -# job_id=function_job.solver_job_id, -# user_id=user_id, -# ) -# return job_outputs_logfile -# else: -# msg = f"Function type {function.function_class} not supported" -# raise TypeError(msg) diff --git a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py index 91adebd2606..10ff8db42dc 100644 --- a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py +++ b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py @@ -121,6 +121,8 @@ LicensedResource, ) +# pylint: disable=too-many-public-methods + _exception_mapper = partial(service_exception_mapper, service_name="WebApiServer") From 7e74a60097c9c5106997039975ea088b3b99bc39 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Thu, 8 May 2025 15:33:07 +0200 Subject: [PATCH 28/69] Add assert checks in functions rpc interface --- .../functions/functions_rpc_interface.py | 103 +++++++++++++----- 1 file changed, 75 insertions(+), 28 deletions(-) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py index 41a020124e2..4d2b8170227 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py @@ -32,11 +32,13 @@ async def register_function( *, function: Function, ) -> Function: - return await rabbitmq_rpc_client.request( + result: Function = await rabbitmq_rpc_client.request( WEBSERVER_RPC_NAMESPACE, TypeAdapter(RPCMethodName).validate_python("register_function"), function=function, ) + assert isinstance(result, Function) # nosec + return result @log_decorator(_logger, level=logging.DEBUG) @@ -45,11 +47,13 @@ async def get_function( *, function_id: FunctionID, ) -> Function: - return await rabbitmq_rpc_client.request( + result: Function = await rabbitmq_rpc_client.request( WEBSERVER_RPC_NAMESPACE, TypeAdapter(RPCMethodName).validate_python("get_function"), function_id=function_id, ) + assert isinstance(result, Function) # nosec + return result @log_decorator(_logger, level=logging.DEBUG) @@ -58,11 +62,13 @@ async def get_function_input_schema( *, function_id: FunctionID, ) -> FunctionInputSchema: - return await rabbitmq_rpc_client.request( + result: FunctionInputSchema = await rabbitmq_rpc_client.request( WEBSERVER_RPC_NAMESPACE, TypeAdapter(RPCMethodName).validate_python("get_function_input_schema"), function_id=function_id, ) + assert isinstance(result, FunctionInputSchema) # nosec + return result @log_decorator(_logger, level=logging.DEBUG) @@ -71,11 +77,13 @@ async def get_function_output_schema( *, function_id: FunctionID, ) -> FunctionOutputSchema: - return await rabbitmq_rpc_client.request( + result: FunctionOutputSchema = await rabbitmq_rpc_client.request( WEBSERVER_RPC_NAMESPACE, TypeAdapter(RPCMethodName).validate_python("get_function_output_schema"), function_id=function_id, ) + assert isinstance(result, FunctionOutputSchema) # nosec + return result @log_decorator(_logger, level=logging.DEBUG) @@ -84,11 +92,13 @@ async def delete_function( *, function_id: FunctionID, ) -> None: - return await rabbitmq_rpc_client.request( + result: None = await rabbitmq_rpc_client.request( WEBSERVER_RPC_NAMESPACE, TypeAdapter(RPCMethodName).validate_python("delete_function"), function_id=function_id, ) + assert result is None # nosec + return result @log_decorator(_logger, level=logging.DEBUG) @@ -98,12 +108,17 @@ async def list_functions( pagination_limit: int, pagination_offset: int, ) -> tuple[list[Function], PageMetaInfoLimitOffset]: - return await rabbitmq_rpc_client.request( - WEBSERVER_RPC_NAMESPACE, - TypeAdapter(RPCMethodName).validate_python("list_functions"), - pagination_offset=pagination_offset, - pagination_limit=pagination_limit, + result: tuple[list[Function], PageMetaInfoLimitOffset] = ( + await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("list_functions"), + pagination_offset=pagination_offset, + pagination_limit=pagination_limit, + ) ) + assert isinstance(result, tuple) + assert len(result) == 2 # nosec + assert isinstance(result[0], list) # nosec @log_decorator(_logger, level=logging.DEBUG) @@ -113,12 +128,19 @@ async def list_function_jobs( pagination_limit: int, pagination_offset: int, ) -> tuple[list[FunctionJob], PageMetaInfoLimitOffset]: - return await rabbitmq_rpc_client.request( - WEBSERVER_RPC_NAMESPACE, - TypeAdapter(RPCMethodName).validate_python("list_function_jobs"), - pagination_offset=pagination_offset, - pagination_limit=pagination_limit, + result: tuple[list[FunctionJob], PageMetaInfoLimitOffset] = ( + await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("list_function_jobs"), + pagination_offset=pagination_offset, + pagination_limit=pagination_limit, + ) ) + assert isinstance(result, tuple) + assert len(result) == 2 # nosec + assert isinstance(result[0], list) # nosec + assert isinstance(result[1], PageMetaInfoLimitOffset) # nosec + return result @log_decorator(_logger, level=logging.DEBUG) @@ -128,12 +150,19 @@ async def list_function_job_collections( pagination_limit: int, pagination_offset: int, ) -> tuple[list[FunctionJobCollection], PageMetaInfoLimitOffset]: - return await rabbitmq_rpc_client.request( - WEBSERVER_RPC_NAMESPACE, - TypeAdapter(RPCMethodName).validate_python("list_function_job_collections"), - pagination_offset=pagination_offset, - pagination_limit=pagination_limit, + result: tuple[list[FunctionJobCollection], PageMetaInfoLimitOffset] = ( + await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("list_function_job_collections"), + pagination_offset=pagination_offset, + pagination_limit=pagination_limit, + ) ) + assert isinstance(result, tuple) + assert len(result) == 2 # nosec + assert isinstance(result[0], list) # nosec + assert isinstance(result[1], PageMetaInfoLimitOffset) # nosec + return result @log_decorator(_logger, level=logging.DEBUG) @@ -143,12 +172,14 @@ async def run_function( function_id: FunctionID, inputs: FunctionInputs, ) -> FunctionJob: - return await rabbitmq_rpc_client.request( + result: FunctionJob = await rabbitmq_rpc_client.request( WEBSERVER_RPC_NAMESPACE, TypeAdapter(RPCMethodName).validate_python("run_function"), function_id=function_id, inputs=inputs, ) + assert isinstance(result, FunctionJob) # nosec + return result @log_decorator(_logger, level=logging.DEBUG) @@ -157,11 +188,13 @@ async def register_function_job( *, function_job: FunctionJob, ) -> FunctionJob: - return await rabbitmq_rpc_client.request( + result: FunctionJob = await rabbitmq_rpc_client.request( WEBSERVER_RPC_NAMESPACE, TypeAdapter(RPCMethodName).validate_python("register_function_job"), function_job=function_job, ) + assert isinstance(result, FunctionJob) # nosec + return result @log_decorator(_logger, level=logging.DEBUG) @@ -170,11 +203,13 @@ async def get_function_job( *, function_job_id: FunctionJobID, ) -> FunctionJob: - return await rabbitmq_rpc_client.request( + result: FunctionJob = await rabbitmq_rpc_client.request( WEBSERVER_RPC_NAMESPACE, TypeAdapter(RPCMethodName).validate_python("get_function_job"), function_job_id=function_job_id, ) + assert isinstance(result, FunctionJob) # nosec + return result @log_decorator(_logger, level=logging.DEBUG) @@ -183,11 +218,13 @@ async def delete_function_job( *, function_job_id: FunctionJobID, ) -> None: - return await rabbitmq_rpc_client.request( + result: None = await rabbitmq_rpc_client.request( WEBSERVER_RPC_NAMESPACE, TypeAdapter(RPCMethodName).validate_python("delete_function_job"), function_job_id=function_job_id, ) + assert result is None # nosec + return result @log_decorator(_logger, level=logging.DEBUG) @@ -197,12 +234,16 @@ async def find_cached_function_job( function_id: FunctionID, inputs: FunctionInputs, ) -> FunctionJob | None: - return await rabbitmq_rpc_client.request( + result: FunctionJob = await rabbitmq_rpc_client.request( WEBSERVER_RPC_NAMESPACE, TypeAdapter(RPCMethodName).validate_python("find_cached_function_job"), function_id=function_id, inputs=inputs, ) + if result is None: + return None + assert isinstance(result, FunctionJob) # nosec + return result @log_decorator(_logger, level=logging.DEBUG) @@ -211,11 +252,13 @@ async def register_function_job_collection( *, function_job_collection: FunctionJobCollection, ) -> FunctionJobCollection: - return await rabbitmq_rpc_client.request( + result: FunctionJobCollection = await rabbitmq_rpc_client.request( WEBSERVER_RPC_NAMESPACE, TypeAdapter(RPCMethodName).validate_python("register_function_job_collection"), function_job_collection=function_job_collection, ) + assert isinstance(result, FunctionJobCollection) # nosec + return result @log_decorator(_logger, level=logging.DEBUG) @@ -224,11 +267,13 @@ async def get_function_job_collection( *, function_job_collection_id: FunctionJobCollectionID, ) -> FunctionJobCollection: - return await rabbitmq_rpc_client.request( + result: FunctionJobCollection = await rabbitmq_rpc_client.request( WEBSERVER_RPC_NAMESPACE, TypeAdapter(RPCMethodName).validate_python("get_function_job_collection"), function_job_collection_id=function_job_collection_id, ) + assert isinstance(result, FunctionJobCollection) # nosec + return result @log_decorator(_logger, level=logging.DEBUG) @@ -237,8 +282,10 @@ async def delete_function_job_collection( *, function_job_collection_id: FunctionJobCollectionID, ) -> None: - return await rabbitmq_rpc_client.request( + result: None = await rabbitmq_rpc_client.request( WEBSERVER_RPC_NAMESPACE, TypeAdapter(RPCMethodName).validate_python("delete_function_job_collection"), function_job_collection_id=function_job_collection_id, ) + assert result is None + return result From c1660c6793cc092655dea56fb76f367bf67a21b7 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Thu, 8 May 2025 16:29:01 +0200 Subject: [PATCH 29/69] Fix gh action tests --- .../functions/functions_rpc_interface.py | 18 +++++++++--------- .../api/routes/functions_routes.py | 14 +++++++++----- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py index 4d2b8170227..6c5c264890a 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py @@ -1,4 +1,5 @@ import logging +from typing import get_origin from models_library.api_schemas_webserver import ( WEBSERVER_RPC_NAMESPACE, @@ -37,7 +38,7 @@ async def register_function( TypeAdapter(RPCMethodName).validate_python("register_function"), function=function, ) - assert isinstance(result, Function) # nosec + assert isinstance(result, get_origin(Function) or Function) # nosec return result @@ -52,7 +53,7 @@ async def get_function( TypeAdapter(RPCMethodName).validate_python("get_function"), function_id=function_id, ) - assert isinstance(result, Function) # nosec + assert isinstance(result, get_origin(Function) or Function) # nosec return result @@ -117,8 +118,9 @@ async def list_functions( ) ) assert isinstance(result, tuple) - assert len(result) == 2 # nosec assert isinstance(result[0], list) # nosec + assert isinstance(result[1], PageMetaInfoLimitOffset) # nosec + return result @log_decorator(_logger, level=logging.DEBUG) @@ -137,7 +139,6 @@ async def list_function_jobs( ) ) assert isinstance(result, tuple) - assert len(result) == 2 # nosec assert isinstance(result[0], list) # nosec assert isinstance(result[1], PageMetaInfoLimitOffset) # nosec return result @@ -159,7 +160,6 @@ async def list_function_job_collections( ) ) assert isinstance(result, tuple) - assert len(result) == 2 # nosec assert isinstance(result[0], list) # nosec assert isinstance(result[1], PageMetaInfoLimitOffset) # nosec return result @@ -178,7 +178,7 @@ async def run_function( function_id=function_id, inputs=inputs, ) - assert isinstance(result, FunctionJob) # nosec + assert isinstance(result, get_origin(FunctionJob) or FunctionJob) # nosec return result @@ -193,7 +193,7 @@ async def register_function_job( TypeAdapter(RPCMethodName).validate_python("register_function_job"), function_job=function_job, ) - assert isinstance(result, FunctionJob) # nosec + assert isinstance(result, get_origin(FunctionJob) or FunctionJob) # nosec return result @@ -208,7 +208,7 @@ async def get_function_job( TypeAdapter(RPCMethodName).validate_python("get_function_job"), function_job_id=function_job_id, ) - assert isinstance(result, FunctionJob) # nosec + assert isinstance(result, get_origin(FunctionJob) or FunctionJob) # nosec return result @@ -242,7 +242,7 @@ async def find_cached_function_job( ) if result is None: return None - assert isinstance(result, FunctionJob) # nosec + assert isinstance(result, get_origin(FunctionJob) or FunctionJob) # nosec return result diff --git a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py index de469bb93e6..9ac80a59771 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py @@ -201,10 +201,10 @@ async def validate_function_inputs( ): function = await wb_api_rpc.get_function(function_id=function_id) - if function.input_schema is None: + if function.input_schema is None or function.input_schema.schema_dict is None: return True, "No input schema defined for this function" try: - jsonschema.validate(instance=inputs, schema=function.input_schema.schema_dict) # type: ignore + jsonschema.validate(instance=inputs, schema=function.input_schema.schema_dict) except ValidationError as err: return False, str(err) return True, "Inputs are valid" @@ -421,7 +421,7 @@ async def function_job_status( ): job_status = await studies_jobs.inspect_study_job( study_id=function.project_id, - job_id=function_job.project_job_id, # type: ignore + job_id=function_job.project_job_id, user_id=user_id, director2_api=director2_api, ) @@ -466,7 +466,7 @@ async def function_job_outputs( ): job_outputs = await studies_jobs.get_study_job_outputs( study_id=function.project_id, - job_id=function_job.project_job_id, # type: ignore + job_id=function_job.project_job_id, user_id=user_id, webserver_api=webserver_api, storage_client=storage_client, @@ -539,7 +539,11 @@ async def map_function( # noqa: PLR0913 uid=None, title="Function job collection of function map", description=function_job_collection_description, - job_ids=[function_job.uid for function_job in function_jobs], # type: ignore + job_ids=[ + function_job.uid + for function_job in function_jobs + if function_job.uid is not None + ], ), ) From d4ef130cbefd968678388a6c262164650997db91 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Thu, 8 May 2025 17:11:04 +0200 Subject: [PATCH 30/69] Add types jsonschema to api-server test requirements --- services/api-server/requirements/_test.in | 1 + services/api-server/requirements/_test.txt | 2 ++ 2 files changed, 3 insertions(+) diff --git a/services/api-server/requirements/_test.in b/services/api-server/requirements/_test.in index 718feaeb205..805e1f7a7af 100644 --- a/services/api-server/requirements/_test.in +++ b/services/api-server/requirements/_test.in @@ -31,3 +31,4 @@ respx sqlalchemy[mypy] # adds Mypy / Pep-484 Support for ORM Mappings SEE https://docs.sqlalchemy.org/en/20/orm/extensions/mypy.html types-aiofiles types-boto3 +types-jsonschema diff --git a/services/api-server/requirements/_test.txt b/services/api-server/requirements/_test.txt index b0708a30202..84d74a5f0e6 100644 --- a/services/api-server/requirements/_test.txt +++ b/services/api-server/requirements/_test.txt @@ -359,6 +359,8 @@ types-awscrt==0.23.10 # via botocore-stubs types-boto3==1.37.4 # via -r requirements/_test.in +types-jsonschema==4.23.0.20241208 + # via -r requirements/_test.in types-s3transfer==0.11.3 # via types-boto3 typing-extensions==4.12.2 From 4e10cc763271b4cfd36613986464ba09f4adfb7e Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Thu, 8 May 2025 18:48:45 +0200 Subject: [PATCH 31/69] Fix functions rpc assert and delete functions.py old schema --- .../api_schemas_api_server/functions.py | 44 ------------------- .../functions/functions_rpc_interface.py | 38 ++++++++++------ 2 files changed, 24 insertions(+), 58 deletions(-) delete mode 100644 packages/models-library/src/models_library/api_schemas_api_server/functions.py diff --git a/packages/models-library/src/models_library/api_schemas_api_server/functions.py b/packages/models-library/src/models_library/api_schemas_api_server/functions.py deleted file mode 100644 index 44678efd539..00000000000 --- a/packages/models-library/src/models_library/api_schemas_api_server/functions.py +++ /dev/null @@ -1,44 +0,0 @@ -from typing import Annotated, Any, Literal, TypeAlias - -from models_library import projects -from pydantic import BaseModel, Field - -FunctionID: TypeAlias = projects.ProjectID - - -class FunctionSchema(BaseModel): - schema_dict: dict[str, Any] | None # JSON Schema - - -class FunctionInputSchema(FunctionSchema): ... - - -class FunctionOutputSchema(FunctionSchema): ... - - -class Function(BaseModel): - uid: FunctionID | None = None - title: str | None = None - description: str | None = None - input_schema: FunctionInputSchema | None = None - output_schema: FunctionOutputSchema | None = None - - # @classmethod - # def compose_resource_name(cls, function_key) -> api_resources.RelativeResourceName: - # return api_resources.compose_resource_name("functions", function_key) - - -class StudyFunction(Function): - function_type: Literal["study"] = "study" - study_url: str - - -class PythonCodeFunction(Function): - function_type: Literal["python_code"] = "python_code" - code_url: str - - -FunctionUnion: TypeAlias = Annotated[ - StudyFunction | PythonCodeFunction, - Field(discriminator="function_type"), -] diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py index 6c5c264890a..be157e0f85a 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py @@ -1,5 +1,4 @@ import logging -from typing import get_origin from models_library.api_schemas_webserver import ( WEBSERVER_RPC_NAMESPACE, @@ -38,7 +37,7 @@ async def register_function( TypeAdapter(RPCMethodName).validate_python("register_function"), function=function, ) - assert isinstance(result, get_origin(Function) or Function) # nosec + TypeAdapter(Function).validate_python(result) # Validates the result as a Function return result @@ -53,7 +52,7 @@ async def get_function( TypeAdapter(RPCMethodName).validate_python("get_function"), function_id=function_id, ) - assert isinstance(result, get_origin(Function) or Function) # nosec + TypeAdapter(Function).validate_python(result) return result @@ -68,7 +67,7 @@ async def get_function_input_schema( TypeAdapter(RPCMethodName).validate_python("get_function_input_schema"), function_id=function_id, ) - assert isinstance(result, FunctionInputSchema) # nosec + TypeAdapter(FunctionInputSchema).validate_python(result) return result @@ -83,7 +82,7 @@ async def get_function_output_schema( TypeAdapter(RPCMethodName).validate_python("get_function_output_schema"), function_id=function_id, ) - assert isinstance(result, FunctionOutputSchema) # nosec + TypeAdapter(FunctionOutputSchema).validate_python(result) return result @@ -118,7 +117,9 @@ async def list_functions( ) ) assert isinstance(result, tuple) - assert isinstance(result[0], list) # nosec + TypeAdapter(list[Function]).validate_python( + result[0] + ) # Validates the result as a list of Functions assert isinstance(result[1], PageMetaInfoLimitOffset) # nosec return result @@ -139,7 +140,9 @@ async def list_function_jobs( ) ) assert isinstance(result, tuple) - assert isinstance(result[0], list) # nosec + TypeAdapter(list[FunctionJob]).validate_python( + result[0] + ) # Validates the result as a list of FunctionJobs assert isinstance(result[1], PageMetaInfoLimitOffset) # nosec return result @@ -160,7 +163,9 @@ async def list_function_job_collections( ) ) assert isinstance(result, tuple) - assert isinstance(result[0], list) # nosec + TypeAdapter(list[FunctionJobCollection]).validate_python( + result[0] + ) # Validates the result as a list of FunctionJobCollections assert isinstance(result[1], PageMetaInfoLimitOffset) # nosec return result @@ -178,7 +183,9 @@ async def run_function( function_id=function_id, inputs=inputs, ) - assert isinstance(result, get_origin(FunctionJob) or FunctionJob) # nosec + TypeAdapter(FunctionJob).validate_python( + result + ) # Validates the result as a FunctionJob return result @@ -193,7 +200,9 @@ async def register_function_job( TypeAdapter(RPCMethodName).validate_python("register_function_job"), function_job=function_job, ) - assert isinstance(result, get_origin(FunctionJob) or FunctionJob) # nosec + TypeAdapter(FunctionJob).validate_python( + result + ) # Validates the result as a FunctionJob return result @@ -208,7 +217,8 @@ async def get_function_job( TypeAdapter(RPCMethodName).validate_python("get_function_job"), function_job_id=function_job_id, ) - assert isinstance(result, get_origin(FunctionJob) or FunctionJob) # nosec + + TypeAdapter(FunctionJob).validate_python(result) return result @@ -242,7 +252,7 @@ async def find_cached_function_job( ) if result is None: return None - assert isinstance(result, get_origin(FunctionJob) or FunctionJob) # nosec + TypeAdapter(FunctionJob).validate_python(result) return result @@ -257,7 +267,7 @@ async def register_function_job_collection( TypeAdapter(RPCMethodName).validate_python("register_function_job_collection"), function_job_collection=function_job_collection, ) - assert isinstance(result, FunctionJobCollection) # nosec + TypeAdapter(FunctionJobCollection).validate_python(result) return result @@ -272,7 +282,7 @@ async def get_function_job_collection( TypeAdapter(RPCMethodName).validate_python("get_function_job_collection"), function_job_collection_id=function_job_collection_id, ) - assert isinstance(result, FunctionJobCollection) # nosec + TypeAdapter(FunctionJobCollection).validate_python(result) return result From 188b0daac243b6a69a8ee9ae5346a3f1466e2aa9 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Thu, 8 May 2025 18:52:17 +0200 Subject: [PATCH 32/69] Add functions api. New commit to clean up db migration --- .../api_schemas_api_server/functions.py | 44 + .../functions_wb_schema.py | 132 ++ .../models/functions_models_db.py | 170 +++ .../webserver/functions/functions.py | 22 - .../functions/functions_rpc_interface.py | 187 +++ services/api-server/openapi.json | 1289 ++++++++++++++++- .../simcore_service_api_server/api/root.py | 19 +- .../api/routes/functions.py | 17 - .../api/routes/functions_routes.py | 453 ++++++ .../api/routes/studies_jobs.py | 8 +- .../models/schemas/functions_api_schema.py | 85 ++ .../services_rpc/wb_api_server.py | 106 +- .../functions/_controller_rpc.py | 21 - .../functions/_functions_controller_rpc.py | 262 ++++ .../functions/_functions_repository.py | 206 +++ .../functions/_service.py | 4 +- .../functions/plugin.py | 4 +- 17 files changed, 2953 insertions(+), 76 deletions(-) create mode 100644 packages/models-library/src/models_library/api_schemas_api_server/functions.py create mode 100644 packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py create mode 100644 packages/postgres-database/src/simcore_postgres_database/models/functions_models_db.py delete mode 100644 packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions.py create mode 100644 packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py delete mode 100644 services/api-server/src/simcore_service_api_server/api/routes/functions.py create mode 100644 services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py create mode 100644 services/api-server/src/simcore_service_api_server/models/schemas/functions_api_schema.py delete mode 100644 services/web/server/src/simcore_service_webserver/functions/_controller_rpc.py create mode 100644 services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py create mode 100644 services/web/server/src/simcore_service_webserver/functions/_functions_repository.py diff --git a/packages/models-library/src/models_library/api_schemas_api_server/functions.py b/packages/models-library/src/models_library/api_schemas_api_server/functions.py new file mode 100644 index 00000000000..44678efd539 --- /dev/null +++ b/packages/models-library/src/models_library/api_schemas_api_server/functions.py @@ -0,0 +1,44 @@ +from typing import Annotated, Any, Literal, TypeAlias + +from models_library import projects +from pydantic import BaseModel, Field + +FunctionID: TypeAlias = projects.ProjectID + + +class FunctionSchema(BaseModel): + schema_dict: dict[str, Any] | None # JSON Schema + + +class FunctionInputSchema(FunctionSchema): ... + + +class FunctionOutputSchema(FunctionSchema): ... + + +class Function(BaseModel): + uid: FunctionID | None = None + title: str | None = None + description: str | None = None + input_schema: FunctionInputSchema | None = None + output_schema: FunctionOutputSchema | None = None + + # @classmethod + # def compose_resource_name(cls, function_key) -> api_resources.RelativeResourceName: + # return api_resources.compose_resource_name("functions", function_key) + + +class StudyFunction(Function): + function_type: Literal["study"] = "study" + study_url: str + + +class PythonCodeFunction(Function): + function_type: Literal["python_code"] = "python_code" + code_url: str + + +FunctionUnion: TypeAlias = Annotated[ + StudyFunction | PythonCodeFunction, + Field(discriminator="function_type"), +] diff --git a/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py b/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py new file mode 100644 index 00000000000..4391a77a658 --- /dev/null +++ b/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py @@ -0,0 +1,132 @@ +from enum import Enum +from typing import Annotated, Any, Literal, TypeAlias +from uuid import UUID + +from models_library import projects +from pydantic import BaseModel, Field + +from ..projects import ProjectID + +FunctionID: TypeAlias = projects.ProjectID +FunctionJobID: TypeAlias = projects.ProjectID +FileID: TypeAlias = UUID + +InputTypes: TypeAlias = FileID | float | int | bool | str | list | None + + +class FunctionSchema(BaseModel): + schema_dict: dict[str, Any] | None # JSON Schema + + +class FunctionInputSchema(FunctionSchema): ... + + +class FunctionOutputSchema(FunctionSchema): ... + + +class FunctionClass(str, Enum): + project = "project" + python_code = "python_code" + + +FunctionClassSpecificData: TypeAlias = dict[str, Any] +FunctionJobClassSpecificData: TypeAlias = FunctionClassSpecificData + + +# TODO, use InputTypes here, but api is throwing weird errors and asking for dict for elements # noqa: FIX002 +FunctionInputs: TypeAlias = dict[str, Any] | None + +FunctionInputsList: TypeAlias = list[FunctionInputs] + +FunctionOutputs: TypeAlias = dict[str, Any] | None + + +class FunctionBase(BaseModel): + uid: FunctionID | None = None + title: str | None = None + description: str | None = None + function_class: FunctionClass + input_schema: FunctionInputSchema | None = None + output_schema: FunctionOutputSchema | None = None + + +class FunctionDB(BaseModel): + uuid: FunctionJobID | None = None + title: str | None = None + description: str | None = None + function_class: FunctionClass + input_schema: FunctionInputSchema | None = None + output_schema: FunctionOutputSchema | None = None + class_specific_data: FunctionClassSpecificData + + +class FunctionJobDB(BaseModel): + uuid: FunctionJobID | None = None + function_uuid: FunctionID + title: str | None = None + inputs: FunctionInputs | None = None + outputs: FunctionOutputs | None = None + class_specific_data: FunctionJobClassSpecificData + function_class: FunctionClass + + +class ProjectFunction(FunctionBase): + function_class: Literal[FunctionClass.project] = FunctionClass.project + project_id: ProjectID + + +class PythonCodeFunction(FunctionBase): + function_class: Literal[FunctionClass.python_code] = FunctionClass.python_code + code_url: str + + +Function: TypeAlias = Annotated[ + ProjectFunction | PythonCodeFunction, + Field(discriminator="function_class"), +] + +FunctionJobCollectionID: TypeAlias = projects.ProjectID + + +class FunctionJobBase(BaseModel): + uid: FunctionJobID | None = None + title: str | None = None + description: str | None = None + function_uid: FunctionID + inputs: FunctionInputs | None = None + outputs: FunctionOutputs | None = None + function_class: FunctionClass + + +class ProjectFunctionJob(FunctionJobBase): + function_class: Literal[FunctionClass.project] = FunctionClass.project + project_job_id: ProjectID + + +class PythonCodeFunctionJob(FunctionJobBase): + function_class: Literal[FunctionClass.python_code] = FunctionClass.python_code + code_url: str + + +FunctionJob: TypeAlias = Annotated[ + ProjectFunctionJob | PythonCodeFunctionJob, + Field(discriminator="function_class"), +] + + +class FunctionJobStatus(BaseModel): + status: str + + +class FunctionJobCollection(BaseModel): + """Model for a collection of function jobs""" + + id: FunctionJobCollectionID + title: str | None + description: str | None + job_ids: list[FunctionJobID] + status: str + + +class FunctionJobCollectionStatus(BaseModel): + status: list[str] diff --git a/packages/postgres-database/src/simcore_postgres_database/models/functions_models_db.py b/packages/postgres-database/src/simcore_postgres_database/models/functions_models_db.py new file mode 100644 index 00000000000..e8a8ba6f2ec --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/models/functions_models_db.py @@ -0,0 +1,170 @@ +"""Functions table + +- List of functions served by the simcore platform +""" + +import uuid + +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import UUID + +from ._common import RefActions +from .base import metadata + +functions = sa.Table( + "functions", + metadata, + sa.Column( + "uuid", + UUID(as_uuid=True), + primary_key=True, + index=True, + default=uuid.uuid4, + doc="Unique id of the function", + ), + sa.Column( + "title", + sa.String, + doc="Name of the function", + ), + sa.Column( + "function_class", + sa.String, + doc="Class of the function", + ), + sa.Column( + "description", + sa.String, + doc="Description of the function", + ), + sa.Column( + "input_schema", + sa.JSON, + doc="Input schema of the function", + ), + sa.Column( + "output_schema", + sa.JSON, + doc="Output schema of the function", + ), + sa.Column( + "system_tags", + sa.JSON, + nullable=True, + doc="System-level tags of the function", + ), + sa.Column( + "user_tags", + sa.JSON, + nullable=True, + doc="User-level tags of the function", + ), + sa.Column( + "class_specific_data", + sa.JSON, + nullable=True, + doc="Fields specific for a function class", + ), + sa.PrimaryKeyConstraint("uuid", name="functions_pk"), +) + +function_jobs = sa.Table( + "function_jobs", + metadata, + sa.Column( + "uuid", + UUID(as_uuid=True), + primary_key=True, + index=True, + default=uuid.uuid4, + doc="Unique id of the function job", + ), + sa.Column( + "title", + sa.String, + doc="Name of the function job", + ), + sa.Column( + "function_uuid", + sa.ForeignKey( + functions.c.uuid, + onupdate=RefActions.CASCADE, + ondelete=RefActions.CASCADE, + name="fk_functions_to_function_jobs_to_function_uuid", + ), + nullable=False, + index=True, + doc="Unique identifier of the function", + ), + sa.Column( + "function_class", + sa.String, + doc="Class of the function", + ), + sa.Column( + "status", + sa.String, + doc="Status of the function job", + ), + sa.Column( + "inputs", + sa.JSON, + doc="Inputs of the function job", + ), + sa.Column( + "outputs", + sa.JSON, + doc="Outputs of the function job", + ), + sa.Column( + "class_specific_data", + sa.JSON, + nullable=True, + doc="Fields specific for a function class", + ), + sa.PrimaryKeyConstraint("uuid", name="function_jobs_pk"), +) + +function_job_collections = sa.Table( + "function_job_collections", + metadata, + sa.Column( + "uuid", + UUID(as_uuid=True), + default=uuid.uuid4, + primary_key=True, + index=True, + doc="Unique id of the function job collection", + ), + sa.Column( + "name", + sa.String, + doc="Name of the function job collection", + ), + sa.PrimaryKeyConstraint("uuid", name="function_job_collections_pk"), +) + +function_job_collections_to_function_jobs = sa.Table( + "function_job_collections_to_function_jobs", + metadata, + sa.Column( + "function_job_collection_uuid", + sa.ForeignKey( + function_job_collections.c.uuid, + onupdate=RefActions.CASCADE, + ondelete=RefActions.CASCADE, + name="fk_func_job_coll_to_func_jobs_to_func_job_coll_uuid", + ), + doc="Unique identifier of the function job collection", + ), + sa.Column( + "function_job_uuid", + sa.ForeignKey( + function_jobs.c.uuid, + onupdate=RefActions.CASCADE, + ondelete=RefActions.CASCADE, + name="fk_func_job_coll_to_func_jobs_to_func_job_uuid", + ), + doc="Unique identifier of the function job", + ), +) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions.py deleted file mode 100644 index d53adfafa66..00000000000 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions.py +++ /dev/null @@ -1,22 +0,0 @@ -import logging - -from models_library.api_schemas_webserver import WEBSERVER_RPC_NAMESPACE -from models_library.rabbitmq_basic_types import RPCMethodName -from pydantic import TypeAdapter - -from .....logging_utils import log_decorator -from .....rabbitmq import RabbitMQRPCClient - -_logger = logging.getLogger(__name__) - - -@log_decorator(_logger, level=logging.DEBUG) -async def ping( - rabbitmq_rpc_client: RabbitMQRPCClient, -) -> str: - result = await rabbitmq_rpc_client.request( - WEBSERVER_RPC_NAMESPACE, - TypeAdapter(RPCMethodName).validate_python("ping"), - ) - assert isinstance(result, str) # nosec - return result diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py new file mode 100644 index 00000000000..7e06bef912c --- /dev/null +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py @@ -0,0 +1,187 @@ +import logging + +from models_library.api_schemas_webserver import ( + WEBSERVER_RPC_NAMESPACE, +) +from models_library.api_schemas_webserver.functions_wb_schema import ( + Function, + FunctionID, + FunctionInputs, + FunctionInputSchema, + FunctionJob, + FunctionJobID, + FunctionOutputSchema, +) +from models_library.rabbitmq_basic_types import RPCMethodName +from pydantic import TypeAdapter + +from .....logging_utils import log_decorator +from .... import RabbitMQRPCClient + +_logger = logging.getLogger(__name__) + + +@log_decorator(_logger, level=logging.DEBUG) +async def ping( + rabbitmq_rpc_client: RabbitMQRPCClient, +) -> str: + result = await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("ping"), + ) + assert isinstance(result, str) # nosec + return result + + +@log_decorator(_logger, level=logging.DEBUG) +async def register_function( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + function: Function, +) -> Function: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("register_function"), + function=function, + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def get_function( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + function_id: FunctionID, +) -> Function: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("get_function"), + function_id=function_id, + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def get_function_input_schema( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + function_id: FunctionID, +) -> FunctionInputSchema: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("get_function_input_schema"), + function_id=function_id, + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def get_function_output_schema( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + function_id: FunctionID, +) -> FunctionOutputSchema: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("get_function_output_schema"), + function_id=function_id, + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def delete_function( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + function_id: FunctionID, +) -> None: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("delete_function"), + function_id=function_id, + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def list_functions( + rabbitmq_rpc_client: RabbitMQRPCClient, +) -> list[Function]: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("list_functions"), + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def run_function( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + function_id: FunctionID, + inputs: FunctionInputs, +) -> FunctionJob: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("run_function"), + function_id=function_id, + inputs=inputs, + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def register_function_job( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + function_job: FunctionJob, +) -> FunctionJob: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("register_function_job"), + function_job=function_job, + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def get_function_job( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + function_job_id: FunctionJobID, +) -> FunctionJob: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("get_function_job"), + function_job_id=function_job_id, + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def list_function_jobs( + rabbitmq_rpc_client: RabbitMQRPCClient, +) -> list[FunctionJob]: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("list_function_jobs"), + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def delete_function_job( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + function_job_id: FunctionJobID, +) -> None: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("delete_function_job"), + function_job_id=function_job_id, + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def find_cached_function_job( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + function_id: FunctionID, + inputs: FunctionInputs, +) -> FunctionJob | None: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("find_cached_function_job"), + function_id=function_id, + inputs=inputs, + ) diff --git a/services/api-server/openapi.json b/services/api-server/openapi.json index e7ff27a22a1..3196a2cee37 100644 --- a/services/api-server/openapi.json +++ b/services/api-server/openapi.json @@ -5276,6 +5276,924 @@ } } }, + "/v0/functions/ping": { + "post": { + "tags": [ + "functions" + ], + "summary": "Ping", + "operationId": "ping", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + } + } + } + }, + "/v0/functions": { + "get": { + "tags": [ + "functions" + ], + "summary": "List Functions", + "description": "List functions", + "operationId": "list_functions", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "items": { + "oneOf": [ + { + "$ref": "#/components/schemas/ProjectFunction" + }, + { + "$ref": "#/components/schemas/PythonCodeFunction" + } + ], + "discriminator": { + "propertyName": "function_class", + "mapping": { + "project": "#/components/schemas/ProjectFunction", + "python_code": "#/components/schemas/PythonCodeFunction" + } + } + }, + "type": "array", + "title": "Response List Functions V0 Functions Get" + } + } + } + } + } + }, + "post": { + "tags": [ + "functions" + ], + "summary": "Register Function", + "description": "Create function", + "operationId": "register_function", + "requestBody": { + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/ProjectFunction" + }, + { + "$ref": "#/components/schemas/PythonCodeFunction" + } + ], + "title": "Function", + "discriminator": { + "propertyName": "function_class", + "mapping": { + "project": "#/components/schemas/ProjectFunction", + "python_code": "#/components/schemas/PythonCodeFunction" + } + } + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/ProjectFunction" + }, + { + "$ref": "#/components/schemas/PythonCodeFunction" + } + ], + "title": "Response Register Function V0 Functions Post", + "discriminator": { + "propertyName": "function_class", + "mapping": { + "project": "#/components/schemas/ProjectFunction", + "python_code": "#/components/schemas/PythonCodeFunction" + } + } + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/functions/{function_id}": { + "get": { + "tags": [ + "functions" + ], + "summary": "Get Function", + "description": "Get function", + "operationId": "get_function", + "parameters": [ + { + "name": "function_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/ProjectFunction" + }, + { + "$ref": "#/components/schemas/PythonCodeFunction" + } + ], + "discriminator": { + "propertyName": "function_class", + "mapping": { + "project": "#/components/schemas/ProjectFunction", + "python_code": "#/components/schemas/PythonCodeFunction" + } + }, + "title": "Response Get Function V0 Functions Function Id Get" + } + } + } + }, + "404": { + "description": "Function not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "delete": { + "tags": [ + "functions" + ], + "summary": "Delete Function", + "description": "Delete function", + "operationId": "delete_function", + "parameters": [ + { + "name": "function_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/ProjectFunction" + }, + { + "$ref": "#/components/schemas/PythonCodeFunction" + } + ], + "discriminator": { + "propertyName": "function_class", + "mapping": { + "project": "#/components/schemas/ProjectFunction", + "python_code": "#/components/schemas/PythonCodeFunction" + } + }, + "title": "Response Delete Function V0 Functions Function Id Delete" + } + } + } + }, + "404": { + "description": "Function not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/functions/{function_id}:run": { + "post": { + "tags": [ + "functions" + ], + "summary": "Run Function", + "description": "Run function", + "operationId": "run_function", + "security": [ + { + "HTTPBasic": [] + } + ], + "parameters": [ + { + "name": "function_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Id" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Function Inputs" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/ProjectFunctionJob" + }, + { + "$ref": "#/components/schemas/PythonCodeFunctionJob" + } + ], + "discriminator": { + "propertyName": "function_class", + "mapping": { + "project": "#/components/schemas/ProjectFunctionJob", + "python_code": "#/components/schemas/PythonCodeFunctionJob" + } + }, + "title": "Response Run Function V0 Functions Function Id Run Post" + } + } + } + }, + "404": { + "description": "Function not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/functions/{function_id}/input_schema": { + "get": { + "tags": [ + "functions" + ], + "summary": "Get Function Input Schema", + "description": "Get function", + "operationId": "get_function_input_schema", + "parameters": [ + { + "name": "function_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FunctionInputSchema" + } + } + } + }, + "404": { + "description": "Function not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/functions/{function_id}/output_schema": { + "get": { + "tags": [ + "functions" + ], + "summary": "Get Function Output Schema", + "description": "Get function", + "operationId": "get_function_output_schema", + "parameters": [ + { + "name": "function_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FunctionOutputSchema" + } + } + } + }, + "404": { + "description": "Function not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/functions/{function_id}:map": { + "post": { + "tags": [ + "functions" + ], + "summary": "Map Function", + "description": "Map function over input parameters", + "operationId": "map_function", + "security": [ + { + "HTTPBasic": [] + } + ], + "parameters": [ + { + "name": "function_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Id" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ] + }, + "title": "Function Inputs List" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "oneOf": [ + { + "$ref": "#/components/schemas/ProjectFunctionJob" + }, + { + "$ref": "#/components/schemas/PythonCodeFunctionJob" + } + ], + "discriminator": { + "propertyName": "function_class", + "mapping": { + "project": "#/components/schemas/ProjectFunctionJob", + "python_code": "#/components/schemas/PythonCodeFunctionJob" + } + } + }, + "title": "Response Map Function V0 Functions Function Id Map Post" + } + } + } + }, + "404": { + "description": "Function not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/function_jobs": { + "get": { + "tags": [ + "function_jobs" + ], + "summary": "List Function Jobs", + "description": "List function jobs", + "operationId": "list_function_jobs", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "items": { + "oneOf": [ + { + "$ref": "#/components/schemas/ProjectFunctionJob" + }, + { + "$ref": "#/components/schemas/PythonCodeFunctionJob" + } + ], + "discriminator": { + "propertyName": "function_class", + "mapping": { + "project": "#/components/schemas/ProjectFunctionJob", + "python_code": "#/components/schemas/PythonCodeFunctionJob" + } + } + }, + "type": "array", + "title": "Response List Function Jobs V0 Function Jobs Get" + } + } + } + } + } + }, + "post": { + "tags": [ + "function_jobs" + ], + "summary": "Register Function Job", + "description": "Create function job", + "operationId": "register_function_job", + "requestBody": { + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/ProjectFunctionJob" + }, + { + "$ref": "#/components/schemas/PythonCodeFunctionJob" + } + ], + "title": "Function Job", + "discriminator": { + "propertyName": "function_class", + "mapping": { + "project": "#/components/schemas/ProjectFunctionJob", + "python_code": "#/components/schemas/PythonCodeFunctionJob" + } + } + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/ProjectFunctionJob" + }, + { + "$ref": "#/components/schemas/PythonCodeFunctionJob" + } + ], + "title": "Response Register Function Job V0 Function Jobs Post", + "discriminator": { + "propertyName": "function_class", + "mapping": { + "project": "#/components/schemas/ProjectFunctionJob", + "python_code": "#/components/schemas/PythonCodeFunctionJob" + } + } + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/function_jobs/{function_job_id}": { + "get": { + "tags": [ + "function_jobs" + ], + "summary": "Get Function Job", + "description": "Get function job", + "operationId": "get_function_job", + "parameters": [ + { + "name": "function_job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Job Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/ProjectFunctionJob" + }, + { + "$ref": "#/components/schemas/PythonCodeFunctionJob" + } + ], + "discriminator": { + "propertyName": "function_class", + "mapping": { + "project": "#/components/schemas/ProjectFunctionJob", + "python_code": "#/components/schemas/PythonCodeFunctionJob" + } + }, + "title": "Response Get Function Job V0 Function Jobs Function Job Id Get" + } + } + } + }, + "404": { + "description": "Function job not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "delete": { + "tags": [ + "function_jobs" + ], + "summary": "Delete Function Job", + "description": "Delete function job", + "operationId": "delete_function_job", + "parameters": [ + { + "name": "function_job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Job Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "404": { + "description": "Function job not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/function_jobs/{function_job_id}/status": { + "get": { + "tags": [ + "function_jobs" + ], + "summary": "Function Job Status", + "description": "Get function job status", + "operationId": "function_job_status", + "security": [ + { + "HTTPBasic": [] + } + ], + "parameters": [ + { + "name": "function_job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Job Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FunctionJobStatus" + } + } + } + }, + "404": { + "description": "Function job not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/function_jobs/{function_job_id}/outputs": { + "get": { + "tags": [ + "function_jobs" + ], + "summary": "Function Job Outputs", + "description": "Get function job outputs", + "operationId": "function_job_outputs", + "security": [ + { + "HTTPBasic": [] + } + ], + "parameters": [ + { + "name": "function_job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Job Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Response Function Job Outputs V0 Function Jobs Function Job Id Outputs Get" + } + } + } + }, + "404": { + "description": "Function job not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, "/v0/wallets/default": { "get": { "tags": [ @@ -6246,11 +7164,64 @@ }, "type": "object", "required": [ - "chunk_size", - "urls", - "links" + "chunk_size", + "urls", + "links" + ], + "title": "FileUploadData" + }, + "FunctionInputSchema": { + "properties": { + "schema_dict": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Schema Dict" + } + }, + "type": "object", + "required": [ + "schema_dict" + ], + "title": "FunctionInputSchema" + }, + "FunctionJobStatus": { + "properties": { + "status": { + "type": "string", + "title": "Status" + } + }, + "type": "object", + "required": [ + "status" + ], + "title": "FunctionJobStatus" + }, + "FunctionOutputSchema": { + "properties": { + "schema_dict": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Schema Dict" + } + }, + "type": "object", + "required": [ + "schema_dict" ], - "title": "FileUploadData" + "title": "FunctionOutputSchema" }, "GetCreditPriceLegacy": { "properties": { @@ -7787,6 +8758,316 @@ "version_display": "8.0.0" } }, + "ProjectFunction": { + "properties": { + "uid": { + "anyOf": [ + { + "type": "string", + "format": "uuid" + }, + { + "type": "null" + } + ], + "title": "Uid" + }, + "title": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Title" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "function_class": { + "type": "string", + "const": "project", + "title": "Function Class", + "default": "project" + }, + "input_schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/FunctionInputSchema" + }, + { + "type": "null" + } + ] + }, + "output_schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/FunctionOutputSchema" + }, + { + "type": "null" + } + ] + }, + "project_id": { + "type": "string", + "format": "uuid", + "title": "Project Id" + } + }, + "type": "object", + "required": [ + "project_id" + ], + "title": "ProjectFunction" + }, + "ProjectFunctionJob": { + "properties": { + "uid": { + "anyOf": [ + { + "type": "string", + "format": "uuid" + }, + { + "type": "null" + } + ], + "title": "Uid" + }, + "title": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Title" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "function_uid": { + "type": "string", + "format": "uuid", + "title": "Function Uid" + }, + "inputs": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Inputs" + }, + "outputs": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Outputs" + }, + "function_class": { + "type": "string", + "const": "project", + "title": "Function Class", + "default": "project" + }, + "project_job_id": { + "type": "string", + "format": "uuid", + "title": "Project Job Id" + } + }, + "type": "object", + "required": [ + "function_uid", + "project_job_id" + ], + "title": "ProjectFunctionJob" + }, + "PythonCodeFunction": { + "properties": { + "uid": { + "anyOf": [ + { + "type": "string", + "format": "uuid" + }, + { + "type": "null" + } + ], + "title": "Uid" + }, + "title": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Title" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "function_class": { + "type": "string", + "const": "python_code", + "title": "Function Class", + "default": "python_code" + }, + "input_schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/FunctionInputSchema" + }, + { + "type": "null" + } + ] + }, + "output_schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/FunctionOutputSchema" + }, + { + "type": "null" + } + ] + }, + "code_url": { + "type": "string", + "title": "Code Url" + } + }, + "type": "object", + "required": [ + "code_url" + ], + "title": "PythonCodeFunction" + }, + "PythonCodeFunctionJob": { + "properties": { + "uid": { + "anyOf": [ + { + "type": "string", + "format": "uuid" + }, + { + "type": "null" + } + ], + "title": "Uid" + }, + "title": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Title" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "function_uid": { + "type": "string", + "format": "uuid", + "title": "Function Uid" + }, + "inputs": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Inputs" + }, + "outputs": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Outputs" + }, + "function_class": { + "type": "string", + "const": "python_code", + "title": "Function Class", + "default": "python_code" + }, + "code_url": { + "type": "string", + "title": "Code Url" + } + }, + "type": "object", + "required": [ + "function_uid", + "code_url" + ], + "title": "PythonCodeFunctionJob" + }, "RunningState": { "type": "string", "enum": [ diff --git a/services/api-server/src/simcore_service_api_server/api/root.py b/services/api-server/src/simcore_service_api_server/api/root.py index 5654601d403..5a1de4a711c 100644 --- a/services/api-server/src/simcore_service_api_server/api/root.py +++ b/services/api-server/src/simcore_service_api_server/api/root.py @@ -6,7 +6,7 @@ from .routes import credits as _credits from .routes import ( files, - functions, + functions_routes, health, licensed_items, meta, @@ -42,12 +42,27 @@ def create_router(settings: ApplicationSettings): ) router.include_router(studies.router, tags=["studies"], prefix="/studies") router.include_router(studies_jobs.router, tags=["studies"], prefix="/studies") + router.include_router( + functions_routes.function_router, tags=["functions"], prefix="/functions" + ) + router.include_router( + functions_routes.function_job_router, + tags=["function_jobs"], + prefix="/function_jobs", + ) + router.include_router( + functions_routes.function_job_collections_router, + tags=["function_job_collections"], + prefix="/function_job_collections", + ) router.include_router(wallets.router, tags=["wallets"], prefix="/wallets") router.include_router(_credits.router, tags=["credits"], prefix="/credits") router.include_router( licensed_items.router, tags=["licensed-items"], prefix="/licensed-items" ) - router.include_router(functions.router, tags=["functions"], prefix="/functions") + router.include_router( + functions_routes.function_router, tags=["functions"], prefix="/functions" + ) # NOTE: multiple-files upload is currently disabled # Web form to upload files at http://localhost:8000/v0/upload-form-view diff --git a/services/api-server/src/simcore_service_api_server/api/routes/functions.py b/services/api-server/src/simcore_service_api_server/api/routes/functions.py deleted file mode 100644 index 6d5c277451d..00000000000 --- a/services/api-server/src/simcore_service_api_server/api/routes/functions.py +++ /dev/null @@ -1,17 +0,0 @@ -from typing import Annotated - -from fastapi import APIRouter, Depends - -from ...services_rpc.wb_api_server import WbApiRpcClient -from ..dependencies.webserver_rpc import ( - get_wb_api_rpc_client, -) - -router = APIRouter() - - -@router.post("/ping", include_in_schema=False) -async def ping( - wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], -): - return await wb_api_rpc.ping() diff --git a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py new file mode 100644 index 00000000000..dc2a80abcf5 --- /dev/null +++ b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py @@ -0,0 +1,453 @@ +from collections.abc import Callable +from typing import Annotated, Final + +from fastapi import APIRouter, Depends, Request, status +from models_library.api_schemas_webserver.functions_wb_schema import ( + Function, + FunctionClass, + FunctionID, + FunctionInputs, + FunctionInputSchema, + FunctionInputsList, + FunctionJob, + FunctionJobID, + FunctionJobStatus, + FunctionOutputs, + FunctionOutputSchema, + ProjectFunctionJob, +) +from pydantic import PositiveInt +from servicelib.fastapi.dependencies import get_reverse_url_mapper + +from ...models.schemas.errors import ErrorGet +from ...models.schemas.jobs import ( + JobInputs, +) +from ...services_http.director_v2 import DirectorV2Api +from ...services_http.storage import StorageApi +from ...services_http.webserver import AuthSession +from ...services_rpc.wb_api_server import WbApiRpcClient +from ..dependencies.authentication import get_current_user_id, get_product_name +from ..dependencies.services import get_api_client +from ..dependencies.webserver_http import get_webserver_session +from ..dependencies.webserver_rpc import ( + get_wb_api_rpc_client, +) +from . import studies_jobs + +function_router = APIRouter() +function_job_router = APIRouter() +function_job_collections_router = APIRouter() + +_COMMON_FUNCTION_ERROR_RESPONSES: Final[dict] = { + status.HTTP_404_NOT_FOUND: { + "description": "Function not found", + "model": ErrorGet, + }, +} + + +@function_router.post("/ping") +async def ping( + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + return await wb_api_rpc.ping() + + +@function_router.get("", response_model=list[Function], description="List functions") +async def list_functions( + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + return await wb_api_rpc.list_functions() + + +@function_router.post("", response_model=Function, description="Create function") +async def register_function( + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], + function: Function, +): + return await wb_api_rpc.register_function(function=function) + + +@function_router.get( + "/{function_id:uuid}", + response_model=Function, + responses={**_COMMON_FUNCTION_ERROR_RESPONSES}, + description="Get function", +) +async def get_function( + function_id: FunctionID, + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + return await wb_api_rpc.get_function(function_id=function_id) + + +@function_router.post( + "/{function_id:uuid}:run", + response_model=FunctionJob, + responses={**_COMMON_FUNCTION_ERROR_RESPONSES}, + description="Run function", +) +async def run_function( + request: Request, + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], + webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], + url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], + director2_api: Annotated[DirectorV2Api, Depends(get_api_client(DirectorV2Api))], + function_id: FunctionID, + function_inputs: FunctionInputs, + user_id: Annotated[PositiveInt, Depends(get_current_user_id)], + product_name: Annotated[str, Depends(get_product_name)], +): + + to_run_function = await wb_api_rpc.get_function(function_id=function_id) + + assert to_run_function.uid is not None + + if cached_function_job := await wb_api_rpc.find_cached_function_job( + function_id=to_run_function.uid, + inputs=function_inputs, + ): + return cached_function_job + + if to_run_function.function_class == FunctionClass.project: + study_job = await studies_jobs.create_study_job( + study_id=to_run_function.project_id, + job_inputs=JobInputs(values=function_inputs or {}), + webserver_api=webserver_api, + wb_api_rpc=wb_api_rpc, + url_for=url_for, + x_simcore_parent_project_uuid=None, + x_simcore_parent_node_id=None, + user_id=user_id, + product_name=product_name, + ) + await studies_jobs.start_study_job( + request=request, + study_id=to_run_function.project_id, + job_id=study_job.id, + user_id=user_id, + webserver_api=webserver_api, + director2_api=director2_api, + ) + return await register_function_job( + wb_api_rpc=wb_api_rpc, + function_job=ProjectFunctionJob( + function_uid=to_run_function.uid, + title=f"Function job of function {to_run_function.uid}", + description=to_run_function.description, + inputs=function_inputs, + outputs=None, + project_job_id=study_job.id, + ), + ) + else: # noqa: RET505 + msg = f"Function type {type(to_run_function)} not supported" + raise TypeError(msg) + + +@function_router.delete( + "/{function_id:uuid}", + response_model=Function, + responses={**_COMMON_FUNCTION_ERROR_RESPONSES}, + description="Delete function", +) +async def delete_function( + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], + function_id: FunctionID, +): + return await wb_api_rpc.delete_function(function_id=function_id) + + +@function_router.get( + "/{function_id:uuid}/input_schema", + response_model=FunctionInputSchema, + responses={**_COMMON_FUNCTION_ERROR_RESPONSES}, + description="Get function", +) +async def get_function_input_schema( + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], + function_id: FunctionID, +): + return await wb_api_rpc.get_function_input_schema(function_id=function_id) + + +@function_router.get( + "/{function_id:uuid}/output_schema", + response_model=FunctionOutputSchema, + responses={**_COMMON_FUNCTION_ERROR_RESPONSES}, + description="Get function", +) +async def get_function_output_schema( + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], + function_id: FunctionID, +): + return await wb_api_rpc.get_function_output_schema(function_id=function_id) + + +_COMMON_FUNCTION_JOB_ERROR_RESPONSES: Final[dict] = { + status.HTTP_404_NOT_FOUND: { + "description": "Function job not found", + "model": ErrorGet, + }, +} + + +@function_job_router.post( + "", response_model=FunctionJob, description="Create function job" +) +async def register_function_job( + function_job: FunctionJob, + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + return await wb_api_rpc.register_function_job(function_job=function_job) + + +@function_job_router.get( + "/{function_job_id:uuid}", + response_model=FunctionJob, + responses={**_COMMON_FUNCTION_JOB_ERROR_RESPONSES}, + description="Get function job", +) +async def get_function_job( + function_job_id: FunctionJobID, + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + return await wb_api_rpc.get_function_job(function_job_id=function_job_id) + + +@function_job_router.get( + "", response_model=list[FunctionJob], description="List function jobs" +) +async def list_function_jobs( + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + return await wb_api_rpc.list_function_jobs() + + +@function_job_router.delete( + "/{function_job_id:uuid}", + response_model=None, + responses={**_COMMON_FUNCTION_JOB_ERROR_RESPONSES}, + description="Delete function job", +) +async def delete_function_job( + function_job_id: FunctionJobID, + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + return await wb_api_rpc.delete_function_job(function_job_id=function_job_id) + + +async def get_function_from_functionjobid( + wb_api_rpc: WbApiRpcClient, + function_job_id: FunctionJobID, +) -> tuple[Function, FunctionJob]: + function_job = await get_function_job( + wb_api_rpc=wb_api_rpc, function_job_id=function_job_id + ) + + return ( + await get_function( + wb_api_rpc=wb_api_rpc, function_id=function_job.function_uid + ), + function_job, + ) + + +@function_job_router.get( + "/{function_job_id:uuid}/status", + response_model=FunctionJobStatus, + responses={**_COMMON_FUNCTION_JOB_ERROR_RESPONSES}, + description="Get function job status", +) +async def function_job_status( + function_job_id: FunctionJobID, + director2_api: Annotated[DirectorV2Api, Depends(get_api_client(DirectorV2Api))], + user_id: Annotated[PositiveInt, Depends(get_current_user_id)], + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + function, function_job = await get_function_from_functionjobid( + wb_api_rpc=wb_api_rpc, function_job_id=function_job_id + ) + + if ( + function.function_class == FunctionClass.project + and function_job.function_class == FunctionClass.project + ): + job_status = await studies_jobs.inspect_study_job( + study_id=function.project_id, + job_id=function_job.project_job_id, + user_id=user_id, + director2_api=director2_api, + ) + return FunctionJobStatus(status=job_status.state) + else: # noqa: RET505 + msg = f"Function type {function.function_class} / Function job type {function_job.function_class} not supported" + raise TypeError(msg) + + +@function_job_router.get( + "/{function_job_id:uuid}/outputs", + response_model=FunctionOutputs, + responses={**_COMMON_FUNCTION_JOB_ERROR_RESPONSES}, + description="Get function job outputs", +) +async def function_job_outputs( + function_job_id: FunctionJobID, + webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], + user_id: Annotated[PositiveInt, Depends(get_current_user_id)], + storage_client: Annotated[StorageApi, Depends(get_api_client(StorageApi))], + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + function, function_job = await get_function_from_functionjobid( + wb_api_rpc=wb_api_rpc, function_job_id=function_job_id + ) + + if ( + function.function_class != FunctionClass.project + or function_job.function_class != FunctionClass.project + ): + msg = f"Function type {function.function_class} not supported" + raise TypeError(msg) + else: # noqa: RET506 + job_outputs = await studies_jobs.get_study_job_outputs( + study_id=function.project_id, + job_id=function_job.project_job_id, + user_id=user_id, + webserver_api=webserver_api, + storage_client=storage_client, + ) + + return job_outputs.results + + +@function_router.post( + "/{function_id:uuid}:map", + response_model=list[FunctionJob], + responses={**_COMMON_FUNCTION_ERROR_RESPONSES}, + description="Map function over input parameters", +) +async def map_function( + function_id: FunctionID, + function_inputs_list: FunctionInputsList, + request: Request, + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], + webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], + url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], + director2_api: Annotated[DirectorV2Api, Depends(get_api_client(DirectorV2Api))], + user_id: Annotated[PositiveInt, Depends(get_current_user_id)], + product_name: Annotated[str, Depends(get_product_name)], +): + function_jobs = [] + for function_inputs in function_inputs_list: + function_jobs = [ + await run_function( + wb_api_rpc=wb_api_rpc, + function_id=function_id, + function_inputs=function_inputs, + product_name=product_name, + user_id=user_id, + webserver_api=webserver_api, + url_for=url_for, + director2_api=director2_api, + request=request, + ) + for function_inputs in function_inputs_list + ] + # TODO poor system can't handle doing this in parallel, get this fixed # noqa: FIX002 + # function_jobs = await asyncio.gather(*function_jobs_tasks) + + return function_jobs + + +# ruff: noqa: ERA001 + + +# _logger = logging.getLogger(__name__) + +# _COMMON_FUNCTION_JOB_COLLECTION_ERROR_RESPONSES: Final[dict] = { +# status.HTTP_404_NOT_FOUND: { +# "description": "Function job collection not found", +# "model": ErrorGet, +# }, +# } + + +# @function_job_collections_router.get( +# "", +# response_model=FunctionJobCollection, +# description="List function job collections", +# ) +# async def list_function_job_collections( +# page_params: Annotated[PaginationParams, Depends()], +# webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], +# ): +# msg = "list function jobs collection not implemented yet" +# raise NotImplementedError(msg) + + +# @function_job_collections_router.post( +# "", response_model=FunctionJobCollection, description="Create function job" +# ) +# async def create_function_job_collection( +# webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], +# job_ids: Annotated[list[FunctionJob], Depends()], +# ): +# msg = "create function job collection not implemented yet" +# raise NotImplementedError(msg) + + +# @function_job_collections_router.get( +# "/{function_job_collection_id:uuid}", +# response_model=FunctionJobCollection, +# responses={**_COMMON_FUNCTION_JOB_COLLECTION_ERROR_RESPONSES}, +# description="Get function job", +# ) +# async def get_function_job_collection( +# function_job_collection_id: FunctionJobCollectionID, +# webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], +# ): +# msg = "get function job collection not implemented yet" +# raise NotImplementedError(msg) + + +# @function_job_collections_router.delete( +# "/{function_job_collection_id:uuid}", +# response_model=FunctionJob, +# responses={**_COMMON_FUNCTION_JOB_COLLECTION_ERROR_RESPONSES}, +# description="Delete function job collection", +# ) +# async def delete_function_job_collection( +# function_job_collection_id: FunctionJobCollectionID, +# webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], +# ): +# msg = "delete function job collection not implemented yet" +# raise NotImplementedError(msg) + + +# @function_job_collections_router.get( +# "/{function_job_collection_id:uuid}/function_jobs", +# response_model=list[FunctionJob], +# responses={**_COMMON_FUNCTION_JOB_COLLECTION_ERROR_RESPONSES}, +# description="Get the function jobs in function job collection", +# ) +# async def function_job_collection_list_function_jobs( +# function_job_collection_id: FunctionJobCollectionID, +# webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], +# ): +# msg = "function job collection listing not implemented yet" +# raise NotImplementedError(msg) + + +# @function_job_collections_router.get( +# "/{function_job_collection_id:uuid}/status", +# response_model=FunctionJobCollectionStatus, +# responses={**_COMMON_FUNCTION_JOB_COLLECTION_ERROR_RESPONSES}, +# description="Get function job collection status", +# ) +# async def function_job_collection_status( +# function_job_collection_id: FunctionJobCollectionID, +# webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], +# ): +# msg = "function job collection status not implemented yet" +# raise NotImplementedError(msg) diff --git a/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py b/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py index 436633b1c18..0da6c51574c 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py @@ -133,7 +133,7 @@ async def create_study_job( url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], user_id: Annotated[PositiveInt, Depends(get_current_user_id)], product_name: Annotated[str, Depends(get_product_name)], - hidden: Annotated[bool, Query()] = True, + hidden: Annotated[bool, Query()] = True, # noqa: FBT002 x_simcore_parent_project_uuid: ProjectID | None = Header(default=None), x_simcore_parent_node_id: NodeID | None = Header(default=None), ) -> Job: @@ -304,13 +304,12 @@ async def start_study_job( return JSONResponse( content=jsonable_encoder(job_status), status_code=status.HTTP_200_OK ) - job_status = await inspect_study_job( + return await inspect_study_job( study_id=study_id, job_id=job_id, user_id=user_id, director2_api=director2_api, ) - return job_status @router.post( @@ -387,10 +386,9 @@ async def get_study_job_output_logfile( level=logging.DEBUG, msg=f"get study job output logfile study_id={study_id!r} job_id={job_id!r}.", ): - log_link_map = await director2_api.get_computation_logs( + return await director2_api.get_computation_logs( user_id=user_id, project_id=job_id ) - return log_link_map @router.get( diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/functions_api_schema.py b/services/api-server/src/simcore_service_api_server/models/schemas/functions_api_schema.py new file mode 100644 index 00000000000..ba316c0e88d --- /dev/null +++ b/services/api-server/src/simcore_service_api_server/models/schemas/functions_api_schema.py @@ -0,0 +1,85 @@ +from enum import Enum +from typing import Annotated, Any, Literal, TypeAlias + +from models_library import projects +from pydantic import BaseModel, Field + +FunctionID: TypeAlias = projects.ProjectID + + +class FunctionSchema(BaseModel): + schema_dict: dict[str, Any] | None # JSON Schema + + +class FunctionInputSchema(FunctionSchema): ... + + +class FunctionOutputSchema(FunctionSchema): ... + + +class FunctionClass(str, Enum): + project = "project" + python_code = "python_code" + + +class FunctionInputs(BaseModel): + inputs_dict: dict[str, Any] | None # JSON Schema + + +class FunctionOutputs(BaseModel): + outputs_dict: dict[str, Any] | None # JSON Schema + + +class Function(BaseModel): + uid: FunctionID | None = None + title: str | None = None + description: str | None = None + input_schema: FunctionInputSchema | None = None + output_schema: FunctionOutputSchema | None = None + + +class StudyFunction(Function): + function_type: Literal["study"] = "study" + study_url: str + + +class PythonCodeFunction(Function): + function_type: Literal["python_code"] = "python_code" + code_url: str + + +FunctionUnion: TypeAlias = Annotated[ + StudyFunction | PythonCodeFunction, + Field(discriminator="function_type"), +] + +FunctionJobID: TypeAlias = projects.ProjectID +FunctionJobCollectionID: TypeAlias = projects.ProjectID + + +class FunctionJob(BaseModel): + uid: FunctionJobID + title: str | None + description: str | None + status: str + function_uid: FunctionID + inputs: FunctionInputs | None + outputs: FunctionOutputs | None + + +class FunctionJobStatus(BaseModel): + status: str + + +class FunctionJobCollection(BaseModel): + """Model for a collection of function jobs""" + + id: FunctionJobCollectionID + title: str | None + description: str | None + job_ids: list[FunctionJobID] + status: str + + +class FunctionJobCollectionStatus(BaseModel): + status: list[str] diff --git a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py index fa9da284649..80139346f25 100644 --- a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py +++ b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py @@ -4,6 +4,15 @@ from fastapi import FastAPI from fastapi_pagination import create_page +from models_library.api_schemas_webserver.functions_wb_schema import ( + Function, + FunctionID, + FunctionInputs, + FunctionInputSchema, + FunctionJob, + FunctionJobID, + FunctionOutputSchema, +) from models_library.api_schemas_webserver.licensed_items import LicensedItemRpcGetPage from models_library.licenses import LicensedItemID from models_library.products import ProductName @@ -29,9 +38,45 @@ NotEnoughAvailableSeatsError, ) from servicelib.rabbitmq.rpc_interfaces.webserver import projects as projects_rpc -from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions import ( +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + delete_function as _delete_function, +) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + delete_function_job as _delete_function_job, +) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + find_cached_function_job as _find_cached_function_job, +) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + get_function as _get_function, +) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + get_function_input_schema as _get_function_input_schema, +) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + get_function_job as _get_function_job, +) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + get_function_output_schema as _get_function_output_schema, +) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + list_function_jobs as _list_function_jobs, +) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + list_functions as _list_functions, +) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( ping as _ping, ) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + register_function as _register_function, +) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + register_function_job as _register_function_job, +) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + run_function as _run_function, +) from servicelib.rabbitmq.rpc_interfaces.webserver.licenses.licensed_items import ( checkout_licensed_item_for_wallet as _checkout_licensed_item_for_wallet, ) @@ -237,6 +282,65 @@ async def list_projects_marked_as_jobs( job_parent_resource_name_prefix=job_parent_resource_name_prefix, ) + async def register_function(self, *, function: Function) -> Function: + function.input_schema = ( + FunctionInputSchema(**function.input_schema.model_dump()) + if function.input_schema is not None + else None + ) + function.output_schema = ( + FunctionOutputSchema(**function.output_schema.model_dump()) + if function.output_schema is not None + else None + ) + return await _register_function( + self._client, + function=function, + ) + + async def get_function(self, *, function_id: FunctionID) -> Function: + return await _get_function(self._client, function_id=function_id) + + async def delete_function(self, *, function_id: FunctionID) -> None: + return await _delete_function(self._client, function_id=function_id) + + async def list_functions(self) -> list[Function]: + return await _list_functions(self._client) + + async def run_function( + self, *, function_id: FunctionID, inputs: FunctionInputs + ) -> FunctionJob: + return await _run_function(self._client, function_id=function_id, inputs=inputs) + + async def get_function_job(self, *, function_job_id: FunctionJobID) -> FunctionJob: + return await _get_function_job(self._client, function_job_id=function_job_id) + + async def delete_function_job(self, *, function_job_id: FunctionJobID) -> None: + return await _delete_function_job(self._client, function_job_id=function_job_id) + + async def register_function_job(self, *, function_job: FunctionJob) -> FunctionJob: + return await _register_function_job(self._client, function_job=function_job) + + async def get_function_input_schema( + self, *, function_id: FunctionID + ) -> FunctionInputSchema: + return await _get_function_input_schema(self._client, function_id=function_id) + + async def get_function_output_schema( + self, *, function_id: FunctionID + ) -> FunctionOutputSchema: + return await _get_function_output_schema(self._client, function_id=function_id) + + async def find_cached_function_job( + self, *, function_id: FunctionID, inputs: FunctionInputs + ) -> FunctionJob | None: + return await _find_cached_function_job( + self._client, function_id=function_id, inputs=inputs + ) + + async def list_function_jobs(self) -> list[FunctionJob]: + return await _list_function_jobs(self._client) + def setup(app: FastAPI, rabbitmq_rmp_client: RabbitMQRPCClient): wb_api_rpc_client = WbApiRpcClient(_client=rabbitmq_rmp_client) diff --git a/services/web/server/src/simcore_service_webserver/functions/_controller_rpc.py b/services/web/server/src/simcore_service_webserver/functions/_controller_rpc.py deleted file mode 100644 index 483fcdc26e3..00000000000 --- a/services/web/server/src/simcore_service_webserver/functions/_controller_rpc.py +++ /dev/null @@ -1,21 +0,0 @@ -from aiohttp import web -from models_library.api_schemas_webserver import WEBSERVER_RPC_NAMESPACE -from servicelib.rabbitmq import RPCRouter - -from ..rabbitmq import get_rabbitmq_rpc_server - -# this is the rpc interface exposed to the api-server -# this interface should call the service layer - -router = RPCRouter() - - -@router.expose() -async def ping(app: web.Application) -> str: - assert app - return "pong from webserver" - - -async def register_rpc_routes_on_startup(app: web.Application): - rpc_server = get_rabbitmq_rpc_server(app) - await rpc_server.register_router(router, WEBSERVER_RPC_NAMESPACE, app) diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py new file mode 100644 index 00000000000..3a689d37ab9 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py @@ -0,0 +1,262 @@ +from aiohttp import web +from models_library.api_schemas_webserver import WEBSERVER_RPC_NAMESPACE +from models_library.api_schemas_webserver.functions_wb_schema import ( + Function, + FunctionClass, + FunctionClassSpecificData, + FunctionDB, + FunctionID, + FunctionInputs, + FunctionInputSchema, + FunctionJob, + FunctionJobClassSpecificData, + FunctionJobDB, + FunctionJobID, + FunctionOutputSchema, + ProjectFunction, + ProjectFunctionJob, +) +from servicelib.rabbitmq import RPCRouter + +from ..rabbitmq import get_rabbitmq_rpc_server +from . import _functions_repository + +router = RPCRouter() + + +@router.expose() +async def ping(app: web.Application) -> str: + assert app + return "pong from webserver" + + +@router.expose() +async def register_function(app: web.Application, *, function: Function) -> Function: + assert app + if function.function_class == FunctionClass.project: + saved_function = await _functions_repository.create_function( + app=app, + function=FunctionDB( + title=function.title, + description=function.description, + input_schema=function.input_schema, + output_schema=function.output_schema, + function_class=function.function_class, + class_specific_data=FunctionClassSpecificData( + { + "project_id": str(function.project_id), + } + ), + ), + ) + return ProjectFunction( + uid=saved_function.uuid, + title=saved_function.title, + description=saved_function.description, + input_schema=saved_function.input_schema, + output_schema=saved_function.output_schema, + project_id=saved_function.class_specific_data["project_id"], + ) + else: # noqa: RET505 + msg = f"Unsupported function class: {function.function_class}" + raise TypeError(msg) + + +def _decode_function( + function: FunctionDB, +) -> Function: + if function.function_class == "project": + return ProjectFunction( + uid=function.uuid, + title=function.title, + description=function.description, + input_schema=function.input_schema, + output_schema=function.output_schema, + project_id=function.class_specific_data["project_id"], + ) + else: # noqa: RET505 + msg = f"Unsupported function class: [{function.function_class}]" + raise TypeError(msg) + + +@router.expose() +async def get_function(app: web.Application, *, function_id: FunctionID) -> Function: + assert app + returned_function = await _functions_repository.get_function( + app=app, + function_id=function_id, + ) + return _decode_function( + returned_function, + ) + + +@router.expose() +async def get_function_job( + app: web.Application, *, function_job_id: FunctionJobID +) -> FunctionJob: + assert app + returned_function_job = await _functions_repository.get_function_job( + app=app, + function_job_id=function_job_id, + ) + assert returned_function_job is not None + + if returned_function_job.function_class == FunctionClass.project: + return ProjectFunctionJob( + uid=returned_function_job.uuid, + title=returned_function_job.title, + description="", + function_uid=returned_function_job.function_uuid, + inputs=returned_function_job.inputs, + outputs=None, + project_job_id=returned_function_job.class_specific_data["project_job_id"], + ) + else: # noqa: RET505 + msg = f"Unsupported function class: [{returned_function_job.function_class}]" + raise TypeError(msg) + + +@router.expose() +async def list_function_jobs(app: web.Application) -> list[FunctionJob]: + assert app + returned_function_jobs = await _functions_repository.list_function_jobs( + app=app, + ) + return [ + ProjectFunctionJob( + uid=returned_function_job.uuid, + title=returned_function_job.title, + description="", + function_uid=returned_function_job.function_uuid, + inputs=returned_function_job.inputs, + outputs=None, + project_job_id=returned_function_job.class_specific_data["project_job_id"], + ) + for returned_function_job in returned_function_jobs + ] + + +@router.expose() +async def get_function_input_schema( + app: web.Application, *, function_id: FunctionID +) -> FunctionInputSchema: + assert app + returned_function = await _functions_repository.get_function( + app=app, + function_id=function_id, + ) + return FunctionInputSchema( + schema_dict=( + returned_function.input_schema.schema_dict + if returned_function.input_schema + else None + ) + ) + + +@router.expose() +async def get_function_output_schema( + app: web.Application, *, function_id: FunctionID +) -> FunctionOutputSchema: + assert app + returned_function = await _functions_repository.get_function( + app=app, + function_id=function_id, + ) + return FunctionOutputSchema( + schema_dict=( + returned_function.output_schema.schema_dict + if returned_function.output_schema + else None + ) + ) + + +@router.expose() +async def list_functions(app: web.Application) -> list[Function]: + assert app + returned_functions = await _functions_repository.list_functions( + app=app, + ) + return [ + _decode_function(returned_function) for returned_function in returned_functions + ] + + +@router.expose() +async def delete_function(app: web.Application, *, function_id: FunctionID) -> None: + assert app + await _functions_repository.delete_function( + app=app, + function_id=function_id, + ) + + +@router.expose() +async def register_function_job( + app: web.Application, *, function_job: FunctionJob +) -> FunctionJob: + assert app + if function_job.function_class == FunctionClass.project: + created_function_job_db = await _functions_repository.register_function_job( + app=app, + function_job=FunctionJobDB( + title=function_job.title, + function_uuid=function_job.function_uid, + inputs=function_job.inputs, + outputs=None, + class_specific_data=FunctionJobClassSpecificData( + { + "project_job_id": str(function_job.project_job_id), + } + ), + function_class=function_job.function_class, + ), + ) + + return ProjectFunctionJob( + uid=created_function_job_db.uuid, + title=created_function_job_db.title, + description="", + function_uid=created_function_job_db.function_uuid, + inputs=created_function_job_db.inputs, + outputs=None, + project_job_id=created_function_job_db.class_specific_data[ + "project_job_id" + ], + ) + else: # noqa: RET505 + msg = f"Unsupported function class: [{function_job.function_class}]" + raise TypeError(msg) + + +@router.expose() +async def find_cached_function_job( + app: web.Application, *, function_id: FunctionID, inputs: FunctionInputs +) -> FunctionJob | None: + assert app + returned_function_job = await _functions_repository.find_cached_function_job( + app=app, function_id=function_id, inputs=inputs + ) + if returned_function_job is None: + return None + + if returned_function_job.function_class == FunctionClass.project: + return ProjectFunctionJob( + uid=returned_function_job.uuid, + title=returned_function_job.title, + description="", + function_uid=returned_function_job.function_uuid, + inputs=returned_function_job.inputs, + outputs=None, + project_job_id=returned_function_job.class_specific_data["project_job_id"], + ) + else: # noqa: RET505 + msg = f"Unsupported function class: [{returned_function_job.function_class}]" + raise TypeError(msg) + + +async def register_rpc_routes_on_startup(app: web.Application): + rpc_server = get_rabbitmq_rpc_server(app) + await rpc_server.register_router(router, WEBSERVER_RPC_NAMESPACE, app) diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py new file mode 100644 index 00000000000..495f93c0c25 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py @@ -0,0 +1,206 @@ +import json + +from aiohttp import web +from models_library.api_schemas_webserver.functions_wb_schema import ( + FunctionDB, + FunctionID, + FunctionInputs, + FunctionJobDB, +) +from simcore_postgres_database.models.functions_models_db import ( + function_jobs as function_jobs_table, +) +from simcore_postgres_database.models.functions_models_db import ( + functions as functions_table, +) +from simcore_postgres_database.utils_repos import ( + get_columns_from_db_model, + transaction_context, +) +from sqlalchemy import Text, cast +from sqlalchemy.ext.asyncio import AsyncConnection + +from ..db.plugin import get_asyncpg_engine + +_FUNCTIONS_TABLE_COLS = get_columns_from_db_model(functions_table, FunctionDB) +_FUNCTION_JOBS_TABLE_COLS = get_columns_from_db_model( + function_jobs_table, FunctionJobDB +) + + +async def create_function( + app: web.Application, + connection: AsyncConnection | None = None, + *, + function: FunctionDB, +) -> FunctionDB: + + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream( + functions_table.insert() + .values( + title=function.title, + description=function.description, + input_schema=( + function.input_schema.model_dump() + if function.input_schema is not None + else None + ), + output_schema=( + function.output_schema.model_dump() + if function.output_schema is not None + else None + ), + function_class=function.function_class, + class_specific_data=function.class_specific_data, + ) + .returning(*_FUNCTIONS_TABLE_COLS) + ) + row = await result.first() + + if row is None: + msg = "No row was returned from the database after creating function." + raise ValueError(msg) + + return FunctionDB.model_validate(dict(row)) + + +async def get_function( + app: web.Application, + connection: AsyncConnection | None = None, + *, + function_id: FunctionID, +) -> FunctionDB: + + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream( + functions_table.select().where(functions_table.c.uuid == function_id) + ) + row = await result.first() + + if row is None: + msg = f"No function found with id {function_id}." + raise web.HTTPNotFound(reason=msg) + + return FunctionDB.model_validate(dict(row)) + + +async def list_functions( + app: web.Application, + connection: AsyncConnection | None = None, +) -> list[FunctionDB]: + + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream(functions_table.select().where()) + rows = await result.all() + if rows is None: + return [] + + return [FunctionDB.model_validate(dict(row)) for row in rows] + + +async def delete_function( + app: web.Application, + connection: AsyncConnection | None = None, + *, + function_id: FunctionID, +) -> None: + + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + await conn.execute( + functions_table.delete().where(functions_table.c.uuid == int(function_id)) + ) + + +async def register_function_job( + app: web.Application, + connection: AsyncConnection | None = None, + *, + function_job: FunctionJobDB, +) -> FunctionJobDB: + + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream( + function_jobs_table.insert() + .values( + function_uuid=function_job.function_uuid, + inputs=function_job.inputs, + function_class=function_job.function_class, + class_specific_data=function_job.class_specific_data, + title=function_job.title, + status="created", + ) + .returning(*_FUNCTION_JOBS_TABLE_COLS) + ) + row = await result.first() + + if row is None: + msg = "No row was returned from the database after creating function job." + raise ValueError(msg) + + return FunctionJobDB.model_validate(dict(row)) + + +async def get_function_job( + app: web.Application, + connection: AsyncConnection | None = None, + *, + function_job_id: FunctionID, +) -> FunctionJobDB: + + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream( + function_jobs_table.select().where( + function_jobs_table.c.uuid == function_job_id + ) + ) + row = await result.first() + + if row is None: + msg = f"No function job found with id {function_job_id}." + raise web.HTTPNotFound(reason=msg) + + return FunctionJobDB.model_validate(dict(row)) + + +async def list_function_jobs( + app: web.Application, + connection: AsyncConnection | None = None, +) -> list[FunctionJobDB]: + + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream(function_jobs_table.select().where()) + rows = await result.all() + if rows is None: + return [] + + return [FunctionJobDB.model_validate(dict(row)) for row in rows] + + +async def find_cached_function_job( + app: web.Application, + connection: AsyncConnection | None = None, + *, + function_id: FunctionID, + inputs: FunctionInputs, +) -> FunctionJobDB | None: + + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream( + function_jobs_table.select().where( + function_jobs_table.c.function_uuid == function_id, + cast(function_jobs_table.c.inputs, Text) == json.dumps(inputs), + ), + ) + + rows = await result.all() + + if rows is None: + return None + + for row in rows: + job = FunctionJobDB.model_validate(dict(row)) + if job.inputs == inputs: + return job + + return None diff --git a/services/web/server/src/simcore_service_webserver/functions/_service.py b/services/web/server/src/simcore_service_webserver/functions/_service.py index 2c967d7c841..899b0b1c681 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_service.py +++ b/services/web/server/src/simcore_service_webserver/functions/_service.py @@ -5,7 +5,7 @@ from aiohttp import web from models_library.users import UserID -from ..projects import _projects_service +from ..projects import projects_service from ..projects.models import ProjectDict @@ -16,6 +16,6 @@ async def get_project_from_function( user_id: UserID, ) -> ProjectDict: - return await _projects_service.get_project_for_user( + return await projects_service.get_project_for_user( app=app, project_uuid=function_uuid, user_id=user_id ) diff --git a/services/web/server/src/simcore_service_webserver/functions/plugin.py b/services/web/server/src/simcore_service_webserver/functions/plugin.py index 364436f4898..8d18e55a034 100644 --- a/services/web/server/src/simcore_service_webserver/functions/plugin.py +++ b/services/web/server/src/simcore_service_webserver/functions/plugin.py @@ -3,7 +3,7 @@ from aiohttp import web from servicelib.aiohttp.application_setup import ModuleCategory, app_module_setup -from . import _controller_rpc +from . import _functions_controller_rpc _logger = logging.getLogger(__name__) @@ -15,4 +15,4 @@ logger=_logger, ) def setup_functions(app: web.Application): - app.on_startup.append(_controller_rpc.register_rpc_routes_on_startup) + app.on_startup.append(_functions_controller_rpc.register_rpc_routes_on_startup) From 2feedeeda2ba1bc82eeb50a42afa8d22f635a642 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Wed, 16 Apr 2025 11:33:23 +0200 Subject: [PATCH 33/69] Add db migration script for functions api --- .../93dbd49553ae_add_function_tables.py | 133 ++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/93dbd49553ae_add_function_tables.py diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/93dbd49553ae_add_function_tables.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/93dbd49553ae_add_function_tables.py new file mode 100644 index 00000000000..d44b8e271e2 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/93dbd49553ae_add_function_tables.py @@ -0,0 +1,133 @@ +"""Add function tables + +Revision ID: 93dbd49553ae +Revises: cf8f743fd0b7 +Create Date: 2025-04-16 09:32:48.976846+00:00 + +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "93dbd49553ae" +down_revision = "cf8f743fd0b7" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "function_job_collections", + sa.Column("uuid", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("name", sa.String(), nullable=True), + sa.PrimaryKeyConstraint("uuid", name="function_job_collections_pk"), + ) + op.create_index( + op.f("ix_function_job_collections_uuid"), + "function_job_collections", + ["uuid"], + unique=False, + ) + op.create_table( + "functions", + sa.Column("uuid", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("title", sa.String(), nullable=True), + sa.Column("function_class", sa.String(), nullable=True), + sa.Column("description", sa.String(), nullable=True), + sa.Column("input_schema", sa.JSON(), nullable=True), + sa.Column("output_schema", sa.JSON(), nullable=True), + sa.Column("system_tags", sa.JSON(), nullable=True), + sa.Column("user_tags", sa.JSON(), nullable=True), + sa.Column("class_specific_data", sa.JSON(), nullable=True), + sa.PrimaryKeyConstraint("uuid", name="functions_pk"), + ) + op.create_index(op.f("ix_functions_uuid"), "functions", ["uuid"], unique=False) + op.create_table( + "function_jobs", + sa.Column("uuid", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("title", sa.String(), nullable=True), + sa.Column("function_uuid", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("function_class", sa.String(), nullable=True), + sa.Column("status", sa.String(), nullable=True), + sa.Column("inputs", sa.JSON(), nullable=True), + sa.Column("outputs", sa.JSON(), nullable=True), + sa.Column("class_specific_data", sa.JSON(), nullable=True), + sa.ForeignKeyConstraint( + ["function_uuid"], + ["functions.uuid"], + name="fk_functions_to_function_jobs_to_function_uuid", + onupdate="CASCADE", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("uuid", name="function_jobs_pk"), + ) + op.create_index( + op.f("ix_function_jobs_function_uuid"), + "function_jobs", + ["function_uuid"], + unique=False, + ) + op.create_index( + op.f("ix_function_jobs_uuid"), "function_jobs", ["uuid"], unique=False + ) + op.create_table( + "function_job_collections_to_function_jobs", + sa.Column( + "function_job_collection_uuid", postgresql.UUID(as_uuid=True), nullable=True + ), + sa.Column("function_job_uuid", postgresql.UUID(as_uuid=True), nullable=True), + sa.ForeignKeyConstraint( + ["function_job_collection_uuid"], + ["function_job_collections.uuid"], + name="fk_func_job_coll_to_func_jobs_to_func_job_coll_uuid", + onupdate="CASCADE", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["function_job_uuid"], + ["function_jobs.uuid"], + name="fk_func_job_coll_to_func_jobs_to_func_job_uuid", + onupdate="CASCADE", + ondelete="CASCADE", + ), + ) + op.drop_index("idx_projects_last_change_date_desc", table_name="projects") + op.create_index( + "idx_projects_last_change_date_desc", + "projects", + ["last_change_date"], + unique=False, + postgresql_using="btree", + postgresql_ops={"last_change_date": "DESC"}, + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index( + "idx_projects_last_change_date_desc", + table_name="projects", + postgresql_using="btree", + postgresql_ops={"last_change_date": "DESC"}, + ) + op.create_index( + "idx_projects_last_change_date_desc", + "projects", + [sa.text("last_change_date DESC")], + unique=False, + ) + op.drop_table("function_job_collections_to_function_jobs") + op.drop_index(op.f("ix_function_jobs_uuid"), table_name="function_jobs") + op.drop_index(op.f("ix_function_jobs_function_uuid"), table_name="function_jobs") + op.drop_table("function_jobs") + op.drop_index(op.f("ix_functions_uuid"), table_name="functions") + op.drop_table("functions") + op.drop_index( + op.f("ix_function_job_collections_uuid"), table_name="function_job_collections" + ) + op.drop_table("function_job_collections") + # ### end Alembic commands ### From f9f4a6723d839d28af3ec6e2fb9f6d3318ccbf02 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Wed, 16 Apr 2025 17:57:43 +0200 Subject: [PATCH 34/69] Add solver functions --- .../api_schemas_api_server/functions.py | 44 ---- .../functions_wb_schema.py | 30 ++- services/api-server/openapi.json | 234 ++++++++++++++++-- .../api/routes/functions_routes.py | 85 ++++++- .../api/routes/solvers_jobs.py | 2 +- .../models/schemas/functions_api_schema.py | 85 ------- .../functions/_functions_controller_rpc.py | 124 ++++++++-- 7 files changed, 416 insertions(+), 188 deletions(-) delete mode 100644 packages/models-library/src/models_library/api_schemas_api_server/functions.py delete mode 100644 services/api-server/src/simcore_service_api_server/models/schemas/functions_api_schema.py diff --git a/packages/models-library/src/models_library/api_schemas_api_server/functions.py b/packages/models-library/src/models_library/api_schemas_api_server/functions.py deleted file mode 100644 index 44678efd539..00000000000 --- a/packages/models-library/src/models_library/api_schemas_api_server/functions.py +++ /dev/null @@ -1,44 +0,0 @@ -from typing import Annotated, Any, Literal, TypeAlias - -from models_library import projects -from pydantic import BaseModel, Field - -FunctionID: TypeAlias = projects.ProjectID - - -class FunctionSchema(BaseModel): - schema_dict: dict[str, Any] | None # JSON Schema - - -class FunctionInputSchema(FunctionSchema): ... - - -class FunctionOutputSchema(FunctionSchema): ... - - -class Function(BaseModel): - uid: FunctionID | None = None - title: str | None = None - description: str | None = None - input_schema: FunctionInputSchema | None = None - output_schema: FunctionOutputSchema | None = None - - # @classmethod - # def compose_resource_name(cls, function_key) -> api_resources.RelativeResourceName: - # return api_resources.compose_resource_name("functions", function_key) - - -class StudyFunction(Function): - function_type: Literal["study"] = "study" - study_url: str - - -class PythonCodeFunction(Function): - function_type: Literal["python_code"] = "python_code" - code_url: str - - -FunctionUnion: TypeAlias = Annotated[ - StudyFunction | PythonCodeFunction, - Field(discriminator="function_type"), -] diff --git a/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py b/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py index 4391a77a658..a17a31ee113 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py @@ -3,7 +3,9 @@ from uuid import UUID from models_library import projects -from pydantic import BaseModel, Field +from models_library.basic_regex import SIMPLE_VERSION_RE +from models_library.services_regex import COMPUTATIONAL_SERVICE_KEY_RE +from pydantic import BaseModel, Field, StringConstraints from ..projects import ProjectID @@ -26,6 +28,7 @@ class FunctionOutputSchema(FunctionSchema): ... class FunctionClass(str, Enum): project = "project" + solver = "solver" python_code = "python_code" @@ -75,13 +78,28 @@ class ProjectFunction(FunctionBase): project_id: ProjectID +SolverKeyId = Annotated[ + str, StringConstraints(strip_whitespace=True, pattern=COMPUTATIONAL_SERVICE_KEY_RE) +] +VersionStr: TypeAlias = Annotated[ + str, StringConstraints(strip_whitespace=True, pattern=SIMPLE_VERSION_RE) +] +SolverJobID: TypeAlias = UUID + + +class SolverFunction(FunctionBase): + function_class: Literal[FunctionClass.solver] = FunctionClass.solver + solver_key: SolverKeyId + solver_version: str + + class PythonCodeFunction(FunctionBase): function_class: Literal[FunctionClass.python_code] = FunctionClass.python_code code_url: str Function: TypeAlias = Annotated[ - ProjectFunction | PythonCodeFunction, + ProjectFunction | PythonCodeFunction | SolverFunction, Field(discriminator="function_class"), ] @@ -103,13 +121,17 @@ class ProjectFunctionJob(FunctionJobBase): project_job_id: ProjectID +class SolverFunctionJob(FunctionJobBase): + function_class: Literal[FunctionClass.solver] = FunctionClass.solver + solver_job_id: ProjectID + + class PythonCodeFunctionJob(FunctionJobBase): function_class: Literal[FunctionClass.python_code] = FunctionClass.python_code - code_url: str FunctionJob: TypeAlias = Annotated[ - ProjectFunctionJob | PythonCodeFunctionJob, + ProjectFunctionJob | PythonCodeFunctionJob | SolverFunctionJob, Field(discriminator="function_class"), ] diff --git a/services/api-server/openapi.json b/services/api-server/openapi.json index 3196a2cee37..2d22489c953 100644 --- a/services/api-server/openapi.json +++ b/services/api-server/openapi.json @@ -5316,13 +5316,17 @@ }, { "$ref": "#/components/schemas/PythonCodeFunction" + }, + { + "$ref": "#/components/schemas/SolverFunction" } ], "discriminator": { "propertyName": "function_class", "mapping": { "project": "#/components/schemas/ProjectFunction", - "python_code": "#/components/schemas/PythonCodeFunction" + "python_code": "#/components/schemas/PythonCodeFunction", + "solver": "#/components/schemas/SolverFunction" } } }, @@ -5351,6 +5355,9 @@ }, { "$ref": "#/components/schemas/PythonCodeFunction" + }, + { + "$ref": "#/components/schemas/SolverFunction" } ], "title": "Function", @@ -5358,7 +5365,8 @@ "propertyName": "function_class", "mapping": { "project": "#/components/schemas/ProjectFunction", - "python_code": "#/components/schemas/PythonCodeFunction" + "python_code": "#/components/schemas/PythonCodeFunction", + "solver": "#/components/schemas/SolverFunction" } } } @@ -5378,6 +5386,9 @@ }, { "$ref": "#/components/schemas/PythonCodeFunction" + }, + { + "$ref": "#/components/schemas/SolverFunction" } ], "title": "Response Register Function V0 Functions Post", @@ -5385,7 +5396,8 @@ "propertyName": "function_class", "mapping": { "project": "#/components/schemas/ProjectFunction", - "python_code": "#/components/schemas/PythonCodeFunction" + "python_code": "#/components/schemas/PythonCodeFunction", + "solver": "#/components/schemas/SolverFunction" } } } @@ -5437,13 +5449,17 @@ }, { "$ref": "#/components/schemas/PythonCodeFunction" + }, + { + "$ref": "#/components/schemas/SolverFunction" } ], "discriminator": { "propertyName": "function_class", "mapping": { "project": "#/components/schemas/ProjectFunction", - "python_code": "#/components/schemas/PythonCodeFunction" + "python_code": "#/components/schemas/PythonCodeFunction", + "solver": "#/components/schemas/SolverFunction" } }, "title": "Response Get Function V0 Functions Function Id Get" @@ -5504,13 +5520,17 @@ }, { "$ref": "#/components/schemas/PythonCodeFunction" + }, + { + "$ref": "#/components/schemas/SolverFunction" } ], "discriminator": { "propertyName": "function_class", "mapping": { "project": "#/components/schemas/ProjectFunction", - "python_code": "#/components/schemas/PythonCodeFunction" + "python_code": "#/components/schemas/PythonCodeFunction", + "solver": "#/components/schemas/SolverFunction" } }, "title": "Response Delete Function V0 Functions Function Id Delete" @@ -5596,13 +5616,17 @@ }, { "$ref": "#/components/schemas/PythonCodeFunctionJob" + }, + { + "$ref": "#/components/schemas/SolverFunctionJob" } ], "discriminator": { "propertyName": "function_class", "mapping": { "project": "#/components/schemas/ProjectFunctionJob", - "python_code": "#/components/schemas/PythonCodeFunctionJob" + "python_code": "#/components/schemas/PythonCodeFunctionJob", + "solver": "#/components/schemas/SolverFunctionJob" } }, "title": "Response Run Function V0 Functions Function Id Run Post" @@ -5801,13 +5825,17 @@ }, { "$ref": "#/components/schemas/PythonCodeFunctionJob" + }, + { + "$ref": "#/components/schemas/SolverFunctionJob" } ], "discriminator": { "propertyName": "function_class", "mapping": { "project": "#/components/schemas/ProjectFunctionJob", - "python_code": "#/components/schemas/PythonCodeFunctionJob" + "python_code": "#/components/schemas/PythonCodeFunctionJob", + "solver": "#/components/schemas/SolverFunctionJob" } } }, @@ -5860,13 +5888,17 @@ }, { "$ref": "#/components/schemas/PythonCodeFunctionJob" + }, + { + "$ref": "#/components/schemas/SolverFunctionJob" } ], "discriminator": { "propertyName": "function_class", "mapping": { "project": "#/components/schemas/ProjectFunctionJob", - "python_code": "#/components/schemas/PythonCodeFunctionJob" + "python_code": "#/components/schemas/PythonCodeFunctionJob", + "solver": "#/components/schemas/SolverFunctionJob" } } }, @@ -5895,6 +5927,9 @@ }, { "$ref": "#/components/schemas/PythonCodeFunctionJob" + }, + { + "$ref": "#/components/schemas/SolverFunctionJob" } ], "title": "Function Job", @@ -5902,7 +5937,8 @@ "propertyName": "function_class", "mapping": { "project": "#/components/schemas/ProjectFunctionJob", - "python_code": "#/components/schemas/PythonCodeFunctionJob" + "python_code": "#/components/schemas/PythonCodeFunctionJob", + "solver": "#/components/schemas/SolverFunctionJob" } } } @@ -5922,6 +5958,9 @@ }, { "$ref": "#/components/schemas/PythonCodeFunctionJob" + }, + { + "$ref": "#/components/schemas/SolverFunctionJob" } ], "title": "Response Register Function Job V0 Function Jobs Post", @@ -5929,7 +5968,8 @@ "propertyName": "function_class", "mapping": { "project": "#/components/schemas/ProjectFunctionJob", - "python_code": "#/components/schemas/PythonCodeFunctionJob" + "python_code": "#/components/schemas/PythonCodeFunctionJob", + "solver": "#/components/schemas/SolverFunctionJob" } } } @@ -5981,13 +6021,17 @@ }, { "$ref": "#/components/schemas/PythonCodeFunctionJob" + }, + { + "$ref": "#/components/schemas/SolverFunctionJob" } ], "discriminator": { "propertyName": "function_class", "mapping": { "project": "#/components/schemas/ProjectFunctionJob", - "python_code": "#/components/schemas/PythonCodeFunctionJob" + "python_code": "#/components/schemas/PythonCodeFunctionJob", + "solver": "#/components/schemas/SolverFunctionJob" } }, "title": "Response Get Function Job V0 Function Jobs Function Job Id Get" @@ -9055,16 +9099,11 @@ "const": "python_code", "title": "Function Class", "default": "python_code" - }, - "code_url": { - "type": "string", - "title": "Code Url" } }, "type": "object", "required": [ - "function_uid", - "code_url" + "function_uid" ], "title": "PythonCodeFunctionJob" }, @@ -9205,6 +9244,167 @@ "version": "2.1.1" } }, + "SolverFunction": { + "properties": { + "uid": { + "anyOf": [ + { + "type": "string", + "format": "uuid" + }, + { + "type": "null" + } + ], + "title": "Uid" + }, + "title": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Title" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "function_class": { + "type": "string", + "const": "solver", + "title": "Function Class", + "default": "solver" + }, + "input_schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/FunctionInputSchema" + }, + { + "type": "null" + } + ] + }, + "output_schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/FunctionOutputSchema" + }, + { + "type": "null" + } + ] + }, + "solver_key": { + "type": "string", + "pattern": "^simcore/services/comp/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", + "title": "Solver Key" + }, + "solver_version": { + "type": "string", + "title": "Solver Version" + } + }, + "type": "object", + "required": [ + "solver_key", + "solver_version" + ], + "title": "SolverFunction" + }, + "SolverFunctionJob": { + "properties": { + "uid": { + "anyOf": [ + { + "type": "string", + "format": "uuid" + }, + { + "type": "null" + } + ], + "title": "Uid" + }, + "title": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Title" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "function_uid": { + "type": "string", + "format": "uuid", + "title": "Function Uid" + }, + "inputs": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Inputs" + }, + "outputs": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Outputs" + }, + "function_class": { + "type": "string", + "const": "solver", + "title": "Function Class", + "default": "solver" + }, + "solver_job_id": { + "type": "string", + "format": "uuid", + "title": "Solver Job Id" + } + }, + "type": "object", + "required": [ + "function_uid", + "solver_job_id" + ], + "title": "SolverFunctionJob" + }, "SolverPort": { "properties": { "key": { diff --git a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py index dc2a80abcf5..e94bf7fd450 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py @@ -15,6 +15,7 @@ FunctionOutputs, FunctionOutputSchema, ProjectFunctionJob, + SolverFunctionJob, ) from pydantic import PositiveInt from servicelib.fastapi.dependencies import get_reverse_url_mapper @@ -23,17 +24,19 @@ from ...models.schemas.jobs import ( JobInputs, ) +from ...services_http.catalog import CatalogApi from ...services_http.director_v2 import DirectorV2Api from ...services_http.storage import StorageApi from ...services_http.webserver import AuthSession from ...services_rpc.wb_api_server import WbApiRpcClient from ..dependencies.authentication import get_current_user_id, get_product_name +from ..dependencies.database import Engine, get_db_engine from ..dependencies.services import get_api_client from ..dependencies.webserver_http import get_webserver_session from ..dependencies.webserver_rpc import ( get_wb_api_rpc_client, ) -from . import studies_jobs +from . import solvers_jobs, solvers_jobs_getters, studies_jobs function_router = APIRouter() function_job_router = APIRouter() @@ -98,6 +101,7 @@ async def run_function( function_inputs: FunctionInputs, user_id: Annotated[PositiveInt, Depends(get_current_user_id)], product_name: Annotated[str, Depends(get_product_name)], + catalog_client: Annotated[CatalogApi, Depends(get_api_client(CatalogApi))], ): to_run_function = await wb_api_rpc.get_function(function_id=function_id) @@ -141,7 +145,41 @@ async def run_function( project_job_id=study_job.id, ), ) - else: # noqa: RET505 + elif to_run_function.function_class == FunctionClass.solver: # noqa: RET505 + solver_job = await solvers_jobs.create_solver_job( + solver_key=to_run_function.solver_key, + version=to_run_function.solver_version, + inputs=JobInputs(values=function_inputs or {}), + webserver_api=webserver_api, + wb_api_rpc=wb_api_rpc, + url_for=url_for, + x_simcore_parent_project_uuid=None, + x_simcore_parent_node_id=None, + user_id=user_id, + product_name=product_name, + catalog_client=catalog_client, + ) + await solvers_jobs.start_job( + request=request, + solver_key=to_run_function.solver_key, + version=to_run_function.solver_version, + job_id=solver_job.id, + user_id=user_id, + webserver_api=webserver_api, + director2_api=director2_api, + ) + return await register_function_job( + wb_api_rpc=wb_api_rpc, + function_job=SolverFunctionJob( + function_uid=to_run_function.uid, + title=f"Function job of function {to_run_function.uid}", + description=to_run_function.description, + inputs=function_inputs, + outputs=None, + solver_job_id=solver_job.id, + ), + ) + else: msg = f"Function type {type(to_run_function)} not supported" raise TypeError(msg) @@ -276,12 +314,23 @@ async def function_job_status( ): job_status = await studies_jobs.inspect_study_job( study_id=function.project_id, - job_id=function_job.project_job_id, + job_id=function_job.project_job_id, # type: ignore + user_id=user_id, + director2_api=director2_api, + ) + return FunctionJobStatus(status=job_status.state) + elif (function.function_class == FunctionClass.solver) and ( # noqa: RET505 + function_job.function_class == FunctionClass.solver + ): + job_status = await solvers_jobs.inspect_job( + solver_key=function.solver_key, + version=function.solver_version, + job_id=function_job.solver_job_id, user_id=user_id, director2_api=director2_api, ) return FunctionJobStatus(status=job_status.state) - else: # noqa: RET505 + else: msg = f"Function type {function.function_class} / Function job type {function_job.function_class} not supported" raise TypeError(msg) @@ -298,27 +347,41 @@ async def function_job_outputs( user_id: Annotated[PositiveInt, Depends(get_current_user_id)], storage_client: Annotated[StorageApi, Depends(get_api_client(StorageApi))], wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], + db_engine: Annotated[Engine, Depends(get_db_engine)], ): function, function_job = await get_function_from_functionjobid( wb_api_rpc=wb_api_rpc, function_job_id=function_job_id ) if ( - function.function_class != FunctionClass.project - or function_job.function_class != FunctionClass.project + function.function_class == FunctionClass.project + and function_job.function_class == FunctionClass.project ): - msg = f"Function type {function.function_class} not supported" - raise TypeError(msg) - else: # noqa: RET506 job_outputs = await studies_jobs.get_study_job_outputs( study_id=function.project_id, - job_id=function_job.project_job_id, + job_id=function_job.project_job_id, # type: ignore user_id=user_id, webserver_api=webserver_api, storage_client=storage_client, ) return job_outputs.results + elif (function.function_class == FunctionClass.solver) and ( # noqa: RET505 + function_job.function_class == FunctionClass.solver + ): + job_outputs = await solvers_jobs_getters.get_job_outputs( + solver_key=function.solver_key, + version=function.solver_version, + job_id=function_job.solver_job_id, + user_id=user_id, + webserver_api=webserver_api, + storage_client=storage_client, + db_engine=db_engine, + ) + return job_outputs.results + else: + msg = f"Function type {function.function_class} not supported" + raise TypeError(msg) @function_router.post( @@ -337,6 +400,7 @@ async def map_function( director2_api: Annotated[DirectorV2Api, Depends(get_api_client(DirectorV2Api))], user_id: Annotated[PositiveInt, Depends(get_current_user_id)], product_name: Annotated[str, Depends(get_product_name)], + catalog_client: Annotated[CatalogApi, Depends(get_api_client(CatalogApi))], ): function_jobs = [] for function_inputs in function_inputs_list: @@ -351,6 +415,7 @@ async def map_function( url_for=url_for, director2_api=director2_api, request=request, + catalog_client=catalog_client, ) for function_inputs in function_inputs_list ] diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py index 7b724de24dc..4490db6e628 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py @@ -86,7 +86,7 @@ def compose_job_resource_name(solver_key, solver_version, job_id) -> str: status_code=status.HTTP_201_CREATED, responses=JOBS_STATUS_CODES, ) -async def create_solver_job( +async def create_solver_job( # noqa: PLR0913 solver_key: SolverKeyId, version: VersionStr, inputs: JobInputs, diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/functions_api_schema.py b/services/api-server/src/simcore_service_api_server/models/schemas/functions_api_schema.py deleted file mode 100644 index ba316c0e88d..00000000000 --- a/services/api-server/src/simcore_service_api_server/models/schemas/functions_api_schema.py +++ /dev/null @@ -1,85 +0,0 @@ -from enum import Enum -from typing import Annotated, Any, Literal, TypeAlias - -from models_library import projects -from pydantic import BaseModel, Field - -FunctionID: TypeAlias = projects.ProjectID - - -class FunctionSchema(BaseModel): - schema_dict: dict[str, Any] | None # JSON Schema - - -class FunctionInputSchema(FunctionSchema): ... - - -class FunctionOutputSchema(FunctionSchema): ... - - -class FunctionClass(str, Enum): - project = "project" - python_code = "python_code" - - -class FunctionInputs(BaseModel): - inputs_dict: dict[str, Any] | None # JSON Schema - - -class FunctionOutputs(BaseModel): - outputs_dict: dict[str, Any] | None # JSON Schema - - -class Function(BaseModel): - uid: FunctionID | None = None - title: str | None = None - description: str | None = None - input_schema: FunctionInputSchema | None = None - output_schema: FunctionOutputSchema | None = None - - -class StudyFunction(Function): - function_type: Literal["study"] = "study" - study_url: str - - -class PythonCodeFunction(Function): - function_type: Literal["python_code"] = "python_code" - code_url: str - - -FunctionUnion: TypeAlias = Annotated[ - StudyFunction | PythonCodeFunction, - Field(discriminator="function_type"), -] - -FunctionJobID: TypeAlias = projects.ProjectID -FunctionJobCollectionID: TypeAlias = projects.ProjectID - - -class FunctionJob(BaseModel): - uid: FunctionJobID - title: str | None - description: str | None - status: str - function_uid: FunctionID - inputs: FunctionInputs | None - outputs: FunctionOutputs | None - - -class FunctionJobStatus(BaseModel): - status: str - - -class FunctionJobCollection(BaseModel): - """Model for a collection of function jobs""" - - id: FunctionJobCollectionID - title: str | None - description: str | None - job_ids: list[FunctionJobID] - status: str - - -class FunctionJobCollectionStatus(BaseModel): - status: list[str] diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py index 3a689d37ab9..eb826058ec2 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py @@ -15,6 +15,8 @@ FunctionOutputSchema, ProjectFunction, ProjectFunctionJob, + SolverFunction, + SolverFunctionJob, ) from servicelib.rabbitmq import RPCRouter @@ -57,7 +59,33 @@ async def register_function(app: web.Application, *, function: Function) -> Func output_schema=saved_function.output_schema, project_id=saved_function.class_specific_data["project_id"], ) - else: # noqa: RET505 + elif function.function_class == FunctionClass.solver: # noqa: RET505 + saved_function = await _functions_repository.create_function( + app=app, + function=FunctionDB( + title=function.title, + description=function.description, + input_schema=function.input_schema, + output_schema=function.output_schema, + function_class=function.function_class, + class_specific_data=FunctionClassSpecificData( + { + "solver_key": str(function.solver_key), + "solver_version": str(function.solver_version), + } + ), + ), + ) + return SolverFunction( + uid=saved_function.uuid, + title=saved_function.title, + description=saved_function.description, + input_schema=saved_function.input_schema, + output_schema=saved_function.output_schema, + solver_key=saved_function.class_specific_data["solver_key"], + solver_version=saved_function.class_specific_data["solver_version"], + ) + else: msg = f"Unsupported function class: {function.function_class}" raise TypeError(msg) @@ -74,7 +102,17 @@ def _decode_function( output_schema=function.output_schema, project_id=function.class_specific_data["project_id"], ) - else: # noqa: RET505 + elif function.function_class == "solver": # noqa: RET505 + return SolverFunction( + uid=function.uuid, + title=function.title, + description=function.description, + input_schema=function.input_schema, + output_schema=function.output_schema, + solver_key=function.class_specific_data["solver_key"], + solver_version=function.class_specific_data["solver_version"], + ) + else: msg = f"Unsupported function class: [{function.function_class}]" raise TypeError(msg) @@ -91,6 +129,34 @@ async def get_function(app: web.Application, *, function_id: FunctionID) -> Func ) +def convert_functionjobdb_to_functionjob( + functionjob_db: FunctionJobDB, +) -> FunctionJob: + if functionjob_db.function_class == FunctionClass.project: + return ProjectFunctionJob( + uid=functionjob_db.uuid, + title=functionjob_db.title, + description="", + function_uid=functionjob_db.function_uuid, + inputs=functionjob_db.inputs, + outputs=None, + project_job_id=functionjob_db.class_specific_data["project_job_id"], + ) + elif functionjob_db.function_class == FunctionClass.solver: # noqa: RET505 + return SolverFunctionJob( + uid=functionjob_db.uuid, + title=functionjob_db.title, + description="", + function_uid=functionjob_db.function_uuid, + inputs=functionjob_db.inputs, + outputs=None, + solver_job_id=functionjob_db.class_specific_data["solver_job_id"], + ) + else: + msg = f"Unsupported function class: [{functionjob_db.function_class}]" + raise TypeError(msg) + + @router.expose() async def get_function_job( app: web.Application, *, function_job_id: FunctionJobID @@ -102,19 +168,7 @@ async def get_function_job( ) assert returned_function_job is not None - if returned_function_job.function_class == FunctionClass.project: - return ProjectFunctionJob( - uid=returned_function_job.uuid, - title=returned_function_job.title, - description="", - function_uid=returned_function_job.function_uuid, - inputs=returned_function_job.inputs, - outputs=None, - project_job_id=returned_function_job.class_specific_data["project_job_id"], - ) - else: # noqa: RET505 - msg = f"Unsupported function class: [{returned_function_job.function_class}]" - raise TypeError(msg) + return convert_functionjobdb_to_functionjob(returned_function_job) @router.expose() @@ -124,15 +178,7 @@ async def list_function_jobs(app: web.Application) -> list[FunctionJob]: app=app, ) return [ - ProjectFunctionJob( - uid=returned_function_job.uuid, - title=returned_function_job.title, - description="", - function_uid=returned_function_job.function_uuid, - inputs=returned_function_job.inputs, - outputs=None, - project_job_id=returned_function_job.class_specific_data["project_job_id"], - ) + convert_functionjobdb_to_functionjob(returned_function_job) for returned_function_job in returned_function_jobs ] @@ -208,13 +254,12 @@ async def register_function_job( outputs=None, class_specific_data=FunctionJobClassSpecificData( { - "project_job_id": str(function_job.project_job_id), + "project_job_id": str(function_job.project_job_id), # type: ignore } ), function_class=function_job.function_class, ), ) - return ProjectFunctionJob( uid=created_function_job_db.uuid, title=created_function_job_db.title, @@ -226,7 +271,32 @@ async def register_function_job( "project_job_id" ], ) - else: # noqa: RET505 + elif function_job.function_class == FunctionClass.solver: # noqa: RET505 + created_function_job_db = await _functions_repository.register_function_job( + app=app, + function_job=FunctionJobDB( + title=function_job.title, + function_uuid=function_job.function_uid, + inputs=function_job.inputs, + outputs=None, + class_specific_data=FunctionJobClassSpecificData( + { + "solver_job_id": str(function_job.solver_job_id), + } + ), + function_class=function_job.function_class, + ), + ) + return SolverFunctionJob( + uid=created_function_job_db.uuid, + title=created_function_job_db.title, + description="", + function_uid=created_function_job_db.function_uuid, + inputs=created_function_job_db.inputs, + outputs=None, + solver_job_id=created_function_job_db.class_specific_data["solver_job_id"], + ) + else: msg = f"Unsupported function class: [{function_job.function_class}]" raise TypeError(msg) From a5b5a76ba7f8ea1a96f557aaa749f9e287e6abcb Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Wed, 16 Apr 2025 18:21:16 +0200 Subject: [PATCH 35/69] Add default inputs to functions --- .../functions_wb_schema.py | 6 +- .../models/functions_models_db.py | 6 + services/api-server/openapi.json | 69 +++++-- .../functions/_functions_controller_rpc.py | 192 ++++++++---------- 4 files changed, 142 insertions(+), 131 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py b/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py index a17a31ee113..5e0f30f6666 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py @@ -45,21 +45,23 @@ class FunctionClass(str, Enum): class FunctionBase(BaseModel): + function_class: FunctionClass uid: FunctionID | None = None title: str | None = None description: str | None = None - function_class: FunctionClass input_schema: FunctionInputSchema | None = None output_schema: FunctionOutputSchema | None = None + default_inputs: FunctionInputs | None = None class FunctionDB(BaseModel): + function_class: FunctionClass uuid: FunctionJobID | None = None title: str | None = None description: str | None = None - function_class: FunctionClass input_schema: FunctionInputSchema | None = None output_schema: FunctionOutputSchema | None = None + default_inputs: FunctionInputs | None = None class_specific_data: FunctionClassSpecificData diff --git a/packages/postgres-database/src/simcore_postgres_database/models/functions_models_db.py b/packages/postgres-database/src/simcore_postgres_database/models/functions_models_db.py index e8a8ba6f2ec..6d9b80b0509 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/functions_models_db.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/functions_models_db.py @@ -65,6 +65,12 @@ nullable=True, doc="Fields specific for a function class", ), + sa.Column( + "default_inputs", + sa.JSON, + nullable=True, + doc="Default inputs of the function", + ), sa.PrimaryKeyConstraint("uuid", name="functions_pk"), ) diff --git a/services/api-server/openapi.json b/services/api-server/openapi.json index 2d22489c953..e28cd250482 100644 --- a/services/api-server/openapi.json +++ b/services/api-server/openapi.json @@ -8804,6 +8804,12 @@ }, "ProjectFunction": { "properties": { + "function_class": { + "type": "string", + "const": "project", + "title": "Function Class", + "default": "project" + }, "uid": { "anyOf": [ { @@ -8838,12 +8844,6 @@ ], "title": "Description" }, - "function_class": { - "type": "string", - "const": "project", - "title": "Function Class", - "default": "project" - }, "input_schema": { "anyOf": [ { @@ -8864,6 +8864,17 @@ } ] }, + "default_inputs": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Default Inputs" + }, "project_id": { "type": "string", "format": "uuid", @@ -8960,6 +8971,12 @@ }, "PythonCodeFunction": { "properties": { + "function_class": { + "type": "string", + "const": "python_code", + "title": "Function Class", + "default": "python_code" + }, "uid": { "anyOf": [ { @@ -8994,12 +9011,6 @@ ], "title": "Description" }, - "function_class": { - "type": "string", - "const": "python_code", - "title": "Function Class", - "default": "python_code" - }, "input_schema": { "anyOf": [ { @@ -9020,6 +9031,17 @@ } ] }, + "default_inputs": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Default Inputs" + }, "code_url": { "type": "string", "title": "Code Url" @@ -9246,6 +9268,12 @@ }, "SolverFunction": { "properties": { + "function_class": { + "type": "string", + "const": "solver", + "title": "Function Class", + "default": "solver" + }, "uid": { "anyOf": [ { @@ -9280,12 +9308,6 @@ ], "title": "Description" }, - "function_class": { - "type": "string", - "const": "solver", - "title": "Function Class", - "default": "solver" - }, "input_schema": { "anyOf": [ { @@ -9306,6 +9328,17 @@ } ] }, + "default_inputs": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Default Inputs" + }, "solver_key": { "type": "string", "pattern": "^simcore/services/comp/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py index eb826058ec2..64bf15a364a 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py @@ -35,59 +35,10 @@ async def ping(app: web.Application) -> str: @router.expose() async def register_function(app: web.Application, *, function: Function) -> Function: assert app - if function.function_class == FunctionClass.project: - saved_function = await _functions_repository.create_function( - app=app, - function=FunctionDB( - title=function.title, - description=function.description, - input_schema=function.input_schema, - output_schema=function.output_schema, - function_class=function.function_class, - class_specific_data=FunctionClassSpecificData( - { - "project_id": str(function.project_id), - } - ), - ), - ) - return ProjectFunction( - uid=saved_function.uuid, - title=saved_function.title, - description=saved_function.description, - input_schema=saved_function.input_schema, - output_schema=saved_function.output_schema, - project_id=saved_function.class_specific_data["project_id"], - ) - elif function.function_class == FunctionClass.solver: # noqa: RET505 - saved_function = await _functions_repository.create_function( - app=app, - function=FunctionDB( - title=function.title, - description=function.description, - input_schema=function.input_schema, - output_schema=function.output_schema, - function_class=function.function_class, - class_specific_data=FunctionClassSpecificData( - { - "solver_key": str(function.solver_key), - "solver_version": str(function.solver_version), - } - ), - ), - ) - return SolverFunction( - uid=saved_function.uuid, - title=saved_function.title, - description=saved_function.description, - input_schema=saved_function.input_schema, - output_schema=saved_function.output_schema, - solver_key=saved_function.class_specific_data["solver_key"], - solver_version=saved_function.class_specific_data["solver_version"], - ) - else: - msg = f"Unsupported function class: {function.function_class}" - raise TypeError(msg) + saved_function = await _functions_repository.create_function( + app=app, function=_encode_function(function) + ) + return _decode_function(saved_function) def _decode_function( @@ -117,6 +68,42 @@ def _decode_function( raise TypeError(msg) +def _encode_function( + function: Function, +) -> FunctionDB: + if function.function_class == FunctionClass.project: + return FunctionDB( + title=function.title, + description=function.description, + input_schema=function.input_schema, + output_schema=function.output_schema, + function_class=function.function_class, + default_inputs=function.default_inputs, + class_specific_data=FunctionClassSpecificData( + { + "project_id": str(function.project_id), + } + ), + ) + elif function.function_class == FunctionClass.solver: # noqa: RET505 + return FunctionDB( + title=function.title, + description=function.description, + input_schema=function.input_schema, + output_schema=function.output_schema, + function_class=function.function_class, + class_specific_data=FunctionClassSpecificData( + { + "solver_key": str(function.solver_key), + "solver_version": str(function.solver_version), + } + ), + ) + else: + msg = f"Unsupported function class: {function.function_class}" + raise TypeError(msg) + + @router.expose() async def get_function(app: web.Application, *, function_id: FunctionID) -> Function: assert app @@ -129,7 +116,7 @@ async def get_function(app: web.Application, *, function_id: FunctionID) -> Func ) -def convert_functionjobdb_to_functionjob( +def _decode_functionjob( functionjob_db: FunctionJobDB, ) -> FunctionJob: if functionjob_db.function_class == FunctionClass.project: @@ -157,6 +144,40 @@ def convert_functionjobdb_to_functionjob( raise TypeError(msg) +def _encode_functionjob( + functionjob: FunctionJob, +) -> FunctionJobDB: + if functionjob.function_class == FunctionClass.project: + return FunctionJobDB( + title=functionjob.title, + function_uuid=functionjob.function_uid, + inputs=functionjob.inputs, + outputs=None, + class_specific_data=FunctionJobClassSpecificData( + { + "project_job_id": str(functionjob.project_job_id), + } + ), + function_class=functionjob.function_class, + ) + elif functionjob.function_class == FunctionClass.solver: # noqa: RET505 + return FunctionJobDB( + title=functionjob.title, + function_uuid=functionjob.function_uid, + inputs=functionjob.inputs, + outputs=None, + class_specific_data=FunctionJobClassSpecificData( + { + "solver_job_id": str(functionjob.solver_job_id), + } + ), + function_class=functionjob.function_class, + ) + else: + msg = f"Unsupported function class: [{functionjob.function_class}]" + raise TypeError(msg) + + @router.expose() async def get_function_job( app: web.Application, *, function_job_id: FunctionJobID @@ -168,7 +189,7 @@ async def get_function_job( ) assert returned_function_job is not None - return convert_functionjobdb_to_functionjob(returned_function_job) + return _decode_functionjob(returned_function_job) @router.expose() @@ -178,7 +199,7 @@ async def list_function_jobs(app: web.Application) -> list[FunctionJob]: app=app, ) return [ - convert_functionjobdb_to_functionjob(returned_function_job) + _decode_functionjob(returned_function_job) for returned_function_job in returned_function_jobs ] @@ -244,61 +265,10 @@ async def register_function_job( app: web.Application, *, function_job: FunctionJob ) -> FunctionJob: assert app - if function_job.function_class == FunctionClass.project: - created_function_job_db = await _functions_repository.register_function_job( - app=app, - function_job=FunctionJobDB( - title=function_job.title, - function_uuid=function_job.function_uid, - inputs=function_job.inputs, - outputs=None, - class_specific_data=FunctionJobClassSpecificData( - { - "project_job_id": str(function_job.project_job_id), # type: ignore - } - ), - function_class=function_job.function_class, - ), - ) - return ProjectFunctionJob( - uid=created_function_job_db.uuid, - title=created_function_job_db.title, - description="", - function_uid=created_function_job_db.function_uuid, - inputs=created_function_job_db.inputs, - outputs=None, - project_job_id=created_function_job_db.class_specific_data[ - "project_job_id" - ], - ) - elif function_job.function_class == FunctionClass.solver: # noqa: RET505 - created_function_job_db = await _functions_repository.register_function_job( - app=app, - function_job=FunctionJobDB( - title=function_job.title, - function_uuid=function_job.function_uid, - inputs=function_job.inputs, - outputs=None, - class_specific_data=FunctionJobClassSpecificData( - { - "solver_job_id": str(function_job.solver_job_id), - } - ), - function_class=function_job.function_class, - ), - ) - return SolverFunctionJob( - uid=created_function_job_db.uuid, - title=created_function_job_db.title, - description="", - function_uid=created_function_job_db.function_uuid, - inputs=created_function_job_db.inputs, - outputs=None, - solver_job_id=created_function_job_db.class_specific_data["solver_job_id"], - ) - else: - msg = f"Unsupported function class: [{function_job.function_class}]" - raise TypeError(msg) + created_function_job_db = await _functions_repository.register_function_job( + app=app, function_job=_encode_functionjob(function_job) + ) + return _decode_functionjob(created_function_job_db) @router.expose() From 1f2448529eba01fcd877d1d5df9f32f3da5a43d7 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Wed, 16 Apr 2025 19:36:21 +0200 Subject: [PATCH 36/69] Add default inputs to functions --- ...94af8f28b25_add_function_default_inputs.py | 49 +++++++++++++++++++ .../api/routes/functions_routes.py | 29 +++++++++-- .../functions/_functions_controller_rpc.py | 45 ++++++++--------- .../functions/_functions_repository.py | 1 + 4 files changed, 94 insertions(+), 30 deletions(-) create mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/d94af8f28b25_add_function_default_inputs.py diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/d94af8f28b25_add_function_default_inputs.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/d94af8f28b25_add_function_default_inputs.py new file mode 100644 index 00000000000..621916c8233 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/d94af8f28b25_add_function_default_inputs.py @@ -0,0 +1,49 @@ +"""Add function default inputs + +Revision ID: d94af8f28b25 +Revises: 93dbd49553ae +Create Date: 2025-04-16 16:23:12.224948+00:00 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "d94af8f28b25" +down_revision = "93dbd49553ae" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column("functions", sa.Column("default_inputs", sa.JSON(), nullable=True)) + op.drop_index("idx_projects_last_change_date_desc", table_name="projects") + op.create_index( + "idx_projects_last_change_date_desc", + "projects", + ["last_change_date"], + unique=False, + postgresql_using="btree", + postgresql_ops={"last_change_date": "DESC"}, + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index( + "idx_projects_last_change_date_desc", + table_name="projects", + postgresql_using="btree", + postgresql_ops={"last_change_date": "DESC"}, + ) + op.create_index( + "idx_projects_last_change_date_desc", + "projects", + [sa.text("last_change_date DESC")], + unique=False, + ) + op.drop_column("functions", "default_inputs") + # ### end Alembic commands ### diff --git a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py index e94bf7fd450..ec0059d3d88 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py @@ -85,6 +85,20 @@ async def get_function( return await wb_api_rpc.get_function(function_id=function_id) +def join_inputs( + default_inputs: FunctionInputs | None, + function_inputs: FunctionInputs | None, +) -> FunctionInputs: + if default_inputs is None: + return function_inputs + + if function_inputs is None: + return default_inputs + + # last dict will override defaults + return {**default_inputs, **function_inputs} + + @function_router.post( "/{function_id:uuid}:run", response_model=FunctionJob, @@ -108,16 +122,21 @@ async def run_function( assert to_run_function.uid is not None + joined_inputs = join_inputs( + to_run_function.default_inputs, + function_inputs, + ) + if cached_function_job := await wb_api_rpc.find_cached_function_job( function_id=to_run_function.uid, - inputs=function_inputs, + inputs=joined_inputs, ): return cached_function_job if to_run_function.function_class == FunctionClass.project: study_job = await studies_jobs.create_study_job( study_id=to_run_function.project_id, - job_inputs=JobInputs(values=function_inputs or {}), + job_inputs=JobInputs(values=joined_inputs or {}), webserver_api=webserver_api, wb_api_rpc=wb_api_rpc, url_for=url_for, @@ -140,7 +159,7 @@ async def run_function( function_uid=to_run_function.uid, title=f"Function job of function {to_run_function.uid}", description=to_run_function.description, - inputs=function_inputs, + inputs=joined_inputs, outputs=None, project_job_id=study_job.id, ), @@ -149,7 +168,7 @@ async def run_function( solver_job = await solvers_jobs.create_solver_job( solver_key=to_run_function.solver_key, version=to_run_function.solver_version, - inputs=JobInputs(values=function_inputs or {}), + inputs=JobInputs(values=joined_inputs or {}), webserver_api=webserver_api, wb_api_rpc=wb_api_rpc, url_for=url_for, @@ -174,7 +193,7 @@ async def run_function( function_uid=to_run_function.uid, title=f"Function job of function {to_run_function.uid}", description=to_run_function.description, - inputs=function_inputs, + inputs=joined_inputs, outputs=None, solver_job_id=solver_job.id, ), diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py index 64bf15a364a..53b3aad7f2b 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py @@ -52,6 +52,7 @@ def _decode_function( input_schema=function.input_schema, output_schema=function.output_schema, project_id=function.class_specific_data["project_id"], + default_inputs=function.default_inputs, ) elif function.function_class == "solver": # noqa: RET505 return SolverFunction( @@ -62,6 +63,7 @@ def _decode_function( output_schema=function.output_schema, solver_key=function.class_specific_data["solver_key"], solver_version=function.class_specific_data["solver_version"], + default_inputs=function.default_inputs, ) else: msg = f"Unsupported function class: [{function.function_class}]" @@ -72,37 +74,30 @@ def _encode_function( function: Function, ) -> FunctionDB: if function.function_class == FunctionClass.project: - return FunctionDB( - title=function.title, - description=function.description, - input_schema=function.input_schema, - output_schema=function.output_schema, - function_class=function.function_class, - default_inputs=function.default_inputs, - class_specific_data=FunctionClassSpecificData( - { - "project_id": str(function.project_id), - } - ), + class_specific_data = FunctionClassSpecificData( + {"project_id": str(function.project_id)} ) - elif function.function_class == FunctionClass.solver: # noqa: RET505 - return FunctionDB( - title=function.title, - description=function.description, - input_schema=function.input_schema, - output_schema=function.output_schema, - function_class=function.function_class, - class_specific_data=FunctionClassSpecificData( - { - "solver_key": str(function.solver_key), - "solver_version": str(function.solver_version), - } - ), + elif function.function_class == FunctionClass.solver: + class_specific_data = FunctionClassSpecificData( + { + "solver_key": str(function.solver_key), + "solver_version": str(function.solver_version), + } ) else: msg = f"Unsupported function class: {function.function_class}" raise TypeError(msg) + return FunctionDB( + title=function.title, + description=function.description, + input_schema=function.input_schema, + output_schema=function.output_schema, + function_class=function.function_class, + default_inputs=function.default_inputs, + class_specific_data=class_specific_data, + ) + @router.expose() async def get_function(app: web.Application, *, function_id: FunctionID) -> Function: diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py index 495f93c0c25..ada1980f4f1 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py @@ -53,6 +53,7 @@ async def create_function( ), function_class=function.function_class, class_specific_data=function.class_specific_data, + default_inputs=function.default_inputs, ) .returning(*_FUNCTIONS_TABLE_COLS) ) From b38daa24f5103b880f1cab210d1a00461a403665 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Tue, 29 Apr 2025 13:11:24 +0200 Subject: [PATCH 37/69] Add function collections --- .../functions_wb_schema.py | 11 +- .../models/functions_models_db.py | 9 +- .../functions/functions_rpc_interface.py | 51 +++ services/api-server/openapi.json | 400 +++++++++++++++++- .../api/routes/functions_routes.py | 259 +++++++----- .../services_rpc/wb_api_server.py | 38 ++ .../functions/_functions_controller_rpc.py | 71 ++++ .../functions/_functions_repository.py | 131 ++++++ 8 files changed, 833 insertions(+), 137 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py b/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py index 5e0f30f6666..53362663569 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py @@ -145,11 +145,18 @@ class FunctionJobStatus(BaseModel): class FunctionJobCollection(BaseModel): """Model for a collection of function jobs""" - id: FunctionJobCollectionID + uid: FunctionJobCollectionID | None title: str | None description: str | None job_ids: list[FunctionJobID] - status: str + + +class FunctionJobCollectionDB(BaseModel): + """Model for a collection of function jobs""" + + uuid: FunctionJobCollectionID | None + title: str | None + description: str | None class FunctionJobCollectionStatus(BaseModel): diff --git a/packages/postgres-database/src/simcore_postgres_database/models/functions_models_db.py b/packages/postgres-database/src/simcore_postgres_database/models/functions_models_db.py index 6d9b80b0509..bfaabb39372 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/functions_models_db.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/functions_models_db.py @@ -143,9 +143,14 @@ doc="Unique id of the function job collection", ), sa.Column( - "name", + "title", + sa.String, + doc="Title of the function job collection", + ), + sa.Column( + "description", sa.String, - doc="Name of the function job collection", + doc="Description of the function job collection", ), sa.PrimaryKeyConstraint("uuid", name="function_job_collections_pk"), ) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py index 7e06bef912c..7bee95d0601 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py @@ -9,6 +9,8 @@ FunctionInputs, FunctionInputSchema, FunctionJob, + FunctionJobCollection, + FunctionJobCollectionID, FunctionJobID, FunctionOutputSchema, ) @@ -185,3 +187,52 @@ async def find_cached_function_job( function_id=function_id, inputs=inputs, ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def list_function_job_collections( + rabbitmq_rpc_client: RabbitMQRPCClient, +) -> list[FunctionJobCollection]: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("list_function_job_collections"), + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def register_function_job_collection( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + function_job_collection: FunctionJobCollection, +) -> FunctionJobCollection: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("register_function_job_collection"), + function_job_collection=function_job_collection, + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def get_function_job_collection( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + function_job_collection_id: FunctionJobCollectionID, +) -> FunctionJobCollection: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("get_function_job_collection"), + function_job_collection_id=function_job_collection_id, + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def delete_function_job_collection( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + function_job_collection_id: FunctionJobCollectionID, +) -> None: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("delete_function_job_collection"), + function_job_collection_id=function_job_collection_id, + ) diff --git a/services/api-server/openapi.json b/services/api-server/openapi.json index e28cd250482..8d3abb3289a 100644 --- a/services/api-server/openapi.json +++ b/services/api-server/openapi.json @@ -5817,29 +5817,7 @@ "content": { "application/json": { "schema": { - "type": "array", - "items": { - "oneOf": [ - { - "$ref": "#/components/schemas/ProjectFunctionJob" - }, - { - "$ref": "#/components/schemas/PythonCodeFunctionJob" - }, - { - "$ref": "#/components/schemas/SolverFunctionJob" - } - ], - "discriminator": { - "propertyName": "function_class", - "mapping": { - "project": "#/components/schemas/ProjectFunctionJob", - "python_code": "#/components/schemas/PythonCodeFunctionJob", - "solver": "#/components/schemas/SolverFunctionJob" - } - } - }, - "title": "Response Map Function V0 Functions Function Id Map Post" + "$ref": "#/components/schemas/FunctionJobCollection" } } } @@ -6238,6 +6216,311 @@ } } }, + "/v0/function_job_collections": { + "get": { + "tags": [ + "function_job_collections" + ], + "summary": "List Function Job Collections", + "description": "List function job collections", + "operationId": "list_function_job_collections", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/FunctionJobCollection" + }, + "type": "array", + "title": "Response List Function Job Collections V0 Function Job Collections Get" + } + } + } + } + } + }, + "post": { + "tags": [ + "function_job_collections" + ], + "summary": "Register Function Job Collection", + "description": "Register function job collection", + "operationId": "register_function_job_collection", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FunctionJobCollection" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FunctionJobCollection" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/function_job_collections/{function_job_collection_id}": { + "get": { + "tags": [ + "function_job_collections" + ], + "summary": "Get Function Job Collection", + "description": "Get function job collection", + "operationId": "get_function_job_collection", + "parameters": [ + { + "name": "function_job_collection_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Job Collection Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FunctionJobCollection" + } + } + } + }, + "404": { + "description": "Function job collection not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "delete": { + "tags": [ + "function_job_collections" + ], + "summary": "Delete Function Job Collection", + "description": "Delete function job collection", + "operationId": "delete_function_job_collection", + "parameters": [ + { + "name": "function_job_collection_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Job Collection Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "404": { + "description": "Function job collection not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/function_job_collections/{function_job_collection_id}/function_jobs": { + "get": { + "tags": [ + "function_job_collections" + ], + "summary": "Function Job Collection List Function Jobs", + "description": "Get the function jobs in function job collection", + "operationId": "function_job_collection_list_function_jobs", + "parameters": [ + { + "name": "function_job_collection_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Job Collection Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "oneOf": [ + { + "$ref": "#/components/schemas/ProjectFunctionJob" + }, + { + "$ref": "#/components/schemas/PythonCodeFunctionJob" + }, + { + "$ref": "#/components/schemas/SolverFunctionJob" + } + ], + "discriminator": { + "propertyName": "function_class", + "mapping": { + "project": "#/components/schemas/ProjectFunctionJob", + "python_code": "#/components/schemas/PythonCodeFunctionJob", + "solver": "#/components/schemas/SolverFunctionJob" + } + } + }, + "title": "Response Function Job Collection List Function Jobs V0 Function Job Collections Function Job Collection Id Function Jobs Get" + } + } + } + }, + "404": { + "description": "Function job collection not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/function_job_collections/{function_job_collection_id}/status": { + "get": { + "tags": [ + "function_job_collections" + ], + "summary": "Function Job Collection Status", + "description": "Get function job collection status", + "operationId": "function_job_collection_status", + "security": [ + { + "HTTPBasic": [] + } + ], + "parameters": [ + { + "name": "function_job_collection_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Job Collection Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FunctionJobCollectionStatus" + } + } + } + }, + "404": { + "description": "Function job collection not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, "/v0/wallets/default": { "get": { "tags": [ @@ -7234,6 +7517,77 @@ ], "title": "FunctionInputSchema" }, + "FunctionJobCollection": { + "properties": { + "uid": { + "anyOf": [ + { + "type": "string", + "format": "uuid" + }, + { + "type": "null" + } + ], + "title": "Uid" + }, + "title": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Title" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "job_ids": { + "items": { + "type": "string", + "format": "uuid" + }, + "type": "array", + "title": "Job Ids" + } + }, + "type": "object", + "required": [ + "uid", + "title", + "description", + "job_ids" + ], + "title": "FunctionJobCollection", + "description": "Model for a collection of function jobs" + }, + "FunctionJobCollectionStatus": { + "properties": { + "status": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Status" + } + }, + "type": "object", + "required": [ + "status" + ], + "title": "FunctionJobCollectionStatus" + }, "FunctionJobStatus": { "properties": { "status": { diff --git a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py index ec0059d3d88..b8aeacc7c75 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py @@ -1,3 +1,4 @@ +import asyncio from collections.abc import Callable from typing import Annotated, Final @@ -10,6 +11,9 @@ FunctionInputSchema, FunctionInputsList, FunctionJob, + FunctionJobCollection, + FunctionJobCollectionID, + FunctionJobCollectionStatus, FunctionJobID, FunctionJobStatus, FunctionOutputs, @@ -405,7 +409,7 @@ async def function_job_outputs( @function_router.post( "/{function_id:uuid}:map", - response_model=list[FunctionJob], + response_model=FunctionJobCollection, responses={**_COMMON_FUNCTION_ERROR_RESPONSES}, description="Map function over input parameters", ) @@ -422,116 +426,151 @@ async def map_function( catalog_client: Annotated[CatalogApi, Depends(get_api_client(CatalogApi))], ): function_jobs = [] - for function_inputs in function_inputs_list: - function_jobs = [ - await run_function( + function_jobs = [ + await run_function( + wb_api_rpc=wb_api_rpc, + function_id=function_id, + function_inputs=function_inputs, + product_name=product_name, + user_id=user_id, + webserver_api=webserver_api, + url_for=url_for, + director2_api=director2_api, + request=request, + catalog_client=catalog_client, + ) + for function_inputs in function_inputs_list + ] + + assert all( + function_job.uid is not None for function_job in function_jobs + ), "Function job uid should not be None" + + return await register_function_job_collection( + wb_api_rpc=wb_api_rpc, + function_job_collection=FunctionJobCollection( + id=None, + title=f"Function job collection of function map {[function_job.uid for function_job in function_jobs]}", + description="", + job_ids=[function_job.uid for function_job in function_jobs], # type: ignore + ), + ) + + +_COMMON_FUNCTION_JOB_COLLECTION_ERROR_RESPONSES: Final[dict] = { + status.HTTP_404_NOT_FOUND: { + "description": "Function job collection not found", + "model": ErrorGet, + }, +} + + +@function_job_collections_router.get( + "", + response_model=list[FunctionJobCollection], + description="List function job collections", +) +async def list_function_job_collections( + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + return await wb_api_rpc.list_function_job_collections() + + +@function_job_collections_router.get( + "/{function_job_collection_id:uuid}", + response_model=FunctionJobCollection, + responses={**_COMMON_FUNCTION_JOB_COLLECTION_ERROR_RESPONSES}, + description="Get function job collection", +) +async def get_function_job_collection( + function_job_collection_id: FunctionJobCollectionID, + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + return await wb_api_rpc.get_function_job_collection( + function_job_collection_id=function_job_collection_id + ) + + +@function_job_collections_router.post( + "", + response_model=FunctionJobCollection, + description="Register function job collection", +) +async def register_function_job_collection( + function_job_collection: FunctionJobCollection, + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + return await wb_api_rpc.register_function_job_collection( + function_job_collection=function_job_collection + ) + + +@function_job_collections_router.delete( + "/{function_job_collection_id:uuid}", + response_model=None, + responses={**_COMMON_FUNCTION_JOB_COLLECTION_ERROR_RESPONSES}, + description="Delete function job collection", +) +async def delete_function_job_collection( + function_job_collection_id: FunctionJobCollectionID, + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + return await wb_api_rpc.delete_function_job_collection( + function_job_collection_id=function_job_collection_id + ) + + +@function_job_collections_router.get( + "/{function_job_collection_id:uuid}/function_jobs", + response_model=list[FunctionJob], + responses={**_COMMON_FUNCTION_JOB_COLLECTION_ERROR_RESPONSES}, + description="Get the function jobs in function job collection", +) +async def function_job_collection_list_function_jobs( + function_job_collection_id: FunctionJobCollectionID, + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + function_job_collection = await get_function_job_collection( + function_job_collection_id=function_job_collection_id, + wb_api_rpc=wb_api_rpc, + ) + return [ + await get_function_job( + job_id, + wb_api_rpc=wb_api_rpc, + ) + for job_id in function_job_collection.job_ids + ] + + +@function_job_collections_router.get( + "/{function_job_collection_id:uuid}/status", + response_model=FunctionJobCollectionStatus, + responses={**_COMMON_FUNCTION_JOB_COLLECTION_ERROR_RESPONSES}, + description="Get function job collection status", +) +async def function_job_collection_status( + function_job_collection_id: FunctionJobCollectionID, + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], + director2_api: Annotated[DirectorV2Api, Depends(get_api_client(DirectorV2Api))], + user_id: Annotated[PositiveInt, Depends(get_current_user_id)], +): + function_job_collection = await get_function_job_collection( + function_job_collection_id=function_job_collection_id, + wb_api_rpc=wb_api_rpc, + ) + + job_statuses = await asyncio.gather( + *[ + function_job_status( + job_id, wb_api_rpc=wb_api_rpc, - function_id=function_id, - function_inputs=function_inputs, - product_name=product_name, - user_id=user_id, - webserver_api=webserver_api, - url_for=url_for, director2_api=director2_api, - request=request, - catalog_client=catalog_client, + user_id=user_id, ) - for function_inputs in function_inputs_list + for job_id in function_job_collection.job_ids ] - # TODO poor system can't handle doing this in parallel, get this fixed # noqa: FIX002 - # function_jobs = await asyncio.gather(*function_jobs_tasks) - - return function_jobs - - -# ruff: noqa: ERA001 - - -# _logger = logging.getLogger(__name__) - -# _COMMON_FUNCTION_JOB_COLLECTION_ERROR_RESPONSES: Final[dict] = { -# status.HTTP_404_NOT_FOUND: { -# "description": "Function job collection not found", -# "model": ErrorGet, -# }, -# } - - -# @function_job_collections_router.get( -# "", -# response_model=FunctionJobCollection, -# description="List function job collections", -# ) -# async def list_function_job_collections( -# page_params: Annotated[PaginationParams, Depends()], -# webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], -# ): -# msg = "list function jobs collection not implemented yet" -# raise NotImplementedError(msg) - - -# @function_job_collections_router.post( -# "", response_model=FunctionJobCollection, description="Create function job" -# ) -# async def create_function_job_collection( -# webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], -# job_ids: Annotated[list[FunctionJob], Depends()], -# ): -# msg = "create function job collection not implemented yet" -# raise NotImplementedError(msg) - - -# @function_job_collections_router.get( -# "/{function_job_collection_id:uuid}", -# response_model=FunctionJobCollection, -# responses={**_COMMON_FUNCTION_JOB_COLLECTION_ERROR_RESPONSES}, -# description="Get function job", -# ) -# async def get_function_job_collection( -# function_job_collection_id: FunctionJobCollectionID, -# webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], -# ): -# msg = "get function job collection not implemented yet" -# raise NotImplementedError(msg) - - -# @function_job_collections_router.delete( -# "/{function_job_collection_id:uuid}", -# response_model=FunctionJob, -# responses={**_COMMON_FUNCTION_JOB_COLLECTION_ERROR_RESPONSES}, -# description="Delete function job collection", -# ) -# async def delete_function_job_collection( -# function_job_collection_id: FunctionJobCollectionID, -# webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], -# ): -# msg = "delete function job collection not implemented yet" -# raise NotImplementedError(msg) - - -# @function_job_collections_router.get( -# "/{function_job_collection_id:uuid}/function_jobs", -# response_model=list[FunctionJob], -# responses={**_COMMON_FUNCTION_JOB_COLLECTION_ERROR_RESPONSES}, -# description="Get the function jobs in function job collection", -# ) -# async def function_job_collection_list_function_jobs( -# function_job_collection_id: FunctionJobCollectionID, -# webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], -# ): -# msg = "function job collection listing not implemented yet" -# raise NotImplementedError(msg) - - -# @function_job_collections_router.get( -# "/{function_job_collection_id:uuid}/status", -# response_model=FunctionJobCollectionStatus, -# responses={**_COMMON_FUNCTION_JOB_COLLECTION_ERROR_RESPONSES}, -# description="Get function job collection status", -# ) -# async def function_job_collection_status( -# function_job_collection_id: FunctionJobCollectionID, -# webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], -# ): -# msg = "function job collection status not implemented yet" -# raise NotImplementedError(msg) + ) + return FunctionJobCollectionStatus( + status=[job_status.status for job_status in job_statuses] + ) diff --git a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py index 80139346f25..c5a35499ef2 100644 --- a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py +++ b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py @@ -10,6 +10,8 @@ FunctionInputs, FunctionInputSchema, FunctionJob, + FunctionJobCollection, + FunctionJobCollectionID, FunctionJobID, FunctionOutputSchema, ) @@ -44,6 +46,9 @@ from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( delete_function_job as _delete_function_job, ) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + delete_function_job_collection as _delete_function_job_collection, +) from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( find_cached_function_job as _find_cached_function_job, ) @@ -56,9 +61,15 @@ from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( get_function_job as _get_function_job, ) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + get_function_job_collection as _get_function_job_collection, +) from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( get_function_output_schema as _get_function_output_schema, ) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + list_function_job_collections as _list_function_job_collections, +) from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( list_function_jobs as _list_function_jobs, ) @@ -74,6 +85,9 @@ from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( register_function_job as _register_function_job, ) +from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( + register_function_job_collection as _register_function_job_collection, +) from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( run_function as _run_function, ) @@ -341,6 +355,30 @@ async def find_cached_function_job( async def list_function_jobs(self) -> list[FunctionJob]: return await _list_function_jobs(self._client) + async def list_function_job_collections(self) -> list[FunctionJobCollection]: + return await _list_function_job_collections(self._client) + + async def get_function_job_collection( + self, *, function_job_collection_id: FunctionJobCollectionID + ) -> FunctionJobCollection: + return await _get_function_job_collection( + self._client, function_job_collection_id=function_job_collection_id + ) + + async def register_function_job_collection( + self, *, function_job_collection: FunctionJobCollection + ) -> FunctionJobCollection: + return await _register_function_job_collection( + self._client, function_job_collection=function_job_collection + ) + + async def delete_function_job_collection( + self, *, function_job_collection_id: FunctionJobCollectionID + ) -> None: + return await _delete_function_job_collection( + self._client, function_job_collection_id=function_job_collection_id + ) + def setup(app: FastAPI, rabbitmq_rmp_client: RabbitMQRPCClient): wb_api_rpc_client = WbApiRpcClient(_client=rabbitmq_rmp_client) diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py index 53b3aad7f2b..c59c444d384 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py @@ -10,6 +10,7 @@ FunctionInputSchema, FunctionJob, FunctionJobClassSpecificData, + FunctionJobCollection, FunctionJobDB, FunctionJobID, FunctionOutputSchema, @@ -292,6 +293,76 @@ async def find_cached_function_job( raise TypeError(msg) +@router.expose() +async def list_function_job_collections( + app: web.Application, +) -> list[FunctionJobCollection]: + assert app + returned_function_job_collections = ( + await _functions_repository.list_function_job_collections( + app=app, + ) + ) + return [ + FunctionJobCollection( + uid=function_job_collection.uuid, + title=function_job_collection.title, + description=function_job_collection.description, + job_ids=job_ids, + ) + for function_job_collection, job_ids in returned_function_job_collections + ] + + +@router.expose() +async def register_function_job_collection( + app: web.Application, *, function_job_collection: FunctionJobCollection +) -> FunctionJobCollection: + assert app + registered_function_job_collection, registered_job_ids = ( + await _functions_repository.register_function_job_collection( + app=app, + function_job_collection=function_job_collection, + ) + ) + return FunctionJobCollection( + uid=registered_function_job_collection.uuid, + title=registered_function_job_collection.title, + description=registered_function_job_collection.description, + job_ids=registered_job_ids, + ) + + +@router.expose() +async def get_function_job_collection( + app: web.Application, *, function_job_collection_id: FunctionJobID +) -> FunctionJobCollection: + assert app + returned_function_job_collection, job_ids = ( + await _functions_repository.get_function_job_collection( + app=app, + function_job_collection_id=function_job_collection_id, + ) + ) + return FunctionJobCollection( + uid=returned_function_job_collection.uuid, + title=returned_function_job_collection.title, + description=returned_function_job_collection.description, + job_ids=job_ids, + ) + + +@router.expose() +async def delete_function_job_collection( + app: web.Application, *, function_job_collection_id: FunctionJobID +) -> None: + assert app + await _functions_repository.delete_function_job_collection( + app=app, + function_job_collection_id=function_job_collection_id, + ) + + async def register_rpc_routes_on_startup(app: web.Application): rpc_server = get_rabbitmq_rpc_server(app) await rpc_server.register_router(router, WEBSERVER_RPC_NAMESPACE, app) diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py index ada1980f4f1..d2397851233 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py @@ -5,7 +5,16 @@ FunctionDB, FunctionID, FunctionInputs, + FunctionJobCollection, + FunctionJobCollectionDB, FunctionJobDB, + FunctionJobID, +) +from simcore_postgres_database.models.functions_models_db import ( + function_job_collections as function_job_collections_table, +) +from simcore_postgres_database.models.functions_models_db import ( + function_job_collections_to_function_jobs as function_job_collections_to_function_jobs_table, ) from simcore_postgres_database.models.functions_models_db import ( function_jobs as function_jobs_table, @@ -26,6 +35,9 @@ _FUNCTION_JOBS_TABLE_COLS = get_columns_from_db_model( function_jobs_table, FunctionJobDB ) +_FUNCTION_JOB_COLLECTIONS_TABLE_COLS = get_columns_from_db_model( + function_jobs_table, FunctionJobCollectionDB +) async def create_function( @@ -205,3 +217,122 @@ async def find_cached_function_job( return job return None + + +async def list_function_job_collections( + app: web.Application, + connection: AsyncConnection | None = None, +) -> list[tuple[FunctionJobCollectionDB, list[FunctionJobID]]]: + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream(function_job_collections_table.select().where()) + rows = await result.all() + if rows is None: + return [] + + collections = [] + for row in rows: + collection = FunctionJobCollection.model_validate(dict(row)) + job_result = await conn.stream( + function_job_collections_to_function_jobs_table.select().where( + function_job_collections_to_function_jobs_table.c.function_job_collection_uuid + == row["uuid"] + ) + ) + job_rows = await job_result.all() + job_ids = ( + [job_row["function_job_uuid"] for job_row in job_rows] + if job_rows + else [] + ) + collections.append((collection, job_ids)) + return collections + + +async def get_function_job_collection( + app: web.Application, + connection: AsyncConnection | None = None, + *, + function_job_collection_id: FunctionID, +) -> tuple[FunctionJobCollectionDB, list[FunctionJobID]]: + + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream( + function_job_collections_table.select().where( + function_job_collections_table.c.uuid == function_job_collection_id + ) + ) + row = await result.first() + + if row is None: + msg = f"No function job collection found with id {function_job_collection_id}." + raise web.HTTPNotFound(reason=msg) + + # Retrieve associated job ids from the join table + job_result = await conn.stream( + function_job_collections_to_function_jobs_table.select().where( + function_job_collections_to_function_jobs_table.c.function_job_collection_uuid + == row["uuid"] + ) + ) + job_rows = await job_result.all() + + job_ids = ( + [job_row["function_job_uuid"] for job_row in job_rows] if job_rows else [] + ) + + job_collection = FunctionJobCollectionDB.model_validate(dict(row)) + return job_collection, job_ids + + +async def register_function_job_collection( + app: web.Application, + connection: AsyncConnection | None = None, + *, + function_job_collection: FunctionJobCollection, +) -> tuple[FunctionJobCollectionDB, list[FunctionJobID]]: + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream( + function_job_collections_table.insert() + .values( + title=function_job_collection.title, + description=function_job_collection.description, + ) + .returning(*_FUNCTION_JOB_COLLECTIONS_TABLE_COLS) + ) + row = await result.first() + + if row is None: + msg = "No row was returned from the database after creating function job collection." + raise ValueError(msg) + + for job_id in function_job_collection.job_ids: + await conn.execute( + function_job_collections_to_function_jobs_table.insert().values( + function_job_collection_uuid=row["uuid"], + function_job_uuid=job_id, + ) + ) + + job_collection = FunctionJobCollectionDB.model_validate(dict(row)) + return job_collection, function_job_collection.job_ids + + +async def delete_function_job_collection( + app: web.Application, + connection: AsyncConnection | None = None, + *, + function_job_collection_id: FunctionID, +) -> None: + + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + await conn.execute( + function_job_collections_table.delete().where( + function_job_collections_table.c.uuid == function_job_collection_id + ) + ) + await conn.execute( + function_job_collections_to_function_jobs_table.delete().where( + function_job_collections_to_function_jobs_table.c.function_job_collection_uuid + == function_job_collection_id + ) + ) From ef3ce7bf48e2480d39910c0e5fb72c1178a7423f Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Tue, 29 Apr 2025 14:45:13 +0200 Subject: [PATCH 38/69] Working project function job collection --- .../functions_wb_schema.py | 2 + services/api-server/openapi.json | 59 +++++++++++++++++++ .../api/routes/functions_routes.py | 51 +++++++++++++++- .../functions/_functions_repository.py | 2 +- 4 files changed, 110 insertions(+), 4 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py b/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py index 53362663569..94b47183b37 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py @@ -43,6 +43,8 @@ class FunctionClass(str, Enum): FunctionOutputs: TypeAlias = dict[str, Any] | None +FunctionOutputsLogfile: TypeAlias = Any + class FunctionBase(BaseModel): function_class: FunctionClass diff --git a/services/api-server/openapi.json b/services/api-server/openapi.json index 8d3abb3289a..f85b6502196 100644 --- a/services/api-server/openapi.json +++ b/services/api-server/openapi.json @@ -6216,6 +6216,65 @@ } } }, + "/v0/function_jobs/{function_job_id}/outputs/logfile": { + "get": { + "tags": [ + "function_jobs" + ], + "summary": "Function Job Logfile", + "description": "Get function job outputs", + "operationId": "function_job_logfile", + "security": [ + { + "HTTPBasic": [] + } + ], + "parameters": [ + { + "name": "function_job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Job Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "title": "Response Function Job Logfile V0 Function Jobs Function Job Id Outputs Logfile Get" + } + } + } + }, + "404": { + "description": "Function job not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, "/v0/function_job_collections": { "get": { "tags": [ diff --git a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py index b8aeacc7c75..f8335552cfc 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py @@ -446,12 +446,13 @@ async def map_function( function_job.uid is not None for function_job in function_jobs ), "Function job uid should not be None" + function_job_collection_description = f"Function job collection of map of function {function_id} with {len(function_inputs_list)} inputs" return await register_function_job_collection( wb_api_rpc=wb_api_rpc, function_job_collection=FunctionJobCollection( - id=None, - title=f"Function job collection of function map {[function_job.uid for function_job in function_jobs]}", - description="", + uid=None, + title="Function job collection of function map", + description=function_job_collection_description, job_ids=[function_job.uid for function_job in function_jobs], # type: ignore ), ) @@ -574,3 +575,47 @@ async def function_job_collection_status( return FunctionJobCollectionStatus( status=[job_status.status for job_status in job_statuses] ) + + +# @function_job_router.get( +# "/{function_job_id:uuid}/outputs/logfile", +# response_model=FunctionOutputsLogfile, +# responses={**_COMMON_FUNCTION_JOB_ERROR_RESPONSES}, +# description="Get function job outputs", +# ) +# async def function_job_logfile( +# function_job_id: FunctionJobID, +# user_id: Annotated[PositiveInt, Depends(get_current_user_id)], +# wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +# director2_api: Annotated[DirectorV2Api, Depends(get_api_client(DirectorV2Api))], +# ): +# function, function_job = await get_function_from_functionjobid( +# wb_api_rpc=wb_api_rpc, function_job_id=function_job_id +# ) + +# if ( +# function.function_class == FunctionClass.project +# and function_job.function_class == FunctionClass.project +# ): +# job_outputs = await studies_jobs.get_study_job_output_logfile( +# study_id=function.project_id, +# job_id=function_job.project_job_id, # type: ignore +# user_id=user_id, +# director2_api=director2_api, +# ) + +# return job_outputs +# elif (function.function_class == FunctionClass.solver) and ( # noqa: RET505 +# function_job.function_class == FunctionClass.solver +# ): +# job_outputs_logfile = await solvers_jobs_getters.get_job_output_logfile( +# director2_api=director2_api, +# solver_key=function.solver_key, +# version=function.solver_version, +# job_id=function_job.solver_job_id, +# user_id=user_id, +# ) +# return job_outputs_logfile +# else: +# msg = f"Function type {function.function_class} not supported" +# raise TypeError(msg) diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py index d2397851233..f9d0cce1c71 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py @@ -36,7 +36,7 @@ function_jobs_table, FunctionJobDB ) _FUNCTION_JOB_COLLECTIONS_TABLE_COLS = get_columns_from_db_model( - function_jobs_table, FunctionJobCollectionDB + function_job_collections_table, FunctionJobCollectionDB ) From 3f5ea5cba55e7f74ae7aa2b2ff2f06803f4e80f9 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Tue, 29 Apr 2025 15:20:29 +0200 Subject: [PATCH 39/69] Add db migration for job collections --- ...b7f433b_fix_function_job_collections_db.py | 60 +++++++++++++++++++ .../functions/_functions_controller_rpc.py | 14 ++++- 2 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/0b64fb7f433b_fix_function_job_collections_db.py diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/0b64fb7f433b_fix_function_job_collections_db.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/0b64fb7f433b_fix_function_job_collections_db.py new file mode 100644 index 00000000000..2fb484396f8 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/0b64fb7f433b_fix_function_job_collections_db.py @@ -0,0 +1,60 @@ +"""Fix function job collections db + +Revision ID: 0b64fb7f433b +Revises: d94af8f28b25 +Create Date: 2025-04-29 11:12:23.529262+00:00 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "0b64fb7f433b" +down_revision = "d94af8f28b25" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "function_job_collections", sa.Column("title", sa.String(), nullable=True) + ) + op.add_column( + "function_job_collections", sa.Column("description", sa.String(), nullable=True) + ) + op.drop_column("function_job_collections", "name") + op.drop_index("idx_projects_last_change_date_desc", table_name="projects") + op.create_index( + "idx_projects_last_change_date_desc", + "projects", + ["last_change_date"], + unique=False, + postgresql_using="btree", + postgresql_ops={"last_change_date": "DESC"}, + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index( + "idx_projects_last_change_date_desc", + table_name="projects", + postgresql_using="btree", + postgresql_ops={"last_change_date": "DESC"}, + ) + op.create_index( + "idx_projects_last_change_date_desc", + "projects", + [sa.text("last_change_date DESC")], + unique=False, + ) + op.add_column( + "function_job_collections", + sa.Column("name", sa.VARCHAR(), autoincrement=False, nullable=True), + ) + op.drop_column("function_job_collections", "description") + op.drop_column("function_job_collections", "title") + # ### end Alembic commands ### diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py index c59c444d384..da4737553d5 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py @@ -288,6 +288,16 @@ async def find_cached_function_job( outputs=None, project_job_id=returned_function_job.class_specific_data["project_job_id"], ) + elif returned_function_job.function_class == FunctionClass.solver: # noqa: RET505 + return SolverFunctionJob( + uid=returned_function_job.uuid, + title=returned_function_job.title, + description="", + function_uid=returned_function_job.function_uuid, + inputs=returned_function_job.inputs, + outputs=None, + solver_job_id=returned_function_job.class_specific_data["solver_job_id"], + ) else: # noqa: RET505 msg = f"Unsupported function class: [{returned_function_job.function_class}]" raise TypeError(msg) @@ -338,7 +348,7 @@ async def get_function_job_collection( app: web.Application, *, function_job_collection_id: FunctionJobID ) -> FunctionJobCollection: assert app - returned_function_job_collection, job_ids = ( + returned_function_job_collection, returned_job_ids = ( await _functions_repository.get_function_job_collection( app=app, function_job_collection_id=function_job_collection_id, @@ -348,7 +358,7 @@ async def get_function_job_collection( uid=returned_function_job_collection.uuid, title=returned_function_job_collection.title, description=returned_function_job_collection.description, - job_ids=job_ids, + job_ids=returned_job_ids, ) From 5031f89ffb5e15b7e1ff1e4fc7adaf6d0ace51c7 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Tue, 29 Apr 2025 15:40:07 +0200 Subject: [PATCH 40/69] Adapt for changes in solver job api --- services/api-server/openapi.json | 59 ------------------- .../api/routes/functions_routes.py | 24 ++++---- 2 files changed, 14 insertions(+), 69 deletions(-) diff --git a/services/api-server/openapi.json b/services/api-server/openapi.json index f85b6502196..8d3abb3289a 100644 --- a/services/api-server/openapi.json +++ b/services/api-server/openapi.json @@ -6216,65 +6216,6 @@ } } }, - "/v0/function_jobs/{function_job_id}/outputs/logfile": { - "get": { - "tags": [ - "function_jobs" - ], - "summary": "Function Job Logfile", - "description": "Get function job outputs", - "operationId": "function_job_logfile", - "security": [ - { - "HTTPBasic": [] - } - ], - "parameters": [ - { - "name": "function_job_id", - "in": "path", - "required": true, - "schema": { - "type": "string", - "format": "uuid", - "title": "Function Job Id" - } - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "title": "Response Function Job Logfile V0 Function Jobs Function Job Id Outputs Logfile Get" - } - } - } - }, - "404": { - "description": "Function job not found", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorGet" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, "/v0/function_job_collections": { "get": { "tags": [ diff --git a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py index f8335552cfc..82bbf6ce9ff 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py @@ -23,18 +23,20 @@ ) from pydantic import PositiveInt from servicelib.fastapi.dependencies import get_reverse_url_mapper +from sqlalchemy.ext.asyncio import AsyncEngine +from ..._service_job import JobService +from ..._service_solvers import SolverService from ...models.schemas.errors import ErrorGet from ...models.schemas.jobs import ( JobInputs, ) -from ...services_http.catalog import CatalogApi from ...services_http.director_v2 import DirectorV2Api from ...services_http.storage import StorageApi from ...services_http.webserver import AuthSession from ...services_rpc.wb_api_server import WbApiRpcClient from ..dependencies.authentication import get_current_user_id, get_product_name -from ..dependencies.database import Engine, get_db_engine +from ..dependencies.database import get_db_asyncpg_engine from ..dependencies.services import get_api_client from ..dependencies.webserver_http import get_webserver_session from ..dependencies.webserver_rpc import ( @@ -119,7 +121,8 @@ async def run_function( function_inputs: FunctionInputs, user_id: Annotated[PositiveInt, Depends(get_current_user_id)], product_name: Annotated[str, Depends(get_product_name)], - catalog_client: Annotated[CatalogApi, Depends(get_api_client(CatalogApi))], + solver_service: Annotated[SolverService, Depends()], + job_service: Annotated[JobService, Depends()], ): to_run_function = await wb_api_rpc.get_function(function_id=function_id) @@ -173,14 +176,13 @@ async def run_function( solver_key=to_run_function.solver_key, version=to_run_function.solver_version, inputs=JobInputs(values=joined_inputs or {}), - webserver_api=webserver_api, - wb_api_rpc=wb_api_rpc, + solver_service=solver_service, + job_service=job_service, url_for=url_for, x_simcore_parent_project_uuid=None, x_simcore_parent_node_id=None, user_id=user_id, product_name=product_name, - catalog_client=catalog_client, ) await solvers_jobs.start_job( request=request, @@ -370,7 +372,7 @@ async def function_job_outputs( user_id: Annotated[PositiveInt, Depends(get_current_user_id)], storage_client: Annotated[StorageApi, Depends(get_api_client(StorageApi))], wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], - db_engine: Annotated[Engine, Depends(get_db_engine)], + async_pg_engine: Annotated[AsyncEngine, Depends(get_db_asyncpg_engine)], ): function, function_job = await get_function_from_functionjobid( wb_api_rpc=wb_api_rpc, function_job_id=function_job_id @@ -399,7 +401,7 @@ async def function_job_outputs( user_id=user_id, webserver_api=webserver_api, storage_client=storage_client, - db_engine=db_engine, + async_pg_engine=async_pg_engine, ) return job_outputs.results else: @@ -423,7 +425,8 @@ async def map_function( director2_api: Annotated[DirectorV2Api, Depends(get_api_client(DirectorV2Api))], user_id: Annotated[PositiveInt, Depends(get_current_user_id)], product_name: Annotated[str, Depends(get_product_name)], - catalog_client: Annotated[CatalogApi, Depends(get_api_client(CatalogApi))], + solver_service: Annotated[SolverService, Depends()], + job_service: Annotated[JobService, Depends()], ): function_jobs = [] function_jobs = [ @@ -437,7 +440,8 @@ async def map_function( url_for=url_for, director2_api=director2_api, request=request, - catalog_client=catalog_client, + solver_service=solver_service, + job_service=job_service, ) for function_inputs in function_inputs_list ] From 695780197608bb4862bea934dead15aca866ff63 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Tue, 29 Apr 2025 16:02:29 +0200 Subject: [PATCH 41/69] Db merge heads functions draft and master rebase --- ...3b85134_merge_742123f0933a_0b64fb7f433b.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/ecd7a3b85134_merge_742123f0933a_0b64fb7f433b.py diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/ecd7a3b85134_merge_742123f0933a_0b64fb7f433b.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/ecd7a3b85134_merge_742123f0933a_0b64fb7f433b.py new file mode 100644 index 00000000000..0118aaa4a77 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/ecd7a3b85134_merge_742123f0933a_0b64fb7f433b.py @@ -0,0 +1,21 @@ +"""merge 742123f0933a 0b64fb7f433b + +Revision ID: ecd7a3b85134 +Revises: 742123f0933a, 0b64fb7f433b +Create Date: 2025-04-29 13:40:28.311099+00:00 + +""" + +# revision identifiers, used by Alembic. +revision = "ecd7a3b85134" +down_revision = ("742123f0933a", "0b64fb7f433b") +branch_labels = None +depends_on = None + + +def upgrade(): + pass + + +def downgrade(): + pass From baa5bfbd43f2dfdd34b171d83e8006143abed6d0 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Wed, 30 Apr 2025 15:11:03 +0200 Subject: [PATCH 42/69] Add tests for functions api server --- .../functions_wb_schema.py | 2 +- services/api-server/openapi.json | 248 ++++--- .../api/routes/functions_routes.py | 132 ++-- .../test_api_routers_functions.py | 660 ++++++++++++++++++ 4 files changed, 895 insertions(+), 147 deletions(-) create mode 100644 services/api-server/tests/unit/api_functions/test_api_routers_functions.py diff --git a/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py b/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py index 94b47183b37..0961b75765b 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py @@ -147,7 +147,7 @@ class FunctionJobStatus(BaseModel): class FunctionJobCollection(BaseModel): """Model for a collection of function jobs""" - uid: FunctionJobCollectionID | None + uid: FunctionJobCollectionID | None = None title: str | None description: str | None job_ids: list[FunctionJobID] diff --git a/services/api-server/openapi.json b/services/api-server/openapi.json index 8d3abb3289a..a2a22f8d5e4 100644 --- a/services/api-server/openapi.json +++ b/services/api-server/openapi.json @@ -5276,25 +5276,6 @@ } } }, - "/v0/functions/ping": { - "post": { - "tags": [ - "functions" - ], - "summary": "Ping", - "operationId": "ping", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": {} - } - } - } - } - } - }, "/v0/functions": { "get": { "tags": [ @@ -5404,6 +5385,16 @@ } } }, + "404": { + "description": "Function not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorGet" + } + } + } + }, "422": { "description": "Validation Error", "content": { @@ -5513,28 +5504,7 @@ "description": "Successful Response", "content": { "application/json": { - "schema": { - "oneOf": [ - { - "$ref": "#/components/schemas/ProjectFunction" - }, - { - "$ref": "#/components/schemas/PythonCodeFunction" - }, - { - "$ref": "#/components/schemas/SolverFunction" - } - ], - "discriminator": { - "propertyName": "function_class", - "mapping": { - "project": "#/components/schemas/ProjectFunction", - "python_code": "#/components/schemas/PythonCodeFunction", - "solver": "#/components/schemas/SolverFunction" - } - }, - "title": "Response Delete Function V0 Functions Function Id Delete" - } + "schema": {} } } }, @@ -5561,19 +5531,14 @@ } } }, - "/v0/functions/{function_id}:run": { - "post": { + "/v0/functions/{function_id}/input_schema": { + "get": { "tags": [ "functions" ], - "summary": "Run Function", - "description": "Run function", - "operationId": "run_function", - "security": [ - { - "HTTPBasic": [] - } - ], + "summary": "Get Function Inputschema", + "description": "Get function input schema", + "operationId": "get_function_inputschema", "parameters": [ { "name": "function_id", @@ -5586,50 +5551,13 @@ } } ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "anyOf": [ - { - "type": "object" - }, - { - "type": "null" - } - ], - "title": "Function Inputs" - } - } - } - }, "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { - "oneOf": [ - { - "$ref": "#/components/schemas/ProjectFunctionJob" - }, - { - "$ref": "#/components/schemas/PythonCodeFunctionJob" - }, - { - "$ref": "#/components/schemas/SolverFunctionJob" - } - ], - "discriminator": { - "propertyName": "function_class", - "mapping": { - "project": "#/components/schemas/ProjectFunctionJob", - "python_code": "#/components/schemas/PythonCodeFunctionJob", - "solver": "#/components/schemas/SolverFunctionJob" - } - }, - "title": "Response Run Function V0 Functions Function Id Run Post" + "$ref": "#/components/schemas/FunctionInputSchema" } } } @@ -5657,14 +5585,14 @@ } } }, - "/v0/functions/{function_id}/input_schema": { + "/v0/functions/{function_id}/output_schema": { "get": { "tags": [ "functions" ], - "summary": "Get Function Input Schema", - "description": "Get function", - "operationId": "get_function_input_schema", + "summary": "Get Function Outputschema", + "description": "Get function input schema", + "operationId": "get_function_outputschema", "parameters": [ { "name": "function_id", @@ -5711,14 +5639,98 @@ } } }, - "/v0/functions/{function_id}/output_schema": { - "get": { + "/v0/functions/{function_id}:validate_inputs": { + "post": { "tags": [ "functions" ], - "summary": "Get Function Output Schema", - "description": "Get function", - "operationId": "get_function_output_schema", + "summary": "Validate Function Inputs", + "description": "Validate inputs against the function's input schema", + "operationId": "validate_function_inputs", + "parameters": [ + { + "name": "function_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid", + "title": "Function Id" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Inputs" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "array", + "prefixItems": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "minItems": 2, + "maxItems": 2, + "title": "Response Validate Function Inputs V0 Functions Function Id Validate Inputs Post" + } + } + } + }, + "400": { + "description": "Invalid inputs" + }, + "404": { + "description": "Function not found" + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/functions/{function_id}:run": { + "post": { + "tags": [ + "functions" + ], + "summary": "Run Function", + "description": "Run function", + "operationId": "run_function", + "security": [ + { + "HTTPBasic": [] + } + ], "parameters": [ { "name": "function_id", @@ -5731,13 +5743,50 @@ } } ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Function Inputs" + } + } + } + }, "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/FunctionOutputSchema" + "oneOf": [ + { + "$ref": "#/components/schemas/ProjectFunctionJob" + }, + { + "$ref": "#/components/schemas/PythonCodeFunctionJob" + }, + { + "$ref": "#/components/schemas/SolverFunctionJob" + } + ], + "discriminator": { + "propertyName": "function_class", + "mapping": { + "project": "#/components/schemas/ProjectFunctionJob", + "python_code": "#/components/schemas/PythonCodeFunctionJob", + "solver": "#/components/schemas/SolverFunctionJob" + } + }, + "title": "Response Run Function V0 Functions Function Id Run Post" } } } @@ -7564,7 +7613,6 @@ }, "type": "object", "required": [ - "uid", "title", "description", "job_ids" diff --git a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py index 82bbf6ce9ff..d14739cdd9d 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py @@ -2,7 +2,9 @@ from collections.abc import Callable from typing import Annotated, Final +import jsonschema from fastapi import APIRouter, Depends, Request, status +from jsonschema import ValidationError from models_library.api_schemas_webserver.functions_wb_schema import ( Function, FunctionClass, @@ -17,7 +19,6 @@ FunctionJobID, FunctionJobStatus, FunctionOutputs, - FunctionOutputSchema, ProjectFunctionJob, SolverFunctionJob, ) @@ -56,21 +57,12 @@ } -@function_router.post("/ping") -async def ping( - wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], -): - return await wb_api_rpc.ping() - - -@function_router.get("", response_model=list[Function], description="List functions") -async def list_functions( - wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], -): - return await wb_api_rpc.list_functions() - - -@function_router.post("", response_model=Function, description="Create function") +@function_router.post( + "", + response_model=Function, + responses={**_COMMON_FUNCTION_ERROR_RESPONSES}, + description="Create function", +) async def register_function( wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], function: Function, @@ -91,6 +83,13 @@ async def get_function( return await wb_api_rpc.get_function(function_id=function_id) +@function_router.get("", response_model=list[Function], description="List functions") +async def list_functions( + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + return await wb_api_rpc.list_functions() + + def join_inputs( default_inputs: FunctionInputs | None, function_inputs: FunctionInputs | None, @@ -105,13 +104,66 @@ def join_inputs( return {**default_inputs, **function_inputs} +@function_router.get( + "/{function_id:uuid}/input_schema", + response_model=FunctionInputSchema, + responses={**_COMMON_FUNCTION_ERROR_RESPONSES}, + description="Get function input schema", +) +async def get_function_inputschema( + function_id: FunctionID, + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + function = await wb_api_rpc.get_function(function_id=function_id) + return function.input_schema + + +@function_router.get( + "/{function_id:uuid}/output_schema", + response_model=FunctionInputSchema, + responses={**_COMMON_FUNCTION_ERROR_RESPONSES}, + description="Get function input schema", +) +async def get_function_outputschema( + function_id: FunctionID, + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + function = await wb_api_rpc.get_function(function_id=function_id) + return function.output_schema + + +@function_router.post( + "/{function_id:uuid}:validate_inputs", + response_model=tuple[bool, str], + responses={ + status.HTTP_400_BAD_REQUEST: {"description": "Invalid inputs"}, + status.HTTP_404_NOT_FOUND: {"description": "Function not found"}, + }, + description="Validate inputs against the function's input schema", +) +async def validate_function_inputs( + function_id: FunctionID, + inputs: FunctionInputs, + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +): + function = await wb_api_rpc.get_function(function_id=function_id) + + if function.input_schema is None: + return True, "No input schema defined for this function" + try: + jsonschema.validate(instance=inputs, schema=function.input_schema.schema_dict) # type: ignore + except ValidationError as err: + return False, str(err) + return True, "Inputs are valid" + + @function_router.post( "/{function_id:uuid}:run", response_model=FunctionJob, responses={**_COMMON_FUNCTION_ERROR_RESPONSES}, description="Run function", ) -async def run_function( +async def run_function( # noqa: PLR0913 request: Request, wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], @@ -134,6 +186,18 @@ async def run_function( function_inputs, ) + if to_run_function.input_schema is not None: + is_valid, validation_str = await validate_function_inputs( + function_id=to_run_function.uid, + inputs=joined_inputs, + wb_api_rpc=wb_api_rpc, + ) + if not is_valid: + msg = ( + f"Function {to_run_function.uid} inputs are not valid: {validation_str}" + ) + raise ValidationError(msg) + if cached_function_job := await wb_api_rpc.find_cached_function_job( function_id=to_run_function.uid, inputs=joined_inputs, @@ -211,7 +275,7 @@ async def run_function( @function_router.delete( "/{function_id:uuid}", - response_model=Function, + response_model=None, responses={**_COMMON_FUNCTION_ERROR_RESPONSES}, description="Delete function", ) @@ -222,32 +286,6 @@ async def delete_function( return await wb_api_rpc.delete_function(function_id=function_id) -@function_router.get( - "/{function_id:uuid}/input_schema", - response_model=FunctionInputSchema, - responses={**_COMMON_FUNCTION_ERROR_RESPONSES}, - description="Get function", -) -async def get_function_input_schema( - wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], - function_id: FunctionID, -): - return await wb_api_rpc.get_function_input_schema(function_id=function_id) - - -@function_router.get( - "/{function_id:uuid}/output_schema", - response_model=FunctionOutputSchema, - responses={**_COMMON_FUNCTION_ERROR_RESPONSES}, - description="Get function", -) -async def get_function_output_schema( - wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], - function_id: FunctionID, -): - return await wb_api_rpc.get_function_output_schema(function_id=function_id) - - _COMMON_FUNCTION_JOB_ERROR_RESPONSES: Final[dict] = { status.HTTP_404_NOT_FOUND: { "description": "Function job not found", @@ -415,7 +453,7 @@ async def function_job_outputs( responses={**_COMMON_FUNCTION_ERROR_RESPONSES}, description="Map function over input parameters", ) -async def map_function( +async def map_function( # noqa: PLR0913 function_id: FunctionID, function_inputs_list: FunctionInputsList, request: Request, @@ -581,6 +619,8 @@ async def function_job_collection_status( ) +# ruff: noqa: ERA001 + # @function_job_router.get( # "/{function_job_id:uuid}/outputs/logfile", # response_model=FunctionOutputsLogfile, @@ -609,7 +649,7 @@ async def function_job_collection_status( # ) # return job_outputs -# elif (function.function_class == FunctionClass.solver) and ( # noqa: RET505 +# elif (function.function_class == FunctionClass.solver) and ( # function_job.function_class == FunctionClass.solver # ): # job_outputs_logfile = await solvers_jobs_getters.get_job_output_logfile( diff --git a/services/api-server/tests/unit/api_functions/test_api_routers_functions.py b/services/api-server/tests/unit/api_functions/test_api_routers_functions.py new file mode 100644 index 00000000000..8223e7e5813 --- /dev/null +++ b/services/api-server/tests/unit/api_functions/test_api_routers_functions.py @@ -0,0 +1,660 @@ +from unittest.mock import MagicMock +from uuid import uuid4 + +import pytest +from fastapi import FastAPI, HTTPException +from fastapi.testclient import TestClient +from models_library.api_schemas_webserver.functions_wb_schema import ( + Function, + FunctionJob, + FunctionJobCollection, +) +from pydantic import TypeAdapter +from simcore_service_api_server.api.routes.functions_routes import ( + function_job_collections_router, + function_job_router, + function_router, + get_current_user_id, + get_wb_api_rpc_client, +) +from sqlalchemy.ext.asyncio import AsyncEngine + + +@pytest.fixture(name="api_app") +def _api_app() -> FastAPI: + fastapi_app = FastAPI() + fastapi_app.include_router(function_router, prefix="/functions") + fastapi_app.include_router(function_job_router, prefix="/function_jobs") + fastapi_app.include_router( + function_job_collections_router, prefix="/function_job_collections" + ) + + # Mock authentication dependency + async def mock_auth_dependency() -> int: + # Mock a valid user ID + return 100 + + fastapi_app.dependency_overrides[get_current_user_id] = mock_auth_dependency + + fake_wb_api_rpc = FakeWbApiRpc() + + async def fake_get_wb_api_rpc_client() -> FakeWbApiRpc: + return fake_wb_api_rpc + + fastapi_app.dependency_overrides[get_wb_api_rpc_client] = fake_get_wb_api_rpc_client + + mock_engine = MagicMock(spec=AsyncEngine) + mock_engine.pool = MagicMock() + mock_engine.pool.checkedin = MagicMock(return_value=[]) + fastapi_app.state.engine = mock_engine + + return fastapi_app + + +class FakeWbApiRpc: + def __init__(self) -> None: + self._functions = {} + self._function_jobs = {} + self._function_job_collections = {} + + async def register_function(self, function: Function) -> Function: + # Mimic returning the same function that was passed and store it for later retrieval + function.uid = uuid4() + self._functions[function.uid] = TypeAdapter(Function).validate_python( + { + "uid": str(function.uid), + "title": function.title, + "function_class": function.function_class, + "project_id": getattr(function, "project_id", None), + "description": function.description, + "input_schema": function.input_schema, + "output_schema": function.output_schema, + "default_inputs": None, + } + ) + return self._functions[function.uid] + + async def get_function(self, function_id: str) -> dict: + # Mimic retrieval of a function based on function_id and raise 404 if not found + if function_id not in self._functions: + raise HTTPException(status_code=404, detail="Function not found") + return self._functions[function_id] + + async def run_function(self, function_id: str, inputs: dict) -> dict: + # Mimic running a function and returning a success status + if function_id not in self._functions: + raise HTTPException( + status_code=404, + detail=f"Function {function_id} not found in {self._functions}", + ) + return {"status": "success", "function_id": function_id, "inputs": inputs} + + async def list_functions(self) -> list: + # Mimic listing all functions + return list(self._functions.values()) + + async def delete_function(self, function_id: str) -> None: + # Mimic deleting a function + if function_id in self._functions: + del self._functions[function_id] + else: + raise HTTPException(status_code=404, detail="Function not found") + + async def register_function_job(self, function_job: FunctionJob) -> FunctionJob: + # Mimic registering a function job + function_job.uid = uuid4() + self._function_jobs[function_job.uid] = TypeAdapter( + FunctionJob + ).validate_python( + { + "uid": str(function_job.uid), + "function_uid": function_job.function_uid, + "title": function_job.title, + "description": function_job.description, + "project_job_id": getattr(function_job, "project_job_id", None), + "inputs": function_job.inputs, + "outputs": function_job.outputs, + "function_class": function_job.function_class, + } + ) + return self._function_jobs[function_job.uid] + + async def get_function_job(self, function_job_id: str) -> dict: + # Mimic retrieval of a function job based on function_job_id and raise 404 if not found + if function_job_id not in self._function_jobs: + raise HTTPException(status_code=404, detail="Function job not found") + return self._function_jobs[function_job_id] + + async def list_function_jobs(self) -> list: + # Mimic listing all function jobs + return list(self._function_jobs.values()) + + async def delete_function_job(self, function_job_id: str) -> None: + # Mimic deleting a function job + if function_job_id in self._function_jobs: + del self._function_jobs[function_job_id] + else: + raise HTTPException(status_code=404, detail="Function job not found") + + async def register_function_job_collection( + self, function_job_collection: FunctionJobCollection + ) -> FunctionJobCollection: + # Mimic registering a function job collection + function_job_collection.uid = uuid4() + self._function_job_collections[function_job_collection.uid] = TypeAdapter( + FunctionJobCollection + ).validate_python( + { + "uid": str(function_job_collection.uid), + "title": function_job_collection.title, + "description": function_job_collection.description, + "job_ids": function_job_collection.job_ids, + } + ) + return self._function_job_collections[function_job_collection.uid] + + async def get_function_job_collection( + self, function_job_collection_id: str + ) -> dict: + # Mimic retrieval of a function job collection based on collection_id and raise 404 if not found + if function_job_collection_id not in self._function_job_collections: + raise HTTPException( + status_code=404, detail="Function job collection not found" + ) + return self._function_job_collections[function_job_collection_id] + + async def list_function_job_collections(self) -> list: + # Mimic listing all function job collections + return list(self._function_job_collections.values()) + + async def delete_function_job_collection( + self, function_job_collection_id: str + ) -> None: + # Mimic deleting a function job collection + if function_job_collection_id in self._function_job_collections: + del self._function_job_collections[function_job_collection_id] + else: + raise HTTPException( + status_code=404, detail="Function job collection not found" + ) + + +def test_register_function(api_app) -> None: + client = TestClient(api_app) + sample_function = { + "title": "test_function", + "function_class": "project", + "project_id": str(uuid4()), + "description": "A test function", + "input_schema": {"schema_dict": {}}, + "output_schema": {"schema_dict": {}}, + } + response = client.post("/functions", json=sample_function) + assert response.status_code == 200 + data = response.json() + assert data["uid"] is not None + assert data["function_class"] == sample_function["function_class"] + assert data["project_id"] == sample_function["project_id"] + assert data["input_schema"] == sample_function["input_schema"] + assert data["output_schema"] == sample_function["output_schema"] + assert data["title"] == sample_function["title"] + assert data["description"] == sample_function["description"] + + +def test_register_function_invalid(api_app: FastAPI) -> None: + client = TestClient(api_app) + invalid_function = { + "title": "test_function", + "function_class": "invalid_class", # Invalid class + "project_id": str(uuid4()), + } + response = client.post("/functions", json=invalid_function) + assert response.status_code == 422 # Unprocessable Entity + assert ( + "Input tag 'invalid_class' found using 'function_class' does no" + in response.json()["detail"][0]["msg"] + ) + + +def test_get_function(api_app: FastAPI) -> None: + client = TestClient(api_app) + project_id = str(uuid4()) + # First, register a sample function so that it exists + sample_function = { + "title": "example_function", + "function_class": "project", + "project_id": project_id, + "description": "An example function", + "input_schema": {"schema_dict": {}}, + "output_schema": {"schema_dict": {}}, + } + post_response = client.post("/functions", json=sample_function) + assert post_response.status_code == 200 + data = post_response.json() + function_id = data["uid"] + + expected_function = { + "uid": function_id, + "title": "example_function", + "description": "An example function", + "function_class": "project", + "project_id": project_id, + "input_schema": {"schema_dict": {}}, + "output_schema": {"schema_dict": {}}, + "default_inputs": None, + } + response = client.get(f"/functions/{function_id}") + assert response.status_code == 200 + data = response.json() + # Exclude the 'project_id' field from both expected and actual results before comparing + assert data == expected_function + + +def test_get_function_not_found(api_app: FastAPI) -> None: + client = TestClient(api_app) + non_existent_function_id = str(uuid4()) + response = client.get(f"/functions/{non_existent_function_id}") + assert response.status_code == 404 + assert response.json() == {"detail": "Function not found"} + + +def test_list_functions(api_app: FastAPI) -> None: + client = TestClient(api_app) + # Register a sample function + sample_function = { + "title": "example_function", + "function_class": "project", + "project_id": str(uuid4()), + "description": "An example function", + "input_schema": {"schema_dict": {}}, + "output_schema": {"schema_dict": {}}, + } + post_response = client.post("/functions", json=sample_function) + assert post_response.status_code == 200 + + # List functions + response = client.get("/functions") + assert response.status_code == 200 + data = response.json() + assert len(data) > 0 + assert data[0]["title"] == sample_function["title"] + + +def test_get_function_input_schema(api_app: FastAPI) -> None: + client = TestClient(api_app) + project_id = str(uuid4()) + # Register a sample function + sample_function = { + "title": "example_function", + "function_class": "project", + "project_id": project_id, + "description": "An example function", + "input_schema": { + "schema_dict": { + "type": "object", + "properties": {"input1": {"type": "integer"}}, + } + }, + "output_schema": {"schema_dict": {}}, + } + post_response = client.post("/functions", json=sample_function) + assert post_response.status_code == 200 + data = post_response.json() + function_id = data["uid"] + + # Get the input schema + # assert f"/functions/{function_id}/input-schema" is None + response = client.get(f"/functions/{function_id}/input_schema") + assert response.status_code == 200 + data = response.json() + assert data["schema_dict"] == sample_function["input_schema"]["schema_dict"] + + +def test_get_function_output_schema(api_app: FastAPI) -> None: + client = TestClient(api_app) + project_id = str(uuid4()) + # Register a sample function + sample_function = { + "title": "example_function", + "function_class": "project", + "project_id": project_id, + "description": "An example function", + "input_schema": {"schema_dict": {}}, + "output_schema": { + "schema_dict": { + "type": "object", + "properties": {"output1": {"type": "string"}}, + } + }, + } + post_response = client.post("/functions", json=sample_function) + assert post_response.status_code == 200 + data = post_response.json() + function_id = data["uid"] + + # Get the output schema + response = client.get(f"/functions/{function_id}/output_schema") + assert response.status_code == 200 + data = response.json() + assert data["schema_dict"] == sample_function["output_schema"]["schema_dict"] + + +def test_validate_function_inputs(api_app: FastAPI) -> None: + client = TestClient(api_app) + project_id = str(uuid4()) + # Register a sample function + sample_function = { + "title": "example_function", + "function_class": "project", + "project_id": project_id, + "description": "An example function", + "input_schema": { + "schema_dict": { + "type": "object", + "properties": {"input1": {"type": "integer"}}, + } + }, + "output_schema": {"schema_dict": {}}, + } + post_response = client.post("/functions", json=sample_function) + assert post_response.status_code == 200 + data = post_response.json() + function_id = data["uid"] + + # Validate inputs + validate_payload = {"input1": 10} + response = client.post( + f"/functions/{function_id}:validate_inputs", json=validate_payload + ) + assert response.status_code == 200 + data = response.json() + assert data == [True, "Inputs are valid"] + + +def test_delete_function(api_app: FastAPI) -> None: + client = TestClient(api_app) + project_id = str(uuid4()) + # Register a sample function + sample_function = { + "title": "example_function", + "function_class": "project", + "project_id": project_id, + "description": "An example function", + "input_schema": {"schema_dict": {}}, + "output_schema": {"schema_dict": {}}, + } + post_response = client.post("/functions", json=sample_function) + assert post_response.status_code == 200 + data = post_response.json() + function_id = data["uid"] + + # Delete the function + response = client.delete(f"/functions/{function_id}") + assert response.status_code == 200 + + +def test_register_function_job(api_app: FastAPI) -> None: + """Test the register_function_job endpoint.""" + + client = TestClient(api_app) + mock_function_job = { + "function_uid": str(uuid4()), + "title": "Test Function Job", + "description": "A test function job", + "inputs": {"key": "value"}, + "outputs": None, + "project_job_id": str(uuid4()), + "function_class": "project", + } + + # Act + response = client.post("/function_jobs", json=mock_function_job) + + # Assert + assert response.status_code == 200 + response_data = response.json() + assert response_data["uid"] is not None + response_data.pop("uid", None) # Remove the uid field + assert response_data == mock_function_job + + +def test_get_function_job(api_app: FastAPI) -> None: + """Test the get_function_job endpoint.""" + + client = TestClient(api_app) + mock_function_job = { + "function_uid": str(uuid4()), + "title": "Test Function Job", + "description": "A test function job", + "inputs": {"key": "value"}, + "outputs": None, + "project_job_id": str(uuid4()), + "function_class": "project", + } + + # First, register a function job + post_response = client.post("/function_jobs", json=mock_function_job) + assert post_response.status_code == 200 + data = post_response.json() + function_job_id = data["uid"] + + # Now, get the function job + response = client.get(f"/function_jobs/{function_job_id}") + assert response.status_code == 200 + data = response.json() + assert data["uid"] == function_job_id + assert data["title"] == mock_function_job["title"] + assert data["description"] == mock_function_job["description"] + assert data["inputs"] == mock_function_job["inputs"] + assert data["outputs"] == mock_function_job["outputs"] + + +def test_list_function_jobs(api_app: FastAPI) -> None: + """Test the list_function_jobs endpoint.""" + + client = TestClient(api_app) + mock_function_job = { + "function_uid": str(uuid4()), + "title": "Test Function Job", + "description": "A test function job", + "inputs": {"key": "value"}, + "outputs": None, + "project_job_id": str(uuid4()), + "function_class": "project", + } + + # First, register a function job + post_response = client.post("/function_jobs", json=mock_function_job) + assert post_response.status_code == 200 + + # Now, list function jobs + response = client.get("/function_jobs") + assert response.status_code == 200 + data = response.json() + assert len(data) > 0 + assert data[0]["title"] == mock_function_job["title"] + + +def test_delete_function_job(api_app: FastAPI) -> None: + """Test the delete_function_job endpoint.""" + + client = TestClient(api_app) + mock_function_job = { + "function_uid": str(uuid4()), + "title": "Test Function Job", + "description": "A test function job", + "inputs": {"key": "value"}, + "outputs": None, + "project_job_id": str(uuid4()), + "function_class": "project", + } + + # First, register a function job + post_response = client.post("/function_jobs", json=mock_function_job) + assert post_response.status_code == 200 + data = post_response.json() + function_job_id = data["uid"] + + # Now, delete the function job + response = client.delete(f"/function_jobs/{function_job_id}") + assert response.status_code == 200 + + +def test_register_function_job_collection(api_app: FastAPI) -> None: + # Arrange + client = TestClient(api_app) + + mock_function_job_collection = { + "title": "Test Collection", + "description": "A test function job collection", + "job_ids": [str(uuid4()), str(uuid4())], + } + + # Act + response = client.post( + "/function_job_collections", json=mock_function_job_collection + ) + + # Assert + assert response.status_code == 200 + response_data = response.json() + assert response_data["uid"] is not None + response_data.pop("uid", None) # Remove the uid field + assert response_data == mock_function_job_collection + + +def test_get_function_job_collection(api_app: FastAPI) -> None: + # Arrange + client = TestClient(api_app) + mock_function_job_collection = { + "title": "Test Collection", + "description": "A test function job collection", + "job_ids": [str(uuid4()), str(uuid4())], + } + + # First, register a function job collection + post_response = client.post( + "/function_job_collections", json=mock_function_job_collection + ) + assert post_response.status_code == 200 + data = post_response.json() + collection_id = data["uid"] + + # Act + response = client.get(f"/function_job_collections/{collection_id}") + + # Assert + assert response.status_code == 200 + data = response.json() + assert data["uid"] == collection_id + assert data["title"] == mock_function_job_collection["title"] + assert data["description"] == mock_function_job_collection["description"] + assert data["job_ids"] == mock_function_job_collection["job_ids"] + + +# def test_run_function_project_class(api_app: FastAPI) -> None: +# client = TestClient(api_app) +# project_id = str(uuid4()) +# # Register a sample function with "project" class +# sample_function = { +# "title": "example_function", +# "function_class": "project", +# "project_id": project_id, +# "description": "An example function", +# "input_schema": {"schema_dict": {"type": "object", "properties": {"input1": {"type": "integer"}}}}, +# "output_schema": {"schema_dict": {}}, +# "default_inputs": {"input1": 5}, +# } +# post_response = client.post("/functions", json=sample_function) +# assert post_response.status_code == 200 +# data = post_response.json() +# function_id = data["uid"] + +# # Run the function +# run_payload = {"input1": 10} +# response = client.post(f"/functions/{function_id}:run", json=run_payload) +# assert response.status_code == 200 +# data = response.json() +# assert data["function_uid"] == function_id +# assert data["inputs"] == {"input1": 10} + +# def test_run_function_solver_class(api_app: FastAPI) -> None: +# client = TestClient(api_app) +# # Register a sample function with "solver" class +# sample_function = { +# "title": "solver_function", +# "function_class": "solver", +# "solver_key": "example_solver", +# "solver_version": "1.0.0", +# "description": "A solver function", +# "input_schema": {"schema_dict": {"type": "object", "properties": {"input1": {"type": "integer"}}}}, +# "output_schema": {"schema_dict": {}}, +# "default_inputs": {"input1": 5}, +# } +# post_response = client.post("/functions", json=sample_function) +# assert post_response.status_code == 200 +# data = post_response.json() +# function_id = data["uid"] + +# # Run the function +# run_payload = {"input1": 15} +# response = client.post(f"/functions/{function_id}:run", json=run_payload) +# assert response.status_code == 200 +# data = response.json() +# assert data["function_uid"] == function_id +# assert data["inputs"] == {"input1": 15} + +# def test_run_function_invalid_inputs(api_app: FastAPI) -> None: +# client = TestClient(api_app) +# # Register a sample function with input schema +# sample_function = { +# "title": "example_function", +# "function_class": "project", +# "project_id": str(uuid4()), +# "description": "An example function", +# "input_schema": {"schema_dict": {"type": "object", "properties": {"input1": {"type": "integer"}}}}, +# "output_schema": {"schema_dict": {}}, +# } +# post_response = client.post("/functions", json=sample_function) +# assert post_response.status_code == 200 +# data = post_response.json() +# function_id = data["uid"] + +# # Run the function with invalid inputs +# run_payload = {"input1": "invalid_value"} +# response = client.post(f"/functions/{function_id}:run", json=run_payload) +# assert response.status_code == 400 +# assert "inputs are not valid" in response.json()["detail"] + +# def test_run_function_not_found(api_app: FastAPI) -> None: +# client = TestClient(api_app) +# non_existent_function_id = str(uuid4()) +# run_payload = {"input1": 10} +# response = client.post(f"/functions/{non_existent_function_id}:run", json=run_payload) +# assert response.status_code == 404 +# assert response.json() == {"detail": "Function not found"} + +# def test_run_function_cached_job(api_app: FastAPI) -> None: +# client = TestClient(api_app) +# # Register a sample function +# sample_function = { +# "title": "example_function", +# "function_class": "project", +# "project_id": str(uuid4()), +# "description": "An example function", +# "input_schema": {"schema_dict": {"type": "object", "properties": {"input1": {"type": "integer"}}}}, +# "output_schema": {"schema_dict": {}}, +# "default_inputs": {"input1": 5}, +# } +# post_response = client.post("/functions", json=sample_function) +# assert post_response.status_code == 200 +# data = post_response.json() +# function_id = data["uid"] + +# # Mimic a cached job +# run_payload = {"input1": 10} +# response = client.post(f"/functions/{function_id}:run", json=run_payload) +# assert response.status_code == 200 +# data = response.json() +# assert data["function_uid"] == function_id +# assert data["inputs"] == {"input1": 10} From c4aad007eff7fdbe7f941ea023d5738cc96cc292 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Wed, 30 Apr 2025 20:06:41 +0200 Subject: [PATCH 43/69] Add function rpc tests --- .../functions/_functions_controller_rpc.py | 21 +- .../functions/_functions_repository.py | 17 +- .../functions/_repo.py | 2 - .../functions/_service.py | 21 - .../05/test_functions_controller_rpc.py | 449 ++++++++++++++++++ 5 files changed, 481 insertions(+), 29 deletions(-) delete mode 100644 services/web/server/src/simcore_service_webserver/functions/_repo.py delete mode 100644 services/web/server/src/simcore_service_webserver/functions/_service.py create mode 100644 services/web/server/tests/unit/with_dbs/05/test_functions_controller_rpc.py diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py index da4737553d5..f8e39796249 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py @@ -122,7 +122,7 @@ def _decode_functionjob( description="", function_uid=functionjob_db.function_uuid, inputs=functionjob_db.inputs, - outputs=None, + outputs=functionjob_db.outputs, project_job_id=functionjob_db.class_specific_data["project_job_id"], ) elif functionjob_db.function_class == FunctionClass.solver: # noqa: RET505 @@ -132,7 +132,7 @@ def _decode_functionjob( description="", function_uid=functionjob_db.function_uuid, inputs=functionjob_db.inputs, - outputs=None, + outputs=functionjob_db.outputs, solver_job_id=functionjob_db.class_specific_data["solver_job_id"], ) else: @@ -148,7 +148,7 @@ def _encode_functionjob( title=functionjob.title, function_uuid=functionjob.function_uid, inputs=functionjob.inputs, - outputs=None, + outputs=functionjob.outputs, class_specific_data=FunctionJobClassSpecificData( { "project_job_id": str(functionjob.project_job_id), @@ -161,7 +161,7 @@ def _encode_functionjob( title=functionjob.title, function_uuid=functionjob.function_uid, inputs=functionjob.inputs, - outputs=None, + outputs=functionjob.outputs, class_specific_data=FunctionJobClassSpecificData( { "solver_job_id": str(functionjob.solver_job_id), @@ -267,6 +267,17 @@ async def register_function_job( return _decode_functionjob(created_function_job_db) +@router.expose() +async def delete_function_job( + app: web.Application, *, function_job_id: FunctionJobID +) -> None: + assert app + await _functions_repository.delete_function_job( + app=app, + function_job_id=function_job_id, + ) + + @router.expose() async def find_cached_function_job( app: web.Application, *, function_id: FunctionID, inputs: FunctionInputs @@ -298,7 +309,7 @@ async def find_cached_function_job( outputs=None, solver_job_id=returned_function_job.class_specific_data["solver_job_id"], ) - else: # noqa: RET505 + else: msg = f"Unsupported function class: [{returned_function_job.function_class}]" raise TypeError(msg) diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py index f9d0cce1c71..6e3fa98b94d 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py @@ -121,7 +121,7 @@ async def delete_function( async with transaction_context(get_asyncpg_engine(app), connection) as conn: await conn.execute( - functions_table.delete().where(functions_table.c.uuid == int(function_id)) + functions_table.delete().where(functions_table.c.uuid == function_id) ) @@ -138,6 +138,7 @@ async def register_function_job( .values( function_uuid=function_job.function_uuid, inputs=function_job.inputs, + outputs=function_job.outputs, function_class=function_job.function_class, class_specific_data=function_job.class_specific_data, title=function_job.title, @@ -190,6 +191,20 @@ async def list_function_jobs( return [FunctionJobDB.model_validate(dict(row)) for row in rows] +async def delete_function_job( + app: web.Application, + connection: AsyncConnection | None = None, + *, + function_job_id: FunctionID, +) -> None: + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + await conn.execute( + function_jobs_table.delete().where( + function_jobs_table.c.uuid == function_job_id + ) + ) + + async def find_cached_function_job( app: web.Application, connection: AsyncConnection | None = None, diff --git a/services/web/server/src/simcore_service_webserver/functions/_repo.py b/services/web/server/src/simcore_service_webserver/functions/_repo.py deleted file mode 100644 index baf76a9ee8d..00000000000 --- a/services/web/server/src/simcore_service_webserver/functions/_repo.py +++ /dev/null @@ -1,2 +0,0 @@ -# the repository layer. Calls directly to the db (function table) should be done here. -# see e.g. licenses/_licensed_resources_repository.py file for an example diff --git a/services/web/server/src/simcore_service_webserver/functions/_service.py b/services/web/server/src/simcore_service_webserver/functions/_service.py deleted file mode 100644 index 899b0b1c681..00000000000 --- a/services/web/server/src/simcore_service_webserver/functions/_service.py +++ /dev/null @@ -1,21 +0,0 @@ -# This is where calls to the business logic is done. -# calls to the `projects` interface should be done here. -# calls to _repo.py should also be done here - -from aiohttp import web -from models_library.users import UserID - -from ..projects import projects_service -from ..projects.models import ProjectDict - - -# example function -async def get_project_from_function( - app: web.Application, - function_uuid: str, - user_id: UserID, -) -> ProjectDict: - - return await projects_service.get_project_for_user( - app=app, project_uuid=function_uuid, user_id=user_id - ) diff --git a/services/web/server/tests/unit/with_dbs/05/test_functions_controller_rpc.py b/services/web/server/tests/unit/with_dbs/05/test_functions_controller_rpc.py new file mode 100644 index 00000000000..400813e43e9 --- /dev/null +++ b/services/web/server/tests/unit/with_dbs/05/test_functions_controller_rpc.py @@ -0,0 +1,449 @@ +# pylint: disable=redefined-outer-name +from uuid import uuid4 + +import pytest +import simcore_service_webserver.functions._functions_controller_rpc as functions_rpc +from aiohttp import web +from models_library.api_schemas_webserver.functions_wb_schema import ( + Function, + FunctionInputSchema, + FunctionJobCollection, + FunctionOutputSchema, + ProjectFunction, + ProjectFunctionJob, +) + + +@pytest.fixture +def mock_function() -> Function: + return ProjectFunction( + title="Test Function", + description="A test function", + input_schema=FunctionInputSchema( + schema_dict={"type": "object", "properties": {"input1": {"type": "string"}}} + ), + output_schema=FunctionOutputSchema( + schema_dict={ + "type": "object", + "properties": {"output1": {"type": "string"}}, + } + ), + project_id=uuid4(), + default_inputs=None, + ) + + +@pytest.mark.asyncio +async def test_register_function(client, mock_function): + # Register the function + registered_function = await functions_rpc.register_function( + app=client.app, function=mock_function + ) + assert registered_function.uid is not None + # Retrieve the function from the repository to verify it was saved + saved_function = await functions_rpc.get_function( + app=client.app, function_id=registered_function.uid + ) + + # Assert the saved function matches the input function + assert saved_function.uid is not None + assert saved_function.title == mock_function.title + assert saved_function.description == mock_function.description + + # Ensure saved_function is of type ProjectFunction before accessing project_id + assert isinstance(saved_function, ProjectFunction) + assert saved_function.project_id == mock_function.project_id + + # Assert the returned function matches the expected result + assert registered_function.title == mock_function.title + assert registered_function.description == mock_function.description + assert isinstance(registered_function, ProjectFunction) + assert registered_function.project_id == mock_function.project_id + + +@pytest.mark.asyncio +async def test_get_function(client, mock_function): + # Register the function first + registered_function = await functions_rpc.register_function( + app=client.app, function=mock_function + ) + assert registered_function.uid is not None + + # Retrieve the function using its ID + retrieved_function = await functions_rpc.get_function( + app=client.app, function_id=registered_function.uid + ) + + # Assert the retrieved function matches the registered function + assert retrieved_function.uid == registered_function.uid + assert retrieved_function.title == registered_function.title + assert retrieved_function.description == registered_function.description + + # Ensure retrieved_function is of type ProjectFunction before accessing project_id + assert isinstance(retrieved_function, ProjectFunction) + assert isinstance(registered_function, ProjectFunction) + assert retrieved_function.project_id == registered_function.project_id + + +@pytest.mark.asyncio +async def test_get_function_not_found(client): + # Attempt to retrieve a function that does not exist + with pytest.raises(web.HTTPNotFound): + await functions_rpc.get_function(app=client.app, function_id=uuid4()) + + +@pytest.mark.asyncio +async def test_list_functions(client): + # Register a function first + mock_function = ProjectFunction( + title="Test Function", + description="A test function", + input_schema=None, + output_schema=None, + project_id=uuid4(), + default_inputs=None, + ) + registered_function = await functions_rpc.register_function( + app=client.app, function=mock_function + ) + assert registered_function.uid is not None + + # List functions + functions = await functions_rpc.list_functions(app=client.app) + + # Assert the list contains the registered function + assert len(functions) > 0 + assert any(f.uid == registered_function.uid for f in functions) + + +@pytest.mark.asyncio +async def test_get_function_input_schema(client, mock_function): + # Register the function first + registered_function = await functions_rpc.register_function( + app=client.app, function=mock_function + ) + assert registered_function.uid is not None + + # Retrieve the input schema using its ID + input_schema = await functions_rpc.get_function_input_schema( + app=client.app, function_id=registered_function.uid + ) + + # Assert the input schema matches the registered function's input schema + assert input_schema == registered_function.input_schema + + +@pytest.mark.asyncio +async def test_get_function_output_schema(client, mock_function): + # Register the function first + registered_function = await functions_rpc.register_function( + app=client.app, function=mock_function + ) + assert registered_function.uid is not None + + # Retrieve the output schema using its ID + output_schema = await functions_rpc.get_function_output_schema( + app=client.app, function_id=registered_function.uid + ) + + # Assert the output schema matches the registered function's output schema + assert output_schema == registered_function.output_schema + + +@pytest.mark.asyncio +async def test_delete_function(client, mock_function): + # Register the function first + registered_function = await functions_rpc.register_function( + app=client.app, function=mock_function + ) + assert registered_function.uid is not None + + # Delete the function using its ID + await functions_rpc.delete_function( + app=client.app, function_id=registered_function.uid + ) + + # Attempt to retrieve the deleted function + with pytest.raises(web.HTTPNotFound): + await functions_rpc.get_function( + app=client.app, function_id=registered_function.uid + ) + + +@pytest.mark.asyncio +async def test_register_function_job(client, mock_function): + # Register the function first + registered_function = await functions_rpc.register_function( + app=client.app, function=mock_function + ) + assert registered_function.uid is not None + + function_job = ProjectFunctionJob( + function_uid=registered_function.uid, + title="Test Function Job", + description="A test function job", + project_job_id=uuid4(), + inputs={"input1": "value1"}, + outputs={"output1": "result1"}, + ) + + # Register the function job + registered_job = await functions_rpc.register_function_job( + app=client.app, function_job=function_job + ) + + # Assert the registered job matches the input job + assert registered_job.function_uid == function_job.function_uid + assert registered_job.inputs == function_job.inputs + assert registered_job.outputs == function_job.outputs + + +@pytest.mark.asyncio +async def test_get_function_job(client, mock_function): + # Register the function first + registered_function = await functions_rpc.register_function( + app=client.app, function=mock_function + ) + assert registered_function.uid is not None + + function_job = ProjectFunctionJob( + function_uid=registered_function.uid, + title="Test Function Job", + description="A test function job", + project_job_id=uuid4(), + inputs={"input1": "value1"}, + outputs={"output1": "result1"}, + ) + + # Register the function job + registered_job = await functions_rpc.register_function_job( + app=client.app, function_job=function_job + ) + assert registered_job.uid is not None + + # Retrieve the function job using its ID + retrieved_job = await functions_rpc.get_function_job( + app=client.app, function_job_id=registered_job.uid + ) + + # Assert the retrieved job matches the registered job + assert retrieved_job.function_uid == registered_job.function_uid + assert retrieved_job.inputs == registered_job.inputs + assert retrieved_job.outputs == registered_job.outputs + + +@pytest.mark.asyncio +async def test_get_function_job_not_found(client): + # Attempt to retrieve a function job that does not exist + with pytest.raises(web.HTTPNotFound): + await functions_rpc.get_function_job(app=client.app, function_job_id=uuid4()) + + +@pytest.mark.asyncio +async def test_list_function_jobs(client, mock_function): + # Register the function first + registered_function = await functions_rpc.register_function( + app=client.app, function=mock_function + ) + assert registered_function.uid is not None + + function_job = ProjectFunctionJob( + function_uid=registered_function.uid, + title="Test Function Job", + description="A test function job", + project_job_id=uuid4(), + inputs={"input1": "value1"}, + outputs={"output1": "result1"}, + ) + + # Register the function job + registered_job = await functions_rpc.register_function_job( + app=client.app, function_job=function_job + ) + + # List function jobs + jobs = await functions_rpc.list_function_jobs(app=client.app) + + # Assert the list contains the registered job + assert len(jobs) > 0 + assert any(j.uid == registered_job.uid for j in jobs) + + +@pytest.mark.asyncio +async def test_delete_function_job(client, mock_function): + # Register the function first + registered_function = await functions_rpc.register_function( + app=client.app, function=mock_function + ) + assert registered_function.uid is not None + + function_job = ProjectFunctionJob( + function_uid=registered_function.uid, + title="Test Function Job", + description="A test function job", + project_job_id=uuid4(), + inputs={"input1": "value1"}, + outputs={"output1": "result1"}, + ) + + # Register the function job + registered_job = await functions_rpc.register_function_job( + app=client.app, function_job=function_job + ) + assert registered_job.uid is not None + + # Delete the function job using its ID + await functions_rpc.delete_function_job( + app=client.app, function_job_id=registered_job.uid + ) + + # Attempt to retrieve the deleted job + with pytest.raises(web.HTTPNotFound): + await functions_rpc.get_function_job( + app=client.app, function_job_id=registered_job.uid + ) + + +@pytest.mark.asyncio +async def test_function_job_collection(client, mock_function): + # Register the function first + registered_function = await functions_rpc.register_function( + app=client.app, function=mock_function + ) + assert registered_function.uid is not None + + registered_function_job = ProjectFunctionJob( + function_uid=registered_function.uid, + title="Test Function Job", + description="A test function job", + project_job_id=uuid4(), + inputs={"input1": "value1"}, + outputs={"output1": "result1"}, + ) + # Register the function job + function_job_ids = [] + for _ in range(3): + registered_function_job = ProjectFunctionJob( + function_uid=registered_function.uid, + title="Test Function Job", + description="A test function job", + project_job_id=uuid4(), + inputs={"input1": "value1"}, + outputs={"output1": "result1"}, + ) + # Register the function job + registered_job = await functions_rpc.register_function_job( + app=client.app, function_job=registered_function_job + ) + assert registered_job.uid is not None + function_job_ids.append(registered_job.uid) + + function_job_collection = FunctionJobCollection( + title="Test Function Job Collection", + description="A test function job collection", + job_ids=function_job_ids, + ) + + # Register the function job collection + registered_collection = await functions_rpc.register_function_job_collection( + app=client.app, function_job_collection=function_job_collection + ) + assert registered_collection.uid is not None + + # Assert the registered collection matches the input collection + assert registered_collection.job_ids == function_job_ids + + await functions_rpc.delete_function_job_collection( + app=client.app, function_job_collection_id=registered_collection.uid + ) + # Attempt to retrieve the deleted collection + with pytest.raises(web.HTTPNotFound): + await functions_rpc.get_function_job( + app=client.app, function_job_id=registered_collection.uid + ) + + +# @pytest.mark.asyncio +# async def test_find_cached_function_job_project_class( +# mock_app, mock_function_id, mock_function_inputs +# ): +# mock_function_job = AsyncMock() +# mock_function_job.function_class = FunctionClass.project +# mock_function_job.uuid = "mock-uuid" +# mock_function_job.title = "mock-title" +# mock_function_job.function_uuid = "mock-function-uuid" +# mock_function_job.inputs = {"key": "value"} +# mock_function_job.class_specific_data = {"project_job_id": "mock-project-job-id"} + +# with patch( +# "simcore_service_webserver.functions._functions_repository.find_cached_function_job", +# return_value=mock_function_job, +# ): +# result = await find_cached_function_job( +# app=mock_app, function_id=mock_function_id, inputs=mock_function_inputs +# ) + +# assert isinstance(result, ProjectFunctionJob) +# assert result.uid == "mock-uuid" +# assert result.title == "mock-title" +# assert result.function_uid == "mock-function-uuid" +# assert result.inputs == {"key": "value"} +# assert result.project_job_id == "mock-project-job-id" + + +# @pytest.mark.asyncio +# async def test_find_cached_function_job_solver_class( +# mock_app, mock_function_id, mock_function_inputs +# ): +# mock_function_job = AsyncMock() +# mock_function_job.function_class = FunctionClass.solver +# mock_function_job.uuid = "mock-uuid" +# mock_function_job.title = "mock-title" +# mock_function_job.function_uuid = "mock-function-uuid" +# mock_function_job.inputs = {"key": "value"} +# mock_function_job.class_specific_data = {"solver_job_id": "mock-solver-job-id"} + +# with patch( +# "simcore_service_webserver.functions._functions_repository.find_cached_function_job", +# return_value=mock_function_job, +# ): +# result = await find_cached_function_job( +# app=mock_app, function_id=mock_function_id, inputs=mock_function_inputs +# ) + +# assert isinstance(result, SolverFunctionJob) +# assert result.uid == "mock-uuid" +# assert result.title == "mock-title" +# assert result.function_uid == "mock-function-uuid" +# assert result.inputs == {"key": "value"} +# assert result.solver_job_id == "mock-solver-job-id" + + +# @pytest.mark.asyncio +# async def test_find_cached_function_job_none(mock_app, mock_function_id, mock_function_inputs): +# with patch( +# "simcore_service_webserver.functions._functions_repository.find_cached_function_job", +# return_value=None, +# ): +# result = await find_cached_function_job( +# app=mock_app, function_id=mock_function_id, inputs=mock_function_inputs +# ) + +# assert result is None + + +# @pytest.mark.asyncio +# async def test_find_cached_function_job_unsupported_class( +# mock_app, mock_function_id, mock_function_inputs +# ): +# mock_function_job = AsyncMock() +# mock_function_job.function_class = "unsupported_class" + +# with patch( +# "simcore_service_webserver.functions._functions_repository.find_cached_function_job", +# return_value=mock_function_job, +# ): +# with pytest.raises(TypeError, match="Unsupported function class:"): +# await find_cached_function_job( +# app=mock_app, function_id=mock_function_id, inputs=mock_function_inputs +# ) From b5421f22117e408adf8eafa78fbbcedbdcc9cfb0 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Wed, 30 Apr 2025 20:17:39 +0200 Subject: [PATCH 44/69] Move function rpc test dir --- .../{05 => functions_rpc}/test_functions_controller_rpc.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename services/web/server/tests/unit/with_dbs/{05 => functions_rpc}/test_functions_controller_rpc.py (100%) diff --git a/services/web/server/tests/unit/with_dbs/05/test_functions_controller_rpc.py b/services/web/server/tests/unit/with_dbs/functions_rpc/test_functions_controller_rpc.py similarity index 100% rename from services/web/server/tests/unit/with_dbs/05/test_functions_controller_rpc.py rename to services/web/server/tests/unit/with_dbs/functions_rpc/test_functions_controller_rpc.py From b061250e74c21c79c8767ddcb4b495d60b47a131 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Wed, 7 May 2025 10:45:27 +0200 Subject: [PATCH 45/69] Remove Nullable fields --- .../functions_wb_schema.py | 58 ++--- services/api-server/openapi.json | 200 ++++++------------ .../api/routes/functions_routes.py | 16 +- .../services_rpc/wb_api_server.py | 6 - 4 files changed, 105 insertions(+), 175 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py b/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py index 0961b75765b..ad005fd8bc2 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py @@ -13,7 +13,7 @@ FunctionJobID: TypeAlias = projects.ProjectID FileID: TypeAlias = UUID -InputTypes: TypeAlias = FileID | float | int | bool | str | list | None +InputTypes: TypeAlias = FileID | float | int | bool | str | list class FunctionSchema(BaseModel): @@ -48,31 +48,31 @@ class FunctionClass(str, Enum): class FunctionBase(BaseModel): function_class: FunctionClass - uid: FunctionID | None = None - title: str | None = None - description: str | None = None - input_schema: FunctionInputSchema | None = None - output_schema: FunctionOutputSchema | None = None - default_inputs: FunctionInputs | None = None + uid: FunctionID | None + title: str = "" + description: str = "" + input_schema: FunctionInputSchema | None + output_schema: FunctionOutputSchema | None + default_inputs: FunctionInputs class FunctionDB(BaseModel): function_class: FunctionClass - uuid: FunctionJobID | None = None - title: str | None = None - description: str | None = None - input_schema: FunctionInputSchema | None = None - output_schema: FunctionOutputSchema | None = None - default_inputs: FunctionInputs | None = None + uuid: FunctionJobID | None + title: str = "" + description: str = "" + input_schema: FunctionInputSchema | None + output_schema: FunctionOutputSchema | None + default_inputs: FunctionInputs class_specific_data: FunctionClassSpecificData class FunctionJobDB(BaseModel): - uuid: FunctionJobID | None = None + uuid: FunctionJobID | None function_uuid: FunctionID - title: str | None = None - inputs: FunctionInputs | None = None - outputs: FunctionOutputs | None = None + title: str = "" + inputs: FunctionInputs + outputs: FunctionOutputs class_specific_data: FunctionJobClassSpecificData function_class: FunctionClass @@ -94,7 +94,7 @@ class ProjectFunction(FunctionBase): class SolverFunction(FunctionBase): function_class: Literal[FunctionClass.solver] = FunctionClass.solver solver_key: SolverKeyId - solver_version: str + solver_version: str = "" class PythonCodeFunction(FunctionBase): @@ -111,12 +111,12 @@ class PythonCodeFunction(FunctionBase): class FunctionJobBase(BaseModel): - uid: FunctionJobID | None = None - title: str | None = None - description: str | None = None + uid: FunctionJobID | None + title: str = "" + description: str = "" function_uid: FunctionID - inputs: FunctionInputs | None = None - outputs: FunctionOutputs | None = None + inputs: FunctionInputs + outputs: FunctionOutputs function_class: FunctionClass @@ -147,18 +147,18 @@ class FunctionJobStatus(BaseModel): class FunctionJobCollection(BaseModel): """Model for a collection of function jobs""" - uid: FunctionJobCollectionID | None = None - title: str | None - description: str | None + uid: FunctionJobCollectionID | None + title: str = "" + description: str = "" job_ids: list[FunctionJobID] class FunctionJobCollectionDB(BaseModel): """Model for a collection of function jobs""" - uuid: FunctionJobCollectionID | None - title: str | None - description: str | None + uuid: FunctionJobCollectionID + title: str = "" + description: str = "" class FunctionJobCollectionStatus(BaseModel): diff --git a/services/api-server/openapi.json b/services/api-server/openapi.json index a2a22f8d5e4..d38dafb42bb 100644 --- a/services/api-server/openapi.json +++ b/services/api-server/openapi.json @@ -7581,26 +7581,14 @@ "title": "Uid" }, "title": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Title" + "type": "string", + "title": "Title", + "default": "" }, "description": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Description" + "type": "string", + "title": "Description", + "default": "" }, "job_ids": { "items": { @@ -7613,8 +7601,7 @@ }, "type": "object", "required": [ - "title", - "description", + "uid", "job_ids" ], "title": "FunctionJobCollection", @@ -9225,26 +9212,14 @@ "title": "Uid" }, "title": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Title" + "type": "string", + "title": "Title", + "default": "" }, "description": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Description" + "type": "string", + "title": "Description", + "default": "" }, "input_schema": { "anyOf": [ @@ -9285,6 +9260,10 @@ }, "type": "object", "required": [ + "uid", + "input_schema", + "output_schema", + "default_inputs", "project_id" ], "title": "ProjectFunction" @@ -9304,26 +9283,14 @@ "title": "Uid" }, "title": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Title" + "type": "string", + "title": "Title", + "default": "" }, "description": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Description" + "type": "string", + "title": "Description", + "default": "" }, "function_uid": { "type": "string", @@ -9366,7 +9333,10 @@ }, "type": "object", "required": [ + "uid", "function_uid", + "inputs", + "outputs", "project_job_id" ], "title": "ProjectFunctionJob" @@ -9392,26 +9362,14 @@ "title": "Uid" }, "title": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Title" + "type": "string", + "title": "Title", + "default": "" }, "description": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Description" + "type": "string", + "title": "Description", + "default": "" }, "input_schema": { "anyOf": [ @@ -9451,6 +9409,10 @@ }, "type": "object", "required": [ + "uid", + "input_schema", + "output_schema", + "default_inputs", "code_url" ], "title": "PythonCodeFunction" @@ -9470,26 +9432,14 @@ "title": "Uid" }, "title": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Title" + "type": "string", + "title": "Title", + "default": "" }, "description": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Description" + "type": "string", + "title": "Description", + "default": "" }, "function_uid": { "type": "string", @@ -9527,7 +9477,10 @@ }, "type": "object", "required": [ - "function_uid" + "uid", + "function_uid", + "inputs", + "outputs" ], "title": "PythonCodeFunctionJob" }, @@ -9689,26 +9642,14 @@ "title": "Uid" }, "title": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Title" + "type": "string", + "title": "Title", + "default": "" }, "description": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Description" + "type": "string", + "title": "Description", + "default": "" }, "input_schema": { "anyOf": [ @@ -9748,13 +9689,17 @@ }, "solver_version": { "type": "string", - "title": "Solver Version" + "title": "Solver Version", + "default": "" } }, "type": "object", "required": [ - "solver_key", - "solver_version" + "uid", + "input_schema", + "output_schema", + "default_inputs", + "solver_key" ], "title": "SolverFunction" }, @@ -9773,26 +9718,14 @@ "title": "Uid" }, "title": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Title" + "type": "string", + "title": "Title", + "default": "" }, "description": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Description" + "type": "string", + "title": "Description", + "default": "" }, "function_uid": { "type": "string", @@ -9835,7 +9768,10 @@ }, "type": "object", "required": [ + "uid", "function_uid", + "inputs", + "outputs", "solver_job_id" ], "title": "SolverFunctionJob" diff --git a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py index d14739cdd9d..a5d31066c8d 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py @@ -24,9 +24,9 @@ ) from pydantic import PositiveInt from servicelib.fastapi.dependencies import get_reverse_url_mapper +from simcore_service_api_server._service_jobs import JobService from sqlalchemy.ext.asyncio import AsyncEngine -from ..._service_job import JobService from ..._service_solvers import SolverService from ...models.schemas.errors import ErrorGet from ...models.schemas.jobs import ( @@ -38,7 +38,7 @@ from ...services_rpc.wb_api_server import WbApiRpcClient from ..dependencies.authentication import get_current_user_id, get_product_name from ..dependencies.database import get_db_asyncpg_engine -from ..dependencies.services import get_api_client +from ..dependencies.services import get_api_client, get_job_service, get_solver_service from ..dependencies.webserver_http import get_webserver_session from ..dependencies.webserver_rpc import ( get_wb_api_rpc_client, @@ -173,8 +173,8 @@ async def run_function( # noqa: PLR0913 function_inputs: FunctionInputs, user_id: Annotated[PositiveInt, Depends(get_current_user_id)], product_name: Annotated[str, Depends(get_product_name)], - solver_service: Annotated[SolverService, Depends()], - job_service: Annotated[JobService, Depends()], + solver_service: Annotated[SolverService, Depends(get_solver_service)], + job_service: Annotated[JobService, Depends(get_job_service)], ): to_run_function = await wb_api_rpc.get_function(function_id=function_id) @@ -227,6 +227,7 @@ async def run_function( # noqa: PLR0913 return await register_function_job( wb_api_rpc=wb_api_rpc, function_job=ProjectFunctionJob( + uid=None, function_uid=to_run_function.uid, title=f"Function job of function {to_run_function.uid}", description=to_run_function.description, @@ -245,8 +246,6 @@ async def run_function( # noqa: PLR0913 url_for=url_for, x_simcore_parent_project_uuid=None, x_simcore_parent_node_id=None, - user_id=user_id, - product_name=product_name, ) await solvers_jobs.start_job( request=request, @@ -260,6 +259,7 @@ async def run_function( # noqa: PLR0913 return await register_function_job( wb_api_rpc=wb_api_rpc, function_job=SolverFunctionJob( + uid=None, function_uid=to_run_function.uid, title=f"Function job of function {to_run_function.uid}", description=to_run_function.description, @@ -463,8 +463,8 @@ async def map_function( # noqa: PLR0913 director2_api: Annotated[DirectorV2Api, Depends(get_api_client(DirectorV2Api))], user_id: Annotated[PositiveInt, Depends(get_current_user_id)], product_name: Annotated[str, Depends(get_product_name)], - solver_service: Annotated[SolverService, Depends()], - job_service: Annotated[JobService, Depends()], + solver_service: Annotated[SolverService, Depends(get_solver_service)], + job_service: Annotated[JobService, Depends(get_job_service)], ): function_jobs = [] function_jobs = [ diff --git a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py index c5a35499ef2..9c8787acc5a 100644 --- a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py +++ b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py @@ -76,9 +76,6 @@ from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( list_functions as _list_functions, ) -from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( - ping as _ping, -) from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions_rpc_interface import ( register_function as _register_function, ) @@ -260,9 +257,6 @@ async def release_licensed_item_for_wallet( num_of_seats=licensed_item_checkout_get.num_of_seats, ) - async def ping(self) -> str: - return await _ping(self._client) - async def mark_project_as_job( self, product_name: ProductName, From e2d1da0f31c00dcb3738436698aa102c33db61a0 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Wed, 7 May 2025 10:46:18 +0200 Subject: [PATCH 46/69] Merge alembic heads after rebase --- ...8debc3e_merge_0d52976dc616_ecd7a3b85134.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/1b5c88debc3e_merge_0d52976dc616_ecd7a3b85134.py diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/1b5c88debc3e_merge_0d52976dc616_ecd7a3b85134.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/1b5c88debc3e_merge_0d52976dc616_ecd7a3b85134.py new file mode 100644 index 00000000000..be5f96cccd6 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/1b5c88debc3e_merge_0d52976dc616_ecd7a3b85134.py @@ -0,0 +1,21 @@ +"""merge 0d52976dc616 ecd7a3b85134 + +Revision ID: 1b5c88debc3e +Revises: 0d52976dc616, ecd7a3b85134 +Create Date: 2025-05-07 08:45:47.779512+00:00 + +""" + +# revision identifiers, used by Alembic. +revision = "1b5c88debc3e" +down_revision = ("0d52976dc616", "ecd7a3b85134") +branch_labels = None +depends_on = None + + +def upgrade(): + pass + + +def downgrade(): + pass From df546b31ac8460b569237202736e2d4a5c70be95 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Wed, 7 May 2025 11:54:13 +0200 Subject: [PATCH 47/69] Fix tests after rebase --- .../test_api_routers_functions.py | 22 +++++++++++++++++++ .../functions/_functions_controller_rpc.py | 5 ++++- .../functions/_functions_repository.py | 6 ++++- .../test_functions_controller_rpc.py | 9 ++++++++ 4 files changed, 40 insertions(+), 2 deletions(-) diff --git a/services/api-server/tests/unit/api_functions/test_api_routers_functions.py b/services/api-server/tests/unit/api_functions/test_api_routers_functions.py index 8223e7e5813..21ac2be2a90 100644 --- a/services/api-server/tests/unit/api_functions/test_api_routers_functions.py +++ b/services/api-server/tests/unit/api_functions/test_api_routers_functions.py @@ -182,12 +182,14 @@ async def delete_function_job_collection( def test_register_function(api_app) -> None: client = TestClient(api_app) sample_function = { + "uid": None, "title": "test_function", "function_class": "project", "project_id": str(uuid4()), "description": "A test function", "input_schema": {"schema_dict": {}}, "output_schema": {"schema_dict": {}}, + "default_inputs": None, } response = client.post("/functions", json=sample_function) assert response.status_code == 200 @@ -221,12 +223,14 @@ def test_get_function(api_app: FastAPI) -> None: project_id = str(uuid4()) # First, register a sample function so that it exists sample_function = { + "uid": None, "title": "example_function", "function_class": "project", "project_id": project_id, "description": "An example function", "input_schema": {"schema_dict": {}}, "output_schema": {"schema_dict": {}}, + "default_inputs": None, } post_response = client.post("/functions", json=sample_function) assert post_response.status_code == 200 @@ -262,12 +266,14 @@ def test_list_functions(api_app: FastAPI) -> None: client = TestClient(api_app) # Register a sample function sample_function = { + "uid": None, "title": "example_function", "function_class": "project", "project_id": str(uuid4()), "description": "An example function", "input_schema": {"schema_dict": {}}, "output_schema": {"schema_dict": {}}, + "default_inputs": None, } post_response = client.post("/functions", json=sample_function) assert post_response.status_code == 200 @@ -285,6 +291,7 @@ def test_get_function_input_schema(api_app: FastAPI) -> None: project_id = str(uuid4()) # Register a sample function sample_function = { + "uid": None, "title": "example_function", "function_class": "project", "project_id": project_id, @@ -296,6 +303,7 @@ def test_get_function_input_schema(api_app: FastAPI) -> None: } }, "output_schema": {"schema_dict": {}}, + "default_inputs": None, } post_response = client.post("/functions", json=sample_function) assert post_response.status_code == 200 @@ -315,6 +323,7 @@ def test_get_function_output_schema(api_app: FastAPI) -> None: project_id = str(uuid4()) # Register a sample function sample_function = { + "uid": None, "title": "example_function", "function_class": "project", "project_id": project_id, @@ -326,6 +335,7 @@ def test_get_function_output_schema(api_app: FastAPI) -> None: "properties": {"output1": {"type": "string"}}, } }, + "default_inputs": None, } post_response = client.post("/functions", json=sample_function) assert post_response.status_code == 200 @@ -344,6 +354,7 @@ def test_validate_function_inputs(api_app: FastAPI) -> None: project_id = str(uuid4()) # Register a sample function sample_function = { + "uid": None, "title": "example_function", "function_class": "project", "project_id": project_id, @@ -355,6 +366,7 @@ def test_validate_function_inputs(api_app: FastAPI) -> None: } }, "output_schema": {"schema_dict": {}}, + "default_inputs": None, } post_response = client.post("/functions", json=sample_function) assert post_response.status_code == 200 @@ -376,12 +388,14 @@ def test_delete_function(api_app: FastAPI) -> None: project_id = str(uuid4()) # Register a sample function sample_function = { + "uid": None, "title": "example_function", "function_class": "project", "project_id": project_id, "description": "An example function", "input_schema": {"schema_dict": {}}, "output_schema": {"schema_dict": {}}, + "default_inputs": None, } post_response = client.post("/functions", json=sample_function) assert post_response.status_code == 200 @@ -398,6 +412,7 @@ def test_register_function_job(api_app: FastAPI) -> None: client = TestClient(api_app) mock_function_job = { + "uid": None, "function_uid": str(uuid4()), "title": "Test Function Job", "description": "A test function job", @@ -415,6 +430,7 @@ def test_register_function_job(api_app: FastAPI) -> None: response_data = response.json() assert response_data["uid"] is not None response_data.pop("uid", None) # Remove the uid field + mock_function_job.pop("uid", None) # Remove the uid field assert response_data == mock_function_job @@ -423,6 +439,7 @@ def test_get_function_job(api_app: FastAPI) -> None: client = TestClient(api_app) mock_function_job = { + "uid": None, "function_uid": str(uuid4()), "title": "Test Function Job", "description": "A test function job", @@ -454,6 +471,7 @@ def test_list_function_jobs(api_app: FastAPI) -> None: client = TestClient(api_app) mock_function_job = { + "uid": None, "function_uid": str(uuid4()), "title": "Test Function Job", "description": "A test function job", @@ -480,6 +498,7 @@ def test_delete_function_job(api_app: FastAPI) -> None: client = TestClient(api_app) mock_function_job = { + "uid": None, "function_uid": str(uuid4()), "title": "Test Function Job", "description": "A test function job", @@ -505,6 +524,7 @@ def test_register_function_job_collection(api_app: FastAPI) -> None: client = TestClient(api_app) mock_function_job_collection = { + "uid": None, "title": "Test Collection", "description": "A test function job collection", "job_ids": [str(uuid4()), str(uuid4())], @@ -520,6 +540,7 @@ def test_register_function_job_collection(api_app: FastAPI) -> None: response_data = response.json() assert response_data["uid"] is not None response_data.pop("uid", None) # Remove the uid field + mock_function_job_collection.pop("uid", None) # Remove the uid field assert response_data == mock_function_job_collection @@ -527,6 +548,7 @@ def test_get_function_job_collection(api_app: FastAPI) -> None: # Arrange client = TestClient(api_app) mock_function_job_collection = { + "uid": None, "title": "Test Collection", "description": "A test function job collection", "job_ids": [str(uuid4()), str(uuid4())], diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py index f8e39796249..90cf67bf71e 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py @@ -36,7 +36,7 @@ async def ping(app: web.Application) -> str: @router.expose() async def register_function(app: web.Application, *, function: Function) -> Function: assert app - saved_function = await _functions_repository.create_function( + saved_function = await _functions_repository.register_function( app=app, function=_encode_function(function) ) return _decode_function(saved_function) @@ -90,6 +90,7 @@ def _encode_function( raise TypeError(msg) return FunctionDB( + uuid=function.uid, title=function.title, description=function.description, input_schema=function.input_schema, @@ -145,6 +146,7 @@ def _encode_functionjob( ) -> FunctionJobDB: if functionjob.function_class == FunctionClass.project: return FunctionJobDB( + uuid=functionjob.uid, title=functionjob.title, function_uuid=functionjob.function_uid, inputs=functionjob.inputs, @@ -158,6 +160,7 @@ def _encode_functionjob( ) elif functionjob.function_class == FunctionClass.solver: # noqa: RET505 return FunctionJobDB( + uuid=functionjob.uid, title=functionjob.title, function_uuid=functionjob.function_uid, inputs=functionjob.inputs, diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py index 6e3fa98b94d..fbcc4ad6c2b 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py @@ -40,13 +40,17 @@ ) -async def create_function( +async def register_function( app: web.Application, connection: AsyncConnection | None = None, *, function: FunctionDB, ) -> FunctionDB: + if function.uuid is not None: + msg = "Function uid is not None. Cannot register function." + raise ValueError(msg) + async with transaction_context(get_asyncpg_engine(app), connection) as conn: result = await conn.stream( functions_table.insert() diff --git a/services/web/server/tests/unit/with_dbs/functions_rpc/test_functions_controller_rpc.py b/services/web/server/tests/unit/with_dbs/functions_rpc/test_functions_controller_rpc.py index 400813e43e9..556885f4b56 100644 --- a/services/web/server/tests/unit/with_dbs/functions_rpc/test_functions_controller_rpc.py +++ b/services/web/server/tests/unit/with_dbs/functions_rpc/test_functions_controller_rpc.py @@ -17,6 +17,7 @@ @pytest.fixture def mock_function() -> Function: return ProjectFunction( + uid=None, title="Test Function", description="A test function", input_schema=FunctionInputSchema( @@ -96,6 +97,7 @@ async def test_get_function_not_found(client): async def test_list_functions(client): # Register a function first mock_function = ProjectFunction( + uid=None, title="Test Function", description="A test function", input_schema=None, @@ -179,6 +181,7 @@ async def test_register_function_job(client, mock_function): assert registered_function.uid is not None function_job = ProjectFunctionJob( + uid=None, function_uid=registered_function.uid, title="Test Function Job", description="A test function job", @@ -207,6 +210,7 @@ async def test_get_function_job(client, mock_function): assert registered_function.uid is not None function_job = ProjectFunctionJob( + uid=None, function_uid=registered_function.uid, title="Test Function Job", description="A test function job", @@ -248,6 +252,7 @@ async def test_list_function_jobs(client, mock_function): assert registered_function.uid is not None function_job = ProjectFunctionJob( + uid=None, function_uid=registered_function.uid, title="Test Function Job", description="A test function job", @@ -278,6 +283,7 @@ async def test_delete_function_job(client, mock_function): assert registered_function.uid is not None function_job = ProjectFunctionJob( + uid=None, function_uid=registered_function.uid, title="Test Function Job", description="A test function job", @@ -313,6 +319,7 @@ async def test_function_job_collection(client, mock_function): assert registered_function.uid is not None registered_function_job = ProjectFunctionJob( + uid=None, function_uid=registered_function.uid, title="Test Function Job", description="A test function job", @@ -324,6 +331,7 @@ async def test_function_job_collection(client, mock_function): function_job_ids = [] for _ in range(3): registered_function_job = ProjectFunctionJob( + uid=None, function_uid=registered_function.uid, title="Test Function Job", description="A test function job", @@ -339,6 +347,7 @@ async def test_function_job_collection(client, mock_function): function_job_ids.append(registered_job.uid) function_job_collection = FunctionJobCollection( + uid=None, title="Test Function Job Collection", description="A test function job collection", job_ids=function_job_ids, From 90dc711bf43d8d070905473db2dddc2ef2ede854 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Wed, 7 May 2025 15:19:27 +0200 Subject: [PATCH 48/69] Add pagenation to function listing --- .../models/functions_models_db.py | 2 +- .../functions/functions_rpc_interface.py | 71 ++++---- .../api/routes/functions_routes.py | 20 ++- .../services_rpc/wb_api_server.py | 50 +++++- .../functions/_functions_controller_rpc.py | 91 ++++++---- .../functions/_functions_repository.py | 157 ++++++++++++------ .../test_functions_controller_rpc.py | 121 +++++++++++++- 7 files changed, 382 insertions(+), 130 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/functions_models_db.py b/packages/postgres-database/src/simcore_postgres_database/models/functions_models_db.py index bfaabb39372..ee8dd1474c0 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/functions_models_db.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/functions_models_db.py @@ -96,7 +96,7 @@ functions.c.uuid, onupdate=RefActions.CASCADE, ondelete=RefActions.CASCADE, - name="fk_functions_to_function_jobs_to_function_uuid", + name="fk_function_jobs_to_function_uuid", ), nullable=False, index=True, diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py index 7bee95d0601..56987fc8693 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py @@ -15,6 +15,9 @@ FunctionOutputSchema, ) from models_library.rabbitmq_basic_types import RPCMethodName +from models_library.rest_pagination import ( + PageMetaInfoLimitOffset, +) from pydantic import TypeAdapter from .....logging_utils import log_decorator @@ -23,18 +26,6 @@ _logger = logging.getLogger(__name__) -@log_decorator(_logger, level=logging.DEBUG) -async def ping( - rabbitmq_rpc_client: RabbitMQRPCClient, -) -> str: - result = await rabbitmq_rpc_client.request( - WEBSERVER_RPC_NAMESPACE, - TypeAdapter(RPCMethodName).validate_python("ping"), - ) - assert isinstance(result, str) # nosec - return result - - @log_decorator(_logger, level=logging.DEBUG) async def register_function( rabbitmq_rpc_client: RabbitMQRPCClient, @@ -103,10 +94,44 @@ async def delete_function( @log_decorator(_logger, level=logging.DEBUG) async def list_functions( rabbitmq_rpc_client: RabbitMQRPCClient, -) -> list[Function]: + *pagination_limit: int, + pagination_offset: int, +) -> tuple[list[Function], PageMetaInfoLimitOffset]: return await rabbitmq_rpc_client.request( WEBSERVER_RPC_NAMESPACE, TypeAdapter(RPCMethodName).validate_python("list_functions"), + pagination_offset=pagination_offset, + pagination_limit=pagination_limit, + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def list_function_jobs( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + pagination_limit: int, + pagination_offset: int, +) -> tuple[list[FunctionJob], PageMetaInfoLimitOffset]: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("list_function_jobs"), + pagination_offset=pagination_offset, + pagination_limit=pagination_limit, + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def list_function_job_collections( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + pagination_limit: int, + pagination_offset: int, +) -> tuple[list[FunctionJobCollection], PageMetaInfoLimitOffset]: + return await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("list_function_job_collections"), + pagination_offset=pagination_offset, + pagination_limit=pagination_limit, ) @@ -151,16 +176,6 @@ async def get_function_job( ) -@log_decorator(_logger, level=logging.DEBUG) -async def list_function_jobs( - rabbitmq_rpc_client: RabbitMQRPCClient, -) -> list[FunctionJob]: - return await rabbitmq_rpc_client.request( - WEBSERVER_RPC_NAMESPACE, - TypeAdapter(RPCMethodName).validate_python("list_function_jobs"), - ) - - @log_decorator(_logger, level=logging.DEBUG) async def delete_function_job( rabbitmq_rpc_client: RabbitMQRPCClient, @@ -189,16 +204,6 @@ async def find_cached_function_job( ) -@log_decorator(_logger, level=logging.DEBUG) -async def list_function_job_collections( - rabbitmq_rpc_client: RabbitMQRPCClient, -) -> list[FunctionJobCollection]: - return await rabbitmq_rpc_client.request( - WEBSERVER_RPC_NAMESPACE, - TypeAdapter(RPCMethodName).validate_python("list_function_job_collections"), - ) - - @log_decorator(_logger, level=logging.DEBUG) async def register_function_job_collection( rabbitmq_rpc_client: RabbitMQRPCClient, diff --git a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py index a5d31066c8d..b25b541058d 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py @@ -4,6 +4,7 @@ import jsonschema from fastapi import APIRouter, Depends, Request, status +from fastapi_pagination.api import create_page from jsonschema import ValidationError from models_library.api_schemas_webserver.functions_wb_schema import ( Function, @@ -28,6 +29,7 @@ from sqlalchemy.ext.asyncio import AsyncEngine from ..._service_solvers import SolverService +from ...models.pagination import PaginationParams from ...models.schemas.errors import ErrorGet from ...models.schemas.jobs import ( JobInputs, @@ -86,8 +88,18 @@ async def get_function( @function_router.get("", response_model=list[Function], description="List functions") async def list_functions( wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], + page_params: Annotated[PaginationParams, Depends()], ): - return await wb_api_rpc.list_functions() + functions_list, meta = await wb_api_rpc.list_functions( + pagination_offset=page_params.offset, + pagination_limit=page_params.limit, + ) + + return create_page( + functions_list, + total=meta.total, + params=page_params, + ) def join_inputs( @@ -322,8 +334,9 @@ async def get_function_job( ) async def list_function_jobs( wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], + page_params: Annotated[PaginationParams, Depends()], ): - return await wb_api_rpc.list_function_jobs() + return await wb_api_rpc.list_function_jobs(page_params=page_params) @function_job_router.delete( @@ -515,8 +528,9 @@ async def map_function( # noqa: PLR0913 ) async def list_function_job_collections( wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], + page_params: Annotated[PaginationParams, Depends()], ): - return await wb_api_rpc.list_function_job_collections() + return await wb_api_rpc.list_function_job_collections(page_params=page_params) @function_job_collections_router.get( diff --git a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py index 9c8787acc5a..91adebd2606 100644 --- a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py +++ b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py @@ -22,6 +22,12 @@ from models_library.resource_tracker_licensed_items_checkouts import ( LicensedItemCheckoutID, ) +from models_library.rest_pagination import ( + DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, + PageLimitInt, + PageMetaInfoLimitOffset, + PageOffsetInt, +) from models_library.services_types import ServiceRunID from models_library.users import UserID from models_library.wallets import WalletID @@ -312,8 +318,42 @@ async def get_function(self, *, function_id: FunctionID) -> Function: async def delete_function(self, *, function_id: FunctionID) -> None: return await _delete_function(self._client, function_id=function_id) - async def list_functions(self) -> list[Function]: - return await _list_functions(self._client) + async def list_functions( + self, + *, + pagination_offset: PageOffsetInt = 0, + pagination_limit: PageLimitInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, + ) -> tuple[list[Function], PageMetaInfoLimitOffset]: + + return await _list_functions( + self._client, + pagination_offset=pagination_offset, + pagination_limit=pagination_limit, + ) + + async def list_function_jobs( + self, + *, + pagination_offset: PageOffsetInt = 0, + pagination_limit: PageLimitInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, + ) -> tuple[list[FunctionJob], PageMetaInfoLimitOffset]: + return await _list_function_jobs( + self._client, + pagination_offset=pagination_offset, + pagination_limit=pagination_limit, + ) + + async def list_function_job_collections( + self, + *, + pagination_offset: PageOffsetInt = 0, + pagination_limit: PageLimitInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, + ) -> tuple[list[FunctionJobCollection], PageMetaInfoLimitOffset]: + return await _list_function_job_collections( + self._client, + pagination_offset=pagination_offset, + pagination_limit=pagination_limit, + ) async def run_function( self, *, function_id: FunctionID, inputs: FunctionInputs @@ -346,12 +386,6 @@ async def find_cached_function_job( self._client, function_id=function_id, inputs=inputs ) - async def list_function_jobs(self) -> list[FunctionJob]: - return await _list_function_jobs(self._client) - - async def list_function_job_collections(self) -> list[FunctionJobCollection]: - return await _list_function_job_collections(self._client) - async def get_function_job_collection( self, *, function_job_collection_id: FunctionJobCollectionID ) -> FunctionJobCollection: diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py index 90cf67bf71e..fa14c220d1f 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py @@ -19,6 +19,9 @@ SolverFunction, SolverFunctionJob, ) +from models_library.rest_pagination import ( + PageMetaInfoLimitOffset, +) from servicelib.rabbitmq import RPCRouter from ..rabbitmq import get_rabbitmq_rpc_server @@ -191,18 +194,6 @@ async def get_function_job( return _decode_functionjob(returned_function_job) -@router.expose() -async def list_function_jobs(app: web.Application) -> list[FunctionJob]: - assert app - returned_function_jobs = await _functions_repository.list_function_jobs( - app=app, - ) - return [ - _decode_functionjob(returned_function_job) - for returned_function_job in returned_function_jobs - ] - - @router.expose() async def get_function_input_schema( app: web.Application, *, function_id: FunctionID @@ -240,14 +231,63 @@ async def get_function_output_schema( @router.expose() -async def list_functions(app: web.Application) -> list[Function]: +async def list_functions( + app: web.Application, + pagination_limit: int, + pagination_offset: int, +) -> tuple[list[Function], PageMetaInfoLimitOffset]: assert app - returned_functions = await _functions_repository.list_functions( + returned_functions, page = await _functions_repository.list_functions( app=app, + pagination_limit=pagination_limit, + pagination_offset=pagination_offset, ) return [ _decode_function(returned_function) for returned_function in returned_functions - ] + ], page + + +@router.expose() +async def list_function_jobs( + app: web.Application, + pagination_limit: int, + pagination_offset: int, +) -> tuple[list[FunctionJob], PageMetaInfoLimitOffset]: + assert app + returned_function_jobs, page = await _functions_repository.list_function_jobs( + app=app, + pagination_limit=pagination_limit, + pagination_offset=pagination_offset, + ) + return [ + _decode_functionjob(returned_function_job) + for returned_function_job in returned_function_jobs + ], page + + +@router.expose() +async def list_function_job_collections( + app: web.Application, + pagination_limit: int, + pagination_offset: int, +) -> tuple[list[FunctionJobCollection], PageMetaInfoLimitOffset]: + assert app + returned_function_job_collections, page = ( + await _functions_repository.list_function_job_collections( + app=app, + pagination_limit=pagination_limit, + pagination_offset=pagination_offset, + ) + ) + return [ + FunctionJobCollection( + uid=function_job_collection.uuid, + title=function_job_collection.title, + description=function_job_collection.description, + job_ids=job_ids, + ) + for function_job_collection, job_ids in returned_function_job_collections + ], page @router.expose() @@ -317,27 +357,6 @@ async def find_cached_function_job( raise TypeError(msg) -@router.expose() -async def list_function_job_collections( - app: web.Application, -) -> list[FunctionJobCollection]: - assert app - returned_function_job_collections = ( - await _functions_repository.list_function_job_collections( - app=app, - ) - ) - return [ - FunctionJobCollection( - uid=function_job_collection.uuid, - title=function_job_collection.title, - description=function_job_collection.description, - job_ids=job_ids, - ) - for function_job_collection, job_ids in returned_function_job_collections - ] - - @router.expose() async def register_function_job_collection( app: web.Application, *, function_job_collection: FunctionJobCollection diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py index fbcc4ad6c2b..5d8e7252064 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py @@ -10,6 +10,9 @@ FunctionJobDB, FunctionJobID, ) +from models_library.rest_pagination import ( + PageMetaInfoLimitOffset, +) from simcore_postgres_database.models.functions_models_db import ( function_job_collections as function_job_collections_table, ) @@ -28,6 +31,7 @@ ) from sqlalchemy import Text, cast from sqlalchemy.ext.asyncio import AsyncConnection +from sqlalchemy.sql import func from ..db.plugin import get_asyncpg_engine @@ -105,15 +109,117 @@ async def get_function( async def list_functions( app: web.Application, connection: AsyncConnection | None = None, -) -> list[FunctionDB]: + *, + pagination_limit: int, + pagination_offset: int, +) -> tuple[list[FunctionDB], PageMetaInfoLimitOffset]: + + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + total_count_result = await conn.scalar( + func.count().select().select_from(functions_table) + ) + result = await conn.stream( + functions_table.select().offset(pagination_offset).limit(pagination_limit) + ) + rows = await result.all() + if rows is None: + return [], PageMetaInfoLimitOffset( + total=0, offset=pagination_offset, limit=pagination_limit, count=0 + ) + + return [ + FunctionDB.model_validate(dict(row)) for row in rows + ], PageMetaInfoLimitOffset( + total=total_count_result, + offset=pagination_offset, + limit=pagination_limit, + count=len(rows), + ) + + +async def list_function_jobs( + app: web.Application, + connection: AsyncConnection | None = None, + *, + pagination_limit: int, + pagination_offset: int, +) -> tuple[list[FunctionJobDB], PageMetaInfoLimitOffset]: async with transaction_context(get_asyncpg_engine(app), connection) as conn: - result = await conn.stream(functions_table.select().where()) + total_count_result = await conn.scalar( + func.count().select().select_from(function_jobs_table) + ) + result = await conn.stream( + function_jobs_table.select() + .offset(pagination_offset) + .limit(pagination_limit) + ) rows = await result.all() if rows is None: - return [] + return [], PageMetaInfoLimitOffset( + total=0, offset=pagination_offset, limit=pagination_limit, count=0 + ) + + return [ + FunctionJobDB.model_validate(dict(row)) for row in rows + ], PageMetaInfoLimitOffset( + total=total_count_result, + offset=pagination_offset, + limit=pagination_limit, + count=len(rows), + ) - return [FunctionDB.model_validate(dict(row)) for row in rows] + +async def list_function_job_collections( + app: web.Application, + connection: AsyncConnection | None = None, + *, + pagination_limit: int, + pagination_offset: int, +) -> tuple[ + list[tuple[FunctionJobCollectionDB, list[FunctionJobID]]], + PageMetaInfoLimitOffset, +]: + """ + Returns a list of function job collections and their associated job ids. + """ + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + total_count_result = await conn.scalar( + func.count().select().select_from(function_job_collections_table) + ) + result = await conn.stream( + function_job_collections_table.select() + .offset(pagination_offset) + .limit(pagination_limit) + ) + rows = await result.all() + if rows is None: + return [], PageMetaInfoLimitOffset( + total=0, offset=pagination_offset, limit=pagination_limit, count=0 + ) + + collections = [] + for row in rows: + collection = FunctionJobCollectionDB.model_validate(dict(row)) + job_result = await conn.stream( + function_job_collections_to_function_jobs_table.select().where( + function_job_collections_to_function_jobs_table.c.function_job_collection_uuid + == row["uuid"] + ) + ) + job_rows = await job_result.all() + job_ids = ( + [job_row["function_job_uuid"] for job_row in job_rows] + if job_rows + else [] + ) + collections.append((collection, job_ids)) + return collections, PageMetaInfoLimitOffset( + total=total_count_result, + offset=pagination_offset, + limit=pagination_limit, + count=len(rows), + ) async def delete_function( @@ -181,20 +287,6 @@ async def get_function_job( return FunctionJobDB.model_validate(dict(row)) -async def list_function_jobs( - app: web.Application, - connection: AsyncConnection | None = None, -) -> list[FunctionJobDB]: - - async with transaction_context(get_asyncpg_engine(app), connection) as conn: - result = await conn.stream(function_jobs_table.select().where()) - rows = await result.all() - if rows is None: - return [] - - return [FunctionJobDB.model_validate(dict(row)) for row in rows] - - async def delete_function_job( app: web.Application, connection: AsyncConnection | None = None, @@ -238,35 +330,6 @@ async def find_cached_function_job( return None -async def list_function_job_collections( - app: web.Application, - connection: AsyncConnection | None = None, -) -> list[tuple[FunctionJobCollectionDB, list[FunctionJobID]]]: - async with transaction_context(get_asyncpg_engine(app), connection) as conn: - result = await conn.stream(function_job_collections_table.select().where()) - rows = await result.all() - if rows is None: - return [] - - collections = [] - for row in rows: - collection = FunctionJobCollection.model_validate(dict(row)) - job_result = await conn.stream( - function_job_collections_to_function_jobs_table.select().where( - function_job_collections_to_function_jobs_table.c.function_job_collection_uuid - == row["uuid"] - ) - ) - job_rows = await job_result.all() - job_ids = ( - [job_row["function_job_uuid"] for job_row in job_rows] - if job_rows - else [] - ) - collections.append((collection, job_ids)) - return collections - - async def get_function_job_collection( app: web.Application, connection: AsyncConnection | None = None, diff --git a/services/web/server/tests/unit/with_dbs/functions_rpc/test_functions_controller_rpc.py b/services/web/server/tests/unit/with_dbs/functions_rpc/test_functions_controller_rpc.py index 556885f4b56..403fe959374 100644 --- a/services/web/server/tests/unit/with_dbs/functions_rpc/test_functions_controller_rpc.py +++ b/services/web/server/tests/unit/with_dbs/functions_rpc/test_functions_controller_rpc.py @@ -111,13 +111,72 @@ async def test_list_functions(client): assert registered_function.uid is not None # List functions - functions = await functions_rpc.list_functions(app=client.app) + functions, _ = await functions_rpc.list_functions( + app=client.app, pagination_limit=10, pagination_offset=0 + ) # Assert the list contains the registered function assert len(functions) > 0 assert any(f.uid == registered_function.uid for f in functions) +async def delete_all_registered_functions(client): + # This function is a placeholder for the actual implementation + # that deletes all registered functions from the database. + functions, _ = await functions_rpc.list_functions( + app=client.app, pagination_limit=100, pagination_offset=0 + ) + for function in functions: + assert function.uid is not None + await functions_rpc.delete_function(app=client.app, function_id=function.uid) + + +@pytest.mark.asyncio +async def test_list_functions_empty(client): + await delete_all_registered_functions(client) + # List functions when none are registered + functions, _ = await functions_rpc.list_functions( + app=client.app, pagination_limit=10, pagination_offset=0 + ) + + # Assert the list is empty + assert len(functions) == 0 + + +@pytest.mark.asyncio +async def test_list_functions_with_pagination(client, mock_function): + await delete_all_registered_functions(client) + + # Register multiple functions + TOTAL_FUNCTIONS = 3 + for _ in range(TOTAL_FUNCTIONS): + await functions_rpc.register_function(app=client.app, function=mock_function) + + functions, page_info = await functions_rpc.list_functions( + app=client.app, pagination_limit=2, pagination_offset=0 + ) + + # List functions with pagination + functions, page_info = await functions_rpc.list_functions( + app=client.app, pagination_limit=2, pagination_offset=0 + ) + + # Assert the list contains the correct number of functions + assert len(functions) == 2 + assert page_info.count == 2 + assert page_info.total == TOTAL_FUNCTIONS + + # List the next page of functions + functions, page_info = await functions_rpc.list_functions( + app=client.app, pagination_limit=2, pagination_offset=2 + ) + + # Assert the list contains the correct number of functions + assert len(functions) == 1 + assert page_info.count == 1 + assert page_info.total == TOTAL_FUNCTIONS + + @pytest.mark.asyncio async def test_get_function_input_schema(client, mock_function): # Register the function first @@ -267,7 +326,9 @@ async def test_list_function_jobs(client, mock_function): ) # List function jobs - jobs = await functions_rpc.list_function_jobs(app=client.app) + jobs, _ = await functions_rpc.list_function_jobs( + app=client.app, pagination_limit=10, pagination_offset=0 + ) # Assert the list contains the registered job assert len(jobs) > 0 @@ -372,6 +433,62 @@ async def test_function_job_collection(client, mock_function): ) +@pytest.mark.asyncio +async def test_list_function_job_collections(client, mock_function): + # Register the function first + registered_function = await functions_rpc.register_function( + app=client.app, function=mock_function + ) + assert registered_function.uid is not None + + # Create a function job collection + function_job_ids = [] + for _ in range(3): + registered_function_job = ProjectFunctionJob( + uid=None, + function_uid=registered_function.uid, + title="Test Function Job", + description="A test function job", + project_job_id=uuid4(), + inputs={"input1": "value1"}, + outputs={"output1": "result1"}, + ) + # Register the function job + registered_job = await functions_rpc.register_function_job( + app=client.app, function_job=registered_function_job + ) + assert registered_job.uid is not None + function_job_ids.append(registered_job.uid) + + function_job_collection = FunctionJobCollection( + uid=None, + title="Test Function Job Collection", + description="A test function job collection", + job_ids=function_job_ids, + ) + + # Register the function job collection + registered_collections = [ + await functions_rpc.register_function_job_collection( + app=client.app, function_job_collection=function_job_collection + ) + for _ in range(3) + ] + assert all( + registered_collection.uid is not None + for registered_collection in registered_collections + ) + + # List function job collections + collections, _ = await functions_rpc.list_function_job_collections( + app=client.app, pagination_limit=1, pagination_offset=1 + ) + + # Assert the list contains the registered collection + assert len(collections) == 1 + assert collections[0].uid == registered_collections[1].uid + + # @pytest.mark.asyncio # async def test_find_cached_function_job_project_class( # mock_app, mock_function_id, mock_function_inputs From d8393d44725ae30d7b02d36185376f1b12424c9a Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Wed, 7 May 2025 15:23:00 +0200 Subject: [PATCH 49/69] Fix function routes --- .../api/routes/functions_routes.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py index b25b541058d..614119542b2 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py @@ -336,7 +336,10 @@ async def list_function_jobs( wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], page_params: Annotated[PaginationParams, Depends()], ): - return await wb_api_rpc.list_function_jobs(page_params=page_params) + return await wb_api_rpc.list_function_jobs( + pagination_offset=page_params.offset, + pagination_limit=page_params.limit, + ) @function_job_router.delete( @@ -530,7 +533,10 @@ async def list_function_job_collections( wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], page_params: Annotated[PaginationParams, Depends()], ): - return await wb_api_rpc.list_function_job_collections(page_params=page_params) + return await wb_api_rpc.list_function_job_collections( + pagination_offset=page_params.offset, + pagination_limit=page_params.limit, + ) @function_job_collections_router.get( From e71932a23547f945637a5698316e3ec40eef61b6 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Wed, 7 May 2025 16:48:48 +0200 Subject: [PATCH 50/69] Fix function pagination api tests --- .github/copilot-instructions.md | 53 --- scripts/common-service.Makefile | 3 +- services/api-server/Makefile | 2 +- services/api-server/openapi.json | 438 ++++++++++++++---- .../api/routes/functions_routes.py | 71 +-- .../test_api_routers_functions.py | 65 ++- 6 files changed, 453 insertions(+), 179 deletions(-) delete mode 100644 .github/copilot-instructions.md diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md deleted file mode 100644 index ebd8c6030a6..00000000000 --- a/.github/copilot-instructions.md +++ /dev/null @@ -1,53 +0,0 @@ -# GitHub Copilot Instructions - -This document provides guidelines and best practices for using GitHub Copilot in the `osparc-simcore` repository and other Python and Node.js projects. - -## General Guidelines - -1. **Use Python 3.11**: Ensure that all Python-related suggestions align with Python 3.11 features and syntax. -2. **Node.js Compatibility**: For Node.js projects, ensure compatibility with the version specified in the project (e.g., Node.js 14 or later). -3. **Follow Coding Conventions**: Adhere to the coding conventions outlined in the `docs/coding-conventions.md` file. -4. **Test-Driven Development**: Write unit tests for all new functions and features. Use `pytest` for Python and appropriate testing frameworks for Node.js. -5. **Environment Variables**: Use environment variables as specified in `docs/env-vars.md` for configuration. Avoid hardcoding sensitive information. -6. **Documentation**: Prefer self-explanatory code; add documentation only if explicitly requested by the developer. - -## Python-Specific Instructions - -- Always use type hints and annotations to improve code clarity and compatibility with tools like `mypy`. - - An exception to that rule is in `test_*` functions return type hint must not be added -- Follow the dependency management practices outlined in `requirements/`. -- Use `ruff` for code formatting and for linting. -- Use `black` for code formatting and `pylint` for linting. -- ensure we use `sqlalchemy` >2 compatible code. -- ensure we use `pydantic` >2 compatible code. -- ensure we use `fastapi` >0.100 compatible code -- use f-string formatting -- Only add comments in function if strictly necessary - - -### Json serialization - -- Generally use `json_dumps`/`json_loads` from `common_library.json_serialization` to built-in `json.dumps` / `json.loads`. -- Prefer Pydantic model methods (e.g., `model.model_dump_json()`) for serialization. - - -## Node.js-Specific Instructions - -- Use ES6+ syntax and features. -- Follow the `package.json` configuration for dependencies and scripts. -- Use `eslint` for linting and `prettier` for code formatting. -- Write modular and reusable code, adhering to the project's structure. - -## Copilot Usage Tips - -1. **Be Specific**: Provide clear and detailed prompts to Copilot for better suggestions. -2. **Iterate**: Review and refine Copilot's suggestions to ensure they meet project standards. -3. **Split Tasks**: Break down complex tasks into smaller, manageable parts for better suggestions. -4. **Test Suggestions**: Always test Copilot-generated code to ensure it works as expected. - -## Additional Resources - -- [Python Coding Conventions](../docs/coding-conventions.md) -- [Environment Variables Guide](../docs/env-vars.md) -- [Steps to Upgrade Python](../docs/steps-to-upgrade-python.md) -- [Node.js Installation Script](../scripts/install_nodejs_14.bash) diff --git a/scripts/common-service.Makefile b/scripts/common-service.Makefile index 57fb6e3b5b4..394f0a861ca 100644 --- a/scripts/common-service.Makefile +++ b/scripts/common-service.Makefile @@ -137,7 +137,7 @@ info: ## displays service info .PHONY: _run-test-dev _run-test-ci -TEST_TARGET := $(if $(target),$(target),$(CURDIR)/tests/unit) +# TEST_TARGET := $(if $(target),$(target),$(CURDIR)/tests/unit) PYTEST_ADDITIONAL_PARAMETERS := $(if $(pytest-parameters),$(pytest-parameters),) _run-test-dev: _check_venv_active # runs tests for development (e.g w/ pdb) @@ -153,7 +153,6 @@ _run-test-dev: _check_venv_active --failed-first \ --junitxml=junit.xml -o junit_family=legacy \ --keep-docker-up \ - --pdb \ -vv \ $(PYTEST_ADDITIONAL_PARAMETERS) \ $(TEST_TARGET) diff --git a/services/api-server/Makefile b/services/api-server/Makefile index e923de11db8..64ff30491ea 100644 --- a/services/api-server/Makefile +++ b/services/api-server/Makefile @@ -89,7 +89,7 @@ APP_URL:=http://$(get_my_ip).nip.io:8006 test-api: ## Runs schemathesis against development server (NOTE: make up-devel first) - @docker run schemathesis/schemathesis:stable run \ + @docker run schemathesis/schemathesis:stable run --experimental=openapi-3.1 \ "$(APP_URL)/api/v0/openapi.json" diff --git a/services/api-server/openapi.json b/services/api-server/openapi.json index d38dafb42bb..65f40210e36 100644 --- a/services/api-server/openapi.json +++ b/services/api-server/openapi.json @@ -5277,48 +5277,6 @@ } }, "/v0/functions": { - "get": { - "tags": [ - "functions" - ], - "summary": "List Functions", - "description": "List functions", - "operationId": "list_functions", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "items": { - "oneOf": [ - { - "$ref": "#/components/schemas/ProjectFunction" - }, - { - "$ref": "#/components/schemas/PythonCodeFunction" - }, - { - "$ref": "#/components/schemas/SolverFunction" - } - ], - "discriminator": { - "propertyName": "function_class", - "mapping": { - "project": "#/components/schemas/ProjectFunction", - "python_code": "#/components/schemas/PythonCodeFunction", - "solver": "#/components/schemas/SolverFunction" - } - } - }, - "type": "array", - "title": "Response List Functions V0 Functions Get" - } - } - } - } - } - }, "post": { "tags": [ "functions" @@ -5327,6 +5285,7 @@ "description": "Create function", "operationId": "register_function", "requestBody": { + "required": true, "content": { "application/json": { "schema": { @@ -5341,7 +5300,6 @@ "$ref": "#/components/schemas/SolverFunction" } ], - "title": "Function", "discriminator": { "propertyName": "function_class", "mapping": { @@ -5349,11 +5307,11 @@ "python_code": "#/components/schemas/PythonCodeFunction", "solver": "#/components/schemas/SolverFunction" } - } + }, + "title": "Function" } } - }, - "required": true + } }, "responses": { "200": { @@ -5372,7 +5330,6 @@ "$ref": "#/components/schemas/SolverFunction" } ], - "title": "Response Register Function V0 Functions Post", "discriminator": { "propertyName": "function_class", "mapping": { @@ -5380,7 +5337,8 @@ "python_code": "#/components/schemas/PythonCodeFunction", "solver": "#/components/schemas/SolverFunction" } - } + }, + "title": "Response Register Function V0 Functions Post" } } } @@ -5406,6 +5364,61 @@ } } } + }, + "get": { + "tags": [ + "functions" + ], + "summary": "List Functions", + "description": "List functions", + "operationId": "list_functions", + "parameters": [ + { + "name": "limit", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "maximum": 50, + "minimum": 1, + "default": 20, + "title": "Limit" + } + }, + { + "name": "offset", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "minimum": 0, + "default": 0, + "title": "Offset" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Page_Annotated_Union_ProjectFunction__PythonCodeFunction__SolverFunction___FieldInfo_annotation_NoneType__required_True__discriminator__function_class____" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } } }, "/v0/functions/{function_id}": { @@ -5902,35 +5915,48 @@ "summary": "List Function Jobs", "description": "List function jobs", "operationId": "list_function_jobs", + "parameters": [ + { + "name": "limit", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "maximum": 50, + "minimum": 1, + "default": 20, + "title": "Limit" + } + }, + { + "name": "offset", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "minimum": 0, + "default": 0, + "title": "Offset" + } + } + ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { - "items": { - "oneOf": [ - { - "$ref": "#/components/schemas/ProjectFunctionJob" - }, - { - "$ref": "#/components/schemas/PythonCodeFunctionJob" - }, - { - "$ref": "#/components/schemas/SolverFunctionJob" - } - ], - "discriminator": { - "propertyName": "function_class", - "mapping": { - "project": "#/components/schemas/ProjectFunctionJob", - "python_code": "#/components/schemas/PythonCodeFunctionJob", - "solver": "#/components/schemas/SolverFunctionJob" - } - } - }, - "type": "array", - "title": "Response List Function Jobs V0 Function Jobs Get" + "$ref": "#/components/schemas/Page_Annotated_Union_ProjectFunctionJob__PythonCodeFunctionJob__SolverFunctionJob___FieldInfo_annotation_NoneType__required_True__discriminator__function_class____" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" } } } @@ -5945,6 +5971,7 @@ "description": "Create function job", "operationId": "register_function_job", "requestBody": { + "required": true, "content": { "application/json": { "schema": { @@ -5959,7 +5986,6 @@ "$ref": "#/components/schemas/SolverFunctionJob" } ], - "title": "Function Job", "discriminator": { "propertyName": "function_class", "mapping": { @@ -5967,11 +5993,11 @@ "python_code": "#/components/schemas/PythonCodeFunctionJob", "solver": "#/components/schemas/SolverFunctionJob" } - } + }, + "title": "Function Job" } } - }, - "required": true + } }, "responses": { "200": { @@ -5990,7 +6016,6 @@ "$ref": "#/components/schemas/SolverFunctionJob" } ], - "title": "Response Register Function Job V0 Function Jobs Post", "discriminator": { "propertyName": "function_class", "mapping": { @@ -5998,7 +6023,8 @@ "python_code": "#/components/schemas/PythonCodeFunctionJob", "solver": "#/components/schemas/SolverFunctionJob" } - } + }, + "title": "Response Register Function Job V0 Function Jobs Post" } } } @@ -6273,17 +6299,48 @@ "summary": "List Function Job Collections", "description": "List function job collections", "operationId": "list_function_job_collections", + "parameters": [ + { + "name": "limit", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "maximum": 50, + "minimum": 1, + "default": 20, + "title": "Limit" + } + }, + { + "name": "offset", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "minimum": 0, + "default": 0, + "title": "Offset" + } + } + ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { - "items": { - "$ref": "#/components/schemas/FunctionJobCollection" - }, - "type": "array", - "title": "Response List Function Job Collections V0 Function Job Collections Get" + "$ref": "#/components/schemas/Page_FunctionJobCollection_" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" } } } @@ -6298,14 +6355,14 @@ "description": "Register function job collection", "operationId": "register_function_job_collection", "requestBody": { + "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/FunctionJobCollection" } } - }, - "required": true + } }, "responses": { "200": { @@ -8710,6 +8767,160 @@ ], "title": "OnePage[StudyPort]" }, + "Page_Annotated_Union_ProjectFunctionJob__PythonCodeFunctionJob__SolverFunctionJob___FieldInfo_annotation_NoneType__required_True__discriminator__function_class____": { + "properties": { + "items": { + "items": { + "oneOf": [ + { + "$ref": "#/components/schemas/ProjectFunctionJob" + }, + { + "$ref": "#/components/schemas/PythonCodeFunctionJob" + }, + { + "$ref": "#/components/schemas/SolverFunctionJob" + } + ], + "discriminator": { + "propertyName": "function_class", + "mapping": { + "project": "#/components/schemas/ProjectFunctionJob", + "python_code": "#/components/schemas/PythonCodeFunctionJob", + "solver": "#/components/schemas/SolverFunctionJob" + } + } + }, + "type": "array", + "title": "Items" + }, + "total": { + "anyOf": [ + { + "type": "integer", + "minimum": 0 + }, + { + "type": "null" + } + ], + "title": "Total" + }, + "limit": { + "anyOf": [ + { + "type": "integer", + "minimum": 1 + }, + { + "type": "null" + } + ], + "title": "Limit" + }, + "offset": { + "anyOf": [ + { + "type": "integer", + "minimum": 0 + }, + { + "type": "null" + } + ], + "title": "Offset" + }, + "links": { + "$ref": "#/components/schemas/Links" + } + }, + "type": "object", + "required": [ + "items", + "total", + "limit", + "offset", + "links" + ], + "title": "Page[Annotated[Union[ProjectFunctionJob, PythonCodeFunctionJob, SolverFunctionJob], FieldInfo(annotation=NoneType, required=True, discriminator='function_class')]]" + }, + "Page_Annotated_Union_ProjectFunction__PythonCodeFunction__SolverFunction___FieldInfo_annotation_NoneType__required_True__discriminator__function_class____": { + "properties": { + "items": { + "items": { + "oneOf": [ + { + "$ref": "#/components/schemas/ProjectFunction" + }, + { + "$ref": "#/components/schemas/PythonCodeFunction" + }, + { + "$ref": "#/components/schemas/SolverFunction" + } + ], + "discriminator": { + "propertyName": "function_class", + "mapping": { + "project": "#/components/schemas/ProjectFunction", + "python_code": "#/components/schemas/PythonCodeFunction", + "solver": "#/components/schemas/SolverFunction" + } + } + }, + "type": "array", + "title": "Items" + }, + "total": { + "anyOf": [ + { + "type": "integer", + "minimum": 0 + }, + { + "type": "null" + } + ], + "title": "Total" + }, + "limit": { + "anyOf": [ + { + "type": "integer", + "minimum": 1 + }, + { + "type": "null" + } + ], + "title": "Limit" + }, + "offset": { + "anyOf": [ + { + "type": "integer", + "minimum": 0 + }, + { + "type": "null" + } + ], + "title": "Offset" + }, + "links": { + "$ref": "#/components/schemas/Links" + } + }, + "type": "object", + "required": [ + "items", + "total", + "limit", + "offset", + "links" + ], + "title": "Page[Annotated[Union[ProjectFunction, PythonCodeFunction, SolverFunction], FieldInfo(annotation=NoneType, required=True, discriminator='function_class')]]" + }, "Page_File_": { "properties": { "items": { @@ -8769,6 +8980,65 @@ ], "title": "Page[File]" }, + "Page_FunctionJobCollection_": { + "properties": { + "items": { + "items": { + "$ref": "#/components/schemas/FunctionJobCollection" + }, + "type": "array", + "title": "Items" + }, + "total": { + "anyOf": [ + { + "type": "integer", + "minimum": 0 + }, + { + "type": "null" + } + ], + "title": "Total" + }, + "limit": { + "anyOf": [ + { + "type": "integer", + "minimum": 1 + }, + { + "type": "null" + } + ], + "title": "Limit" + }, + "offset": { + "anyOf": [ + { + "type": "integer", + "minimum": 0 + }, + { + "type": "null" + } + ], + "title": "Offset" + }, + "links": { + "$ref": "#/components/schemas/Links" + } + }, + "type": "object", + "required": [ + "items", + "total", + "limit", + "offset", + "links" + ], + "title": "Page[FunctionJobCollection]" + }, "Page_Job_": { "properties": { "items": { diff --git a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py index 614119542b2..a03fa39fbdf 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py @@ -29,7 +29,7 @@ from sqlalchemy.ext.asyncio import AsyncEngine from ..._service_solvers import SolverService -from ...models.pagination import PaginationParams +from ...models.pagination import Page, PaginationParams from ...models.schemas.errors import ErrorGet from ...models.schemas.jobs import ( JobInputs, @@ -85,7 +85,7 @@ async def get_function( return await wb_api_rpc.get_function(function_id=function_id) -@function_router.get("", response_model=list[Function], description="List functions") +@function_router.get("", response_model=Page[Function], description="List functions") async def list_functions( wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], page_params: Annotated[PaginationParams, Depends()], @@ -102,6 +102,45 @@ async def list_functions( ) +@function_job_router.get( + "", response_model=Page[FunctionJob], description="List function jobs" +) +async def list_function_jobs( + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], + page_params: Annotated[PaginationParams, Depends()], +): + function_jobs_list, meta = await wb_api_rpc.list_function_jobs( + pagination_offset=page_params.offset, + pagination_limit=page_params.limit, + ) + + return create_page( + function_jobs_list, + total=meta.total, + params=page_params, + ) + + +@function_job_collections_router.get( + "", + response_model=Page[FunctionJobCollection], + description="List function job collections", +) +async def list_function_job_collections( + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], + page_params: Annotated[PaginationParams, Depends()], +): + function_job_collection_list, meta = await wb_api_rpc.list_function_job_collections( + pagination_offset=page_params.offset, + pagination_limit=page_params.limit, + ) + return create_page( + function_job_collection_list, + total=meta.total, + params=page_params, + ) + + def join_inputs( default_inputs: FunctionInputs | None, function_inputs: FunctionInputs | None, @@ -329,19 +368,6 @@ async def get_function_job( return await wb_api_rpc.get_function_job(function_job_id=function_job_id) -@function_job_router.get( - "", response_model=list[FunctionJob], description="List function jobs" -) -async def list_function_jobs( - wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], - page_params: Annotated[PaginationParams, Depends()], -): - return await wb_api_rpc.list_function_jobs( - pagination_offset=page_params.offset, - pagination_limit=page_params.limit, - ) - - @function_job_router.delete( "/{function_job_id:uuid}", response_model=None, @@ -524,21 +550,6 @@ async def map_function( # noqa: PLR0913 } -@function_job_collections_router.get( - "", - response_model=list[FunctionJobCollection], - description="List function job collections", -) -async def list_function_job_collections( - wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], - page_params: Annotated[PaginationParams, Depends()], -): - return await wb_api_rpc.list_function_job_collections( - pagination_offset=page_params.offset, - pagination_limit=page_params.limit, - ) - - @function_job_collections_router.get( "/{function_job_collection_id:uuid}", response_model=FunctionJobCollection, diff --git a/services/api-server/tests/unit/api_functions/test_api_routers_functions.py b/services/api-server/tests/unit/api_functions/test_api_routers_functions.py index 21ac2be2a90..cfb993093b0 100644 --- a/services/api-server/tests/unit/api_functions/test_api_routers_functions.py +++ b/services/api-server/tests/unit/api_functions/test_api_routers_functions.py @@ -4,11 +4,15 @@ import pytest from fastapi import FastAPI, HTTPException from fastapi.testclient import TestClient +from fastapi_pagination import add_pagination from models_library.api_schemas_webserver.functions_wb_schema import ( Function, FunctionJob, FunctionJobCollection, ) +from models_library.rest_pagination import ( + PageMetaInfoLimitOffset, +) from pydantic import TypeAdapter from simcore_service_api_server.api.routes.functions_routes import ( function_job_collections_router, @@ -28,6 +32,7 @@ def _api_app() -> FastAPI: fastapi_app.include_router( function_job_collections_router, prefix="/function_job_collections" ) + add_pagination(fastapi_app) # Mock authentication dependency async def mock_auth_dependency() -> int: @@ -89,9 +94,23 @@ async def run_function(self, function_id: str, inputs: dict) -> dict: ) return {"status": "success", "function_id": function_id, "inputs": inputs} - async def list_functions(self) -> list: + async def list_functions( + self, + pagination_offset: int, + pagination_limit: int, + ) -> tuple[list[Function], PageMetaInfoLimitOffset]: # Mimic listing all functions - return list(self._functions.values()) + functions_list = list(self._functions.values())[ + pagination_offset : pagination_offset + pagination_limit + ] + total_count = len(self._functions) + page_meta_info = PageMetaInfoLimitOffset( + total=total_count, + limit=pagination_limit, + offset=pagination_offset, + count=len(functions_list), + ) + return functions_list, page_meta_info async def delete_function(self, function_id: str) -> None: # Mimic deleting a function @@ -125,9 +144,23 @@ async def get_function_job(self, function_job_id: str) -> dict: raise HTTPException(status_code=404, detail="Function job not found") return self._function_jobs[function_job_id] - async def list_function_jobs(self) -> list: + async def list_function_jobs( + self, + pagination_offset: int, + pagination_limit: int, + ) -> tuple[list[FunctionJob], PageMetaInfoLimitOffset]: # Mimic listing all function jobs - return list(self._function_jobs.values()) + function_jobs_list = list(self._function_jobs.values())[ + pagination_offset : pagination_offset + pagination_limit + ] + total_count = len(self._function_jobs) + page_meta_info = PageMetaInfoLimitOffset( + total=total_count, + limit=pagination_limit, + offset=pagination_offset, + count=len(function_jobs_list), + ) + return function_jobs_list, page_meta_info async def delete_function_job(self, function_job_id: str) -> None: # Mimic deleting a function job @@ -163,9 +196,23 @@ async def get_function_job_collection( ) return self._function_job_collections[function_job_collection_id] - async def list_function_job_collections(self) -> list: + async def list_function_job_collections( + self, + pagination_offset: int, + pagination_limit: int, + ) -> tuple[list[FunctionJobCollection], PageMetaInfoLimitOffset]: # Mimic listing all function job collections - return list(self._function_job_collections.values()) + function_job_collections_list = list(self._function_job_collections.values())[ + pagination_offset : pagination_offset + pagination_limit + ] + total_count = len(self._function_job_collections) + page_meta_info = PageMetaInfoLimitOffset( + total=total_count, + limit=pagination_limit, + offset=pagination_offset, + count=len(function_job_collections_list), + ) + return function_job_collections_list, page_meta_info async def delete_function_job_collection( self, function_job_collection_id: str @@ -279,9 +326,9 @@ def test_list_functions(api_app: FastAPI) -> None: assert post_response.status_code == 200 # List functions - response = client.get("/functions") + response = client.get("/functions", params={"limit": 10, "offset": 0}) assert response.status_code == 200 - data = response.json() + data = response.json()["items"] assert len(data) > 0 assert data[0]["title"] == sample_function["title"] @@ -488,7 +535,7 @@ def test_list_function_jobs(api_app: FastAPI) -> None: # Now, list function jobs response = client.get("/function_jobs") assert response.status_code == 200 - data = response.json() + data = response.json()["items"] assert len(data) > 0 assert data[0]["title"] == mock_function_job["title"] From af697f21cec98794c21db908c3ce2aecea441bab Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Wed, 7 May 2025 17:03:34 +0200 Subject: [PATCH 51/69] Fix pagination of functions again --- .../webserver/functions/functions_rpc_interface.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py index 56987fc8693..41a020124e2 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py @@ -94,7 +94,8 @@ async def delete_function( @log_decorator(_logger, level=logging.DEBUG) async def list_functions( rabbitmq_rpc_client: RabbitMQRPCClient, - *pagination_limit: int, + *, + pagination_limit: int, pagination_offset: int, ) -> tuple[list[Function], PageMetaInfoLimitOffset]: return await rabbitmq_rpc_client.request( From bede7bd06f1082b0b0f456a7496a82cbacdb136b Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Thu, 8 May 2025 08:46:59 +0200 Subject: [PATCH 52/69] Restore some files from master --- .github/copilot-instructions.md | 53 +++++++++++++++++++++++++++++++++ scripts/common-service.Makefile | 3 +- services/api-server/Makefile | 2 +- 3 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 .github/copilot-instructions.md diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 00000000000..ebd8c6030a6 --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,53 @@ +# GitHub Copilot Instructions + +This document provides guidelines and best practices for using GitHub Copilot in the `osparc-simcore` repository and other Python and Node.js projects. + +## General Guidelines + +1. **Use Python 3.11**: Ensure that all Python-related suggestions align with Python 3.11 features and syntax. +2. **Node.js Compatibility**: For Node.js projects, ensure compatibility with the version specified in the project (e.g., Node.js 14 or later). +3. **Follow Coding Conventions**: Adhere to the coding conventions outlined in the `docs/coding-conventions.md` file. +4. **Test-Driven Development**: Write unit tests for all new functions and features. Use `pytest` for Python and appropriate testing frameworks for Node.js. +5. **Environment Variables**: Use environment variables as specified in `docs/env-vars.md` for configuration. Avoid hardcoding sensitive information. +6. **Documentation**: Prefer self-explanatory code; add documentation only if explicitly requested by the developer. + +## Python-Specific Instructions + +- Always use type hints and annotations to improve code clarity and compatibility with tools like `mypy`. + - An exception to that rule is in `test_*` functions return type hint must not be added +- Follow the dependency management practices outlined in `requirements/`. +- Use `ruff` for code formatting and for linting. +- Use `black` for code formatting and `pylint` for linting. +- ensure we use `sqlalchemy` >2 compatible code. +- ensure we use `pydantic` >2 compatible code. +- ensure we use `fastapi` >0.100 compatible code +- use f-string formatting +- Only add comments in function if strictly necessary + + +### Json serialization + +- Generally use `json_dumps`/`json_loads` from `common_library.json_serialization` to built-in `json.dumps` / `json.loads`. +- Prefer Pydantic model methods (e.g., `model.model_dump_json()`) for serialization. + + +## Node.js-Specific Instructions + +- Use ES6+ syntax and features. +- Follow the `package.json` configuration for dependencies and scripts. +- Use `eslint` for linting and `prettier` for code formatting. +- Write modular and reusable code, adhering to the project's structure. + +## Copilot Usage Tips + +1. **Be Specific**: Provide clear and detailed prompts to Copilot for better suggestions. +2. **Iterate**: Review and refine Copilot's suggestions to ensure they meet project standards. +3. **Split Tasks**: Break down complex tasks into smaller, manageable parts for better suggestions. +4. **Test Suggestions**: Always test Copilot-generated code to ensure it works as expected. + +## Additional Resources + +- [Python Coding Conventions](../docs/coding-conventions.md) +- [Environment Variables Guide](../docs/env-vars.md) +- [Steps to Upgrade Python](../docs/steps-to-upgrade-python.md) +- [Node.js Installation Script](../scripts/install_nodejs_14.bash) diff --git a/scripts/common-service.Makefile b/scripts/common-service.Makefile index 394f0a861ca..57fb6e3b5b4 100644 --- a/scripts/common-service.Makefile +++ b/scripts/common-service.Makefile @@ -137,7 +137,7 @@ info: ## displays service info .PHONY: _run-test-dev _run-test-ci -# TEST_TARGET := $(if $(target),$(target),$(CURDIR)/tests/unit) +TEST_TARGET := $(if $(target),$(target),$(CURDIR)/tests/unit) PYTEST_ADDITIONAL_PARAMETERS := $(if $(pytest-parameters),$(pytest-parameters),) _run-test-dev: _check_venv_active # runs tests for development (e.g w/ pdb) @@ -153,6 +153,7 @@ _run-test-dev: _check_venv_active --failed-first \ --junitxml=junit.xml -o junit_family=legacy \ --keep-docker-up \ + --pdb \ -vv \ $(PYTEST_ADDITIONAL_PARAMETERS) \ $(TEST_TARGET) diff --git a/services/api-server/Makefile b/services/api-server/Makefile index 64ff30491ea..e923de11db8 100644 --- a/services/api-server/Makefile +++ b/services/api-server/Makefile @@ -89,7 +89,7 @@ APP_URL:=http://$(get_my_ip).nip.io:8006 test-api: ## Runs schemathesis against development server (NOTE: make up-devel first) - @docker run schemathesis/schemathesis:stable run --experimental=openapi-3.1 \ + @docker run schemathesis/schemathesis:stable run \ "$(APP_URL)/api/v0/openapi.json" From f772f04b63efde6f157b54d775232f7f67183bcb Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Thu, 8 May 2025 09:33:02 +0200 Subject: [PATCH 53/69] Fix pylint --- .../functions/_functions_controller_rpc.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py index fa14c220d1f..0909f5e68b3 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py @@ -29,6 +29,8 @@ router = RPCRouter() +# pylint: disable=no-else-return + @router.expose() async def ping(app: web.Application) -> str: From fae85530ca500932af91e00a3cbc6dc5aee46125 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Thu, 8 May 2025 10:48:07 +0200 Subject: [PATCH 54/69] Changes based on Mads comments wrt functions api --- .../functions_wb_schema.py | 51 +++++++++++++++++ .../functions/_functions_controller_rpc.py | 29 +++++----- .../functions/_functions_repository.py | 56 +++++++++++-------- .../test_functions_controller_rpc.py | 13 +++-- 4 files changed, 104 insertions(+), 45 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py b/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py index ad005fd8bc2..d237413f8f3 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py @@ -17,6 +17,8 @@ class FunctionSchema(BaseModel): + """Schema for function input/output""" + schema_dict: dict[str, Any] | None # JSON Schema @@ -163,3 +165,52 @@ class FunctionJobCollectionDB(BaseModel): class FunctionJobCollectionStatus(BaseModel): status: list[str] + + +class FunctionNotFoundError(Exception): + """Exception raised when a function is not found""" + + def __init__(self, function_id: FunctionID): + self.function_id = function_id + super().__init__(f"Function {function_id} not found") + + +class FunctionJobNotFoundError(Exception): + """Exception raised when a function job is not found""" + + def __init__(self, function_job_id: FunctionJobID): + self.function_job_id = function_job_id + super().__init__(f"Function job {function_job_id} not found") + + +class FunctionJobCollectionNotFoundError(Exception): + """Exception raised when a function job collection is not found""" + + def __init__(self, function_job_collection_id: FunctionJobCollectionID): + self.function_job_collection_id = function_job_collection_id + super().__init__( + f"Function job collection {function_job_collection_id} not found" + ) + + +class RegisterFunctionWithUIDError(Exception): + """Exception raised when registering a function with a UID""" + + def __init__(self): + super().__init__("Cannot register Function with a UID") + + +class UnsupportedFunctionClassError(Exception): + """Exception raised when a function class is not supported""" + + def __init__(self, function_class: str): + self.function_class = function_class + super().__init__(f"Function class {function_class} is not supported") + + +class UnsupportedFunctionJobClassError(Exception): + """Exception raised when a function job class is not supported""" + + def __init__(self, function_job_class: str): + self.function_job_class = function_job_class + super().__init__(f"Function job class {function_job_class} is not supported") diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py index 0909f5e68b3..6d43c8da66a 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py @@ -18,6 +18,8 @@ ProjectFunctionJob, SolverFunction, SolverFunctionJob, + UnsupportedFunctionClassError, + UnsupportedFunctionJobClassError, ) from models_library.rest_pagination import ( PageMetaInfoLimitOffset, @@ -32,12 +34,6 @@ # pylint: disable=no-else-return -@router.expose() -async def ping(app: web.Application) -> str: - assert app - return "pong from webserver" - - @router.expose() async def register_function(app: web.Application, *, function: Function) -> Function: assert app @@ -72,8 +68,7 @@ def _decode_function( default_inputs=function.default_inputs, ) else: - msg = f"Unsupported function class: [{function.function_class}]" - raise TypeError(msg) + raise UnsupportedFunctionClassError(function_class=function.function_class) def _encode_function( @@ -91,8 +86,7 @@ def _encode_function( } ) else: - msg = f"Unsupported function class: {function.function_class}" - raise TypeError(msg) + raise UnsupportedFunctionClassError(function_class=function.function_class) return FunctionDB( uuid=function.uid, @@ -142,8 +136,9 @@ def _decode_functionjob( solver_job_id=functionjob_db.class_specific_data["solver_job_id"], ) else: - msg = f"Unsupported function class: [{functionjob_db.function_class}]" - raise TypeError(msg) + raise UnsupportedFunctionJobClassError( + function_job_class=functionjob_db.function_class + ) def _encode_functionjob( @@ -178,8 +173,9 @@ def _encode_functionjob( function_class=functionjob.function_class, ) else: - msg = f"Unsupported function class: [{functionjob.function_class}]" - raise TypeError(msg) + raise UnsupportedFunctionJobClassError( + function_job_class=functionjob.function_class + ) @router.expose() @@ -355,8 +351,9 @@ async def find_cached_function_job( solver_job_id=returned_function_job.class_specific_data["solver_job_id"], ) else: - msg = f"Unsupported function class: [{returned_function_job.function_class}]" - raise TypeError(msg) + raise UnsupportedFunctionJobClassError( + function_job_class=returned_function_job.function_class + ) @router.expose() diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py index 5d8e7252064..1de44feb96b 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py @@ -7,8 +7,12 @@ FunctionInputs, FunctionJobCollection, FunctionJobCollectionDB, + FunctionJobCollectionNotFoundError, FunctionJobDB, FunctionJobID, + FunctionJobNotFoundError, + FunctionNotFoundError, + RegisterFunctionWithUIDError, ) from models_library.rest_pagination import ( PageMetaInfoLimitOffset, @@ -52,8 +56,7 @@ async def register_function( ) -> FunctionDB: if function.uuid is not None: - msg = "Function uid is not None. Cannot register function." - raise ValueError(msg) + raise RegisterFunctionWithUIDError async with transaction_context(get_asyncpg_engine(app), connection) as conn: result = await conn.stream( @@ -79,9 +82,10 @@ async def register_function( ) row = await result.first() - if row is None: - msg = "No row was returned from the database after creating function." - raise ValueError(msg) + assert row is not None, ( + "No row was returned from the database after creating function." + f" Function: {function}" + ) # nosec return FunctionDB.model_validate(dict(row)) @@ -100,9 +104,7 @@ async def get_function( row = await result.first() if row is None: - msg = f"No function found with id {function_id}." - raise web.HTTPNotFound(reason=msg) - + raise FunctionNotFoundError(function_id=function_id) return FunctionDB.model_validate(dict(row)) @@ -258,9 +260,10 @@ async def register_function_job( ) row = await result.first() - if row is None: - msg = "No row was returned from the database after creating function job." - raise ValueError(msg) + assert row is not None, ( + "No row was returned from the database after creating function job." + f" Function job: {function_job}" + ) # nosec return FunctionJobDB.model_validate(dict(row)) @@ -281,8 +284,7 @@ async def get_function_job( row = await result.first() if row is None: - msg = f"No function job found with id {function_job_id}." - raise web.HTTPNotFound(reason=msg) + raise FunctionJobNotFoundError(function_job_id=function_job_id) return FunctionJobDB.model_validate(dict(row)) @@ -319,13 +321,19 @@ async def find_cached_function_job( rows = await result.all() - if rows is None: + if rows is None or len(rows) == 0: return None - for row in rows: - job = FunctionJobDB.model_validate(dict(row)) - if job.inputs == inputs: - return job + assert len(rows) == 1, ( + "More than one function job found with the same function id and inputs." + f" Function id: {function_id}, Inputs: {inputs}" + ) # nosec + + row = rows[0] + + job = FunctionJobDB.model_validate(dict(row)) + if job.inputs == inputs: + return job return None @@ -346,8 +354,9 @@ async def get_function_job_collection( row = await result.first() if row is None: - msg = f"No function job collection found with id {function_job_collection_id}." - raise web.HTTPNotFound(reason=msg) + raise FunctionJobCollectionNotFoundError( + function_job_collection_id=function_job_collection_id + ) # Retrieve associated job ids from the join table job_result = await conn.stream( @@ -383,9 +392,10 @@ async def register_function_job_collection( ) row = await result.first() - if row is None: - msg = "No row was returned from the database after creating function job collection." - raise ValueError(msg) + assert row is not None, ( + "No row was returned from the database after creating function job collection." + f" Function job collection: {function_job_collection}" + ) # nosec for job_id in function_job_collection.job_ids: await conn.execute( diff --git a/services/web/server/tests/unit/with_dbs/functions_rpc/test_functions_controller_rpc.py b/services/web/server/tests/unit/with_dbs/functions_rpc/test_functions_controller_rpc.py index 403fe959374..2a5122268f0 100644 --- a/services/web/server/tests/unit/with_dbs/functions_rpc/test_functions_controller_rpc.py +++ b/services/web/server/tests/unit/with_dbs/functions_rpc/test_functions_controller_rpc.py @@ -3,11 +3,12 @@ import pytest import simcore_service_webserver.functions._functions_controller_rpc as functions_rpc -from aiohttp import web from models_library.api_schemas_webserver.functions_wb_schema import ( Function, FunctionInputSchema, FunctionJobCollection, + FunctionJobNotFoundError, + FunctionNotFoundError, FunctionOutputSchema, ProjectFunction, ProjectFunctionJob, @@ -89,7 +90,7 @@ async def test_get_function(client, mock_function): @pytest.mark.asyncio async def test_get_function_not_found(client): # Attempt to retrieve a function that does not exist - with pytest.raises(web.HTTPNotFound): + with pytest.raises(FunctionNotFoundError): await functions_rpc.get_function(app=client.app, function_id=uuid4()) @@ -225,7 +226,7 @@ async def test_delete_function(client, mock_function): ) # Attempt to retrieve the deleted function - with pytest.raises(web.HTTPNotFound): + with pytest.raises(FunctionNotFoundError): await functions_rpc.get_function( app=client.app, function_id=registered_function.uid ) @@ -298,7 +299,7 @@ async def test_get_function_job(client, mock_function): @pytest.mark.asyncio async def test_get_function_job_not_found(client): # Attempt to retrieve a function job that does not exist - with pytest.raises(web.HTTPNotFound): + with pytest.raises(FunctionJobNotFoundError): await functions_rpc.get_function_job(app=client.app, function_job_id=uuid4()) @@ -365,7 +366,7 @@ async def test_delete_function_job(client, mock_function): ) # Attempt to retrieve the deleted job - with pytest.raises(web.HTTPNotFound): + with pytest.raises(FunctionJobNotFoundError): await functions_rpc.get_function_job( app=client.app, function_job_id=registered_job.uid ) @@ -427,7 +428,7 @@ async def test_function_job_collection(client, mock_function): app=client.app, function_job_collection_id=registered_collection.uid ) # Attempt to retrieve the deleted collection - with pytest.raises(web.HTTPNotFound): + with pytest.raises(FunctionJobNotFoundError): await functions_rpc.get_function_job( app=client.app, function_job_id=registered_collection.uid ) From a8fbdf12ae7fe6e0becd7ac481d6a7ea214fb0e5 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Thu, 8 May 2025 14:55:45 +0200 Subject: [PATCH 55/69] Mention explicit exceptions in functions rpc --- .../functions_wb_schema.py | 22 +- .../functions/_functions_controller_rpc.py | 411 +++++++++--------- .../functions/_functions_repository.py | 49 ++- .../test_functions_controller_rpc.py | 100 +---- 4 files changed, 278 insertions(+), 304 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py b/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py index d237413f8f3..806201cc4de 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py @@ -167,7 +167,7 @@ class FunctionJobCollectionStatus(BaseModel): status: list[str] -class FunctionNotFoundError(Exception): +class FunctionIDNotFoundError(Exception): """Exception raised when a function is not found""" def __init__(self, function_id: FunctionID): @@ -175,7 +175,7 @@ def __init__(self, function_id: FunctionID): super().__init__(f"Function {function_id} not found") -class FunctionJobNotFoundError(Exception): +class FunctionJobIDNotFoundError(Exception): """Exception raised when a function job is not found""" def __init__(self, function_job_id: FunctionJobID): @@ -183,7 +183,7 @@ def __init__(self, function_job_id: FunctionJobID): super().__init__(f"Function job {function_job_id} not found") -class FunctionJobCollectionNotFoundError(Exception): +class FunctionJobCollectionIDNotFoundError(Exception): """Exception raised when a function job collection is not found""" def __init__(self, function_job_collection_id: FunctionJobCollectionID): @@ -193,13 +193,27 @@ def __init__(self, function_job_collection_id: FunctionJobCollectionID): ) -class RegisterFunctionWithUIDError(Exception): +class RegisterFunctionWithIDError(Exception): """Exception raised when registering a function with a UID""" def __init__(self): super().__init__("Cannot register Function with a UID") +class RegisterFunctionJobWithIDError(Exception): + """Exception raised when registering a function job with a UID""" + + def __init__(self): + super().__init__("Cannot register FunctionJob with a UID") + + +class RegisterFunctionJobCollectionWithIDError(Exception): + """Exception raised when registering a function job collection with a UID""" + + def __init__(self): + super().__init__("Cannot register FunctionJobCollection with a UID") + + class UnsupportedFunctionClassError(Exception): """Exception raised when a function class is not supported""" diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py index 6d43c8da66a..5e5cb4d7155 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py @@ -6,16 +6,22 @@ FunctionClassSpecificData, FunctionDB, FunctionID, + FunctionIDNotFoundError, FunctionInputs, FunctionInputSchema, FunctionJob, FunctionJobClassSpecificData, FunctionJobCollection, + FunctionJobCollectionIDNotFoundError, FunctionJobDB, FunctionJobID, + FunctionJobIDNotFoundError, FunctionOutputSchema, ProjectFunction, ProjectFunctionJob, + RegisterFunctionJobCollectionWithIDError, + RegisterFunctionJobWithIDError, + RegisterFunctionWithIDError, SolverFunction, SolverFunctionJob, UnsupportedFunctionClassError, @@ -34,7 +40,9 @@ # pylint: disable=no-else-return -@router.expose() +@router.expose( + reraise_if_error_type=(UnsupportedFunctionClassError, RegisterFunctionWithIDError) +) async def register_function(app: web.Application, *, function: Function) -> Function: assert app saved_function = await _functions_repository.register_function( @@ -43,64 +51,42 @@ async def register_function(app: web.Application, *, function: Function) -> Func return _decode_function(saved_function) -def _decode_function( - function: FunctionDB, -) -> Function: - if function.function_class == "project": - return ProjectFunction( - uid=function.uuid, - title=function.title, - description=function.description, - input_schema=function.input_schema, - output_schema=function.output_schema, - project_id=function.class_specific_data["project_id"], - default_inputs=function.default_inputs, - ) - elif function.function_class == "solver": # noqa: RET505 - return SolverFunction( - uid=function.uuid, - title=function.title, - description=function.description, - input_schema=function.input_schema, - output_schema=function.output_schema, - solver_key=function.class_specific_data["solver_key"], - solver_version=function.class_specific_data["solver_version"], - default_inputs=function.default_inputs, - ) - else: - raise UnsupportedFunctionClassError(function_class=function.function_class) +@router.expose( + reraise_if_error_type=( + UnsupportedFunctionJobClassError, + RegisterFunctionJobWithIDError, + ) +) +async def register_function_job( + app: web.Application, *, function_job: FunctionJob +) -> FunctionJob: + assert app + created_function_job_db = await _functions_repository.register_function_job( + app=app, function_job=_encode_functionjob(function_job) + ) + return _decode_functionjob(created_function_job_db) -def _encode_function( - function: Function, -) -> FunctionDB: - if function.function_class == FunctionClass.project: - class_specific_data = FunctionClassSpecificData( - {"project_id": str(function.project_id)} - ) - elif function.function_class == FunctionClass.solver: - class_specific_data = FunctionClassSpecificData( - { - "solver_key": str(function.solver_key), - "solver_version": str(function.solver_version), - } +@router.expose(reraise_if_error_type=(RegisterFunctionJobCollectionWithIDError,)) +async def register_function_job_collection( + app: web.Application, *, function_job_collection: FunctionJobCollection +) -> FunctionJobCollection: + assert app + registered_function_job_collection, registered_job_ids = ( + await _functions_repository.register_function_job_collection( + app=app, + function_job_collection=function_job_collection, ) - else: - raise UnsupportedFunctionClassError(function_class=function.function_class) - - return FunctionDB( - uuid=function.uid, - title=function.title, - description=function.description, - input_schema=function.input_schema, - output_schema=function.output_schema, - function_class=function.function_class, - default_inputs=function.default_inputs, - class_specific_data=class_specific_data, + ) + return FunctionJobCollection( + uid=registered_function_job_collection.uuid, + title=registered_function_job_collection.title, + description=registered_function_job_collection.description, + job_ids=registered_job_ids, ) -@router.expose() +@router.expose(reraise_if_error_type=(FunctionIDNotFoundError,)) async def get_function(app: web.Application, *, function_id: FunctionID) -> Function: assert app returned_function = await _functions_repository.get_function( @@ -112,73 +98,7 @@ async def get_function(app: web.Application, *, function_id: FunctionID) -> Func ) -def _decode_functionjob( - functionjob_db: FunctionJobDB, -) -> FunctionJob: - if functionjob_db.function_class == FunctionClass.project: - return ProjectFunctionJob( - uid=functionjob_db.uuid, - title=functionjob_db.title, - description="", - function_uid=functionjob_db.function_uuid, - inputs=functionjob_db.inputs, - outputs=functionjob_db.outputs, - project_job_id=functionjob_db.class_specific_data["project_job_id"], - ) - elif functionjob_db.function_class == FunctionClass.solver: # noqa: RET505 - return SolverFunctionJob( - uid=functionjob_db.uuid, - title=functionjob_db.title, - description="", - function_uid=functionjob_db.function_uuid, - inputs=functionjob_db.inputs, - outputs=functionjob_db.outputs, - solver_job_id=functionjob_db.class_specific_data["solver_job_id"], - ) - else: - raise UnsupportedFunctionJobClassError( - function_job_class=functionjob_db.function_class - ) - - -def _encode_functionjob( - functionjob: FunctionJob, -) -> FunctionJobDB: - if functionjob.function_class == FunctionClass.project: - return FunctionJobDB( - uuid=functionjob.uid, - title=functionjob.title, - function_uuid=functionjob.function_uid, - inputs=functionjob.inputs, - outputs=functionjob.outputs, - class_specific_data=FunctionJobClassSpecificData( - { - "project_job_id": str(functionjob.project_job_id), - } - ), - function_class=functionjob.function_class, - ) - elif functionjob.function_class == FunctionClass.solver: # noqa: RET505 - return FunctionJobDB( - uuid=functionjob.uid, - title=functionjob.title, - function_uuid=functionjob.function_uid, - inputs=functionjob.inputs, - outputs=functionjob.outputs, - class_specific_data=FunctionJobClassSpecificData( - { - "solver_job_id": str(functionjob.solver_job_id), - } - ), - function_class=functionjob.function_class, - ) - else: - raise UnsupportedFunctionJobClassError( - function_job_class=functionjob.function_class - ) - - -@router.expose() +@router.expose(reraise_if_error_type=(FunctionJobIDNotFoundError,)) async def get_function_job( app: web.Application, *, function_job_id: FunctionJobID ) -> FunctionJob: @@ -192,39 +112,22 @@ async def get_function_job( return _decode_functionjob(returned_function_job) -@router.expose() -async def get_function_input_schema( - app: web.Application, *, function_id: FunctionID -) -> FunctionInputSchema: +@router.expose(reraise_if_error_type=(FunctionJobCollectionIDNotFoundError,)) +async def get_function_job_collection( + app: web.Application, *, function_job_collection_id: FunctionJobID +) -> FunctionJobCollection: assert app - returned_function = await _functions_repository.get_function( - app=app, - function_id=function_id, - ) - return FunctionInputSchema( - schema_dict=( - returned_function.input_schema.schema_dict - if returned_function.input_schema - else None + returned_function_job_collection, returned_job_ids = ( + await _functions_repository.get_function_job_collection( + app=app, + function_job_collection_id=function_job_collection_id, ) ) - - -@router.expose() -async def get_function_output_schema( - app: web.Application, *, function_id: FunctionID -) -> FunctionOutputSchema: - assert app - returned_function = await _functions_repository.get_function( - app=app, - function_id=function_id, - ) - return FunctionOutputSchema( - schema_dict=( - returned_function.output_schema.schema_dict - if returned_function.output_schema - else None - ) + return FunctionJobCollection( + uid=returned_function_job_collection.uuid, + title=returned_function_job_collection.title, + description=returned_function_job_collection.description, + job_ids=returned_job_ids, ) @@ -288,7 +191,7 @@ async def list_function_job_collections( ], page -@router.expose() +@router.expose(reraise_if_error_type=(FunctionIDNotFoundError,)) async def delete_function(app: web.Application, *, function_id: FunctionID) -> None: assert app await _functions_repository.delete_function( @@ -297,18 +200,7 @@ async def delete_function(app: web.Application, *, function_id: FunctionID) -> N ) -@router.expose() -async def register_function_job( - app: web.Application, *, function_job: FunctionJob -) -> FunctionJob: - assert app - created_function_job_db = await _functions_repository.register_function_job( - app=app, function_job=_encode_functionjob(function_job) - ) - return _decode_functionjob(created_function_job_db) - - -@router.expose() +@router.expose(reraise_if_error_type=(FunctionJobIDNotFoundError,)) async def delete_function_job( app: web.Application, *, function_job_id: FunctionJobID ) -> None: @@ -319,6 +211,17 @@ async def delete_function_job( ) +@router.expose(reraise_if_error_type=(FunctionJobCollectionIDNotFoundError,)) +async def delete_function_job_collection( + app: web.Application, *, function_job_collection_id: FunctionJobID +) -> None: + assert app + await _functions_repository.delete_function_job_collection( + app=app, + function_job_collection_id=function_job_collection_id, + ) + + @router.expose() async def find_cached_function_job( app: web.Application, *, function_id: FunctionID, inputs: FunctionInputs @@ -356,55 +259,165 @@ async def find_cached_function_job( ) -@router.expose() -async def register_function_job_collection( - app: web.Application, *, function_job_collection: FunctionJobCollection -) -> FunctionJobCollection: +@router.expose(reraise_if_error_type=(FunctionIDNotFoundError,)) +async def get_function_input_schema( + app: web.Application, *, function_id: FunctionID +) -> FunctionInputSchema: assert app - registered_function_job_collection, registered_job_ids = ( - await _functions_repository.register_function_job_collection( - app=app, - function_job_collection=function_job_collection, - ) + returned_function = await _functions_repository.get_function( + app=app, + function_id=function_id, ) - return FunctionJobCollection( - uid=registered_function_job_collection.uuid, - title=registered_function_job_collection.title, - description=registered_function_job_collection.description, - job_ids=registered_job_ids, + return FunctionInputSchema( + schema_dict=( + returned_function.input_schema.schema_dict + if returned_function.input_schema + else None + ) ) -@router.expose() -async def get_function_job_collection( - app: web.Application, *, function_job_collection_id: FunctionJobID -) -> FunctionJobCollection: +@router.expose(reraise_if_error_type=(FunctionIDNotFoundError,)) +async def get_function_output_schema( + app: web.Application, *, function_id: FunctionID +) -> FunctionOutputSchema: assert app - returned_function_job_collection, returned_job_ids = ( - await _functions_repository.get_function_job_collection( - app=app, - function_job_collection_id=function_job_collection_id, - ) + returned_function = await _functions_repository.get_function( + app=app, + function_id=function_id, ) - return FunctionJobCollection( - uid=returned_function_job_collection.uuid, - title=returned_function_job_collection.title, - description=returned_function_job_collection.description, - job_ids=returned_job_ids, + return FunctionOutputSchema( + schema_dict=( + returned_function.output_schema.schema_dict + if returned_function.output_schema + else None + ) ) -@router.expose() -async def delete_function_job_collection( - app: web.Application, *, function_job_collection_id: FunctionJobID -) -> None: - assert app - await _functions_repository.delete_function_job_collection( - app=app, - function_job_collection_id=function_job_collection_id, +def _decode_function( + function: FunctionDB, +) -> Function: + if function.function_class == "project": + return ProjectFunction( + uid=function.uuid, + title=function.title, + description=function.description, + input_schema=function.input_schema, + output_schema=function.output_schema, + project_id=function.class_specific_data["project_id"], + default_inputs=function.default_inputs, + ) + elif function.function_class == "solver": # noqa: RET505 + return SolverFunction( + uid=function.uuid, + title=function.title, + description=function.description, + input_schema=function.input_schema, + output_schema=function.output_schema, + solver_key=function.class_specific_data["solver_key"], + solver_version=function.class_specific_data["solver_version"], + default_inputs=function.default_inputs, + ) + else: + raise UnsupportedFunctionClassError(function_class=function.function_class) + + +def _encode_function( + function: Function, +) -> FunctionDB: + if function.function_class == FunctionClass.project: + class_specific_data = FunctionClassSpecificData( + {"project_id": str(function.project_id)} + ) + elif function.function_class == FunctionClass.solver: + class_specific_data = FunctionClassSpecificData( + { + "solver_key": str(function.solver_key), + "solver_version": str(function.solver_version), + } + ) + else: + raise UnsupportedFunctionClassError(function_class=function.function_class) + + return FunctionDB( + uuid=function.uid, + title=function.title, + description=function.description, + input_schema=function.input_schema, + output_schema=function.output_schema, + function_class=function.function_class, + default_inputs=function.default_inputs, + class_specific_data=class_specific_data, ) +def _encode_functionjob( + functionjob: FunctionJob, +) -> FunctionJobDB: + if functionjob.function_class == FunctionClass.project: + return FunctionJobDB( + uuid=functionjob.uid, + title=functionjob.title, + function_uuid=functionjob.function_uid, + inputs=functionjob.inputs, + outputs=functionjob.outputs, + class_specific_data=FunctionJobClassSpecificData( + { + "project_job_id": str(functionjob.project_job_id), + } + ), + function_class=functionjob.function_class, + ) + elif functionjob.function_class == FunctionClass.solver: # noqa: RET505 + return FunctionJobDB( + uuid=functionjob.uid, + title=functionjob.title, + function_uuid=functionjob.function_uid, + inputs=functionjob.inputs, + outputs=functionjob.outputs, + class_specific_data=FunctionJobClassSpecificData( + { + "solver_job_id": str(functionjob.solver_job_id), + } + ), + function_class=functionjob.function_class, + ) + else: + raise UnsupportedFunctionJobClassError( + function_job_class=functionjob.function_class + ) + + +def _decode_functionjob( + functionjob_db: FunctionJobDB, +) -> FunctionJob: + if functionjob_db.function_class == FunctionClass.project: + return ProjectFunctionJob( + uid=functionjob_db.uuid, + title=functionjob_db.title, + description="", + function_uid=functionjob_db.function_uuid, + inputs=functionjob_db.inputs, + outputs=functionjob_db.outputs, + project_job_id=functionjob_db.class_specific_data["project_job_id"], + ) + elif functionjob_db.function_class == FunctionClass.solver: # noqa: RET505 + return SolverFunctionJob( + uid=functionjob_db.uuid, + title=functionjob_db.title, + description="", + function_uid=functionjob_db.function_uuid, + inputs=functionjob_db.inputs, + outputs=functionjob_db.outputs, + solver_job_id=functionjob_db.class_specific_data["solver_job_id"], + ) + else: + raise UnsupportedFunctionJobClassError( + function_job_class=functionjob_db.function_class + ) + + async def register_rpc_routes_on_startup(app: web.Application): rpc_server = get_rabbitmq_rpc_server(app) await rpc_server.register_router(router, WEBSERVER_RPC_NAMESPACE, app) diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py index 1de44feb96b..faa226f7107 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py @@ -4,15 +4,15 @@ from models_library.api_schemas_webserver.functions_wb_schema import ( FunctionDB, FunctionID, + FunctionIDNotFoundError, FunctionInputs, FunctionJobCollection, FunctionJobCollectionDB, - FunctionJobCollectionNotFoundError, + FunctionJobCollectionIDNotFoundError, FunctionJobDB, FunctionJobID, - FunctionJobNotFoundError, - FunctionNotFoundError, - RegisterFunctionWithUIDError, + FunctionJobIDNotFoundError, + RegisterFunctionWithIDError, ) from models_library.rest_pagination import ( PageMetaInfoLimitOffset, @@ -56,7 +56,7 @@ async def register_function( ) -> FunctionDB: if function.uuid is not None: - raise RegisterFunctionWithUIDError + raise RegisterFunctionWithIDError async with transaction_context(get_asyncpg_engine(app), connection) as conn: result = await conn.stream( @@ -104,7 +104,7 @@ async def get_function( row = await result.first() if row is None: - raise FunctionNotFoundError(function_id=function_id) + raise FunctionIDNotFoundError(function_id=function_id) return FunctionDB.model_validate(dict(row)) @@ -232,6 +232,16 @@ async def delete_function( ) -> None: async with transaction_context(get_asyncpg_engine(app), connection) as conn: + # Check if the function exists + result = await conn.stream( + functions_table.select().where(functions_table.c.uuid == function_id) + ) + row = await result.first() + + if row is None: + raise FunctionIDNotFoundError(function_id=function_id) + + # Proceed with deletion await conn.execute( functions_table.delete().where(functions_table.c.uuid == function_id) ) @@ -284,7 +294,7 @@ async def get_function_job( row = await result.first() if row is None: - raise FunctionJobNotFoundError(function_job_id=function_job_id) + raise FunctionJobIDNotFoundError(function_job_id=function_job_id) return FunctionJobDB.model_validate(dict(row)) @@ -296,6 +306,17 @@ async def delete_function_job( function_job_id: FunctionID, ) -> None: async with transaction_context(get_asyncpg_engine(app), connection) as conn: + # Check if the function job exists + result = await conn.stream( + function_jobs_table.select().where( + function_jobs_table.c.uuid == function_job_id + ) + ) + row = await result.first() + if row is None: + raise FunctionJobIDNotFoundError(function_job_id=function_job_id) + + # Proceed with deletion await conn.execute( function_jobs_table.delete().where( function_jobs_table.c.uuid == function_job_id @@ -354,7 +375,7 @@ async def get_function_job_collection( row = await result.first() if row is None: - raise FunctionJobCollectionNotFoundError( + raise FunctionJobCollectionIDNotFoundError( function_job_collection_id=function_job_collection_id ) @@ -417,6 +438,18 @@ async def delete_function_job_collection( ) -> None: async with transaction_context(get_asyncpg_engine(app), connection) as conn: + # Check if the function job collection exists + result = await conn.stream( + function_job_collections_table.select().where( + function_job_collections_table.c.uuid == function_job_collection_id + ) + ) + row = await result.first() + if row is None: + raise FunctionJobCollectionIDNotFoundError( + function_job_collection_id=function_job_collection_id + ) + # Proceed with deletion await conn.execute( function_job_collections_table.delete().where( function_job_collections_table.c.uuid == function_job_collection_id diff --git a/services/web/server/tests/unit/with_dbs/functions_rpc/test_functions_controller_rpc.py b/services/web/server/tests/unit/with_dbs/functions_rpc/test_functions_controller_rpc.py index 2a5122268f0..8e78093115a 100644 --- a/services/web/server/tests/unit/with_dbs/functions_rpc/test_functions_controller_rpc.py +++ b/services/web/server/tests/unit/with_dbs/functions_rpc/test_functions_controller_rpc.py @@ -5,10 +5,10 @@ import simcore_service_webserver.functions._functions_controller_rpc as functions_rpc from models_library.api_schemas_webserver.functions_wb_schema import ( Function, + FunctionIDNotFoundError, FunctionInputSchema, FunctionJobCollection, - FunctionJobNotFoundError, - FunctionNotFoundError, + FunctionJobIDNotFoundError, FunctionOutputSchema, ProjectFunction, ProjectFunctionJob, @@ -90,7 +90,7 @@ async def test_get_function(client, mock_function): @pytest.mark.asyncio async def test_get_function_not_found(client): # Attempt to retrieve a function that does not exist - with pytest.raises(FunctionNotFoundError): + with pytest.raises(FunctionIDNotFoundError): await functions_rpc.get_function(app=client.app, function_id=uuid4()) @@ -226,7 +226,7 @@ async def test_delete_function(client, mock_function): ) # Attempt to retrieve the deleted function - with pytest.raises(FunctionNotFoundError): + with pytest.raises(FunctionIDNotFoundError): await functions_rpc.get_function( app=client.app, function_id=registered_function.uid ) @@ -299,7 +299,7 @@ async def test_get_function_job(client, mock_function): @pytest.mark.asyncio async def test_get_function_job_not_found(client): # Attempt to retrieve a function job that does not exist - with pytest.raises(FunctionJobNotFoundError): + with pytest.raises(FunctionJobIDNotFoundError): await functions_rpc.get_function_job(app=client.app, function_job_id=uuid4()) @@ -366,7 +366,7 @@ async def test_delete_function_job(client, mock_function): ) # Attempt to retrieve the deleted job - with pytest.raises(FunctionJobNotFoundError): + with pytest.raises(FunctionJobIDNotFoundError): await functions_rpc.get_function_job( app=client.app, function_job_id=registered_job.uid ) @@ -428,7 +428,7 @@ async def test_function_job_collection(client, mock_function): app=client.app, function_job_collection_id=registered_collection.uid ) # Attempt to retrieve the deleted collection - with pytest.raises(FunctionJobNotFoundError): + with pytest.raises(FunctionJobIDNotFoundError): await functions_rpc.get_function_job( app=client.app, function_job_id=registered_collection.uid ) @@ -488,89 +488,3 @@ async def test_list_function_job_collections(client, mock_function): # Assert the list contains the registered collection assert len(collections) == 1 assert collections[0].uid == registered_collections[1].uid - - -# @pytest.mark.asyncio -# async def test_find_cached_function_job_project_class( -# mock_app, mock_function_id, mock_function_inputs -# ): -# mock_function_job = AsyncMock() -# mock_function_job.function_class = FunctionClass.project -# mock_function_job.uuid = "mock-uuid" -# mock_function_job.title = "mock-title" -# mock_function_job.function_uuid = "mock-function-uuid" -# mock_function_job.inputs = {"key": "value"} -# mock_function_job.class_specific_data = {"project_job_id": "mock-project-job-id"} - -# with patch( -# "simcore_service_webserver.functions._functions_repository.find_cached_function_job", -# return_value=mock_function_job, -# ): -# result = await find_cached_function_job( -# app=mock_app, function_id=mock_function_id, inputs=mock_function_inputs -# ) - -# assert isinstance(result, ProjectFunctionJob) -# assert result.uid == "mock-uuid" -# assert result.title == "mock-title" -# assert result.function_uid == "mock-function-uuid" -# assert result.inputs == {"key": "value"} -# assert result.project_job_id == "mock-project-job-id" - - -# @pytest.mark.asyncio -# async def test_find_cached_function_job_solver_class( -# mock_app, mock_function_id, mock_function_inputs -# ): -# mock_function_job = AsyncMock() -# mock_function_job.function_class = FunctionClass.solver -# mock_function_job.uuid = "mock-uuid" -# mock_function_job.title = "mock-title" -# mock_function_job.function_uuid = "mock-function-uuid" -# mock_function_job.inputs = {"key": "value"} -# mock_function_job.class_specific_data = {"solver_job_id": "mock-solver-job-id"} - -# with patch( -# "simcore_service_webserver.functions._functions_repository.find_cached_function_job", -# return_value=mock_function_job, -# ): -# result = await find_cached_function_job( -# app=mock_app, function_id=mock_function_id, inputs=mock_function_inputs -# ) - -# assert isinstance(result, SolverFunctionJob) -# assert result.uid == "mock-uuid" -# assert result.title == "mock-title" -# assert result.function_uid == "mock-function-uuid" -# assert result.inputs == {"key": "value"} -# assert result.solver_job_id == "mock-solver-job-id" - - -# @pytest.mark.asyncio -# async def test_find_cached_function_job_none(mock_app, mock_function_id, mock_function_inputs): -# with patch( -# "simcore_service_webserver.functions._functions_repository.find_cached_function_job", -# return_value=None, -# ): -# result = await find_cached_function_job( -# app=mock_app, function_id=mock_function_id, inputs=mock_function_inputs -# ) - -# assert result is None - - -# @pytest.mark.asyncio -# async def test_find_cached_function_job_unsupported_class( -# mock_app, mock_function_id, mock_function_inputs -# ): -# mock_function_job = AsyncMock() -# mock_function_job.function_class = "unsupported_class" - -# with patch( -# "simcore_service_webserver.functions._functions_repository.find_cached_function_job", -# return_value=mock_function_job, -# ): -# with pytest.raises(TypeError, match="Unsupported function class:"): -# await find_cached_function_job( -# app=mock_app, function_id=mock_function_id, inputs=mock_function_inputs -# ) From b683bc569c6fc1d3a35ec66b80f6e08a96b85957 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Thu, 8 May 2025 15:10:40 +0200 Subject: [PATCH 56/69] Fix linting --- .../api/routes/functions_routes.py | 48 +------------------ .../services_rpc/wb_api_server.py | 2 + 2 files changed, 4 insertions(+), 46 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py index a03fa39fbdf..de469bb93e6 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py @@ -47,6 +47,8 @@ ) from . import solvers_jobs, solvers_jobs_getters, studies_jobs +# pylint: disable=too-many-arguments,no-else-return + function_router = APIRouter() function_job_router = APIRouter() function_job_collections_router = APIRouter() @@ -648,49 +650,3 @@ async def function_job_collection_status( return FunctionJobCollectionStatus( status=[job_status.status for job_status in job_statuses] ) - - -# ruff: noqa: ERA001 - -# @function_job_router.get( -# "/{function_job_id:uuid}/outputs/logfile", -# response_model=FunctionOutputsLogfile, -# responses={**_COMMON_FUNCTION_JOB_ERROR_RESPONSES}, -# description="Get function job outputs", -# ) -# async def function_job_logfile( -# function_job_id: FunctionJobID, -# user_id: Annotated[PositiveInt, Depends(get_current_user_id)], -# wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], -# director2_api: Annotated[DirectorV2Api, Depends(get_api_client(DirectorV2Api))], -# ): -# function, function_job = await get_function_from_functionjobid( -# wb_api_rpc=wb_api_rpc, function_job_id=function_job_id -# ) - -# if ( -# function.function_class == FunctionClass.project -# and function_job.function_class == FunctionClass.project -# ): -# job_outputs = await studies_jobs.get_study_job_output_logfile( -# study_id=function.project_id, -# job_id=function_job.project_job_id, # type: ignore -# user_id=user_id, -# director2_api=director2_api, -# ) - -# return job_outputs -# elif (function.function_class == FunctionClass.solver) and ( -# function_job.function_class == FunctionClass.solver -# ): -# job_outputs_logfile = await solvers_jobs_getters.get_job_output_logfile( -# director2_api=director2_api, -# solver_key=function.solver_key, -# version=function.solver_version, -# job_id=function_job.solver_job_id, -# user_id=user_id, -# ) -# return job_outputs_logfile -# else: -# msg = f"Function type {function.function_class} not supported" -# raise TypeError(msg) diff --git a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py index 91adebd2606..10ff8db42dc 100644 --- a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py +++ b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py @@ -121,6 +121,8 @@ LicensedResource, ) +# pylint: disable=too-many-public-methods + _exception_mapper = partial(service_exception_mapper, service_name="WebApiServer") From e0d186be5f46ce5fd28c8aea153590a76fbf4399 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Thu, 8 May 2025 15:33:07 +0200 Subject: [PATCH 57/69] Add assert checks in functions rpc interface --- .../functions/functions_rpc_interface.py | 103 +++++++++++++----- 1 file changed, 75 insertions(+), 28 deletions(-) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py index 41a020124e2..4d2b8170227 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py @@ -32,11 +32,13 @@ async def register_function( *, function: Function, ) -> Function: - return await rabbitmq_rpc_client.request( + result: Function = await rabbitmq_rpc_client.request( WEBSERVER_RPC_NAMESPACE, TypeAdapter(RPCMethodName).validate_python("register_function"), function=function, ) + assert isinstance(result, Function) # nosec + return result @log_decorator(_logger, level=logging.DEBUG) @@ -45,11 +47,13 @@ async def get_function( *, function_id: FunctionID, ) -> Function: - return await rabbitmq_rpc_client.request( + result: Function = await rabbitmq_rpc_client.request( WEBSERVER_RPC_NAMESPACE, TypeAdapter(RPCMethodName).validate_python("get_function"), function_id=function_id, ) + assert isinstance(result, Function) # nosec + return result @log_decorator(_logger, level=logging.DEBUG) @@ -58,11 +62,13 @@ async def get_function_input_schema( *, function_id: FunctionID, ) -> FunctionInputSchema: - return await rabbitmq_rpc_client.request( + result: FunctionInputSchema = await rabbitmq_rpc_client.request( WEBSERVER_RPC_NAMESPACE, TypeAdapter(RPCMethodName).validate_python("get_function_input_schema"), function_id=function_id, ) + assert isinstance(result, FunctionInputSchema) # nosec + return result @log_decorator(_logger, level=logging.DEBUG) @@ -71,11 +77,13 @@ async def get_function_output_schema( *, function_id: FunctionID, ) -> FunctionOutputSchema: - return await rabbitmq_rpc_client.request( + result: FunctionOutputSchema = await rabbitmq_rpc_client.request( WEBSERVER_RPC_NAMESPACE, TypeAdapter(RPCMethodName).validate_python("get_function_output_schema"), function_id=function_id, ) + assert isinstance(result, FunctionOutputSchema) # nosec + return result @log_decorator(_logger, level=logging.DEBUG) @@ -84,11 +92,13 @@ async def delete_function( *, function_id: FunctionID, ) -> None: - return await rabbitmq_rpc_client.request( + result: None = await rabbitmq_rpc_client.request( WEBSERVER_RPC_NAMESPACE, TypeAdapter(RPCMethodName).validate_python("delete_function"), function_id=function_id, ) + assert result is None # nosec + return result @log_decorator(_logger, level=logging.DEBUG) @@ -98,12 +108,17 @@ async def list_functions( pagination_limit: int, pagination_offset: int, ) -> tuple[list[Function], PageMetaInfoLimitOffset]: - return await rabbitmq_rpc_client.request( - WEBSERVER_RPC_NAMESPACE, - TypeAdapter(RPCMethodName).validate_python("list_functions"), - pagination_offset=pagination_offset, - pagination_limit=pagination_limit, + result: tuple[list[Function], PageMetaInfoLimitOffset] = ( + await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("list_functions"), + pagination_offset=pagination_offset, + pagination_limit=pagination_limit, + ) ) + assert isinstance(result, tuple) + assert len(result) == 2 # nosec + assert isinstance(result[0], list) # nosec @log_decorator(_logger, level=logging.DEBUG) @@ -113,12 +128,19 @@ async def list_function_jobs( pagination_limit: int, pagination_offset: int, ) -> tuple[list[FunctionJob], PageMetaInfoLimitOffset]: - return await rabbitmq_rpc_client.request( - WEBSERVER_RPC_NAMESPACE, - TypeAdapter(RPCMethodName).validate_python("list_function_jobs"), - pagination_offset=pagination_offset, - pagination_limit=pagination_limit, + result: tuple[list[FunctionJob], PageMetaInfoLimitOffset] = ( + await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("list_function_jobs"), + pagination_offset=pagination_offset, + pagination_limit=pagination_limit, + ) ) + assert isinstance(result, tuple) + assert len(result) == 2 # nosec + assert isinstance(result[0], list) # nosec + assert isinstance(result[1], PageMetaInfoLimitOffset) # nosec + return result @log_decorator(_logger, level=logging.DEBUG) @@ -128,12 +150,19 @@ async def list_function_job_collections( pagination_limit: int, pagination_offset: int, ) -> tuple[list[FunctionJobCollection], PageMetaInfoLimitOffset]: - return await rabbitmq_rpc_client.request( - WEBSERVER_RPC_NAMESPACE, - TypeAdapter(RPCMethodName).validate_python("list_function_job_collections"), - pagination_offset=pagination_offset, - pagination_limit=pagination_limit, + result: tuple[list[FunctionJobCollection], PageMetaInfoLimitOffset] = ( + await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("list_function_job_collections"), + pagination_offset=pagination_offset, + pagination_limit=pagination_limit, + ) ) + assert isinstance(result, tuple) + assert len(result) == 2 # nosec + assert isinstance(result[0], list) # nosec + assert isinstance(result[1], PageMetaInfoLimitOffset) # nosec + return result @log_decorator(_logger, level=logging.DEBUG) @@ -143,12 +172,14 @@ async def run_function( function_id: FunctionID, inputs: FunctionInputs, ) -> FunctionJob: - return await rabbitmq_rpc_client.request( + result: FunctionJob = await rabbitmq_rpc_client.request( WEBSERVER_RPC_NAMESPACE, TypeAdapter(RPCMethodName).validate_python("run_function"), function_id=function_id, inputs=inputs, ) + assert isinstance(result, FunctionJob) # nosec + return result @log_decorator(_logger, level=logging.DEBUG) @@ -157,11 +188,13 @@ async def register_function_job( *, function_job: FunctionJob, ) -> FunctionJob: - return await rabbitmq_rpc_client.request( + result: FunctionJob = await rabbitmq_rpc_client.request( WEBSERVER_RPC_NAMESPACE, TypeAdapter(RPCMethodName).validate_python("register_function_job"), function_job=function_job, ) + assert isinstance(result, FunctionJob) # nosec + return result @log_decorator(_logger, level=logging.DEBUG) @@ -170,11 +203,13 @@ async def get_function_job( *, function_job_id: FunctionJobID, ) -> FunctionJob: - return await rabbitmq_rpc_client.request( + result: FunctionJob = await rabbitmq_rpc_client.request( WEBSERVER_RPC_NAMESPACE, TypeAdapter(RPCMethodName).validate_python("get_function_job"), function_job_id=function_job_id, ) + assert isinstance(result, FunctionJob) # nosec + return result @log_decorator(_logger, level=logging.DEBUG) @@ -183,11 +218,13 @@ async def delete_function_job( *, function_job_id: FunctionJobID, ) -> None: - return await rabbitmq_rpc_client.request( + result: None = await rabbitmq_rpc_client.request( WEBSERVER_RPC_NAMESPACE, TypeAdapter(RPCMethodName).validate_python("delete_function_job"), function_job_id=function_job_id, ) + assert result is None # nosec + return result @log_decorator(_logger, level=logging.DEBUG) @@ -197,12 +234,16 @@ async def find_cached_function_job( function_id: FunctionID, inputs: FunctionInputs, ) -> FunctionJob | None: - return await rabbitmq_rpc_client.request( + result: FunctionJob = await rabbitmq_rpc_client.request( WEBSERVER_RPC_NAMESPACE, TypeAdapter(RPCMethodName).validate_python("find_cached_function_job"), function_id=function_id, inputs=inputs, ) + if result is None: + return None + assert isinstance(result, FunctionJob) # nosec + return result @log_decorator(_logger, level=logging.DEBUG) @@ -211,11 +252,13 @@ async def register_function_job_collection( *, function_job_collection: FunctionJobCollection, ) -> FunctionJobCollection: - return await rabbitmq_rpc_client.request( + result: FunctionJobCollection = await rabbitmq_rpc_client.request( WEBSERVER_RPC_NAMESPACE, TypeAdapter(RPCMethodName).validate_python("register_function_job_collection"), function_job_collection=function_job_collection, ) + assert isinstance(result, FunctionJobCollection) # nosec + return result @log_decorator(_logger, level=logging.DEBUG) @@ -224,11 +267,13 @@ async def get_function_job_collection( *, function_job_collection_id: FunctionJobCollectionID, ) -> FunctionJobCollection: - return await rabbitmq_rpc_client.request( + result: FunctionJobCollection = await rabbitmq_rpc_client.request( WEBSERVER_RPC_NAMESPACE, TypeAdapter(RPCMethodName).validate_python("get_function_job_collection"), function_job_collection_id=function_job_collection_id, ) + assert isinstance(result, FunctionJobCollection) # nosec + return result @log_decorator(_logger, level=logging.DEBUG) @@ -237,8 +282,10 @@ async def delete_function_job_collection( *, function_job_collection_id: FunctionJobCollectionID, ) -> None: - return await rabbitmq_rpc_client.request( + result: None = await rabbitmq_rpc_client.request( WEBSERVER_RPC_NAMESPACE, TypeAdapter(RPCMethodName).validate_python("delete_function_job_collection"), function_job_collection_id=function_job_collection_id, ) + assert result is None + return result From 9b2ff02258d3eeeed2158b17d603903f825f30dd Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Thu, 8 May 2025 16:29:01 +0200 Subject: [PATCH 58/69] Fix gh action tests --- .../functions/functions_rpc_interface.py | 18 +++++++++--------- .../api/routes/functions_routes.py | 14 +++++++++----- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py index 4d2b8170227..6c5c264890a 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py @@ -1,4 +1,5 @@ import logging +from typing import get_origin from models_library.api_schemas_webserver import ( WEBSERVER_RPC_NAMESPACE, @@ -37,7 +38,7 @@ async def register_function( TypeAdapter(RPCMethodName).validate_python("register_function"), function=function, ) - assert isinstance(result, Function) # nosec + assert isinstance(result, get_origin(Function) or Function) # nosec return result @@ -52,7 +53,7 @@ async def get_function( TypeAdapter(RPCMethodName).validate_python("get_function"), function_id=function_id, ) - assert isinstance(result, Function) # nosec + assert isinstance(result, get_origin(Function) or Function) # nosec return result @@ -117,8 +118,9 @@ async def list_functions( ) ) assert isinstance(result, tuple) - assert len(result) == 2 # nosec assert isinstance(result[0], list) # nosec + assert isinstance(result[1], PageMetaInfoLimitOffset) # nosec + return result @log_decorator(_logger, level=logging.DEBUG) @@ -137,7 +139,6 @@ async def list_function_jobs( ) ) assert isinstance(result, tuple) - assert len(result) == 2 # nosec assert isinstance(result[0], list) # nosec assert isinstance(result[1], PageMetaInfoLimitOffset) # nosec return result @@ -159,7 +160,6 @@ async def list_function_job_collections( ) ) assert isinstance(result, tuple) - assert len(result) == 2 # nosec assert isinstance(result[0], list) # nosec assert isinstance(result[1], PageMetaInfoLimitOffset) # nosec return result @@ -178,7 +178,7 @@ async def run_function( function_id=function_id, inputs=inputs, ) - assert isinstance(result, FunctionJob) # nosec + assert isinstance(result, get_origin(FunctionJob) or FunctionJob) # nosec return result @@ -193,7 +193,7 @@ async def register_function_job( TypeAdapter(RPCMethodName).validate_python("register_function_job"), function_job=function_job, ) - assert isinstance(result, FunctionJob) # nosec + assert isinstance(result, get_origin(FunctionJob) or FunctionJob) # nosec return result @@ -208,7 +208,7 @@ async def get_function_job( TypeAdapter(RPCMethodName).validate_python("get_function_job"), function_job_id=function_job_id, ) - assert isinstance(result, FunctionJob) # nosec + assert isinstance(result, get_origin(FunctionJob) or FunctionJob) # nosec return result @@ -242,7 +242,7 @@ async def find_cached_function_job( ) if result is None: return None - assert isinstance(result, FunctionJob) # nosec + assert isinstance(result, get_origin(FunctionJob) or FunctionJob) # nosec return result diff --git a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py index de469bb93e6..9ac80a59771 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py @@ -201,10 +201,10 @@ async def validate_function_inputs( ): function = await wb_api_rpc.get_function(function_id=function_id) - if function.input_schema is None: + if function.input_schema is None or function.input_schema.schema_dict is None: return True, "No input schema defined for this function" try: - jsonschema.validate(instance=inputs, schema=function.input_schema.schema_dict) # type: ignore + jsonschema.validate(instance=inputs, schema=function.input_schema.schema_dict) except ValidationError as err: return False, str(err) return True, "Inputs are valid" @@ -421,7 +421,7 @@ async def function_job_status( ): job_status = await studies_jobs.inspect_study_job( study_id=function.project_id, - job_id=function_job.project_job_id, # type: ignore + job_id=function_job.project_job_id, user_id=user_id, director2_api=director2_api, ) @@ -466,7 +466,7 @@ async def function_job_outputs( ): job_outputs = await studies_jobs.get_study_job_outputs( study_id=function.project_id, - job_id=function_job.project_job_id, # type: ignore + job_id=function_job.project_job_id, user_id=user_id, webserver_api=webserver_api, storage_client=storage_client, @@ -539,7 +539,11 @@ async def map_function( # noqa: PLR0913 uid=None, title="Function job collection of function map", description=function_job_collection_description, - job_ids=[function_job.uid for function_job in function_jobs], # type: ignore + job_ids=[ + function_job.uid + for function_job in function_jobs + if function_job.uid is not None + ], ), ) From 61dcec32feb951055f70bd049eabf73c02499a18 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Thu, 8 May 2025 17:11:04 +0200 Subject: [PATCH 59/69] Add types jsonschema to api-server test requirements --- services/api-server/requirements/_test.in | 1 + services/api-server/requirements/_test.txt | 2 ++ 2 files changed, 3 insertions(+) diff --git a/services/api-server/requirements/_test.in b/services/api-server/requirements/_test.in index 718feaeb205..805e1f7a7af 100644 --- a/services/api-server/requirements/_test.in +++ b/services/api-server/requirements/_test.in @@ -31,3 +31,4 @@ respx sqlalchemy[mypy] # adds Mypy / Pep-484 Support for ORM Mappings SEE https://docs.sqlalchemy.org/en/20/orm/extensions/mypy.html types-aiofiles types-boto3 +types-jsonschema diff --git a/services/api-server/requirements/_test.txt b/services/api-server/requirements/_test.txt index b0708a30202..84d74a5f0e6 100644 --- a/services/api-server/requirements/_test.txt +++ b/services/api-server/requirements/_test.txt @@ -359,6 +359,8 @@ types-awscrt==0.23.10 # via botocore-stubs types-boto3==1.37.4 # via -r requirements/_test.in +types-jsonschema==4.23.0.20241208 + # via -r requirements/_test.in types-s3transfer==0.11.3 # via types-boto3 typing-extensions==4.12.2 From 9b1e0a750b884c06879e41531b80422ae21fcc06 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Thu, 8 May 2025 18:48:45 +0200 Subject: [PATCH 60/69] Fix functions rpc assert and delete functions.py old schema --- .../functions/functions_rpc_interface.py | 38 ++++++++++++------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py index 6c5c264890a..be157e0f85a 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py @@ -1,5 +1,4 @@ import logging -from typing import get_origin from models_library.api_schemas_webserver import ( WEBSERVER_RPC_NAMESPACE, @@ -38,7 +37,7 @@ async def register_function( TypeAdapter(RPCMethodName).validate_python("register_function"), function=function, ) - assert isinstance(result, get_origin(Function) or Function) # nosec + TypeAdapter(Function).validate_python(result) # Validates the result as a Function return result @@ -53,7 +52,7 @@ async def get_function( TypeAdapter(RPCMethodName).validate_python("get_function"), function_id=function_id, ) - assert isinstance(result, get_origin(Function) or Function) # nosec + TypeAdapter(Function).validate_python(result) return result @@ -68,7 +67,7 @@ async def get_function_input_schema( TypeAdapter(RPCMethodName).validate_python("get_function_input_schema"), function_id=function_id, ) - assert isinstance(result, FunctionInputSchema) # nosec + TypeAdapter(FunctionInputSchema).validate_python(result) return result @@ -83,7 +82,7 @@ async def get_function_output_schema( TypeAdapter(RPCMethodName).validate_python("get_function_output_schema"), function_id=function_id, ) - assert isinstance(result, FunctionOutputSchema) # nosec + TypeAdapter(FunctionOutputSchema).validate_python(result) return result @@ -118,7 +117,9 @@ async def list_functions( ) ) assert isinstance(result, tuple) - assert isinstance(result[0], list) # nosec + TypeAdapter(list[Function]).validate_python( + result[0] + ) # Validates the result as a list of Functions assert isinstance(result[1], PageMetaInfoLimitOffset) # nosec return result @@ -139,7 +140,9 @@ async def list_function_jobs( ) ) assert isinstance(result, tuple) - assert isinstance(result[0], list) # nosec + TypeAdapter(list[FunctionJob]).validate_python( + result[0] + ) # Validates the result as a list of FunctionJobs assert isinstance(result[1], PageMetaInfoLimitOffset) # nosec return result @@ -160,7 +163,9 @@ async def list_function_job_collections( ) ) assert isinstance(result, tuple) - assert isinstance(result[0], list) # nosec + TypeAdapter(list[FunctionJobCollection]).validate_python( + result[0] + ) # Validates the result as a list of FunctionJobCollections assert isinstance(result[1], PageMetaInfoLimitOffset) # nosec return result @@ -178,7 +183,9 @@ async def run_function( function_id=function_id, inputs=inputs, ) - assert isinstance(result, get_origin(FunctionJob) or FunctionJob) # nosec + TypeAdapter(FunctionJob).validate_python( + result + ) # Validates the result as a FunctionJob return result @@ -193,7 +200,9 @@ async def register_function_job( TypeAdapter(RPCMethodName).validate_python("register_function_job"), function_job=function_job, ) - assert isinstance(result, get_origin(FunctionJob) or FunctionJob) # nosec + TypeAdapter(FunctionJob).validate_python( + result + ) # Validates the result as a FunctionJob return result @@ -208,7 +217,8 @@ async def get_function_job( TypeAdapter(RPCMethodName).validate_python("get_function_job"), function_job_id=function_job_id, ) - assert isinstance(result, get_origin(FunctionJob) or FunctionJob) # nosec + + TypeAdapter(FunctionJob).validate_python(result) return result @@ -242,7 +252,7 @@ async def find_cached_function_job( ) if result is None: return None - assert isinstance(result, get_origin(FunctionJob) or FunctionJob) # nosec + TypeAdapter(FunctionJob).validate_python(result) return result @@ -257,7 +267,7 @@ async def register_function_job_collection( TypeAdapter(RPCMethodName).validate_python("register_function_job_collection"), function_job_collection=function_job_collection, ) - assert isinstance(result, FunctionJobCollection) # nosec + TypeAdapter(FunctionJobCollection).validate_python(result) return result @@ -272,7 +282,7 @@ async def get_function_job_collection( TypeAdapter(RPCMethodName).validate_python("get_function_job_collection"), function_job_collection_id=function_job_collection_id, ) - assert isinstance(result, FunctionJobCollection) # nosec + TypeAdapter(FunctionJobCollection).validate_python(result) return result From f1041e1916cf9247dd339b4e318da0e99c390d66 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Fri, 9 May 2025 12:20:07 +0200 Subject: [PATCH 61/69] Changes suggested by Sylvain wrt to functions api+new function schema --- .../functions_wb_schema.py | 84 +++++-- services/api-server/openapi.json | 207 +++++++++++------- .../api/routes/functions_routes.py | 44 ++-- .../services_rpc/wb_api_server.py | 10 - .../test_api_routers_functions.py | 60 +++-- .../functions/_functions_controller_rpc.py | 16 +- .../test_functions_controller_rpc.py | 19 +- 7 files changed, 264 insertions(+), 176 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py b/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py index 806201cc4de..5893caa8846 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/functions_wb_schema.py @@ -1,31 +1,58 @@ +from collections.abc import Mapping from enum import Enum from typing import Annotated, Any, Literal, TypeAlias from uuid import UUID from models_library import projects -from models_library.basic_regex import SIMPLE_VERSION_RE -from models_library.services_regex import COMPUTATIONAL_SERVICE_KEY_RE -from pydantic import BaseModel, Field, StringConstraints +from models_library.services_types import ServiceKey, ServiceVersion +from pydantic import BaseModel, Field from ..projects import ProjectID -FunctionID: TypeAlias = projects.ProjectID -FunctionJobID: TypeAlias = projects.ProjectID +FunctionID: TypeAlias = UUID +FunctionJobID: TypeAlias = UUID FileID: TypeAlias = UUID InputTypes: TypeAlias = FileID | float | int | bool | str | list -class FunctionSchema(BaseModel): - """Schema for function input/output""" +class FunctionSchemaClass(str, Enum): + json_schema = "application/schema+json" - schema_dict: dict[str, Any] | None # JSON Schema +class FunctionSchemaBase(BaseModel): + schema_content: Any = Field(default=None) + schema_class: FunctionSchemaClass -class FunctionInputSchema(FunctionSchema): ... +class JSONFunctionSchema(FunctionSchemaBase): + schema_content: Mapping[str, Any] = Field( + default={}, description="JSON Schema", title="JSON Schema" + ) # json-schema library defines a schema as Mapping[str, Any] + schema_class: FunctionSchemaClass = FunctionSchemaClass.json_schema -class FunctionOutputSchema(FunctionSchema): ... + +class JSONFunctionInputSchema(JSONFunctionSchema): + schema_class: Literal[FunctionSchemaClass.json_schema] = ( + FunctionSchemaClass.json_schema + ) + + +class JSONFunctionOutputSchema(JSONFunctionSchema): + schema_class: Literal[FunctionSchemaClass.json_schema] = ( + FunctionSchemaClass.json_schema + ) + + +FunctionInputSchema: TypeAlias = Annotated[ + JSONFunctionInputSchema, + Field(discriminator="schema_class"), +] + +FunctionOutputSchema: TypeAlias = Annotated[ + JSONFunctionOutputSchema, + Field(discriminator="schema_class"), +] class FunctionClass(str, Enum): @@ -53,8 +80,8 @@ class FunctionBase(BaseModel): uid: FunctionID | None title: str = "" description: str = "" - input_schema: FunctionInputSchema | None - output_schema: FunctionOutputSchema | None + input_schema: FunctionInputSchema + output_schema: FunctionOutputSchema default_inputs: FunctionInputs @@ -63,8 +90,8 @@ class FunctionDB(BaseModel): uuid: FunctionJobID | None title: str = "" description: str = "" - input_schema: FunctionInputSchema | None - output_schema: FunctionOutputSchema | None + input_schema: FunctionInputSchema + output_schema: FunctionOutputSchema default_inputs: FunctionInputs class_specific_data: FunctionClassSpecificData @@ -84,19 +111,13 @@ class ProjectFunction(FunctionBase): project_id: ProjectID -SolverKeyId = Annotated[ - str, StringConstraints(strip_whitespace=True, pattern=COMPUTATIONAL_SERVICE_KEY_RE) -] -VersionStr: TypeAlias = Annotated[ - str, StringConstraints(strip_whitespace=True, pattern=SIMPLE_VERSION_RE) -] SolverJobID: TypeAlias = UUID class SolverFunction(FunctionBase): function_class: Literal[FunctionClass.solver] = FunctionClass.solver - solver_key: SolverKeyId - solver_version: str = "" + solver_key: ServiceKey + solver_version: ServiceVersion class PythonCodeFunction(FunctionBase): @@ -228,3 +249,22 @@ class UnsupportedFunctionJobClassError(Exception): def __init__(self, function_job_class: str): self.function_job_class = function_job_class super().__init__(f"Function job class {function_job_class} is not supported") + + +class UnsupportedFunctionFunctionJobClassCombinationError(Exception): + """Exception raised when a function / function job class combination is not supported""" + + def __init__(self, function_class: str, function_job_class: str): + self.function_class = function_class + self.function_job_class = function_job_class + super().__init__( + f"Function class {function_class} and function job class {function_job_class} combination is not supported" + ) + + +class FunctionInputsValidationError(Exception): + """Exception raised when validating function inputs""" + + def __init__(self, error: str): + self.errors = error + super().__init__(f"Function inputs validation failed: {error}") diff --git a/services/api-server/openapi.json b/services/api-server/openapi.json index 65f40210e36..1f5fd6654b9 100644 --- a/services/api-server/openapi.json +++ b/services/api-server/openapi.json @@ -5570,7 +5570,18 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/FunctionInputSchema" + "oneOf": [ + { + "$ref": "#/components/schemas/JSONFunctionInputSchema" + } + ], + "discriminator": { + "propertyName": "schema_class", + "mapping": { + "application/schema+json": "#/components/schemas/JSONFunctionInputSchema" + } + }, + "title": "Response Get Function Inputschema V0 Functions Function Id Input Schema Get" } } } @@ -5624,7 +5635,18 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/FunctionInputSchema" + "oneOf": [ + { + "$ref": "#/components/schemas/JSONFunctionInputSchema" + } + ], + "discriminator": { + "propertyName": "schema_class", + "mapping": { + "application/schema+json": "#/components/schemas/JSONFunctionInputSchema" + } + }, + "title": "Response Get Function Outputschema V0 Functions Function Id Output Schema Get" } } } @@ -7603,26 +7625,6 @@ ], "title": "FileUploadData" }, - "FunctionInputSchema": { - "properties": { - "schema_dict": { - "anyOf": [ - { - "type": "object" - }, - { - "type": "null" - } - ], - "title": "Schema Dict" - } - }, - "type": "object", - "required": [ - "schema_dict" - ], - "title": "FunctionInputSchema" - }, "FunctionJobCollection": { "properties": { "uid": { @@ -7693,26 +7695,6 @@ ], "title": "FunctionJobStatus" }, - "FunctionOutputSchema": { - "properties": { - "schema_dict": { - "anyOf": [ - { - "type": "object" - }, - { - "type": "null" - } - ], - "title": "Schema Dict" - } - }, - "type": "object", - "required": [ - "schema_dict" - ], - "title": "FunctionOutputSchema" - }, "GetCreditPriceLegacy": { "properties": { "productName": { @@ -7798,6 +7780,42 @@ "type": "object", "title": "HTTPValidationError" }, + "JSONFunctionInputSchema": { + "properties": { + "schema_content": { + "type": "object", + "title": "JSON Schema", + "description": "JSON Schema", + "default": {} + }, + "schema_class": { + "type": "string", + "const": "application/schema+json", + "title": "Schema Class", + "default": "application/schema+json" + } + }, + "type": "object", + "title": "JSONFunctionInputSchema" + }, + "JSONFunctionOutputSchema": { + "properties": { + "schema_content": { + "type": "object", + "title": "JSON Schema", + "description": "JSON Schema", + "default": {} + }, + "schema_class": { + "type": "string", + "const": "application/schema+json", + "title": "Schema Class", + "default": "application/schema+json" + } + }, + "type": "object", + "title": "JSONFunctionOutputSchema" + }, "Job": { "properties": { "id": { @@ -9492,24 +9510,32 @@ "default": "" }, "input_schema": { - "anyOf": [ + "oneOf": [ { - "$ref": "#/components/schemas/FunctionInputSchema" - }, - { - "type": "null" + "$ref": "#/components/schemas/JSONFunctionInputSchema" } - ] + ], + "title": "Input Schema", + "discriminator": { + "propertyName": "schema_class", + "mapping": { + "application/schema+json": "#/components/schemas/JSONFunctionInputSchema" + } + } }, "output_schema": { - "anyOf": [ - { - "$ref": "#/components/schemas/FunctionOutputSchema" - }, + "oneOf": [ { - "type": "null" + "$ref": "#/components/schemas/JSONFunctionOutputSchema" } - ] + ], + "title": "Output Schema", + "discriminator": { + "propertyName": "schema_class", + "mapping": { + "application/schema+json": "#/components/schemas/JSONFunctionOutputSchema" + } + } }, "default_inputs": { "anyOf": [ @@ -9642,24 +9668,32 @@ "default": "" }, "input_schema": { - "anyOf": [ - { - "$ref": "#/components/schemas/FunctionInputSchema" - }, + "oneOf": [ { - "type": "null" + "$ref": "#/components/schemas/JSONFunctionInputSchema" } - ] + ], + "title": "Input Schema", + "discriminator": { + "propertyName": "schema_class", + "mapping": { + "application/schema+json": "#/components/schemas/JSONFunctionInputSchema" + } + } }, "output_schema": { - "anyOf": [ + "oneOf": [ { - "$ref": "#/components/schemas/FunctionOutputSchema" - }, - { - "type": "null" + "$ref": "#/components/schemas/JSONFunctionOutputSchema" } - ] + ], + "title": "Output Schema", + "discriminator": { + "propertyName": "schema_class", + "mapping": { + "application/schema+json": "#/components/schemas/JSONFunctionOutputSchema" + } + } }, "default_inputs": { "anyOf": [ @@ -9922,24 +9956,32 @@ "default": "" }, "input_schema": { - "anyOf": [ + "oneOf": [ { - "$ref": "#/components/schemas/FunctionInputSchema" - }, - { - "type": "null" + "$ref": "#/components/schemas/JSONFunctionInputSchema" } - ] + ], + "title": "Input Schema", + "discriminator": { + "propertyName": "schema_class", + "mapping": { + "application/schema+json": "#/components/schemas/JSONFunctionInputSchema" + } + } }, "output_schema": { - "anyOf": [ - { - "$ref": "#/components/schemas/FunctionOutputSchema" - }, + "oneOf": [ { - "type": "null" + "$ref": "#/components/schemas/JSONFunctionOutputSchema" } - ] + ], + "title": "Output Schema", + "discriminator": { + "propertyName": "schema_class", + "mapping": { + "application/schema+json": "#/components/schemas/JSONFunctionOutputSchema" + } + } }, "default_inputs": { "anyOf": [ @@ -9954,13 +9996,13 @@ }, "solver_key": { "type": "string", - "pattern": "^simcore/services/comp/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", + "pattern": "^simcore/services/((comp|dynamic|frontend))/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", "title": "Solver Key" }, "solver_version": { "type": "string", - "title": "Solver Version", - "default": "" + "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$", + "title": "Solver Version" } }, "type": "object", @@ -9969,7 +10011,8 @@ "input_schema", "output_schema", "default_inputs", - "solver_key" + "solver_key", + "solver_version" ], "title": "SolverFunction" }, diff --git a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py index 9ac80a59771..4217987d193 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py @@ -13,6 +13,7 @@ FunctionInputs, FunctionInputSchema, FunctionInputsList, + FunctionInputsValidationError, FunctionJob, FunctionJobCollection, FunctionJobCollectionID, @@ -20,8 +21,11 @@ FunctionJobID, FunctionJobStatus, FunctionOutputs, + FunctionSchemaClass, ProjectFunctionJob, SolverFunctionJob, + UnsupportedFunctionClassError, + UnsupportedFunctionFunctionJobClassCombinationError, ) from pydantic import PositiveInt from servicelib.fastapi.dependencies import get_reverse_url_mapper @@ -201,13 +205,22 @@ async def validate_function_inputs( ): function = await wb_api_rpc.get_function(function_id=function_id) - if function.input_schema is None or function.input_schema.schema_dict is None: + if function.input_schema is None or function.input_schema.schema_content is None: return True, "No input schema defined for this function" - try: - jsonschema.validate(instance=inputs, schema=function.input_schema.schema_dict) - except ValidationError as err: - return False, str(err) - return True, "Inputs are valid" + + if function.input_schema.schema_class == FunctionSchemaClass.json_schema: + try: + jsonschema.validate( + instance=inputs, schema=function.input_schema.schema_content + ) + except ValidationError as err: + return False, str(err) + return True, "Inputs are valid" + + return ( + False, + f"Unsupported function schema class {function.input_schema.schema_class}", + ) @function_router.post( @@ -246,10 +259,7 @@ async def run_function( # noqa: PLR0913 wb_api_rpc=wb_api_rpc, ) if not is_valid: - msg = ( - f"Function {to_run_function.uid} inputs are not valid: {validation_str}" - ) - raise ValidationError(msg) + raise FunctionInputsValidationError(error=validation_str) if cached_function_job := await wb_api_rpc.find_cached_function_job( function_id=to_run_function.uid, @@ -322,8 +332,9 @@ async def run_function( # noqa: PLR0913 ), ) else: - msg = f"Function type {type(to_run_function)} not supported" - raise TypeError(msg) + raise UnsupportedFunctionClassError( + function_class=to_run_function.function_class, + ) @function_router.delete( @@ -438,8 +449,10 @@ async def function_job_status( ) return FunctionJobStatus(status=job_status.state) else: - msg = f"Function type {function.function_class} / Function job type {function_job.function_class} not supported" - raise TypeError(msg) + raise UnsupportedFunctionFunctionJobClassCombinationError( + function_class=function.function_class, + function_job_class=function_job.function_class, + ) @function_job_router.get( @@ -487,8 +500,7 @@ async def function_job_outputs( ) return job_outputs.results else: - msg = f"Function type {function.function_class} not supported" - raise TypeError(msg) + raise UnsupportedFunctionClassError(function_class=function.function_class) @function_router.post( diff --git a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py index 10ff8db42dc..f208edda841 100644 --- a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py +++ b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py @@ -299,16 +299,6 @@ async def list_projects_marked_as_jobs( ) async def register_function(self, *, function: Function) -> Function: - function.input_schema = ( - FunctionInputSchema(**function.input_schema.model_dump()) - if function.input_schema is not None - else None - ) - function.output_schema = ( - FunctionOutputSchema(**function.output_schema.model_dump()) - if function.output_schema is not None - else None - ) return await _register_function( self._client, function=function, diff --git a/services/api-server/tests/unit/api_functions/test_api_routers_functions.py b/services/api-server/tests/unit/api_functions/test_api_routers_functions.py index cfb993093b0..852cb4c1928 100644 --- a/services/api-server/tests/unit/api_functions/test_api_routers_functions.py +++ b/services/api-server/tests/unit/api_functions/test_api_routers_functions.py @@ -9,6 +9,8 @@ Function, FunctionJob, FunctionJobCollection, + JSONFunctionInputSchema, + JSONFunctionOutputSchema, ) from models_library.rest_pagination import ( PageMetaInfoLimitOffset, @@ -234,8 +236,8 @@ def test_register_function(api_app) -> None: "function_class": "project", "project_id": str(uuid4()), "description": "A test function", - "input_schema": {"schema_dict": {}}, - "output_schema": {"schema_dict": {}}, + "input_schema": JSONFunctionInputSchema().model_dump(), + "output_schema": JSONFunctionOutputSchema().model_dump(), "default_inputs": None, } response = client.post("/functions", json=sample_function) @@ -275,8 +277,8 @@ def test_get_function(api_app: FastAPI) -> None: "function_class": "project", "project_id": project_id, "description": "An example function", - "input_schema": {"schema_dict": {}}, - "output_schema": {"schema_dict": {}}, + "input_schema": JSONFunctionInputSchema().model_dump(), + "output_schema": JSONFunctionOutputSchema().model_dump(), "default_inputs": None, } post_response = client.post("/functions", json=sample_function) @@ -290,8 +292,8 @@ def test_get_function(api_app: FastAPI) -> None: "description": "An example function", "function_class": "project", "project_id": project_id, - "input_schema": {"schema_dict": {}}, - "output_schema": {"schema_dict": {}}, + "input_schema": JSONFunctionInputSchema().model_dump(), + "output_schema": JSONFunctionOutputSchema().model_dump(), "default_inputs": None, } response = client.get(f"/functions/{function_id}") @@ -318,8 +320,8 @@ def test_list_functions(api_app: FastAPI) -> None: "function_class": "project", "project_id": str(uuid4()), "description": "An example function", - "input_schema": {"schema_dict": {}}, - "output_schema": {"schema_dict": {}}, + "input_schema": JSONFunctionInputSchema().model_dump(), + "output_schema": JSONFunctionOutputSchema().model_dump(), "default_inputs": None, } post_response = client.post("/functions", json=sample_function) @@ -343,13 +345,18 @@ def test_get_function_input_schema(api_app: FastAPI) -> None: "function_class": "project", "project_id": project_id, "description": "An example function", - "input_schema": { - "schema_dict": { + "input_schema": JSONFunctionInputSchema( + schema_content={ "type": "object", "properties": {"input1": {"type": "integer"}}, } - }, - "output_schema": {"schema_dict": {}}, + ).model_dump(), + "output_schema": JSONFunctionOutputSchema( + schema_content={ + "type": "object", + "properties": {"output1": {"type": "string"}}, + } + ).model_dump(), "default_inputs": None, } post_response = client.post("/functions", json=sample_function) @@ -362,7 +369,7 @@ def test_get_function_input_schema(api_app: FastAPI) -> None: response = client.get(f"/functions/{function_id}/input_schema") assert response.status_code == 200 data = response.json() - assert data["schema_dict"] == sample_function["input_schema"]["schema_dict"] + assert data["schema_content"] == sample_function["input_schema"]["schema_content"] def test_get_function_output_schema(api_app: FastAPI) -> None: @@ -375,13 +382,13 @@ def test_get_function_output_schema(api_app: FastAPI) -> None: "function_class": "project", "project_id": project_id, "description": "An example function", - "input_schema": {"schema_dict": {}}, - "output_schema": { - "schema_dict": { + "input_schema": JSONFunctionInputSchema().model_dump(), + "output_schema": JSONFunctionOutputSchema( + schema_content={ "type": "object", "properties": {"output1": {"type": "string"}}, } - }, + ).model_dump(), "default_inputs": None, } post_response = client.post("/functions", json=sample_function) @@ -393,7 +400,7 @@ def test_get_function_output_schema(api_app: FastAPI) -> None: response = client.get(f"/functions/{function_id}/output_schema") assert response.status_code == 200 data = response.json() - assert data["schema_dict"] == sample_function["output_schema"]["schema_dict"] + assert data["schema_content"] == sample_function["output_schema"]["schema_content"] def test_validate_function_inputs(api_app: FastAPI) -> None: @@ -406,13 +413,18 @@ def test_validate_function_inputs(api_app: FastAPI) -> None: "function_class": "project", "project_id": project_id, "description": "An example function", - "input_schema": { - "schema_dict": { + "input_schema": JSONFunctionInputSchema( + schema_content={ "type": "object", "properties": {"input1": {"type": "integer"}}, } - }, - "output_schema": {"schema_dict": {}}, + ).model_dump(), + "output_schema": JSONFunctionOutputSchema( + schema_content={ + "type": "object", + "properties": {"output1": {"type": "string"}}, + } + ).model_dump(), "default_inputs": None, } post_response = client.post("/functions", json=sample_function) @@ -440,8 +452,8 @@ def test_delete_function(api_app: FastAPI) -> None: "function_class": "project", "project_id": project_id, "description": "An example function", - "input_schema": {"schema_dict": {}}, - "output_schema": {"schema_dict": {}}, + "input_schema": JSONFunctionInputSchema().model_dump(), + "output_schema": JSONFunctionOutputSchema().model_dump(), "default_inputs": None, } post_response = client.post("/functions", json=sample_function) diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py index 5e5cb4d7155..a893d91c79d 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py @@ -268,13 +268,7 @@ async def get_function_input_schema( app=app, function_id=function_id, ) - return FunctionInputSchema( - schema_dict=( - returned_function.input_schema.schema_dict - if returned_function.input_schema - else None - ) - ) + return _decode_function(returned_function).input_schema @router.expose(reraise_if_error_type=(FunctionIDNotFoundError,)) @@ -286,13 +280,7 @@ async def get_function_output_schema( app=app, function_id=function_id, ) - return FunctionOutputSchema( - schema_dict=( - returned_function.output_schema.schema_dict - if returned_function.output_schema - else None - ) - ) + return _decode_function(returned_function).output_schema def _decode_function( diff --git a/services/web/server/tests/unit/with_dbs/functions_rpc/test_functions_controller_rpc.py b/services/web/server/tests/unit/with_dbs/functions_rpc/test_functions_controller_rpc.py index 8e78093115a..54720e8a49f 100644 --- a/services/web/server/tests/unit/with_dbs/functions_rpc/test_functions_controller_rpc.py +++ b/services/web/server/tests/unit/with_dbs/functions_rpc/test_functions_controller_rpc.py @@ -6,10 +6,10 @@ from models_library.api_schemas_webserver.functions_wb_schema import ( Function, FunctionIDNotFoundError, - FunctionInputSchema, FunctionJobCollection, FunctionJobIDNotFoundError, - FunctionOutputSchema, + JSONFunctionInputSchema, + JSONFunctionOutputSchema, ProjectFunction, ProjectFunctionJob, ) @@ -21,11 +21,14 @@ def mock_function() -> Function: uid=None, title="Test Function", description="A test function", - input_schema=FunctionInputSchema( - schema_dict={"type": "object", "properties": {"input1": {"type": "string"}}} + input_schema=JSONFunctionInputSchema( + schema_content={ + "type": "object", + "properties": {"input1": {"type": "string"}}, + } ), - output_schema=FunctionOutputSchema( - schema_dict={ + output_schema=JSONFunctionOutputSchema( + schema_content={ "type": "object", "properties": {"output1": {"type": "string"}}, } @@ -101,8 +104,8 @@ async def test_list_functions(client): uid=None, title="Test Function", description="A test function", - input_schema=None, - output_schema=None, + input_schema=JSONFunctionInputSchema(), + output_schema=JSONFunctionOutputSchema(), project_id=uuid4(), default_inputs=None, ) From 1c221433eb74bafbf16e0891866eaeccd75c8e32 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Fri, 9 May 2025 13:07:01 +0200 Subject: [PATCH 62/69] Refactor function db files based on SA's suggestion --- .../funcapi_function_job_collections_table.py | 35 ++++ ..._job_collections_to_function_jobs_table.py | 36 ++++ .../models/funcapi_function_jobs_table.py | 70 +++++++ .../models/funcapi_functions_table.py | 74 +++++++ .../models/functions_models_db.py | 181 ------------------ 5 files changed, 215 insertions(+), 181 deletions(-) create mode 100644 packages/postgres-database/src/simcore_postgres_database/models/funcapi_function_job_collections_table.py create mode 100644 packages/postgres-database/src/simcore_postgres_database/models/funcapi_function_job_collections_to_function_jobs_table.py create mode 100644 packages/postgres-database/src/simcore_postgres_database/models/funcapi_function_jobs_table.py create mode 100644 packages/postgres-database/src/simcore_postgres_database/models/funcapi_functions_table.py delete mode 100644 packages/postgres-database/src/simcore_postgres_database/models/functions_models_db.py diff --git a/packages/postgres-database/src/simcore_postgres_database/models/funcapi_function_job_collections_table.py b/packages/postgres-database/src/simcore_postgres_database/models/funcapi_function_job_collections_table.py new file mode 100644 index 00000000000..457d59f78bc --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/models/funcapi_function_job_collections_table.py @@ -0,0 +1,35 @@ +"""Functions table + +- List of functions served by the simcore platform +""" + +import uuid + +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import UUID + +from .base import metadata + +function_job_collections_table = sa.Table( + "funcapi_function_job_collections", + metadata, + sa.Column( + "uuid", + UUID(as_uuid=True), + default=uuid.uuid4, + primary_key=True, + index=True, + doc="Unique id of the function job collection", + ), + sa.Column( + "title", + sa.String, + doc="Title of the function job collection", + ), + sa.Column( + "description", + sa.String, + doc="Description of the function job collection", + ), + sa.PrimaryKeyConstraint("uuid", name="function_job_collections_pk"), +) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/funcapi_function_job_collections_to_function_jobs_table.py b/packages/postgres-database/src/simcore_postgres_database/models/funcapi_function_job_collections_to_function_jobs_table.py new file mode 100644 index 00000000000..8c2f1f7cb25 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/models/funcapi_function_job_collections_to_function_jobs_table.py @@ -0,0 +1,36 @@ +"""Functions table + +- List of functions served by the simcore platform +""" + +import sqlalchemy as sa + +from ._common import RefActions +from .base import metadata +from .funcapi_function_job_collections_table import function_job_collections_table +from .funcapi_function_jobs_table import function_jobs_table + +function_job_collections_to_function_jobs_table = sa.Table( + "funcapi_function_job_collections_to_function_jobs", + metadata, + sa.Column( + "function_job_collection_uuid", + sa.ForeignKey( + function_job_collections_table.c.uuid, + onupdate=RefActions.CASCADE, + ondelete=RefActions.CASCADE, + name="fk_func_job_coll_to_func_jobs_to_func_job_coll_uuid", + ), + doc="Unique identifier of the function job collection", + ), + sa.Column( + "function_job_uuid", + sa.ForeignKey( + function_jobs_table.c.uuid, + onupdate=RefActions.CASCADE, + ondelete=RefActions.CASCADE, + name="fk_func_job_coll_to_func_jobs_to_func_job_uuid", + ), + doc="Unique identifier of the function job", + ), +) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/funcapi_function_jobs_table.py b/packages/postgres-database/src/simcore_postgres_database/models/funcapi_function_jobs_table.py new file mode 100644 index 00000000000..6a58b11857d --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/models/funcapi_function_jobs_table.py @@ -0,0 +1,70 @@ +"""Functions table + +- List of functions served by the simcore platform +""" + +import uuid + +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import UUID + +from ._common import RefActions +from .base import metadata +from .funcapi_functions_table import functions_table + +function_jobs_table = sa.Table( + "funcapi_function_jobs", + metadata, + sa.Column( + "uuid", + UUID(as_uuid=True), + primary_key=True, + index=True, + default=uuid.uuid4, + doc="Unique id of the function job", + ), + sa.Column( + "title", + sa.String, + doc="Name of the function job", + ), + sa.Column( + "function_uuid", + sa.ForeignKey( + functions_table.c.uuid, + onupdate=RefActions.CASCADE, + ondelete=RefActions.CASCADE, + name="fk_function_jobs_to_function_uuid", + ), + nullable=False, + index=True, + doc="Unique identifier of the function", + ), + sa.Column( + "function_class", + sa.String, + doc="Class of the function", + ), + sa.Column( + "status", + sa.String, + doc="Status of the function job", + ), + sa.Column( + "inputs", + sa.JSON, + doc="Inputs of the function job", + ), + sa.Column( + "outputs", + sa.JSON, + doc="Outputs of the function job", + ), + sa.Column( + "class_specific_data", + sa.JSON, + nullable=True, + doc="Fields specific for a function class", + ), + sa.PrimaryKeyConstraint("uuid", name="function_jobs_pk"), +) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/funcapi_functions_table.py b/packages/postgres-database/src/simcore_postgres_database/models/funcapi_functions_table.py new file mode 100644 index 00000000000..49ee64c564a --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/models/funcapi_functions_table.py @@ -0,0 +1,74 @@ +"""Functions table + +- List of functions served by the simcore platform +""" + +import uuid + +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import UUID + +from .base import metadata + +functions_table = sa.Table( + "funcapi_functions", + metadata, + sa.Column( + "uuid", + UUID(as_uuid=True), + primary_key=True, + index=True, + default=uuid.uuid4, + doc="Unique id of the function", + ), + sa.Column( + "title", + sa.String, + doc="Name of the function", + ), + sa.Column( + "function_class", + sa.String, + doc="Class of the function", + ), + sa.Column( + "description", + sa.String, + doc="Description of the function", + ), + sa.Column( + "input_schema", + sa.JSON, + doc="Input schema of the function", + ), + sa.Column( + "output_schema", + sa.JSON, + doc="Output schema of the function", + ), + sa.Column( + "system_tags", + sa.JSON, + nullable=True, + doc="System-level tags of the function", + ), + sa.Column( + "user_tags", + sa.JSON, + nullable=True, + doc="User-level tags of the function", + ), + sa.Column( + "class_specific_data", + sa.JSON, + nullable=True, + doc="Fields specific for a function class", + ), + sa.Column( + "default_inputs", + sa.JSON, + nullable=True, + doc="Default inputs of the function", + ), + sa.PrimaryKeyConstraint("uuid", name="functions_pk"), +) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/functions_models_db.py b/packages/postgres-database/src/simcore_postgres_database/models/functions_models_db.py deleted file mode 100644 index ee8dd1474c0..00000000000 --- a/packages/postgres-database/src/simcore_postgres_database/models/functions_models_db.py +++ /dev/null @@ -1,181 +0,0 @@ -"""Functions table - -- List of functions served by the simcore platform -""" - -import uuid - -import sqlalchemy as sa -from sqlalchemy.dialects.postgresql import UUID - -from ._common import RefActions -from .base import metadata - -functions = sa.Table( - "functions", - metadata, - sa.Column( - "uuid", - UUID(as_uuid=True), - primary_key=True, - index=True, - default=uuid.uuid4, - doc="Unique id of the function", - ), - sa.Column( - "title", - sa.String, - doc="Name of the function", - ), - sa.Column( - "function_class", - sa.String, - doc="Class of the function", - ), - sa.Column( - "description", - sa.String, - doc="Description of the function", - ), - sa.Column( - "input_schema", - sa.JSON, - doc="Input schema of the function", - ), - sa.Column( - "output_schema", - sa.JSON, - doc="Output schema of the function", - ), - sa.Column( - "system_tags", - sa.JSON, - nullable=True, - doc="System-level tags of the function", - ), - sa.Column( - "user_tags", - sa.JSON, - nullable=True, - doc="User-level tags of the function", - ), - sa.Column( - "class_specific_data", - sa.JSON, - nullable=True, - doc="Fields specific for a function class", - ), - sa.Column( - "default_inputs", - sa.JSON, - nullable=True, - doc="Default inputs of the function", - ), - sa.PrimaryKeyConstraint("uuid", name="functions_pk"), -) - -function_jobs = sa.Table( - "function_jobs", - metadata, - sa.Column( - "uuid", - UUID(as_uuid=True), - primary_key=True, - index=True, - default=uuid.uuid4, - doc="Unique id of the function job", - ), - sa.Column( - "title", - sa.String, - doc="Name of the function job", - ), - sa.Column( - "function_uuid", - sa.ForeignKey( - functions.c.uuid, - onupdate=RefActions.CASCADE, - ondelete=RefActions.CASCADE, - name="fk_function_jobs_to_function_uuid", - ), - nullable=False, - index=True, - doc="Unique identifier of the function", - ), - sa.Column( - "function_class", - sa.String, - doc="Class of the function", - ), - sa.Column( - "status", - sa.String, - doc="Status of the function job", - ), - sa.Column( - "inputs", - sa.JSON, - doc="Inputs of the function job", - ), - sa.Column( - "outputs", - sa.JSON, - doc="Outputs of the function job", - ), - sa.Column( - "class_specific_data", - sa.JSON, - nullable=True, - doc="Fields specific for a function class", - ), - sa.PrimaryKeyConstraint("uuid", name="function_jobs_pk"), -) - -function_job_collections = sa.Table( - "function_job_collections", - metadata, - sa.Column( - "uuid", - UUID(as_uuid=True), - default=uuid.uuid4, - primary_key=True, - index=True, - doc="Unique id of the function job collection", - ), - sa.Column( - "title", - sa.String, - doc="Title of the function job collection", - ), - sa.Column( - "description", - sa.String, - doc="Description of the function job collection", - ), - sa.PrimaryKeyConstraint("uuid", name="function_job_collections_pk"), -) - -function_job_collections_to_function_jobs = sa.Table( - "function_job_collections_to_function_jobs", - metadata, - sa.Column( - "function_job_collection_uuid", - sa.ForeignKey( - function_job_collections.c.uuid, - onupdate=RefActions.CASCADE, - ondelete=RefActions.CASCADE, - name="fk_func_job_coll_to_func_jobs_to_func_job_coll_uuid", - ), - doc="Unique identifier of the function job collection", - ), - sa.Column( - "function_job_uuid", - sa.ForeignKey( - function_jobs.c.uuid, - onupdate=RefActions.CASCADE, - ondelete=RefActions.CASCADE, - name="fk_func_job_coll_to_func_jobs_to_func_job_uuid", - ), - doc="Unique identifier of the function job", - ), -) From c54829322bb46d760652d80f2e411cbe79de838a Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Fri, 9 May 2025 13:11:20 +0200 Subject: [PATCH 63/69] Fix db names in function repo --- .../functions/_functions_repository.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py index faa226f7107..829fec295fc 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py @@ -17,18 +17,16 @@ from models_library.rest_pagination import ( PageMetaInfoLimitOffset, ) -from simcore_postgres_database.models.functions_models_db import ( - function_job_collections as function_job_collections_table, +from simcore_postgres_database.models.funcapi_function_job_collections_table import ( + function_job_collections_table, ) -from simcore_postgres_database.models.functions_models_db import ( - function_job_collections_to_function_jobs as function_job_collections_to_function_jobs_table, +from simcore_postgres_database.models.funcapi_function_job_collections_to_function_jobs_table import ( + function_job_collections_to_function_jobs_table, ) -from simcore_postgres_database.models.functions_models_db import ( - function_jobs as function_jobs_table, -) -from simcore_postgres_database.models.functions_models_db import ( - functions as functions_table, +from simcore_postgres_database.models.funcapi_function_jobs_table import ( + function_jobs_table, ) +from simcore_postgres_database.models.funcapi_functions_table import functions_table from simcore_postgres_database.utils_repos import ( get_columns_from_db_model, transaction_context, From 489dd9284a9c312537283d5523b166f7576c89f0 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Fri, 9 May 2025 13:17:41 +0200 Subject: [PATCH 64/69] Rename function tables primary keys --- .../models/funcapi_function_job_collections_table.py | 2 +- ...uncapi_function_job_collections_to_function_jobs_table.py | 5 +++++ .../models/funcapi_function_jobs_table.py | 2 +- .../models/funcapi_functions_table.py | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/funcapi_function_job_collections_table.py b/packages/postgres-database/src/simcore_postgres_database/models/funcapi_function_job_collections_table.py index 457d59f78bc..49e882b6d63 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/funcapi_function_job_collections_table.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/funcapi_function_job_collections_table.py @@ -31,5 +31,5 @@ sa.String, doc="Description of the function job collection", ), - sa.PrimaryKeyConstraint("uuid", name="function_job_collections_pk"), + sa.PrimaryKeyConstraint("uuid", name="funcapi_function_job_collections_pk"), ) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/funcapi_function_job_collections_to_function_jobs_table.py b/packages/postgres-database/src/simcore_postgres_database/models/funcapi_function_job_collections_to_function_jobs_table.py index 8c2f1f7cb25..2cc264da9bd 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/funcapi_function_job_collections_to_function_jobs_table.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/funcapi_function_job_collections_to_function_jobs_table.py @@ -33,4 +33,9 @@ ), doc="Unique identifier of the function job", ), + sa.PrimaryKeyConstraint( + "function_job_collection_uuid", + "function_job_uuid", + name="funcapi_function_job_collections_to_function_jobs_pk", + ), ) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/funcapi_function_jobs_table.py b/packages/postgres-database/src/simcore_postgres_database/models/funcapi_function_jobs_table.py index 6a58b11857d..24cd014e319 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/funcapi_function_jobs_table.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/funcapi_function_jobs_table.py @@ -66,5 +66,5 @@ nullable=True, doc="Fields specific for a function class", ), - sa.PrimaryKeyConstraint("uuid", name="function_jobs_pk"), + sa.PrimaryKeyConstraint("uuid", name="funcapi_function_jobs_pk"), ) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/funcapi_functions_table.py b/packages/postgres-database/src/simcore_postgres_database/models/funcapi_functions_table.py index 49ee64c564a..a7cb97740a9 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/funcapi_functions_table.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/funcapi_functions_table.py @@ -70,5 +70,5 @@ nullable=True, doc="Default inputs of the function", ), - sa.PrimaryKeyConstraint("uuid", name="functions_pk"), + sa.PrimaryKeyConstraint("uuid", name="funcapi_functions_pk"), ) From f1e9a63bf83c9a1f5ad9fbc4ce8272f5bbf72568 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Fri, 9 May 2025 13:28:14 +0200 Subject: [PATCH 65/69] Remove pk constraints from a functions table --- ...uncapi_function_job_collections_to_function_jobs_table.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/funcapi_function_job_collections_to_function_jobs_table.py b/packages/postgres-database/src/simcore_postgres_database/models/funcapi_function_job_collections_to_function_jobs_table.py index 2cc264da9bd..8c2f1f7cb25 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/funcapi_function_job_collections_to_function_jobs_table.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/funcapi_function_job_collections_to_function_jobs_table.py @@ -33,9 +33,4 @@ ), doc="Unique identifier of the function job", ), - sa.PrimaryKeyConstraint( - "function_job_collection_uuid", - "function_job_uuid", - name="funcapi_function_job_collections_to_function_jobs_pk", - ), ) From df84db9a857d75bc6f3e8c6cb679b337219f228c Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Fri, 9 May 2025 14:18:50 +0200 Subject: [PATCH 66/69] Add migration script for renaming funcapi tables --- .../f31eefd27b2d_rename_funcapi_tables.py | 263 ++++++++++++++++++ 1 file changed, 263 insertions(+) create mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/f31eefd27b2d_rename_funcapi_tables.py diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/f31eefd27b2d_rename_funcapi_tables.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/f31eefd27b2d_rename_funcapi_tables.py new file mode 100644 index 00000000000..74ba53e6520 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/f31eefd27b2d_rename_funcapi_tables.py @@ -0,0 +1,263 @@ +"""Rename funcapi tables + +Revision ID: f31eefd27b2d +Revises: 1b5c88debc3e +Create Date: 2025-05-09 11:29:37.040127+00:00 + +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "f31eefd27b2d" +down_revision = "1b5c88debc3e" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "funcapi_function_job_collections", + sa.Column("uuid", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("title", sa.String(), nullable=True), + sa.Column("description", sa.String(), nullable=True), + sa.PrimaryKeyConstraint("uuid", name="funcapi_function_job_collections_pk"), + ) + op.create_index( + op.f("ix_funcapi_function_job_collections_uuid"), + "funcapi_function_job_collections", + ["uuid"], + unique=False, + ) + op.create_table( + "funcapi_functions", + sa.Column("uuid", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("title", sa.String(), nullable=True), + sa.Column("function_class", sa.String(), nullable=True), + sa.Column("description", sa.String(), nullable=True), + sa.Column("input_schema", sa.JSON(), nullable=True), + sa.Column("output_schema", sa.JSON(), nullable=True), + sa.Column("system_tags", sa.JSON(), nullable=True), + sa.Column("user_tags", sa.JSON(), nullable=True), + sa.Column("class_specific_data", sa.JSON(), nullable=True), + sa.Column("default_inputs", sa.JSON(), nullable=True), + sa.PrimaryKeyConstraint("uuid", name="funcapi_functions_pk"), + ) + op.create_index( + op.f("ix_funcapi_functions_uuid"), "funcapi_functions", ["uuid"], unique=False + ) + op.create_table( + "funcapi_function_jobs", + sa.Column("uuid", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("title", sa.String(), nullable=True), + sa.Column("function_uuid", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("function_class", sa.String(), nullable=True), + sa.Column("status", sa.String(), nullable=True), + sa.Column("inputs", sa.JSON(), nullable=True), + sa.Column("outputs", sa.JSON(), nullable=True), + sa.Column("class_specific_data", sa.JSON(), nullable=True), + sa.ForeignKeyConstraint( + ["function_uuid"], + ["funcapi_functions.uuid"], + name="fk_function_jobs_to_function_uuid", + onupdate="CASCADE", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("uuid", name="funcapi_function_jobs_pk"), + ) + op.create_index( + op.f("ix_funcapi_function_jobs_function_uuid"), + "funcapi_function_jobs", + ["function_uuid"], + unique=False, + ) + op.create_index( + op.f("ix_funcapi_function_jobs_uuid"), + "funcapi_function_jobs", + ["uuid"], + unique=False, + ) + op.create_table( + "funcapi_function_job_collections_to_function_jobs", + sa.Column( + "function_job_collection_uuid", postgresql.UUID(as_uuid=True), nullable=True + ), + sa.Column("function_job_uuid", postgresql.UUID(as_uuid=True), nullable=True), + sa.ForeignKeyConstraint( + ["function_job_collection_uuid"], + ["funcapi_function_job_collections.uuid"], + name="fk_func_job_coll_to_func_jobs_to_func_job_coll_uuid", + onupdate="CASCADE", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["function_job_uuid"], + ["funcapi_function_jobs.uuid"], + name="fk_func_job_coll_to_func_jobs_to_func_job_uuid", + onupdate="CASCADE", + ondelete="CASCADE", + ), + ) + op.drop_table("function_job_collections_to_function_jobs") + op.drop_index( + "ix_function_job_collections_uuid", table_name="function_job_collections" + ) + op.drop_table("function_job_collections") + op.drop_index("ix_function_jobs_function_uuid", table_name="function_jobs") + op.drop_index("ix_function_jobs_uuid", table_name="function_jobs") + op.drop_table("function_jobs") + op.drop_index("ix_functions_uuid", table_name="functions") + op.drop_table("functions") + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "function_job_collections_to_function_jobs", + sa.Column( + "function_job_collection_uuid", + postgresql.UUID(), + autoincrement=False, + nullable=True, + ), + sa.Column( + "function_job_uuid", postgresql.UUID(), autoincrement=False, nullable=True + ), + sa.ForeignKeyConstraint( + ["function_job_collection_uuid"], + ["function_job_collections.uuid"], + name="fk_func_job_coll_to_func_jobs_to_func_job_coll_uuid", + onupdate="CASCADE", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["function_job_uuid"], + ["function_jobs.uuid"], + name="fk_func_job_coll_to_func_jobs_to_func_job_uuid", + onupdate="CASCADE", + ondelete="CASCADE", + ), + ) + op.create_table( + "function_job_collections", + sa.Column("uuid", postgresql.UUID(), autoincrement=False, nullable=False), + sa.Column("title", sa.VARCHAR(), autoincrement=False, nullable=True), + sa.Column("description", sa.VARCHAR(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint("uuid", name="function_job_collections_pk"), + ) + op.create_index( + "ix_function_job_collections_uuid", + "function_job_collections", + ["uuid"], + unique=False, + ) + op.create_table( + "functions", + sa.Column("uuid", postgresql.UUID(), autoincrement=False, nullable=False), + sa.Column("title", sa.VARCHAR(), autoincrement=False, nullable=True), + sa.Column("function_class", sa.VARCHAR(), autoincrement=False, nullable=True), + sa.Column("description", sa.VARCHAR(), autoincrement=False, nullable=True), + sa.Column( + "input_schema", + postgresql.JSON(astext_type=sa.Text()), + autoincrement=False, + nullable=True, + ), + sa.Column( + "output_schema", + postgresql.JSON(astext_type=sa.Text()), + autoincrement=False, + nullable=True, + ), + sa.Column( + "system_tags", + postgresql.JSON(astext_type=sa.Text()), + autoincrement=False, + nullable=True, + ), + sa.Column( + "user_tags", + postgresql.JSON(astext_type=sa.Text()), + autoincrement=False, + nullable=True, + ), + sa.Column( + "class_specific_data", + postgresql.JSON(astext_type=sa.Text()), + autoincrement=False, + nullable=True, + ), + sa.Column( + "default_inputs", + postgresql.JSON(astext_type=sa.Text()), + autoincrement=False, + nullable=True, + ), + sa.PrimaryKeyConstraint("uuid", name="functions_pk"), + postgresql_ignore_search_path=False, + ) + op.create_index("ix_functions_uuid", "functions", ["uuid"], unique=False) + op.create_table( + "function_jobs", + sa.Column("uuid", postgresql.UUID(), autoincrement=False, nullable=False), + sa.Column("title", sa.VARCHAR(), autoincrement=False, nullable=True), + sa.Column( + "function_uuid", postgresql.UUID(), autoincrement=False, nullable=False + ), + sa.Column("function_class", sa.VARCHAR(), autoincrement=False, nullable=True), + sa.Column("status", sa.VARCHAR(), autoincrement=False, nullable=True), + sa.Column( + "inputs", + postgresql.JSON(astext_type=sa.Text()), + autoincrement=False, + nullable=True, + ), + sa.Column( + "outputs", + postgresql.JSON(astext_type=sa.Text()), + autoincrement=False, + nullable=True, + ), + sa.Column( + "class_specific_data", + postgresql.JSON(astext_type=sa.Text()), + autoincrement=False, + nullable=True, + ), + sa.ForeignKeyConstraint( + ["function_uuid"], + ["functions.uuid"], + name="fk_functions_to_function_jobs_to_function_uuid", + onupdate="CASCADE", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("uuid", name="function_jobs_pk"), + ) + op.create_index("ix_function_jobs_uuid", "function_jobs", ["uuid"], unique=False) + op.create_index( + "ix_function_jobs_function_uuid", + "function_jobs", + ["function_uuid"], + unique=False, + ) + op.drop_table("funcapi_function_job_collections_to_function_jobs") + op.drop_index( + op.f("ix_funcapi_function_job_collections_uuid"), + table_name="funcapi_function_job_collections", + ) + op.drop_table("funcapi_function_job_collections") + op.drop_index( + op.f("ix_funcapi_function_jobs_uuid"), table_name="funcapi_function_jobs" + ) + op.drop_index( + op.f("ix_funcapi_function_jobs_function_uuid"), + table_name="funcapi_function_jobs", + ) + op.drop_table("funcapi_function_jobs") + op.drop_index(op.f("ix_funcapi_functions_uuid"), table_name="funcapi_functions") + op.drop_table("funcapi_functions") + # ### end Alembic commands ### From 810a1fa9cfafccae40f41a1d59975cbc36fde4db Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Fri, 9 May 2025 15:08:29 +0200 Subject: [PATCH 67/69] Delete db migrates for funcapi to cleanup --- ...b7f433b_fix_function_job_collections_db.py | 60 ---- ...8debc3e_merge_0d52976dc616_ecd7a3b85134.py | 21 -- .../93dbd49553ae_add_function_tables.py | 133 --------- ...94af8f28b25_add_function_default_inputs.py | 49 ---- ...3b85134_merge_742123f0933a_0b64fb7f433b.py | 21 -- .../f31eefd27b2d_rename_funcapi_tables.py | 263 ------------------ 6 files changed, 547 deletions(-) delete mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/0b64fb7f433b_fix_function_job_collections_db.py delete mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/1b5c88debc3e_merge_0d52976dc616_ecd7a3b85134.py delete mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/93dbd49553ae_add_function_tables.py delete mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/d94af8f28b25_add_function_default_inputs.py delete mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/ecd7a3b85134_merge_742123f0933a_0b64fb7f433b.py delete mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/f31eefd27b2d_rename_funcapi_tables.py diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/0b64fb7f433b_fix_function_job_collections_db.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/0b64fb7f433b_fix_function_job_collections_db.py deleted file mode 100644 index 2fb484396f8..00000000000 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/0b64fb7f433b_fix_function_job_collections_db.py +++ /dev/null @@ -1,60 +0,0 @@ -"""Fix function job collections db - -Revision ID: 0b64fb7f433b -Revises: d94af8f28b25 -Create Date: 2025-04-29 11:12:23.529262+00:00 - -""" - -import sqlalchemy as sa -from alembic import op - -# revision identifiers, used by Alembic. -revision = "0b64fb7f433b" -down_revision = "d94af8f28b25" -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.add_column( - "function_job_collections", sa.Column("title", sa.String(), nullable=True) - ) - op.add_column( - "function_job_collections", sa.Column("description", sa.String(), nullable=True) - ) - op.drop_column("function_job_collections", "name") - op.drop_index("idx_projects_last_change_date_desc", table_name="projects") - op.create_index( - "idx_projects_last_change_date_desc", - "projects", - ["last_change_date"], - unique=False, - postgresql_using="btree", - postgresql_ops={"last_change_date": "DESC"}, - ) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_index( - "idx_projects_last_change_date_desc", - table_name="projects", - postgresql_using="btree", - postgresql_ops={"last_change_date": "DESC"}, - ) - op.create_index( - "idx_projects_last_change_date_desc", - "projects", - [sa.text("last_change_date DESC")], - unique=False, - ) - op.add_column( - "function_job_collections", - sa.Column("name", sa.VARCHAR(), autoincrement=False, nullable=True), - ) - op.drop_column("function_job_collections", "description") - op.drop_column("function_job_collections", "title") - # ### end Alembic commands ### diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/1b5c88debc3e_merge_0d52976dc616_ecd7a3b85134.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/1b5c88debc3e_merge_0d52976dc616_ecd7a3b85134.py deleted file mode 100644 index be5f96cccd6..00000000000 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/1b5c88debc3e_merge_0d52976dc616_ecd7a3b85134.py +++ /dev/null @@ -1,21 +0,0 @@ -"""merge 0d52976dc616 ecd7a3b85134 - -Revision ID: 1b5c88debc3e -Revises: 0d52976dc616, ecd7a3b85134 -Create Date: 2025-05-07 08:45:47.779512+00:00 - -""" - -# revision identifiers, used by Alembic. -revision = "1b5c88debc3e" -down_revision = ("0d52976dc616", "ecd7a3b85134") -branch_labels = None -depends_on = None - - -def upgrade(): - pass - - -def downgrade(): - pass diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/93dbd49553ae_add_function_tables.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/93dbd49553ae_add_function_tables.py deleted file mode 100644 index d44b8e271e2..00000000000 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/93dbd49553ae_add_function_tables.py +++ /dev/null @@ -1,133 +0,0 @@ -"""Add function tables - -Revision ID: 93dbd49553ae -Revises: cf8f743fd0b7 -Create Date: 2025-04-16 09:32:48.976846+00:00 - -""" - -import sqlalchemy as sa -from alembic import op -from sqlalchemy.dialects import postgresql - -# revision identifiers, used by Alembic. -revision = "93dbd49553ae" -down_revision = "cf8f743fd0b7" -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table( - "function_job_collections", - sa.Column("uuid", postgresql.UUID(as_uuid=True), nullable=False), - sa.Column("name", sa.String(), nullable=True), - sa.PrimaryKeyConstraint("uuid", name="function_job_collections_pk"), - ) - op.create_index( - op.f("ix_function_job_collections_uuid"), - "function_job_collections", - ["uuid"], - unique=False, - ) - op.create_table( - "functions", - sa.Column("uuid", postgresql.UUID(as_uuid=True), nullable=False), - sa.Column("title", sa.String(), nullable=True), - sa.Column("function_class", sa.String(), nullable=True), - sa.Column("description", sa.String(), nullable=True), - sa.Column("input_schema", sa.JSON(), nullable=True), - sa.Column("output_schema", sa.JSON(), nullable=True), - sa.Column("system_tags", sa.JSON(), nullable=True), - sa.Column("user_tags", sa.JSON(), nullable=True), - sa.Column("class_specific_data", sa.JSON(), nullable=True), - sa.PrimaryKeyConstraint("uuid", name="functions_pk"), - ) - op.create_index(op.f("ix_functions_uuid"), "functions", ["uuid"], unique=False) - op.create_table( - "function_jobs", - sa.Column("uuid", postgresql.UUID(as_uuid=True), nullable=False), - sa.Column("title", sa.String(), nullable=True), - sa.Column("function_uuid", postgresql.UUID(as_uuid=True), nullable=False), - sa.Column("function_class", sa.String(), nullable=True), - sa.Column("status", sa.String(), nullable=True), - sa.Column("inputs", sa.JSON(), nullable=True), - sa.Column("outputs", sa.JSON(), nullable=True), - sa.Column("class_specific_data", sa.JSON(), nullable=True), - sa.ForeignKeyConstraint( - ["function_uuid"], - ["functions.uuid"], - name="fk_functions_to_function_jobs_to_function_uuid", - onupdate="CASCADE", - ondelete="CASCADE", - ), - sa.PrimaryKeyConstraint("uuid", name="function_jobs_pk"), - ) - op.create_index( - op.f("ix_function_jobs_function_uuid"), - "function_jobs", - ["function_uuid"], - unique=False, - ) - op.create_index( - op.f("ix_function_jobs_uuid"), "function_jobs", ["uuid"], unique=False - ) - op.create_table( - "function_job_collections_to_function_jobs", - sa.Column( - "function_job_collection_uuid", postgresql.UUID(as_uuid=True), nullable=True - ), - sa.Column("function_job_uuid", postgresql.UUID(as_uuid=True), nullable=True), - sa.ForeignKeyConstraint( - ["function_job_collection_uuid"], - ["function_job_collections.uuid"], - name="fk_func_job_coll_to_func_jobs_to_func_job_coll_uuid", - onupdate="CASCADE", - ondelete="CASCADE", - ), - sa.ForeignKeyConstraint( - ["function_job_uuid"], - ["function_jobs.uuid"], - name="fk_func_job_coll_to_func_jobs_to_func_job_uuid", - onupdate="CASCADE", - ondelete="CASCADE", - ), - ) - op.drop_index("idx_projects_last_change_date_desc", table_name="projects") - op.create_index( - "idx_projects_last_change_date_desc", - "projects", - ["last_change_date"], - unique=False, - postgresql_using="btree", - postgresql_ops={"last_change_date": "DESC"}, - ) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_index( - "idx_projects_last_change_date_desc", - table_name="projects", - postgresql_using="btree", - postgresql_ops={"last_change_date": "DESC"}, - ) - op.create_index( - "idx_projects_last_change_date_desc", - "projects", - [sa.text("last_change_date DESC")], - unique=False, - ) - op.drop_table("function_job_collections_to_function_jobs") - op.drop_index(op.f("ix_function_jobs_uuid"), table_name="function_jobs") - op.drop_index(op.f("ix_function_jobs_function_uuid"), table_name="function_jobs") - op.drop_table("function_jobs") - op.drop_index(op.f("ix_functions_uuid"), table_name="functions") - op.drop_table("functions") - op.drop_index( - op.f("ix_function_job_collections_uuid"), table_name="function_job_collections" - ) - op.drop_table("function_job_collections") - # ### end Alembic commands ### diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/d94af8f28b25_add_function_default_inputs.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/d94af8f28b25_add_function_default_inputs.py deleted file mode 100644 index 621916c8233..00000000000 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/d94af8f28b25_add_function_default_inputs.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Add function default inputs - -Revision ID: d94af8f28b25 -Revises: 93dbd49553ae -Create Date: 2025-04-16 16:23:12.224948+00:00 - -""" - -import sqlalchemy as sa -from alembic import op - -# revision identifiers, used by Alembic. -revision = "d94af8f28b25" -down_revision = "93dbd49553ae" -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.add_column("functions", sa.Column("default_inputs", sa.JSON(), nullable=True)) - op.drop_index("idx_projects_last_change_date_desc", table_name="projects") - op.create_index( - "idx_projects_last_change_date_desc", - "projects", - ["last_change_date"], - unique=False, - postgresql_using="btree", - postgresql_ops={"last_change_date": "DESC"}, - ) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_index( - "idx_projects_last_change_date_desc", - table_name="projects", - postgresql_using="btree", - postgresql_ops={"last_change_date": "DESC"}, - ) - op.create_index( - "idx_projects_last_change_date_desc", - "projects", - [sa.text("last_change_date DESC")], - unique=False, - ) - op.drop_column("functions", "default_inputs") - # ### end Alembic commands ### diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/ecd7a3b85134_merge_742123f0933a_0b64fb7f433b.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/ecd7a3b85134_merge_742123f0933a_0b64fb7f433b.py deleted file mode 100644 index 0118aaa4a77..00000000000 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/ecd7a3b85134_merge_742123f0933a_0b64fb7f433b.py +++ /dev/null @@ -1,21 +0,0 @@ -"""merge 742123f0933a 0b64fb7f433b - -Revision ID: ecd7a3b85134 -Revises: 742123f0933a, 0b64fb7f433b -Create Date: 2025-04-29 13:40:28.311099+00:00 - -""" - -# revision identifiers, used by Alembic. -revision = "ecd7a3b85134" -down_revision = ("742123f0933a", "0b64fb7f433b") -branch_labels = None -depends_on = None - - -def upgrade(): - pass - - -def downgrade(): - pass diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/f31eefd27b2d_rename_funcapi_tables.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/f31eefd27b2d_rename_funcapi_tables.py deleted file mode 100644 index 74ba53e6520..00000000000 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/f31eefd27b2d_rename_funcapi_tables.py +++ /dev/null @@ -1,263 +0,0 @@ -"""Rename funcapi tables - -Revision ID: f31eefd27b2d -Revises: 1b5c88debc3e -Create Date: 2025-05-09 11:29:37.040127+00:00 - -""" - -import sqlalchemy as sa -from alembic import op -from sqlalchemy.dialects import postgresql - -# revision identifiers, used by Alembic. -revision = "f31eefd27b2d" -down_revision = "1b5c88debc3e" -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table( - "funcapi_function_job_collections", - sa.Column("uuid", postgresql.UUID(as_uuid=True), nullable=False), - sa.Column("title", sa.String(), nullable=True), - sa.Column("description", sa.String(), nullable=True), - sa.PrimaryKeyConstraint("uuid", name="funcapi_function_job_collections_pk"), - ) - op.create_index( - op.f("ix_funcapi_function_job_collections_uuid"), - "funcapi_function_job_collections", - ["uuid"], - unique=False, - ) - op.create_table( - "funcapi_functions", - sa.Column("uuid", postgresql.UUID(as_uuid=True), nullable=False), - sa.Column("title", sa.String(), nullable=True), - sa.Column("function_class", sa.String(), nullable=True), - sa.Column("description", sa.String(), nullable=True), - sa.Column("input_schema", sa.JSON(), nullable=True), - sa.Column("output_schema", sa.JSON(), nullable=True), - sa.Column("system_tags", sa.JSON(), nullable=True), - sa.Column("user_tags", sa.JSON(), nullable=True), - sa.Column("class_specific_data", sa.JSON(), nullable=True), - sa.Column("default_inputs", sa.JSON(), nullable=True), - sa.PrimaryKeyConstraint("uuid", name="funcapi_functions_pk"), - ) - op.create_index( - op.f("ix_funcapi_functions_uuid"), "funcapi_functions", ["uuid"], unique=False - ) - op.create_table( - "funcapi_function_jobs", - sa.Column("uuid", postgresql.UUID(as_uuid=True), nullable=False), - sa.Column("title", sa.String(), nullable=True), - sa.Column("function_uuid", postgresql.UUID(as_uuid=True), nullable=False), - sa.Column("function_class", sa.String(), nullable=True), - sa.Column("status", sa.String(), nullable=True), - sa.Column("inputs", sa.JSON(), nullable=True), - sa.Column("outputs", sa.JSON(), nullable=True), - sa.Column("class_specific_data", sa.JSON(), nullable=True), - sa.ForeignKeyConstraint( - ["function_uuid"], - ["funcapi_functions.uuid"], - name="fk_function_jobs_to_function_uuid", - onupdate="CASCADE", - ondelete="CASCADE", - ), - sa.PrimaryKeyConstraint("uuid", name="funcapi_function_jobs_pk"), - ) - op.create_index( - op.f("ix_funcapi_function_jobs_function_uuid"), - "funcapi_function_jobs", - ["function_uuid"], - unique=False, - ) - op.create_index( - op.f("ix_funcapi_function_jobs_uuid"), - "funcapi_function_jobs", - ["uuid"], - unique=False, - ) - op.create_table( - "funcapi_function_job_collections_to_function_jobs", - sa.Column( - "function_job_collection_uuid", postgresql.UUID(as_uuid=True), nullable=True - ), - sa.Column("function_job_uuid", postgresql.UUID(as_uuid=True), nullable=True), - sa.ForeignKeyConstraint( - ["function_job_collection_uuid"], - ["funcapi_function_job_collections.uuid"], - name="fk_func_job_coll_to_func_jobs_to_func_job_coll_uuid", - onupdate="CASCADE", - ondelete="CASCADE", - ), - sa.ForeignKeyConstraint( - ["function_job_uuid"], - ["funcapi_function_jobs.uuid"], - name="fk_func_job_coll_to_func_jobs_to_func_job_uuid", - onupdate="CASCADE", - ondelete="CASCADE", - ), - ) - op.drop_table("function_job_collections_to_function_jobs") - op.drop_index( - "ix_function_job_collections_uuid", table_name="function_job_collections" - ) - op.drop_table("function_job_collections") - op.drop_index("ix_function_jobs_function_uuid", table_name="function_jobs") - op.drop_index("ix_function_jobs_uuid", table_name="function_jobs") - op.drop_table("function_jobs") - op.drop_index("ix_functions_uuid", table_name="functions") - op.drop_table("functions") - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table( - "function_job_collections_to_function_jobs", - sa.Column( - "function_job_collection_uuid", - postgresql.UUID(), - autoincrement=False, - nullable=True, - ), - sa.Column( - "function_job_uuid", postgresql.UUID(), autoincrement=False, nullable=True - ), - sa.ForeignKeyConstraint( - ["function_job_collection_uuid"], - ["function_job_collections.uuid"], - name="fk_func_job_coll_to_func_jobs_to_func_job_coll_uuid", - onupdate="CASCADE", - ondelete="CASCADE", - ), - sa.ForeignKeyConstraint( - ["function_job_uuid"], - ["function_jobs.uuid"], - name="fk_func_job_coll_to_func_jobs_to_func_job_uuid", - onupdate="CASCADE", - ondelete="CASCADE", - ), - ) - op.create_table( - "function_job_collections", - sa.Column("uuid", postgresql.UUID(), autoincrement=False, nullable=False), - sa.Column("title", sa.VARCHAR(), autoincrement=False, nullable=True), - sa.Column("description", sa.VARCHAR(), autoincrement=False, nullable=True), - sa.PrimaryKeyConstraint("uuid", name="function_job_collections_pk"), - ) - op.create_index( - "ix_function_job_collections_uuid", - "function_job_collections", - ["uuid"], - unique=False, - ) - op.create_table( - "functions", - sa.Column("uuid", postgresql.UUID(), autoincrement=False, nullable=False), - sa.Column("title", sa.VARCHAR(), autoincrement=False, nullable=True), - sa.Column("function_class", sa.VARCHAR(), autoincrement=False, nullable=True), - sa.Column("description", sa.VARCHAR(), autoincrement=False, nullable=True), - sa.Column( - "input_schema", - postgresql.JSON(astext_type=sa.Text()), - autoincrement=False, - nullable=True, - ), - sa.Column( - "output_schema", - postgresql.JSON(astext_type=sa.Text()), - autoincrement=False, - nullable=True, - ), - sa.Column( - "system_tags", - postgresql.JSON(astext_type=sa.Text()), - autoincrement=False, - nullable=True, - ), - sa.Column( - "user_tags", - postgresql.JSON(astext_type=sa.Text()), - autoincrement=False, - nullable=True, - ), - sa.Column( - "class_specific_data", - postgresql.JSON(astext_type=sa.Text()), - autoincrement=False, - nullable=True, - ), - sa.Column( - "default_inputs", - postgresql.JSON(astext_type=sa.Text()), - autoincrement=False, - nullable=True, - ), - sa.PrimaryKeyConstraint("uuid", name="functions_pk"), - postgresql_ignore_search_path=False, - ) - op.create_index("ix_functions_uuid", "functions", ["uuid"], unique=False) - op.create_table( - "function_jobs", - sa.Column("uuid", postgresql.UUID(), autoincrement=False, nullable=False), - sa.Column("title", sa.VARCHAR(), autoincrement=False, nullable=True), - sa.Column( - "function_uuid", postgresql.UUID(), autoincrement=False, nullable=False - ), - sa.Column("function_class", sa.VARCHAR(), autoincrement=False, nullable=True), - sa.Column("status", sa.VARCHAR(), autoincrement=False, nullable=True), - sa.Column( - "inputs", - postgresql.JSON(astext_type=sa.Text()), - autoincrement=False, - nullable=True, - ), - sa.Column( - "outputs", - postgresql.JSON(astext_type=sa.Text()), - autoincrement=False, - nullable=True, - ), - sa.Column( - "class_specific_data", - postgresql.JSON(astext_type=sa.Text()), - autoincrement=False, - nullable=True, - ), - sa.ForeignKeyConstraint( - ["function_uuid"], - ["functions.uuid"], - name="fk_functions_to_function_jobs_to_function_uuid", - onupdate="CASCADE", - ondelete="CASCADE", - ), - sa.PrimaryKeyConstraint("uuid", name="function_jobs_pk"), - ) - op.create_index("ix_function_jobs_uuid", "function_jobs", ["uuid"], unique=False) - op.create_index( - "ix_function_jobs_function_uuid", - "function_jobs", - ["function_uuid"], - unique=False, - ) - op.drop_table("funcapi_function_job_collections_to_function_jobs") - op.drop_index( - op.f("ix_funcapi_function_job_collections_uuid"), - table_name="funcapi_function_job_collections", - ) - op.drop_table("funcapi_function_job_collections") - op.drop_index( - op.f("ix_funcapi_function_jobs_uuid"), table_name="funcapi_function_jobs" - ) - op.drop_index( - op.f("ix_funcapi_function_jobs_function_uuid"), - table_name="funcapi_function_jobs", - ) - op.drop_table("funcapi_function_jobs") - op.drop_index(op.f("ix_funcapi_functions_uuid"), table_name="funcapi_functions") - op.drop_table("funcapi_functions") - # ### end Alembic commands ### From 8bfaa2a41d54b7b57fe3516934347b917a7edd9a Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Fri, 9 May 2025 15:15:30 +0200 Subject: [PATCH 68/69] Add db migration script for functions_api --- .../44f40f1069aa_add_function_api_tables.py | 125 ++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/44f40f1069aa_add_function_api_tables.py diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/44f40f1069aa_add_function_api_tables.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/44f40f1069aa_add_function_api_tables.py new file mode 100644 index 00000000000..a95d60ec226 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/44f40f1069aa_add_function_api_tables.py @@ -0,0 +1,125 @@ +"""Add function api tables + +Revision ID: 44f40f1069aa +Revises: 0d52976dc616 +Create Date: 2025-05-09 13:12:31.423832+00:00 + +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "44f40f1069aa" +down_revision = "0d52976dc616" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "funcapi_function_job_collections", + sa.Column("uuid", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("title", sa.String(), nullable=True), + sa.Column("description", sa.String(), nullable=True), + sa.PrimaryKeyConstraint("uuid", name="funcapi_function_job_collections_pk"), + ) + op.create_index( + op.f("ix_funcapi_function_job_collections_uuid"), + "funcapi_function_job_collections", + ["uuid"], + unique=False, + ) + op.create_table( + "funcapi_functions", + sa.Column("uuid", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("title", sa.String(), nullable=True), + sa.Column("function_class", sa.String(), nullable=True), + sa.Column("description", sa.String(), nullable=True), + sa.Column("input_schema", sa.JSON(), nullable=True), + sa.Column("output_schema", sa.JSON(), nullable=True), + sa.Column("system_tags", sa.JSON(), nullable=True), + sa.Column("user_tags", sa.JSON(), nullable=True), + sa.Column("class_specific_data", sa.JSON(), nullable=True), + sa.Column("default_inputs", sa.JSON(), nullable=True), + sa.PrimaryKeyConstraint("uuid", name="funcapi_functions_pk"), + ) + op.create_index( + op.f("ix_funcapi_functions_uuid"), "funcapi_functions", ["uuid"], unique=False + ) + op.create_table( + "funcapi_function_jobs", + sa.Column("uuid", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("title", sa.String(), nullable=True), + sa.Column("function_uuid", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("function_class", sa.String(), nullable=True), + sa.Column("status", sa.String(), nullable=True), + sa.Column("inputs", sa.JSON(), nullable=True), + sa.Column("outputs", sa.JSON(), nullable=True), + sa.Column("class_specific_data", sa.JSON(), nullable=True), + sa.ForeignKeyConstraint( + ["function_uuid"], + ["funcapi_functions.uuid"], + name="fk_function_jobs_to_function_uuid", + onupdate="CASCADE", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("uuid", name="funcapi_function_jobs_pk"), + ) + op.create_index( + op.f("ix_funcapi_function_jobs_function_uuid"), + "funcapi_function_jobs", + ["function_uuid"], + unique=False, + ) + op.create_index( + op.f("ix_funcapi_function_jobs_uuid"), + "funcapi_function_jobs", + ["uuid"], + unique=False, + ) + op.create_table( + "funcapi_function_job_collections_to_function_jobs", + sa.Column( + "function_job_collection_uuid", postgresql.UUID(as_uuid=True), nullable=True + ), + sa.Column("function_job_uuid", postgresql.UUID(as_uuid=True), nullable=True), + sa.ForeignKeyConstraint( + ["function_job_collection_uuid"], + ["funcapi_function_job_collections.uuid"], + name="fk_func_job_coll_to_func_jobs_to_func_job_coll_uuid", + onupdate="CASCADE", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["function_job_uuid"], + ["funcapi_function_jobs.uuid"], + name="fk_func_job_coll_to_func_jobs_to_func_job_uuid", + onupdate="CASCADE", + ondelete="CASCADE", + ), + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("funcapi_function_job_collections_to_function_jobs") + op.drop_index( + op.f("ix_funcapi_function_jobs_uuid"), table_name="funcapi_function_jobs" + ) + op.drop_index( + op.f("ix_funcapi_function_jobs_function_uuid"), + table_name="funcapi_function_jobs", + ) + op.drop_table("funcapi_function_jobs") + op.drop_index(op.f("ix_funcapi_functions_uuid"), table_name="funcapi_functions") + op.drop_table("funcapi_functions") + op.drop_index( + op.f("ix_funcapi_function_job_collections_uuid"), + table_name="funcapi_function_job_collections", + ) + op.drop_table("funcapi_function_job_collections") + # ### end Alembic commands ### From 98fa06674de9c3de86b4d2f81e98e8ed61e0d77b Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Fri, 9 May 2025 16:27:55 +0200 Subject: [PATCH 69/69] Run isort on function files --- .../webserver/functions/functions_rpc_interface.py | 8 ++------ .../api/routes/functions_routes.py | 8 ++------ .../unit/api_functions/test_api_routers_functions.py | 4 +--- .../functions/_functions_controller_rpc.py | 4 +--- .../functions/_functions_repository.py | 4 +--- 5 files changed, 7 insertions(+), 21 deletions(-) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py index be157e0f85a..6ff008a75ce 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py @@ -1,8 +1,6 @@ import logging -from models_library.api_schemas_webserver import ( - WEBSERVER_RPC_NAMESPACE, -) +from models_library.api_schemas_webserver import WEBSERVER_RPC_NAMESPACE from models_library.api_schemas_webserver.functions_wb_schema import ( Function, FunctionID, @@ -15,9 +13,7 @@ FunctionOutputSchema, ) from models_library.rabbitmq_basic_types import RPCMethodName -from models_library.rest_pagination import ( - PageMetaInfoLimitOffset, -) +from models_library.rest_pagination import PageMetaInfoLimitOffset from pydantic import TypeAdapter from .....logging_utils import log_decorator diff --git a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py index 4217987d193..526f54a8555 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py @@ -35,9 +35,7 @@ from ..._service_solvers import SolverService from ...models.pagination import Page, PaginationParams from ...models.schemas.errors import ErrorGet -from ...models.schemas.jobs import ( - JobInputs, -) +from ...models.schemas.jobs import JobInputs from ...services_http.director_v2 import DirectorV2Api from ...services_http.storage import StorageApi from ...services_http.webserver import AuthSession @@ -46,9 +44,7 @@ from ..dependencies.database import get_db_asyncpg_engine from ..dependencies.services import get_api_client, get_job_service, get_solver_service from ..dependencies.webserver_http import get_webserver_session -from ..dependencies.webserver_rpc import ( - get_wb_api_rpc_client, -) +from ..dependencies.webserver_rpc import get_wb_api_rpc_client from . import solvers_jobs, solvers_jobs_getters, studies_jobs # pylint: disable=too-many-arguments,no-else-return diff --git a/services/api-server/tests/unit/api_functions/test_api_routers_functions.py b/services/api-server/tests/unit/api_functions/test_api_routers_functions.py index 852cb4c1928..c442eaf22d6 100644 --- a/services/api-server/tests/unit/api_functions/test_api_routers_functions.py +++ b/services/api-server/tests/unit/api_functions/test_api_routers_functions.py @@ -12,9 +12,7 @@ JSONFunctionInputSchema, JSONFunctionOutputSchema, ) -from models_library.rest_pagination import ( - PageMetaInfoLimitOffset, -) +from models_library.rest_pagination import PageMetaInfoLimitOffset from pydantic import TypeAdapter from simcore_service_api_server.api.routes.functions_routes import ( function_job_collections_router, diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py index a893d91c79d..84ac179eef3 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_controller_rpc.py @@ -27,9 +27,7 @@ UnsupportedFunctionClassError, UnsupportedFunctionJobClassError, ) -from models_library.rest_pagination import ( - PageMetaInfoLimitOffset, -) +from models_library.rest_pagination import PageMetaInfoLimitOffset from servicelib.rabbitmq import RPCRouter from ..rabbitmq import get_rabbitmq_rpc_server diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py index 829fec295fc..89c2cfd1a85 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_repository.py @@ -14,9 +14,7 @@ FunctionJobIDNotFoundError, RegisterFunctionWithIDError, ) -from models_library.rest_pagination import ( - PageMetaInfoLimitOffset, -) +from models_library.rest_pagination import PageMetaInfoLimitOffset from simcore_postgres_database.models.funcapi_function_job_collections_table import ( function_job_collections_table, )