From 80b542854e1adf7e836cafdedec863a4f637c580 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 20 Feb 2025 14:04:20 +0100 Subject: [PATCH 01/15] refactor --- .../rabbitmq/rpc_interfaces/dynamic_sidecar/disk_usage.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_sidecar/disk_usage.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_sidecar/disk_usage.py index dbace2f1f4b..d21a71f7d9b 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_sidecar/disk_usage.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_sidecar/disk_usage.py @@ -1,5 +1,4 @@ import logging -from typing import Final from models_library.api_schemas_dynamic_sidecar.telemetry import DiskUsage from models_library.projects_nodes_io import NodeID @@ -11,10 +10,6 @@ _logger = logging.getLogger(__name__) -_UPDATE_DISK_USAGE: Final[RPCMethodName] = TypeAdapter(RPCMethodName).validate_python( - "update_disk_usage" -) - @log_decorator(_logger, level=logging.DEBUG) async def update_disk_usage( @@ -28,7 +23,7 @@ async def update_disk_usage( ) result = await rabbitmq_rpc_client.request( rpc_namespace, - _UPDATE_DISK_USAGE, + TypeAdapter(RPCMethodName).validate_python("update_disk_usage"), usage=usage, ) assert result is None # nosec From b06d5e28f5c1ac6da7cd23548b5d1f99d71ef1ad Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 20 Feb 2025 14:06:29 +0100 Subject: [PATCH 02/15] refactor --- .../{test_api_rpc__disk_usage.py => api/rpc/test__disk_usage.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename services/dynamic-sidecar/tests/unit/{test_api_rpc__disk_usage.py => api/rpc/test__disk_usage.py} (100%) diff --git a/services/dynamic-sidecar/tests/unit/test_api_rpc__disk_usage.py b/services/dynamic-sidecar/tests/unit/api/rpc/test__disk_usage.py similarity index 100% rename from services/dynamic-sidecar/tests/unit/test_api_rpc__disk_usage.py rename to services/dynamic-sidecar/tests/unit/api/rpc/test__disk_usage.py From d366d04f3b3c2da4cf9e0b5e0e606834c3a893d0 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 20 Feb 2025 14:06:44 +0100 Subject: [PATCH 03/15] refactor --- .../dynamic-sidecar/tests/unit/api/rpc/test__disk_usage.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/services/dynamic-sidecar/tests/unit/api/rpc/test__disk_usage.py b/services/dynamic-sidecar/tests/unit/api/rpc/test__disk_usage.py index 1383f165416..8baea608014 100644 --- a/services/dynamic-sidecar/tests/unit/api/rpc/test__disk_usage.py +++ b/services/dynamic-sidecar/tests/unit/api/rpc/test__disk_usage.py @@ -2,8 +2,7 @@ # pylint:disable=redefined-outer-name # pylint:disable=unused-argument -from collections.abc import Awaitable, Callable -from typing import AsyncIterable +from collections.abc import AsyncIterable, Awaitable, Callable from unittest.mock import AsyncMock import pytest From 1877b130d53247864389f9a75a38bb56d25ee368 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 20 Feb 2025 14:14:09 +0100 Subject: [PATCH 04/15] disk is now exposed on the RPC interface --- .../rpc_interfaces/dynamic_sidecar/disk.py | 26 +++++++ .../api/rest/disk.py | 2 +- .../api/rpc/_disk.py | 11 +++ .../api/rpc/routes.py | 3 +- .../services/__init__.py | 0 .../services/disk.py | 1 + .../rest/test_disk.py} | 0 .../tests/unit/api/rpc/test__disk.py | 74 +++++++++++++++++++ 8 files changed, 115 insertions(+), 2 deletions(-) create mode 100644 packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_sidecar/disk.py create mode 100644 services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/_disk.py create mode 100644 services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/services/__init__.py create mode 100644 services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/services/disk.py rename services/dynamic-sidecar/tests/unit/{test_api_rest_disk.py => api/rest/test_disk.py} (100%) create mode 100644 services/dynamic-sidecar/tests/unit/api/rpc/test__disk.py diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_sidecar/disk.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_sidecar/disk.py new file mode 100644 index 00000000000..fbacd1e905a --- /dev/null +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_sidecar/disk.py @@ -0,0 +1,26 @@ +import logging + +from models_library.projects_nodes_io import NodeID +from models_library.rabbitmq_basic_types import RPCMethodName, RPCNamespace +from pydantic import TypeAdapter + +from ....logging_utils import log_decorator +from ... import RabbitMQRPCClient + +_logger = logging.getLogger(__name__) + + +@log_decorator(_logger, level=logging.DEBUG) +async def free_reserved_disk_space( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + node_id: NodeID, +) -> None: + rpc_namespace = RPCNamespace.from_entries( + {"service": "dy-sidecar", "node_id": f"{node_id}"} + ) + result = await rabbitmq_rpc_client.request( + rpc_namespace, + TypeAdapter(RPCMethodName).validate_python("free_reserved_disk_space"), + ) + assert result is None # nosec diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/disk.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/disk.py index f8ce581024a..00d9d5a6f8e 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/disk.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/disk.py @@ -1,6 +1,6 @@ from fastapi import APIRouter, status -from ...core.reserved_space import remove_reserved_disk_space +from ...services.disk import remove_reserved_disk_space router = APIRouter() diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/_disk.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/_disk.py new file mode 100644 index 00000000000..25a8ca3b68c --- /dev/null +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/_disk.py @@ -0,0 +1,11 @@ +from fastapi import FastAPI +from servicelib.rabbitmq import RPCRouter + +from ...services.disk import remove_reserved_disk_space + +router = RPCRouter() + + +@router.expose() +async def free_reserved_disk_space(_: FastAPI) -> None: + remove_reserved_disk_space() diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/routes.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/routes.py index d1c0b0c2a1e..ae5c1099e86 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/routes.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/routes.py @@ -4,10 +4,11 @@ from ...core.rabbitmq import get_rabbitmq_rpc_server from ...core.settings import ApplicationSettings -from . import _disk_usage +from . import _disk, _disk_usage ROUTERS: list[RPCRouter] = [ _disk_usage.router, + _disk.router, ] diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/services/__init__.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/services/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/services/disk.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/services/disk.py new file mode 100644 index 00000000000..d8ec62a609c --- /dev/null +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/services/disk.py @@ -0,0 +1 @@ +__all__: tuple[str, ...] = ("remove_reserved_disk_space",) diff --git a/services/dynamic-sidecar/tests/unit/test_api_rest_disk.py b/services/dynamic-sidecar/tests/unit/api/rest/test_disk.py similarity index 100% rename from services/dynamic-sidecar/tests/unit/test_api_rest_disk.py rename to services/dynamic-sidecar/tests/unit/api/rest/test_disk.py diff --git a/services/dynamic-sidecar/tests/unit/api/rpc/test__disk.py b/services/dynamic-sidecar/tests/unit/api/rpc/test__disk.py new file mode 100644 index 00000000000..22f069005ec --- /dev/null +++ b/services/dynamic-sidecar/tests/unit/api/rpc/test__disk.py @@ -0,0 +1,74 @@ +# pylint:disable=protected-access +# pylint:disable=redefined-outer-name +# pylint:disable=unused-argument + +from collections.abc import AsyncIterable, Awaitable, Callable +from unittest.mock import AsyncMock + +import pytest +from asgi_lifespan import LifespanManager +from fastapi import FastAPI +from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict +from servicelib.rabbitmq import RabbitMQRPCClient +from servicelib.rabbitmq.rpc_interfaces.dynamic_sidecar import disk +from settings_library.rabbit import RabbitSettings +from simcore_service_dynamic_sidecar.core.application import create_app +from simcore_service_dynamic_sidecar.core.reserved_space import ( + _RESERVED_DISK_SPACE_NAME, +) +from simcore_service_dynamic_sidecar.core.settings import ApplicationSettings + +pytest_simcore_core_services_selection = [ + "rabbit", +] + + +@pytest.fixture +def mock_environment( + monkeypatch: pytest.MonkeyPatch, + rabbit_service: RabbitSettings, + mock_environment: EnvVarsDict, + mock_registry_service: AsyncMock, +) -> EnvVarsDict: + return setenvs_from_dict( + monkeypatch, + { + "DY_SIDECAR_SYSTEM_MONITOR_TELEMETRY_ENABLE": "true", + "RABBIT_HOST": rabbit_service.RABBIT_HOST, + "RABBIT_PASSWORD": rabbit_service.RABBIT_PASSWORD.get_secret_value(), + "RABBIT_PORT": f"{rabbit_service.RABBIT_PORT}", + "RABBIT_SECURE": f"{rabbit_service.RABBIT_SECURE}", + "RABBIT_USER": rabbit_service.RABBIT_USER, + }, + ) + + +@pytest.fixture +async def app(mock_environment: EnvVarsDict) -> AsyncIterable[FastAPI]: + app = create_app() + async with LifespanManager(app): + yield app + + +@pytest.fixture +async def rpc_client( + app: FastAPI, + rabbitmq_rpc_client: Callable[[str], Awaitable[RabbitMQRPCClient]], +) -> RabbitMQRPCClient: + return await rabbitmq_rpc_client("client") + + +async def test_free_reserved_disk_space( + cleanup_reserved_disk_space: None, app: FastAPI, rpc_client: RabbitMQRPCClient +): + assert _RESERVED_DISK_SPACE_NAME.exists() + + settings: ApplicationSettings = app.state.settings + + result = await disk.free_reserved_disk_space( + rpc_client, + node_id=settings.DY_SIDECAR_NODE_ID, + ) + assert result is None + + assert not _RESERVED_DISK_SPACE_NAME.exists() From ddf32c89d7d5c295905d949871d837e5bf3a1988 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 20 Feb 2025 14:24:41 +0100 Subject: [PATCH 05/15] fixed tests --- .../services/disk.py | 4 ++ .../tests/unit/api/rpc/conftest.py | 45 +++++++++++++++++++ .../tests/unit/api/rpc/test__disk.py | 44 ------------------ .../tests/unit/api/rpc/test__disk_usage.py | 43 ++---------------- 4 files changed, 52 insertions(+), 84 deletions(-) create mode 100644 services/dynamic-sidecar/tests/unit/api/rpc/conftest.py diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/services/disk.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/services/disk.py index d8ec62a609c..316a86d6036 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/services/disk.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/services/disk.py @@ -1 +1,5 @@ +from ..core.reserved_space import remove_reserved_disk_space + __all__: tuple[str, ...] = ("remove_reserved_disk_space",) + +# nopycln: file diff --git a/services/dynamic-sidecar/tests/unit/api/rpc/conftest.py b/services/dynamic-sidecar/tests/unit/api/rpc/conftest.py new file mode 100644 index 00000000000..48cd8a047a6 --- /dev/null +++ b/services/dynamic-sidecar/tests/unit/api/rpc/conftest.py @@ -0,0 +1,45 @@ +from collections.abc import AsyncIterable, Awaitable, Callable +from unittest.mock import AsyncMock + +import pytest +from asgi_lifespan import LifespanManager +from fastapi import FastAPI +from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict +from servicelib.rabbitmq import RabbitMQRPCClient +from settings_library.rabbit import RabbitSettings +from simcore_service_dynamic_sidecar.core.application import create_app + + +@pytest.fixture +def mock_environment( + monkeypatch: pytest.MonkeyPatch, + rabbit_service: RabbitSettings, + mock_environment: EnvVarsDict, + mock_registry_service: AsyncMock, +) -> EnvVarsDict: + return setenvs_from_dict( + monkeypatch, + { + "DY_SIDECAR_SYSTEM_MONITOR_TELEMETRY_ENABLE": "true", + "RABBIT_HOST": rabbit_service.RABBIT_HOST, + "RABBIT_PASSWORD": rabbit_service.RABBIT_PASSWORD.get_secret_value(), + "RABBIT_PORT": f"{rabbit_service.RABBIT_PORT}", + "RABBIT_SECURE": f"{rabbit_service.RABBIT_SECURE}", + "RABBIT_USER": rabbit_service.RABBIT_USER, + }, + ) + + +@pytest.fixture +async def app(mock_environment: EnvVarsDict) -> AsyncIterable[FastAPI]: + app = create_app() + async with LifespanManager(app): + yield app + + +@pytest.fixture +async def rpc_client( + app: FastAPI, + rabbitmq_rpc_client: Callable[[str], Awaitable[RabbitMQRPCClient]], +) -> RabbitMQRPCClient: + return await rabbitmq_rpc_client("client") diff --git a/services/dynamic-sidecar/tests/unit/api/rpc/test__disk.py b/services/dynamic-sidecar/tests/unit/api/rpc/test__disk.py index 22f069005ec..7ed592976e7 100644 --- a/services/dynamic-sidecar/tests/unit/api/rpc/test__disk.py +++ b/services/dynamic-sidecar/tests/unit/api/rpc/test__disk.py @@ -1,18 +1,9 @@ -# pylint:disable=protected-access -# pylint:disable=redefined-outer-name # pylint:disable=unused-argument -from collections.abc import AsyncIterable, Awaitable, Callable -from unittest.mock import AsyncMock -import pytest -from asgi_lifespan import LifespanManager from fastapi import FastAPI -from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict from servicelib.rabbitmq import RabbitMQRPCClient from servicelib.rabbitmq.rpc_interfaces.dynamic_sidecar import disk -from settings_library.rabbit import RabbitSettings -from simcore_service_dynamic_sidecar.core.application import create_app from simcore_service_dynamic_sidecar.core.reserved_space import ( _RESERVED_DISK_SPACE_NAME, ) @@ -23,41 +14,6 @@ ] -@pytest.fixture -def mock_environment( - monkeypatch: pytest.MonkeyPatch, - rabbit_service: RabbitSettings, - mock_environment: EnvVarsDict, - mock_registry_service: AsyncMock, -) -> EnvVarsDict: - return setenvs_from_dict( - monkeypatch, - { - "DY_SIDECAR_SYSTEM_MONITOR_TELEMETRY_ENABLE": "true", - "RABBIT_HOST": rabbit_service.RABBIT_HOST, - "RABBIT_PASSWORD": rabbit_service.RABBIT_PASSWORD.get_secret_value(), - "RABBIT_PORT": f"{rabbit_service.RABBIT_PORT}", - "RABBIT_SECURE": f"{rabbit_service.RABBIT_SECURE}", - "RABBIT_USER": rabbit_service.RABBIT_USER, - }, - ) - - -@pytest.fixture -async def app(mock_environment: EnvVarsDict) -> AsyncIterable[FastAPI]: - app = create_app() - async with LifespanManager(app): - yield app - - -@pytest.fixture -async def rpc_client( - app: FastAPI, - rabbitmq_rpc_client: Callable[[str], Awaitable[RabbitMQRPCClient]], -) -> RabbitMQRPCClient: - return await rabbitmq_rpc_client("client") - - async def test_free_reserved_disk_space( cleanup_reserved_disk_space: None, app: FastAPI, rpc_client: RabbitMQRPCClient ): diff --git a/services/dynamic-sidecar/tests/unit/api/rpc/test__disk_usage.py b/services/dynamic-sidecar/tests/unit/api/rpc/test__disk_usage.py index 8baea608014..95946212bf1 100644 --- a/services/dynamic-sidecar/tests/unit/api/rpc/test__disk_usage.py +++ b/services/dynamic-sidecar/tests/unit/api/rpc/test__disk_usage.py @@ -1,21 +1,13 @@ -# pylint:disable=protected-access -# pylint:disable=redefined-outer-name # pylint:disable=unused-argument -from collections.abc import AsyncIterable, Awaitable, Callable -from unittest.mock import AsyncMock - import pytest -from asgi_lifespan import LifespanManager from fastapi import FastAPI from models_library.api_schemas_dynamic_sidecar.telemetry import DiskUsage from pydantic import ByteSize -from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict +from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict from servicelib.rabbitmq import RabbitMQRPCClient from servicelib.rabbitmq.rpc_interfaces.dynamic_sidecar import disk_usage -from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings -from simcore_service_dynamic_sidecar.core.application import create_app from simcore_service_dynamic_sidecar.core.settings import ApplicationSettings from simcore_service_dynamic_sidecar.modules.system_monitor._disk_usage import ( get_disk_usage_monitor, @@ -29,38 +21,9 @@ @pytest.fixture def mock_environment( - monkeypatch: pytest.MonkeyPatch, - rabbit_service: RabbitSettings, - redis_service: RedisSettings, - mock_environment: EnvVarsDict, - mock_registry_service: AsyncMock, + redis_service: RedisSettings, mock_environment: EnvVarsDict ) -> EnvVarsDict: - return setenvs_from_dict( - monkeypatch, - { - "DY_SIDECAR_SYSTEM_MONITOR_TELEMETRY_ENABLE": "true", - "RABBIT_HOST": rabbit_service.RABBIT_HOST, - "RABBIT_PASSWORD": rabbit_service.RABBIT_PASSWORD.get_secret_value(), - "RABBIT_PORT": f"{rabbit_service.RABBIT_PORT}", - "RABBIT_SECURE": f"{rabbit_service.RABBIT_SECURE}", - "RABBIT_USER": rabbit_service.RABBIT_USER, - }, - ) - - -@pytest.fixture -async def app(mock_environment: EnvVarsDict) -> AsyncIterable[FastAPI]: - app = create_app() - async with LifespanManager(app): - yield app - - -@pytest.fixture -async def rpc_client( - app: FastAPI, - rabbitmq_rpc_client: Callable[[str], Awaitable[RabbitMQRPCClient]], -) -> RabbitMQRPCClient: - return await rabbitmq_rpc_client("client") + return mock_environment async def test_get_state(app: FastAPI, rpc_client: RabbitMQRPCClient): From 940593b280a80d496756d095bad38f41431beb9c Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 20 Feb 2025 14:25:43 +0100 Subject: [PATCH 06/15] pylint --- services/dynamic-sidecar/tests/unit/api/rpc/conftest.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/services/dynamic-sidecar/tests/unit/api/rpc/conftest.py b/services/dynamic-sidecar/tests/unit/api/rpc/conftest.py index 48cd8a047a6..6097c7a6d43 100644 --- a/services/dynamic-sidecar/tests/unit/api/rpc/conftest.py +++ b/services/dynamic-sidecar/tests/unit/api/rpc/conftest.py @@ -1,3 +1,6 @@ +# pylint:disable=redefined-outer-name +# pylint:disable=unused-argument + from collections.abc import AsyncIterable, Awaitable, Callable from unittest.mock import AsyncMock From c532b6f9ba27dcd32b0e53bd69c8b73396d73086 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 20 Feb 2025 14:26:40 +0100 Subject: [PATCH 07/15] pylint --- services/dynamic-sidecar/tests/unit/api/rpc/test__disk_usage.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/dynamic-sidecar/tests/unit/api/rpc/test__disk_usage.py b/services/dynamic-sidecar/tests/unit/api/rpc/test__disk_usage.py index 95946212bf1..105d4a85ccc 100644 --- a/services/dynamic-sidecar/tests/unit/api/rpc/test__disk_usage.py +++ b/services/dynamic-sidecar/tests/unit/api/rpc/test__disk_usage.py @@ -1,3 +1,5 @@ +# pylint:disable=protected-access +# pylint:disable=redefined-outer-name # pylint:disable=unused-argument import pytest From 328152422e180cf04bf548ebbb3b3d2c037cd961 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 20 Feb 2025 14:32:12 +0100 Subject: [PATCH 08/15] moved test --- .../unit/{test_api_rest_volumes.py => api/rest/test_volumes.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename services/dynamic-sidecar/tests/unit/{test_api_rest_volumes.py => api/rest/test_volumes.py} (100%) diff --git a/services/dynamic-sidecar/tests/unit/test_api_rest_volumes.py b/services/dynamic-sidecar/tests/unit/api/rest/test_volumes.py similarity index 100% rename from services/dynamic-sidecar/tests/unit/test_api_rest_volumes.py rename to services/dynamic-sidecar/tests/unit/api/rest/test_volumes.py From d8fa3808f7223fd23fa04ee5230cb85f44e86bc4 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 20 Feb 2025 14:32:28 +0100 Subject: [PATCH 09/15] using annotated --- .../src/simcore_service_dynamic_sidecar/api/rest/volumes.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/volumes.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/volumes.py index ac2e833a3ab..7deb3b7de83 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/volumes.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/volumes.py @@ -1,3 +1,5 @@ +from typing import Annotated + from fastapi import APIRouter, Depends from fastapi import Path as PathParam from fastapi import status @@ -21,8 +23,8 @@ class PutVolumeItem(BaseModel): ) async def put_volume_state( item: PutVolumeItem, - volume_category: VolumeCategory = PathParam(..., alias="id"), - shared_store: SharedStore = Depends(get_shared_store), + volume_category: Annotated[VolumeCategory, PathParam(..., alias="id")], + shared_store: Annotated[SharedStore, Depends(get_shared_store)], ) -> None: async with shared_store: shared_store.volume_states[volume_category] = VolumeState(status=item.status) From 3bf7431b096cc5338247faa7c9be8a634ef3d646 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 20 Feb 2025 15:09:37 +0100 Subject: [PATCH 10/15] added rpc volumes endpoint --- .../rpc_interfaces/dynamic_sidecar/volumes.py | 31 ++++++++ .../api/rest/disk.py | 4 +- .../api/rest/volumes.py | 13 ++-- .../api/rpc/_disk.py | 4 +- .../api/rpc/_disk_usage.py | 2 + .../api/rpc/_volumes.py | 16 ++++ .../api/rpc/routes.py | 3 +- .../models/shared_store.py | 5 ++ .../services/volumes.py | 12 +++ .../tests/unit/api/rpc/test__volumes.py | 74 +++++++++++++++++++ 10 files changed, 152 insertions(+), 12 deletions(-) create mode 100644 packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_sidecar/volumes.py create mode 100644 services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/_volumes.py create mode 100644 services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/services/volumes.py create mode 100644 services/dynamic-sidecar/tests/unit/api/rpc/test__volumes.py diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_sidecar/volumes.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_sidecar/volumes.py new file mode 100644 index 00000000000..4858f3e927f --- /dev/null +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_sidecar/volumes.py @@ -0,0 +1,31 @@ +import logging + +from models_library.projects_nodes_io import NodeID +from models_library.rabbitmq_basic_types import RPCMethodName, RPCNamespace +from models_library.sidecar_volumes import VolumeCategory, VolumeStatus +from pydantic import TypeAdapter + +from ....logging_utils import log_decorator +from ... import RabbitMQRPCClient + +_logger = logging.getLogger(__name__) + + +@log_decorator(_logger, level=logging.DEBUG) +async def save_volume_state( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + node_id: NodeID, + status: VolumeStatus, + category: VolumeCategory, +) -> None: + rpc_namespace = RPCNamespace.from_entries( + {"service": "dy-sidecar", "node_id": f"{node_id}"} + ) + result = await rabbitmq_rpc_client.request( + rpc_namespace, + TypeAdapter(RPCMethodName).validate_python("save_volume_state"), + status=status, + category=category, + ) + assert result is None # nosec diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/disk.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/disk.py index 00d9d5a6f8e..4ff0cc2dbca 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/disk.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/disk.py @@ -1,6 +1,6 @@ from fastapi import APIRouter, status -from ...services.disk import remove_reserved_disk_space +from ...services import disk router = APIRouter() @@ -11,4 +11,4 @@ status_code=status.HTTP_204_NO_CONTENT, ) async def free_reserved_disk_space() -> None: - remove_reserved_disk_space() + disk.remove_reserved_disk_space() diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/volumes.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/volumes.py index 7deb3b7de83..793fbc687e9 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/volumes.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/volumes.py @@ -1,13 +1,13 @@ from typing import Annotated -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Depends, FastAPI from fastapi import Path as PathParam from fastapi import status -from models_library.sidecar_volumes import VolumeCategory, VolumeState, VolumeStatus +from models_library.sidecar_volumes import VolumeCategory, VolumeStatus from pydantic import BaseModel -from ...models.shared_store import SharedStore -from ._dependencies import get_shared_store +from ...services import volumes +from ._dependencies import get_application router = APIRouter() @@ -23,8 +23,7 @@ class PutVolumeItem(BaseModel): ) async def put_volume_state( item: PutVolumeItem, + app: Annotated[FastAPI, Depends(get_application)], volume_category: Annotated[VolumeCategory, PathParam(..., alias="id")], - shared_store: Annotated[SharedStore, Depends(get_shared_store)], ) -> None: - async with shared_store: - shared_store.volume_states[volume_category] = VolumeState(status=item.status) + await volumes.save_volume_state(app, status=item.status, category=volume_category) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/_disk.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/_disk.py index 25a8ca3b68c..f3bba913ac9 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/_disk.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/_disk.py @@ -1,11 +1,11 @@ from fastapi import FastAPI from servicelib.rabbitmq import RPCRouter -from ...services.disk import remove_reserved_disk_space +from ...services import disk router = RPCRouter() @router.expose() async def free_reserved_disk_space(_: FastAPI) -> None: - remove_reserved_disk_space() + disk.remove_reserved_disk_space() diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/_disk_usage.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/_disk_usage.py index 6968250005c..8e658b769fa 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/_disk_usage.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/_disk_usage.py @@ -1,5 +1,6 @@ from fastapi import FastAPI from models_library.api_schemas_dynamic_sidecar.telemetry import DiskUsage +from pydantic import validate_call from servicelib.rabbitmq import RPCRouter from ...modules.system_monitor import get_disk_usage_monitor @@ -8,5 +9,6 @@ @router.expose() +@validate_call(config={"arbitrary_types_allowed": True}) async def update_disk_usage(app: FastAPI, *, usage: dict[str, DiskUsage]) -> None: get_disk_usage_monitor(app).set_disk_usage_for_path(usage) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/_volumes.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/_volumes.py new file mode 100644 index 00000000000..6f51d7ff629 --- /dev/null +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/_volumes.py @@ -0,0 +1,16 @@ +from fastapi import FastAPI +from models_library.sidecar_volumes import VolumeCategory, VolumeStatus +from pydantic import validate_call +from servicelib.rabbitmq import RPCRouter + +from ...services import volumes + +router = RPCRouter() + + +@router.expose() +@validate_call(config={"arbitrary_types_allowed": True}) +async def save_volume_state( + app: FastAPI, *, status: VolumeStatus, category: VolumeCategory +) -> None: + await volumes.save_volume_state(app, status=status, category=category) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/routes.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/routes.py index ae5c1099e86..1b020c03c37 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/routes.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/routes.py @@ -4,11 +4,12 @@ from ...core.rabbitmq import get_rabbitmq_rpc_server from ...core.settings import ApplicationSettings -from . import _disk, _disk_usage +from . import _disk, _disk_usage, _volumes ROUTERS: list[RPCRouter] = [ _disk_usage.router, _disk.router, + _volumes.router, ] diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/shared_store.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/shared_store.py index 0ca422a3390..822b0a959fd 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/shared_store.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/shared_store.py @@ -119,3 +119,8 @@ async def on_startup() -> None: ) app.add_event_handler("startup", on_startup) + + +def get_shared_store(app: FastAPI) -> SharedStore: + shared_store: SharedStore = app.state.shared_store + return shared_store diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/services/volumes.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/services/volumes.py new file mode 100644 index 00000000000..366276bbaed --- /dev/null +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/services/volumes.py @@ -0,0 +1,12 @@ +from fastapi import FastAPI +from models_library.sidecar_volumes import VolumeCategory, VolumeState, VolumeStatus + +from ..models.shared_store import get_shared_store + + +async def save_volume_state( + app: FastAPI, *, status: VolumeStatus, category: VolumeCategory +) -> None: + shared_store = get_shared_store(app) + async with shared_store: + shared_store.volume_states[category] = VolumeState(status=status) diff --git a/services/dynamic-sidecar/tests/unit/api/rpc/test__volumes.py b/services/dynamic-sidecar/tests/unit/api/rpc/test__volumes.py new file mode 100644 index 00000000000..2fae849d9fc --- /dev/null +++ b/services/dynamic-sidecar/tests/unit/api/rpc/test__volumes.py @@ -0,0 +1,74 @@ +# pylint:disable=protected-access +# pylint:disable=redefined-outer-name +# pylint:disable=unused-argument + +from pathlib import Path + +import pytest +from fastapi import FastAPI +from models_library.sidecar_volumes import VolumeCategory, VolumeState, VolumeStatus +from servicelib.rabbitmq import RabbitMQRPCClient +from servicelib.rabbitmq._errors import RPCServerError +from servicelib.rabbitmq.rpc_interfaces.dynamic_sidecar import volumes +from simcore_service_dynamic_sidecar.core.settings import ApplicationSettings +from simcore_service_dynamic_sidecar.models.shared_store import SharedStore + +pytest_simcore_core_services_selection = [ + "rabbit", +] + + +@pytest.mark.parametrize( + "volume_category, initial_expected_status", + [ + (VolumeCategory.STATES, VolumeStatus.CONTENT_NEEDS_TO_BE_SAVED), + (VolumeCategory.OUTPUTS, VolumeStatus.CONTENT_NEEDS_TO_BE_SAVED), + (VolumeCategory.INPUTS, VolumeStatus.CONTENT_NO_SAVE_REQUIRED), + (VolumeCategory.SHARED_STORE, VolumeStatus.CONTENT_NO_SAVE_REQUIRED), + ], +) +async def test_volumes_state_saved_ok( + ensure_shared_store_dir: Path, + app: FastAPI, + rpc_client: RabbitMQRPCClient, + volume_category: VolumeCategory, + initial_expected_status: VolumeStatus, +): + shared_store: SharedStore = app.state.shared_store + settings: ApplicationSettings = app.state.settings + + # check that initial status is as expected + assert shared_store.volume_states[volume_category] == VolumeState( + status=initial_expected_status + ) + + await volumes.save_volume_state( + rpc_client, + node_id=settings.DY_SIDECAR_NODE_ID, + status=VolumeStatus.CONTENT_WAS_SAVED, + category=volume_category, + ) + + # check that + assert shared_store.volume_states[volume_category] == VolumeState( + status=VolumeStatus.CONTENT_WAS_SAVED + ) + + +@pytest.mark.parametrize("invalid_volume_category", ["outputs", "outputS"]) +async def test_volumes_state_saved_error( + ensure_shared_store_dir: Path, + app: FastAPI, + rpc_client: RabbitMQRPCClient, + invalid_volume_category: VolumeCategory, +): + + settings: ApplicationSettings = app.state.settings + + with pytest.raises(RPCServerError, match="ValidationError"): + await volumes.save_volume_state( + rpc_client, + node_id=settings.DY_SIDECAR_NODE_ID, + status=VolumeStatus.CONTENT_WAS_SAVED, + category=invalid_volume_category, + ) From 4ba963ac5d823d38a451c1f0ea41645f91f6bc27 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 20 Feb 2025 15:11:40 +0100 Subject: [PATCH 11/15] refactor --- services/dynamic-sidecar/tests/unit/api/rpc/test__volumes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/dynamic-sidecar/tests/unit/api/rpc/test__volumes.py b/services/dynamic-sidecar/tests/unit/api/rpc/test__volumes.py index 2fae849d9fc..e19b50916c1 100644 --- a/services/dynamic-sidecar/tests/unit/api/rpc/test__volumes.py +++ b/services/dynamic-sidecar/tests/unit/api/rpc/test__volumes.py @@ -49,7 +49,7 @@ async def test_volumes_state_saved_ok( category=volume_category, ) - # check that + # check that content was saved assert shared_store.volume_states[volume_category] == VolumeState( status=VolumeStatus.CONTENT_WAS_SAVED ) From 74521a5f54c976e2d89e2868062e79e959a5555c Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 21 Feb 2025 13:40:46 +0100 Subject: [PATCH 12/15] updated notes --- .../api/rest/containers_extension.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers_extension.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers_extension.py index d5cf21b8723..ddcaba4cfe8 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers_extension.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers_extension.py @@ -135,7 +135,7 @@ async def attach_container_to_network( ) return - # NOTE: A docker network is only visible on a docker node when it is + # NOTE: A docker overlay network is only visible on a docker node when it is # used by a container network = DockerNetwork(docker=docker, id_=item.network_id) await network.connect( @@ -172,7 +172,7 @@ async def detach_container_from_network( ) return - # NOTE: A docker network is only visible on a docker node when it is + # NOTE: A docker overlay network is only visible on a docker node when it is # used by a container network = DockerNetwork(docker=docker, id_=item.network_id) await network.disconnect({"Container": container_id, "Force": True}) From 06af534b44b8251dff73c3414389067a1291b515 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 24 Feb 2025 11:06:32 +0100 Subject: [PATCH 13/15] added new type --- .../models/schemas/containers.py | 6 +++++- .../unit/test_api_rest_containers_long_running_tasks.py | 5 +++-- .../tests/unit/test_api_rest_prometheus_metrics.py | 5 +++-- .../tests/unit/test_api_rest_workflow_service_metrics.py | 5 +++-- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/schemas/containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/schemas/containers.py index e374c924070..1e3e9366494 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/schemas/containers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/schemas/containers.py @@ -1,9 +1,13 @@ +from typing import TypeAlias + from models_library.services_creation import CreateServiceMetricsAdditionalParams from pydantic import BaseModel +DcokerComposeYamlStr: TypeAlias = str + class ContainersComposeSpec(BaseModel): - docker_compose_yaml: str + docker_compose_yaml: DcokerComposeYamlStr class ContainersCreate(BaseModel): diff --git a/services/dynamic-sidecar/tests/unit/test_api_rest_containers_long_running_tasks.py b/services/dynamic-sidecar/tests/unit/test_api_rest_containers_long_running_tasks.py index 9c050ae8a0e..9b1d987203f 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_rest_containers_long_running_tasks.py +++ b/services/dynamic-sidecar/tests/unit/test_api_rest_containers_long_running_tasks.py @@ -41,6 +41,7 @@ from simcore_service_dynamic_sidecar.models.schemas.containers import ( ContainersComposeSpec, ContainersCreate, + DcokerComposeYamlStr, ) from simcore_service_dynamic_sidecar.models.shared_store import SharedStore from simcore_service_dynamic_sidecar.modules.inputs import enable_inputs_pulling @@ -150,7 +151,7 @@ def dynamic_sidecar_network_name() -> str: }, ] ) -def compose_spec(request: pytest.FixtureRequest) -> str: +def compose_spec(request: pytest.FixtureRequest) -> DcokerComposeYamlStr: spec_dict: dict[str, Any] = request.param # type: ignore return json.dumps(spec_dict) @@ -282,7 +283,7 @@ async def _get_task_id_pull_user_servcices_docker_images( async def _get_task_id_create_service_containers( httpx_async_client: AsyncClient, - compose_spec: str, + compose_spec: DcokerComposeYamlStr, mock_metrics_params: CreateServiceMetricsAdditionalParams, *args, **kwargs, diff --git a/services/dynamic-sidecar/tests/unit/test_api_rest_prometheus_metrics.py b/services/dynamic-sidecar/tests/unit/test_api_rest_prometheus_metrics.py index 89bb741fc30..893f328360e 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_rest_prometheus_metrics.py +++ b/services/dynamic-sidecar/tests/unit/test_api_rest_prometheus_metrics.py @@ -26,6 +26,7 @@ from simcore_service_dynamic_sidecar.models.schemas.containers import ( ContainersComposeSpec, ContainersCreate, + DcokerComposeYamlStr, ) from simcore_service_dynamic_sidecar.modules.prometheus_metrics import ( _USER_SERVICES_NOT_STARTED, @@ -85,7 +86,7 @@ def client( @pytest.fixture -def compose_spec() -> str: +def compose_spec() -> DcokerComposeYamlStr: return json.dumps( { "version": "3", @@ -101,7 +102,7 @@ def compose_spec() -> str: async def _get_task_id_create_service_containers( httpx_async_client: AsyncClient, - compose_spec: str, + compose_spec: DcokerComposeYamlStr, mock_metrics_params: CreateServiceMetricsAdditionalParams, ) -> TaskId: ctontainers_compose_spec = ContainersComposeSpec( diff --git a/services/dynamic-sidecar/tests/unit/test_api_rest_workflow_service_metrics.py b/services/dynamic-sidecar/tests/unit/test_api_rest_workflow_service_metrics.py index 662b7033b88..3db4573a731 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_rest_workflow_service_metrics.py +++ b/services/dynamic-sidecar/tests/unit/test_api_rest_workflow_service_metrics.py @@ -43,6 +43,7 @@ from simcore_service_dynamic_sidecar.models.schemas.containers import ( ContainersComposeSpec, ContainersCreate, + DcokerComposeYamlStr, ) from simcore_service_dynamic_sidecar.models.shared_store import SharedStore from tenacity import AsyncRetrying, TryAgain @@ -73,7 +74,7 @@ def raw_compose_spec(container_names: list[str]) -> dict[str, Any]: @pytest.fixture -def compose_spec(raw_compose_spec: dict[str, Any]) -> str: +def compose_spec(raw_compose_spec: dict[str, Any]) -> DcokerComposeYamlStr: return json.dumps(raw_compose_spec) @@ -145,7 +146,7 @@ def mock_user_services_fail_to_stop(mocker: MockerFixture) -> None: async def _get_task_id_create_service_containers( httpx_async_client: AsyncClient, - compose_spec: str, + compose_spec: DcokerComposeYamlStr, mock_metrics_params: CreateServiceMetricsAdditionalParams, ) -> TaskId: containers_compose_spec = ContainersComposeSpec( From 396db38b3703b2b964b80b18d4e09189dec9bd05 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 28 Feb 2025 13:40:32 +0100 Subject: [PATCH 14/15] moved store_compose_spec to rpc --- .../api_schemas_dynamic_sidecar/containers.py | 2 + .../dynamic_sidecar/containers.py | 29 +++++++ .../api/rest/containers.py | 39 +++------ .../api/rpc/_containers.py | 16 ++++ .../api/rpc/routes.py | 3 +- .../models/schemas/containers.py | 5 +- .../models/shared_store.py | 3 +- .../services/containers.py | 41 ++++++++++ .../rest/test_containers.py} | 8 +- .../tests/unit/api/rpc/test__containers.py | 79 +++++++++++++++++++ 10 files changed, 185 insertions(+), 40 deletions(-) create mode 100644 packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_sidecar/containers.py create mode 100644 services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/_containers.py create mode 100644 services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/services/containers.py rename services/dynamic-sidecar/tests/unit/{test_api_rest_containers.py => api/rest/test_containers.py} (99%) create mode 100644 services/dynamic-sidecar/tests/unit/api/rpc/test__containers.py diff --git a/packages/models-library/src/models_library/api_schemas_dynamic_sidecar/containers.py b/packages/models-library/src/models_library/api_schemas_dynamic_sidecar/containers.py index 2e14ed62c16..9412f0111ee 100644 --- a/packages/models-library/src/models_library/api_schemas_dynamic_sidecar/containers.py +++ b/packages/models-library/src/models_library/api_schemas_dynamic_sidecar/containers.py @@ -16,3 +16,5 @@ class ActivityInfo(BaseModel): ActivityInfoOrNone: TypeAlias = ActivityInfo | None + +DcokerComposeYamlStr: TypeAlias = str diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_sidecar/containers.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_sidecar/containers.py new file mode 100644 index 00000000000..3de2d56ecd9 --- /dev/null +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_sidecar/containers.py @@ -0,0 +1,29 @@ +import logging + +from models_library.api_schemas_dynamic_sidecar.containers import DcokerComposeYamlStr +from models_library.projects_nodes_io import NodeID +from models_library.rabbitmq_basic_types import RPCMethodName, RPCNamespace +from pydantic import TypeAdapter + +from ....logging_utils import log_decorator +from ... import RabbitMQRPCClient + +_logger = logging.getLogger(__name__) + + +@log_decorator(_logger, level=logging.DEBUG) +async def store_compose_spec( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + node_id: NodeID, + docker_compose_yaml: DcokerComposeYamlStr, +) -> None: + rpc_namespace = RPCNamespace.from_entries( + {"service": "dy-sidecar", "node_id": f"{node_id}"} + ) + result = await rabbitmq_rpc_client.request( + rpc_namespace, + TypeAdapter(RPCMethodName).validate_python("store_compose_spec"), + docker_compose_yaml=docker_compose_yaml, + ) + assert result is None # nosec diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers.py index 6ada9de83de..e2bf933a937 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers.py @@ -5,7 +5,7 @@ from asyncio import Lock from typing import Annotated, Any, Final -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends, FastAPI, HTTPException from fastapi import Path as PathParam from fastapi import Query, Request, status from models_library.api_schemas_dynamic_sidecar.containers import ( @@ -22,18 +22,14 @@ ContainerExecTimeoutError, ) from ...core.settings import ApplicationSettings -from ...core.validation import ( - ComposeSpecValidation, - parse_compose_spec, - validate_compose_spec, -) +from ...core.validation import parse_compose_spec from ...models.schemas.containers import ContainersComposeSpec from ...models.shared_store import SharedStore from ...modules.container_utils import run_command_in_container -from ...modules.mounted_fs import MountedVolumes +from ...services import containers from ._dependencies import ( + get_application, get_container_restart_lock, - get_mounted_volumes, get_settings, get_shared_store, ) @@ -65,31 +61,14 @@ def _raise_if_container_is_missing( @cancel_on_disconnect async def store_compose_spec( request: Request, - settings: Annotated[ApplicationSettings, Depends(get_settings)], containers_compose_spec: ContainersComposeSpec, - shared_store: Annotated[SharedStore, Depends(get_shared_store)], - mounted_volumes: Annotated[MountedVolumes, Depends(get_mounted_volumes)], + app: Annotated[FastAPI, Depends(get_application)], ): - """ - Validates and stores the docker compose spec for the user services. - """ _ = request - - async with shared_store: - compose_spec_validation: ComposeSpecValidation = await validate_compose_spec( - settings=settings, - compose_file_content=containers_compose_spec.docker_compose_yaml, - mounted_volumes=mounted_volumes, - ) - shared_store.compose_spec = compose_spec_validation.compose_spec - shared_store.container_names = compose_spec_validation.current_container_names - shared_store.original_to_container_names = ( - compose_spec_validation.original_to_current_container_names - ) - - _logger.info("Validated compose-spec:\n%s", f"{shared_store.compose_spec}") - - assert shared_store.compose_spec # nosec + await containers.store_conpose_spec( + app, + docker_compose_yaml=containers_compose_spec.docker_compose_yaml, + ) @router.get( diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/_containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/_containers.py new file mode 100644 index 00000000000..0e1f6e3ab86 --- /dev/null +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/_containers.py @@ -0,0 +1,16 @@ +from fastapi import FastAPI +from models_library.api_schemas_dynamic_sidecar.containers import DcokerComposeYamlStr +from pydantic import validate_call +from servicelib.rabbitmq import RPCRouter + +from ...services import containers + +router = RPCRouter() + + +@router.expose() +@validate_call(config={"arbitrary_types_allowed": True}) +async def store_compose_spec( + app: FastAPI, *, docker_compose_yaml: DcokerComposeYamlStr +) -> None: + await containers.store_conpose_spec(app, docker_compose_yaml=docker_compose_yaml) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/routes.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/routes.py index 1b020c03c37..18fbafda339 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/routes.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/routes.py @@ -4,9 +4,10 @@ from ...core.rabbitmq import get_rabbitmq_rpc_server from ...core.settings import ApplicationSettings -from . import _disk, _disk_usage, _volumes +from . import _containers, _disk, _disk_usage, _volumes ROUTERS: list[RPCRouter] = [ + _containers.router, _disk_usage.router, _disk.router, _volumes.router, diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/schemas/containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/schemas/containers.py index 1e3e9366494..91e57c05854 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/schemas/containers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/schemas/containers.py @@ -1,10 +1,7 @@ -from typing import TypeAlias - +from models_library.api_schemas_dynamic_sidecar.containers import DcokerComposeYamlStr from models_library.services_creation import CreateServiceMetricsAdditionalParams from pydantic import BaseModel -DcokerComposeYamlStr: TypeAlias = str - class ContainersComposeSpec(BaseModel): docker_compose_yaml: DcokerComposeYamlStr diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/shared_store.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/shared_store.py index 822b0a959fd..fff75a8816c 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/shared_store.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/shared_store.py @@ -4,6 +4,7 @@ import aiofiles from fastapi import FastAPI +from models_library.api_schemas_dynamic_sidecar.containers import DcokerComposeYamlStr from models_library.sidecar_volumes import VolumeCategory, VolumeState, VolumeStatus from pydantic import BaseModel, Field, PrivateAttr @@ -50,7 +51,7 @@ class SharedStore(_StoreMixin): shared_store.container_names = copied_list """ - compose_spec: str | None = Field( + compose_spec: DcokerComposeYamlStr | None = Field( default=None, description="stores the stringified compose spec" ) container_names: list[ContainerNameStr] = Field( diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/services/containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/services/containers.py new file mode 100644 index 00000000000..41c8107fed6 --- /dev/null +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/services/containers.py @@ -0,0 +1,41 @@ +import logging + +from fastapi import FastAPI + +from ..core.settings import ApplicationSettings +from ..core.utils import MountedVolumes +from ..core.validation import ComposeSpecValidation, validate_compose_spec +from ..models.schemas.containers import DcokerComposeYamlStr +from ..models.shared_store import SharedStore, get_shared_store + +_logger = logging.getLogger(__name__) + + +async def store_conpose_spec( + app: FastAPI, + *, + docker_compose_yaml: DcokerComposeYamlStr, +) -> None: + """ + Validates and stores the docker compose spec for the user services. + """ + + settings: ApplicationSettings = app.state.settings + mounted_volumes: MountedVolumes = app.state.mounted_volumes + shared_store: SharedStore = get_shared_store(app) + + async with shared_store: + compose_spec_validation: ComposeSpecValidation = await validate_compose_spec( + settings=settings, + compose_file_content=docker_compose_yaml, + mounted_volumes=mounted_volumes, + ) + shared_store.compose_spec = compose_spec_validation.compose_spec + shared_store.container_names = compose_spec_validation.current_container_names + shared_store.original_to_container_names = ( + compose_spec_validation.original_to_current_container_names + ) + + _logger.info("Validated compose-spec:\n%s", f"{shared_store.compose_spec}") + + assert shared_store.compose_spec # nosec diff --git a/services/dynamic-sidecar/tests/unit/test_api_rest_containers.py b/services/dynamic-sidecar/tests/unit/api/rest/test_containers.py similarity index 99% rename from services/dynamic-sidecar/tests/unit/test_api_rest_containers.py rename to services/dynamic-sidecar/tests/unit/api/rest/test_containers.py index 27ec615b631..777dc32fc90 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_rest_containers.py +++ b/services/dynamic-sidecar/tests/unit/api/rest/test_containers.py @@ -5,10 +5,10 @@ import asyncio import json -from collections.abc import AsyncIterable +from collections.abc import AsyncIterable, AsyncIterator from inspect import signature from pathlib import Path -from typing import Any, AsyncIterator, Final +from typing import Any, Final from unittest.mock import AsyncMock, Mock from uuid import uuid4 @@ -267,10 +267,10 @@ def not_started_containers() -> list[str]: def mock_outputs_labels() -> dict[str, ServiceOutput]: return { "output_port_1": TypeAdapter(ServiceOutput).validate_python( - ServiceOutput.model_config["json_schema_extra"]["examples"][3] + ServiceOutput.model_json_schema()["examples"][3] ), "output_port_2": TypeAdapter(ServiceOutput).validate_python( - ServiceOutput.model_config["json_schema_extra"]["examples"][3] + ServiceOutput.model_json_schema()["examples"][3] ), } diff --git a/services/dynamic-sidecar/tests/unit/api/rpc/test__containers.py b/services/dynamic-sidecar/tests/unit/api/rpc/test__containers.py new file mode 100644 index 00000000000..d40d5904bf5 --- /dev/null +++ b/services/dynamic-sidecar/tests/unit/api/rpc/test__containers.py @@ -0,0 +1,79 @@ +# pylint:disable=protected-access +# pylint:disable=redefined-outer-name +# pylint:disable=unused-argument + +import pytest +import yaml +from fastapi import FastAPI +from models_library.api_schemas_dynamic_sidecar.containers import DcokerComposeYamlStr +from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict +from servicelib.rabbitmq import RabbitMQRPCClient +from servicelib.rabbitmq.rpc_interfaces.dynamic_sidecar import containers +from settings_library.redis import RedisSettings +from simcore_service_dynamic_sidecar.core.settings import ApplicationSettings +from simcore_service_dynamic_sidecar.models.shared_store import ( + SharedStore, + get_shared_store, +) + +pytest_simcore_core_services_selection = [ + "redis", + "rabbit", +] + + +@pytest.fixture +def mock_environment( + redis_service: RedisSettings, mock_environment: EnvVarsDict +) -> EnvVarsDict: + return mock_environment + + +@pytest.fixture +def dynamic_sidecar_network_name() -> str: + return "entrypoint_container_network" + + +@pytest.fixture +def docker_compose_yaml(dynamic_sidecar_network_name: str) -> DcokerComposeYamlStr: + return yaml.dump( + { + "version": "3", + "services": { + "first-box": { + "image": "busybox:latest", + "networks": { + dynamic_sidecar_network_name: None, + }, + "labels": {"io.osparc.test-label": "mark-entrypoint"}, + }, + "second-box": {"image": "busybox:latest"}, + "egress": { + "image": "busybox:latest", + "networks": { + dynamic_sidecar_network_name: None, + }, + }, + }, + "networks": {dynamic_sidecar_network_name: None}, + } + ) + + +async def test_store_compose_spec( + app: FastAPI, + rpc_client: RabbitMQRPCClient, + docker_compose_yaml: DcokerComposeYamlStr, + ensure_external_volumes: None, +): + settings: ApplicationSettings = app.state.settings + + result = await containers.store_compose_spec( + rpc_client, + node_id=settings.DY_SIDECAR_NODE_ID, + docker_compose_yaml=docker_compose_yaml, + ) + assert result is None + + shared_store: SharedStore = get_shared_store(app) + assert shared_store.compose_spec is not None From dcdb645852a0846a8087b8d67f9e1bd8032bbba5 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 28 Feb 2025 13:59:08 +0100 Subject: [PATCH 15/15] fixed imports --- .../simcore_service_dynamic_sidecar/api/rest/containers.py | 2 ++ .../simcore_service_dynamic_sidecar/services/containers.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers.py index e2bf933a937..c697844c1c8 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers.py @@ -64,6 +64,8 @@ async def store_compose_spec( containers_compose_spec: ContainersComposeSpec, app: Annotated[FastAPI, Depends(get_application)], ): + """Validates and stores the docker compose spec for the user services.""" + _ = request await containers.store_conpose_spec( app, diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/services/containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/services/containers.py index 41c8107fed6..cbd6af4db52 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/services/containers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/services/containers.py @@ -1,12 +1,12 @@ import logging from fastapi import FastAPI +from models_library.api_schemas_dynamic_sidecar.containers import DcokerComposeYamlStr from ..core.settings import ApplicationSettings -from ..core.utils import MountedVolumes from ..core.validation import ComposeSpecValidation, validate_compose_spec -from ..models.schemas.containers import DcokerComposeYamlStr from ..models.shared_store import SharedStore, get_shared_store +from ..modules.mounted_fs import MountedVolumes _logger = logging.getLogger(__name__)