DM-51488 : Create API Routes for v2 Campaigns and Manifests

.github/workflows/ci.yaml

@@ -12,6 +12,7 @@ name: "CI"
 
 env:
   UV_FROZEN: "1"
+  TEST__LOCAL_DB: "1"  # signals test fixtures to not use testcontainers
 
 jobs:
   rebase-checker:

pyproject.toml

@@ -169,7 +169,7 @@ max-doc-length = 79
 convention = "numpy"
 
 [tool.pytest.ini_options]
-asyncio_mode = "strict"
+asyncio_mode = "auto"
 asyncio_default_fixture_loop_scope="function"
 # The python_files setting is not for test detection (pytest will pick up any
 # test files named *_test.py without this setting) but to enable special

src/lsst/cmservice/common/jsonpatch.py

@@ -0,0 +1,224 @@
+"""Module implementing functions to support json-patch operations on Python
+objects based on RFC6902.
+"""
+
+import operator
+from collections.abc import MutableMapping, MutableSequence
+from functools import reduce
+from typing import TYPE_CHECKING, Any, Literal
+
+from pydantic import AliasChoices, BaseModel, Field
+
+type AnyMutable = MutableMapping | MutableSequence
+
+
+class JSONPatchError(Exception):
+    """Exception raised when a JSON patch operation cannot be completed."""
+
+    pass
+
+
+class JSONPatch(BaseModel):
+    """Model representing a PATCH operation using RFC6902.
+
+    This model will generally be accepted as a ``Sequence[JSONPatch]``.
+    """
+
+    op: Literal["add", "remove", "replace", "move", "copy", "test"]
+    path: str = Field(
+        description="An RFC6901 JSON Pointer", pattern=r"^\/(metadata|spec|configuration|metadata_|data)\/.*$"
+    )
+    value: Any | None = None
+    from_: str | None = Field(
+        default=None,
+        pattern=r"^\/(metadata|spec|configuration|metadata_|data)\/.*$",
+        validation_alias=AliasChoices("from", "from_"),
+    )
+
+
+def apply_json_patch[T: MutableMapping](op: JSONPatch, o: T) -> T:
+    """Applies a jsonpatch to an object, returning the modified object.
+
+    Modifications are made in-place (i.e., the input object is not copied).
+
+    Notes
+    -----
+    While this JSON Patch operation nominally implements RFC6902, there are
+    some edge cases inappropriate to the application that are supported by the
+    RFC but disallowed through lack of support:
+
+    - Unsupported: JSON pointer values that refer to object/dict keys that are
+      numeric, e.g., {"1": "first", "2": "second"}
+    - Unsupported: JSON pointer values that refer to an entire object, e.g.,
+      "" -- the JSON Patch must have a root element ("/") per the model.
+    - Unsupported: JSON pointer values taht refer to a nameless object, e.g.,
+      "/" -- JSON allows object keys to be the empty string ("") but this is
+      disallowed by the application.
+    """
+    # The JSON Pointer root value is discarded as the rest of the pointer is
+    # split into parts
+    op_path = op.path.split("/")[1:]
+
+    # The terminal path part is either the name of a key or an index in a list
+    # FIXME this assumes that an "integer-string" in the path is always refers
+    #       to a list index, although it could just as well be a key in a dict
+    #       like ``{"1": "first, "2": "second"}`` which is complicated by the
+    #       fact that Python dict keys can be either ints or strs but this is
+    #       not allowed in JSON (i.e., object keys MUST be strings)
+    # FIXME this doesn't support, e.g., nested lists with multiple index values
+    #       in the path, e.g., ``[["a", "A"], ["b", "B"]]``
+    target_key_or_index: str | None = op_path.pop()
+    if target_key_or_index is None:
+        raise JSONPatchError("JSON Patch operations on empty keys not allowed.")
+
+    reference_token: int | str
+    # the reference token is referring to a an array index if the token is
+    # numeric or is the single character "-"
+    if target_key_or_index == "-":
+        reference_token = target_key_or_index
+    elif target_key_or_index.isnumeric():
+        reference_token = int(target_key_or_index)
+    else:
+        reference_token = str(target_key_or_index)
+
+    # The remaining parts of the path are a pointer to the object needing
+    # modification, which should reduce to either a dict or a list
+    try:
+        op_target: AnyMutable = reduce(operator.getitem, op_path, o)
+    except KeyError:
+        raise JSONPatchError(f"Path {op.path} not found in object")
+
+    match op:
+        case JSONPatch(op="add", value=new_value):
+            if reference_token == "-" and isinstance(op_target, MutableSequence):
+                # The "-" reference token is unique to the add operation and
+                # means the next element beyond the end of the current list
+                op_target.append(new_value)
+            elif isinstance(reference_token, int) and isinstance(op_target, MutableSequence):
+                op_target.insert(reference_token, new_value)
+            elif isinstance(reference_token, str) and isinstance(op_target, MutableMapping):
+                op_target[reference_token] = new_value
+
+        case JSONPatch(op="replace", value=new_value):
+            # The main difference between replace and add is that replace will
+            # not create new properties or elements in the target
+            if reference_token == "-":
+                raise JSONPatchError("Cannot use reference token `-` with replace operation.")
+            elif isinstance(op_target, MutableMapping):
+                try:
+                    assert reference_token in op_target.keys()
+                except AssertionError:
+                    raise JSONPatchError(f"Cannot replace missing key {reference_token} in object")
+            elif isinstance(reference_token, int) and isinstance(op_target, MutableSequence):
+                try:
+                    assert reference_token < len(op_target)
+                except AssertionError:
+                    raise JSONPatchError(f"Cannot replace missing index {reference_token} in object")
+
+            if TYPE_CHECKING:
+                assert isinstance(op_target, MutableMapping)
+            op_target[reference_token] = new_value
+
+        case JSONPatch(op="remove"):
+            if isinstance(reference_token, str) and isinstance(op_target, MutableMapping):
+                if reference_token == "-":
+                    raise JSONPatchError("Removal operations not allowed on `-` reference token")
+                _ = op_target.pop(reference_token, None)
+            elif isinstance(reference_token, int):
+                try:
+                    _ = op_target.pop(reference_token)
+                except IndexError:
+                    # The index we are meant to remove does not exist, but that
+                    # is not an error (idempotence)
+                    pass
+            else:
+                # This should be unreachable
+                raise ValueError("Reference token in JSON Patch must be int | str")
+
+        case JSONPatch(op="move", from_=from_location):
+            # the move operation is equivalent to a remove(from) + add(target)
+            if TYPE_CHECKING:
+                assert from_location is not None
+
+            # Handle the from_location with the same logic as the op.path
+            from_path = from_location.split("/")[1:]
+
+            # Is the last element of the from_path an index or a key?
+            from_target: str | int = from_path.pop()
+            try:
+                from_target = int(from_target)
+            except ValueError:
+                pass
+
+            try:
+                from_object = reduce(operator.getitem, from_path, o)
+                value = from_object[from_target]
+            except (KeyError, IndexError):
+                raise JSONPatchError(f"Path {from_location} not found in object")
+
+            # add the value to the new location
+            op_target[reference_token] = value  # type: ignore[index]
+            # and remove it from the old
+            _ = from_object.pop(from_target)
+
+        case JSONPatch(op="copy", from_=from_location):
+            # The copy op is the same as the move op except the original is not
+            # removed
+            if TYPE_CHECKING:
+                assert from_location is not None
+
+            # Handle the from_location with the same logic as the op.path
+            from_path = from_location.split("/")[1:]
+
+            # Is the last element of the from_path an index or a key?
+            from_target = from_path.pop()
+            try:
+                from_target = int(from_target)
+            except ValueError:
+                pass
+
+            try:
+                from_object = reduce(operator.getitem, from_path, o)
+                value = from_object[from_target]
+            except (KeyError, IndexError):
+                raise JSONPatchError(f"Path {from_location} not found in object")
+
+            # add the value to the new location
+            op_target[reference_token] = value  # type: ignore[index]
+
+        case JSONPatch(op="test", value=assert_value):
+            # assert that the patch value is present at the patch path
+            # The main difference between test and replace is that test does
+            # not make any modifications after its assertions
+            if reference_token == "-":
+                raise JSONPatchError("Cannot use reference token `-` with test operation.")
+            elif isinstance(op_target, MutableMapping):
+                try:
+                    assert reference_token in op_target.keys()
+                except AssertionError:
+                    raise JSONPatchError(
+                        f"Test operation assertion failed: Key {reference_token} does not exist at {op.path}"
+                    )
+            elif isinstance(reference_token, int) and isinstance(op_target, MutableSequence):
+                try:
+                    assert reference_token < len(op_target)
+                except AssertionError:
+                    raise JSONPatchError(
+                        f"Test operation assertion failed: "
+                        f"Index {reference_token} does not exist at {op.path}"
+                    )
+
+            if TYPE_CHECKING:
+                assert isinstance(op_target, MutableMapping)
+            try:
+                assert op_target[reference_token] == assert_value
+            except AssertionError:
+                raise JSONPatchError(
+                    f"Test operation assertion failed: {op.path} does not match value {assert_value}"
+                )
+
+        case _:
+            # Model validation should prevent this from ever happening
+            raise JSONPatchError(f"Unknown JSON Patch operation: {op.op}")
+
+    return o

src/lsst/cmservice/common/types.py

@@ -1,12 +1,28 @@
+from typing import Annotated
+
 from sqlalchemy.ext.asyncio import AsyncSession as AsyncSessionSA
 from sqlalchemy.ext.asyncio import async_scoped_session
 from sqlmodel.ext.asyncio.session import AsyncSession
 
 from .. import models
+from ..models.serde import EnumSerializer, ManifestKindEnumValidator, StatusEnumValidator
+from .enums import ManifestKind, StatusEnum
 
 type AnyAsyncSession = AsyncSession | AsyncSessionSA | async_scoped_session
 """A type union of async database sessions the application may use"""
 
 
 type AnyCampaignElement = models.Group | models.Campaign | models.Step | models.Job
 """A type union of Campaign elements"""
+
+
+type StatusField = Annotated[StatusEnum, StatusEnumValidator, EnumSerializer]
+"""A type for fields representing a Status with a custom validator tuned for
+enums operations.
+"""
+
+
+type KindField = Annotated[ManifestKind, ManifestKindEnumValidator, EnumSerializer]
+"""A type for fields representing a Kind with a custom validator tuned for
+enums operations.
+"""

src/lsst/cmservice/config.py

@@ -8,6 +8,7 @@
     AliasChoices,
     BaseModel,
     Field,
+    SecretStr,
     computed_field,
     field_serializer,
     field_validator,
@@ -456,6 +457,11 @@ class AsgiConfiguration(BaseModel):
         default="/cm-service",
     )
 
+    enable_frontend: bool = Field(
+        description="Whether to run the frontend web app",
+        default=True,
+    )
+
     frontend_prefix: str = Field(
         description="The URL prefix for the frontend web app",
         default="/web_app",
@@ -535,13 +541,13 @@ class DatabaseConfiguration(BaseModel):
         description="The URL for the cm-service database",
     )
 
-    password: str | None = Field(
+    password: SecretStr | None = Field(
         default=None,
         description="The password for the cm-service database",
     )
 
-    table_schema: str | None = Field(
-        default=None,
+    table_schema: str = Field(
+        default="public",
         description="Schema to use for cm-service database",
     )
 
@@ -550,6 +556,31 @@ class DatabaseConfiguration(BaseModel):
         description="SQLAlchemy engine echo setting for the cm-service database",
     )
 
+    max_overflow: int = Field(
+        default=10,
+        description="Maximum connection overflow allowed for QueuePool.",
+    )
+
+    pool_size: int = Field(
+        default=5,
+        description="Number of open connections kept in the QueuePool",
+    )
+
+    pool_recycle: int = Field(
+        default=-1,
+        description="Timeout in seconds before connections are recycled",
+    )
+
+    pool_timeout: int = Field(
+        default=30,
+        description="Wait timeout for acquiring a connection from the pool",
+    )
+
+    pool_fields: set[str] = Field(
+        default={"max_overflow", "pool_size", "pool_recycle", "pool_timeout"},
+        description="Set of fields used for connection pool configuration",
+    )
+
 
 class Configuration(BaseSettings):
     """Configuration for cm-service.

src/lsst/cmservice/db/__init__.py

@@ -1,5 +1,6 @@
 """Database table definitions and utility functions"""
 
+from . import campaigns_v2
 from .base import Base
 from .campaign import Campaign
 from .element import ElementMixin