Merge pull request #743 from tisnik/lcore-628-openapi-integration-tests

LCORE-628: OpenAPI integration tests

Author: tisnik

tests/configuration/lightspeed-stack-proper-name.yaml

@@ -0,0 +1,44 @@
+name: Lightspeed Core Service (LCS)
+service:
+  host: localhost
+  port: 8080
+  auth_enabled: false
+  workers: 1
+  color_log: true
+  access_log: true
+  cors:
+    allow_origins:
+      - foo_origin
+      - bar_origin
+      - baz_origin
+    allow_credentials: false
+    allow_methods:
+      - foo_method
+      - bar_method
+      - baz_method
+    allow_headers:
+      - foo_header
+      - bar_header
+      - baz_header
+llama_stack:
+  # Uses a remote llama-stack service
+  # The instance would have already been started with a llama-stack-run.yaml file
+  use_as_library_client: false
+  # Alternative for "as library use"
+  # use_as_library_client: true
+  # library_client_config_path: <path-to-llama-stack-run.yaml-file>
+  url: http://localhost:8321
+  api_key: xyzzy
+user_data_collection:
+  feedback_enabled: true
+  feedback_storage: "/tmp/data/feedback"
+mcp_servers:
+  - name: "server1"
+    provider_id: "provider1"
+    url: "http://url.com:1"
+  - name: "server2"
+    provider_id: "provider2"
+    url: "http://url.com:2"
+  - name: "server3"
+    provider_id: "provider3"
+    url: "http://url.com:3"

tests/integration/test_openapi_json.py

@@ -1,21 +1,27 @@
 """Tests the OpenAPI specification that is to be stored in docs/openapi.json."""
 
+from typing import Any
 import json
 from pathlib import Path
 
 import pytest
+import requests
+
+from fastapi.testclient import TestClient
+from configuration import configuration
 
 # Strategy:
-# - Load the OpenAPI document from docs/openapi.json
+# - Load the OpenAPI document from docs/openapi.json and from endpoint handler
 # - Validate critical structure based on the PR diff:
 #   * openapi version, info, servers
 #   * presence of paths/methods and key response codes
 #   * presence and key attributes of important component schemas (enums, required fields)
 
 OPENAPI_FILE = "docs/openapi.json"
+URL = "/openapi.json"
 
 
-def _load_openapi_spec() -> dict:
+def _load_openapi_spec_from_file() -> dict[str, Any]:
     """Load OpenAPI specification from configured path."""
     path = Path(OPENAPI_FILE)
     if path.is_file():
@@ -26,14 +32,38 @@ def _load_openapi_spec() -> dict:
     return {}
 
 
-@pytest.fixture(scope="module", name="spec")
-def open_api_spec() -> dict:
+def _load_openapi_spec_from_url() -> dict[str, Any]:
+    """Load OpenAPI specification from URL."""
+    configuration_filename = "tests/configuration/lightspeed-stack-proper-name.yaml"
+    cfg = configuration
+    cfg.load_configuration(configuration_filename)
+    from app.main import app  # pylint: disable=C0415
+
+    client = TestClient(app)
+    response = client.get("/openapi.json")
+    assert response.status_code == requests.codes.ok  # pylint: disable=no-member
+
+    # this line ensures that response payload contains proper JSON
+    payload = response.json()
+    assert payload is not None, "Incorrect response"
+
+    return payload
+
+
+@pytest.fixture(scope="module", name="spec_from_file")
+def open_api_spec_from_file() -> dict[str, Any]:
     """Fixture containing OpenAPI specification represented as a dictionary."""
-    return _load_openapi_spec()
+    return _load_openapi_spec_from_file()
 
 
-def test_openapi_top_level_info(spec: dict) -> None:
-    """Test all top level informations stored in OpenAPI specification."""
+@pytest.fixture(scope="module", name="spec_from_url")
+def open_api_spec_from_url() -> dict[str, Any]:
+    """Fixture containing OpenAPI specification represented as a dictionary."""
+    return _load_openapi_spec_from_url()
+
+
+def _check_openapi_top_level_info(spec: dict[str, Any]) -> None:
+    """Check all top level informations stored in OpenAPI specification."""
     assert spec.get("openapi") == "3.1.0"
 
     info = spec.get("info") or {}
@@ -48,20 +78,59 @@ def test_openapi_top_level_info(spec: dict) -> None:
     assert "apache.org/licenses" in (license_info.get("url") or "")
 
 
-def test_servers_section_present(spec: dict) -> None:
-    """Test the servers section stored in OpenAPI specification."""
+def _check_server_section_present(spec: dict[str, Any]) -> None:
+    """Check if the servers section stored in OpenAPI specification."""
     servers = spec.get("servers")
     assert isinstance(servers, list) and servers, "servers must be a non-empty list"
 
 
+def _check_paths_and_responses_exist(
+    spec: dict, path: str, method: str, expected_codes: set[str]
+) -> None:
+    paths = spec.get("paths") or {}
+    assert path in paths, f"Missing path: {path}"
+    op = (paths[path] or {}).get(method)
+    assert isinstance(op, dict), f"Missing method {method.upper()} for path {path}"
+    responses = op.get("responses") or {}
+    got_codes = set(responses.keys())
+    for code in expected_codes:
+        assert (
+            code in got_codes
+        ), f"Missing response code {code} for {method.upper()} {path}"
+
+
+def test_openapi_top_level_info_from_file(spec_from_file: dict[str, Any]) -> None:
+    """Test all top level informations stored in OpenAPI specification."""
+    _check_openapi_top_level_info(spec_from_file)
+
+
+def test_openapi_top_level_info_from_url(spec_from_url: dict[str, Any]) -> None:
+    """Test all top level informations stored in OpenAPI specification."""
+    _check_openapi_top_level_info(spec_from_url)
+
+
+def test_servers_section_present_from_file(spec_from_file: dict[str, Any]) -> None:
+    """Test the servers section stored in OpenAPI specification."""
+    _check_server_section_present(spec_from_file)
+
+
+def test_servers_section_present_from_url(spec_from_url: dict[str, Any]) -> None:
+    """Test the servers section stored in OpenAPI specification."""
+    _check_server_section_present(spec_from_url)
+
+
 @pytest.mark.parametrize(
     "path,method,expected_codes",
     [
         ("/", "get", {"200"}),
         ("/v1/info", "get", {"200", "500"}),
         ("/v1/models", "get", {"200", "500"}),
+        ("/v1/tools", "get", {"200", "500"}),
+        ("/v1/shields", "get", {"200", "500"}),
+        ("/v1/providers", "get", {"200", "500"}),
+        ("/v1/providers/{provider_id}", "get", {"200", "404", "422", "500"}),
         ("/v1/query", "post", {"200", "400", "403", "500", "422"}),
-        ("/v1/streaming_query", "post", {"200", "422"}),
+        ("/v1/streaming_query", "post", {"200", "400", "401", "403", "422", "500"}),
         ("/v1/config", "get", {"200", "503"}),
         ("/v1/feedback", "post", {"200", "401", "403", "500", "422"}),
         ("/v1/feedback/status", "get", {"200"}),
@@ -99,17 +168,64 @@ def test_servers_section_present(spec: dict) -> None:
         ("/metrics", "get", {"200"}),
     ],
 )
-def test_paths_and_responses_exist(
-    spec: dict, path: str, method: str, expected_codes: set[str]
+def test_paths_and_responses_exist_from_file(
+    spec_from_file: dict, path: str, method: str, expected_codes: set[str]
 ) -> None:
     """Tests all paths defined in OpenAPI specification."""
-    paths = spec.get("paths") or {}
-    assert path in paths, f"Missing path: {path}"
-    op = (paths[path] or {}).get(method)
-    assert isinstance(op, dict), f"Missing method {method.upper()} for path {path}"
-    responses = op.get("responses") or {}
-    got_codes = set(responses.keys())
-    for code in expected_codes:
-        assert (
-            code in got_codes
-        ), f"Missing response code {code} for {method.upper()} {path}"
+    _check_paths_and_responses_exist(spec_from_file, path, method, expected_codes)
+
+
+@pytest.mark.parametrize(
+    "path,method,expected_codes",
+    [
+        ("/", "get", {"200"}),
+        ("/v1/info", "get", {"200", "500"}),
+        ("/v1/models", "get", {"200", "500"}),
+        ("/v1/tools", "get", {"200", "500"}),
+        ("/v1/shields", "get", {"200", "500"}),
+        ("/v1/providers", "get", {"200", "500"}),
+        ("/v1/providers/{provider_id}", "get", {"200", "404", "422", "500"}),
+        ("/v1/query", "post", {"200", "400", "403", "500", "422"}),
+        ("/v1/streaming_query", "post", {"200", "400", "401", "403", "422", "500"}),
+        ("/v1/config", "get", {"200", "503"}),
+        ("/v1/feedback", "post", {"200", "401", "403", "500", "422"}),
+        ("/v1/feedback/status", "get", {"200"}),
+        ("/v1/feedback/status", "put", {"200", "422"}),
+        ("/v1/conversations", "get", {"200", "401", "503"}),
+        (
+            "/v1/conversations/{conversation_id}",
+            "get",
+            {"200", "400", "401", "404", "503", "422"},
+        ),
+        (
+            "/v1/conversations/{conversation_id}",
+            "delete",
+            {"200", "400", "401", "404", "503", "422"},
+        ),
+        ("/v2/conversations", "get", {"200"}),
+        (
+            "/v2/conversations/{conversation_id}",
+            "get",
+            {"200", "400", "401", "404", "422"},
+        ),
+        (
+            "/v2/conversations/{conversation_id}",
+            "delete",
+            {"200", "400", "401", "404", "422"},
+        ),
+        (
+            "/v2/conversations/{conversation_id}",
+            "put",
+            {"200", "400", "401", "404", "422"},
+        ),
+        ("/readiness", "get", {"200", "503"}),
+        ("/liveness", "get", {"200"}),
+        ("/authorized", "post", {"200", "400", "401", "403"}),
+        ("/metrics", "get", {"200"}),
+    ],
+)
+def test_paths_and_responses_exist_from_url(
+    spec_from_url: dict, path: str, method: str, expected_codes: set[str]
+) -> None:
+    """Tests all paths defined in OpenAPI specification."""
+    _check_paths_and_responses_exist(spec_from_url, path, method, expected_codes)