LCORE-598: Add authorization e2e tests

This commit adds end-to-end tests for the authorization functionality.

# Implementation details

- Modified `.gitignore` to ignore leftover config backup files
  (unrelated to this change but useful)
- Updated testing documentation to include instructions for running e2e
  tests
- Added a JWK server container to `docker-compose.yaml` which is needed
  for e2e testing JWK auth
- Added a `tests/e2e/configuration/lightspeed-stack-auth-jwk.yaml`
  config file for lightspeed-stack with JWK auth enabled to be used in
  the JWK e2e tests
- Removed hard dependency on the `docker` command in e2e tests and
  instead use the `CONTAINER_CMD` environment variable if set (to allow
  using `podman` instead of `docker`)
- Added `tests/e2e/features/authorization_jwk.feature` which contains
  the actual e2e tests for JWK authz
- Added `tests/e2e/features/steps/jwk_auth.py` which implements the
  steps for the JWK authz tests
- Modified `tests/e2e/features/environment.py` to handle the new JWK
  authz tests, including creating a temporary JWK key pair for the
  tests and writing the public key to a file served by the JWK server
  container (which lightspeed-stack is directed to access through the
  config file mentioned above)
  functionality to functions that can be reused in the e2e tests
- Added `tests/e2e/configuration/test_jwk/.gitignore` to ignore
  generated JWK files
- Updated `tests/e2e/test_list.txt` to include the new JWK authz tests

Author: omertuc

.gitignore

@@ -188,3 +188,6 @@ dev/
 
 # Database files
 *.sqlite
+
+# A file sometimes leftover by the e2e tests
+lightspeed-stack.yaml.backup

docker-compose.yaml

@@ -42,6 +42,23 @@ services:
       retries: 3      # how many times to retry before marking as unhealthy
       start_period: 5s # time to wait before starting checks
 
+  # This server is used for the e2e JWK tests
+  test-jwk-server:
+    # Conveniently uses the same Containerfile as lightspeed-stack as it has
+    # Python installed
+    build:
+      context: .
+      dockerfile: Containerfile
+    container_name: test-jwk-server
+    ports:
+      - "16161:16161"
+    networks:
+      - lightspeednet
+    working_dir: /app-root/test_jwk
+    entrypoint: ["python3", "-m", "http.server", "16161"]
+    volumes:
+      - ./tests/e2e/configuration/test_jwk:/app-root/test_jwk:Z
+
 networks:
   lightspeednet:
     driver: bridge

docs/testing.md

@@ -141,7 +141,40 @@ End to end tests are based on [Behave](https://behave.readthedocs.io/en/stable/)
 
 * Defined in [tests/e2e](https://github.com/lightspeed-core/lightspeed-stack/tree/main/tests/e2e)
 
+### Prerequisites for E2E tests
 
+E2E tests require running services. You may run them using `podman-compose` or `docker-compose`.
+
+Some tests also require an OpenAI API key. You can run specific tests that do not require an OpenAI API key without it.
+
+First you need to build the images:
+
+```bash
+podman-compose build
+```
+
+Then to start the services:
+
+```bash
+OPENAI_API_KEY=your-api-key podman-compose up
+```
+
+And finally to run the actual tests:
+
+```bash
+make test-e2e
+```
+
+If using `podman`, set `CONTAINER_CMD` accordingly:
+
+```bash
+CONTAINER_CMD=podman make test-e2e
+```
+
+**Example of running a particular test with podman and live output:**
+```bash
+CONTAINER_CMD=podman uv run behave tests/e2e/features/authorization_jwk.feature --verbose --no-capture --no-capture-stderr
+```
 
 ## Tips and hints
 

tests/e2e/configuration/lightspeed-stack-auth-jwk.yaml

@@ -0,0 +1,52 @@
+name: Lightspeed Core Service (LCS)
+service:
+  host: 0.0.0.0
+  port: 8080
+  auth_enabled: false
+  workers: 1
+  color_log: true
+  access_log: true
+llama_stack:
+  # Uses a remote llama-stack service
+  # The instance would have already been started with a llama-stack-run.yaml file
+  use_as_library_client: false
+  # Alternative for "as library use"
+  # use_as_library_client: true
+  # library_client_config_path: <path-to-llama-stack-run.yaml-file>
+  url: http://llama-stack:8321
+  api_key: xyzzy
+user_data_collection:
+  feedback_enabled: true
+  feedback_storage: "/tmp/data/feedback"
+  transcripts_enabled: true
+  transcripts_storage: "/tmp/data/transcripts"
+
+authentication:
+  module: "jwk-token"
+  jwk_config:
+    url: "http://test-jwk-server:16161/jwk.json"
+    jwt_configuration:
+      user_id_claim: "user_id"
+      username_claim: "username"
+      role_rules:
+        - jsonpath: "$.roles[*]"
+          operator: "contains"
+          value: "admin"
+          roles: ["administrator"]
+        - jsonpath: "$.roles[*]"
+          operator: "contains"
+          value: "config"
+          roles: ["config"]
+        - jsonpath: "$.roles[*]"
+          operator: "contains"
+          value: "readonly"
+          roles: ["readonly"]
+
+authorization:
+  access_rules:
+    - role: "administrator"
+      actions: ["admin"]
+    - role: "config"
+      actions: ["get_config", "info"]
+    - role: "readonly"
+      actions: ["info"]

tests/e2e/configuration/test_jwk/.gitignore

@@ -0,0 +1,2 @@
+# Ignore JWKs created for e2e tests
+*.json

tests/e2e/features/authorization_jwk.feature

@@ -0,0 +1,39 @@
+@JWKAuth
+Feature: JWK authorization enforcement
+
+  Background:
+    Given The service is started locally
+      And REST API service hostname is localhost
+      And REST API service port is 8080
+      And REST API service prefix is /v1
+
+  Scenario: A user with the admin role can access the info endpoint
+    Given I have a valid JWT token with the admin role
+     When I access REST API endpoint "info" using HTTP GET method
+     Then The status code of the response is 200
+
+  Scenario: A user with the admin role can access the config endpoint
+    Given I have a valid JWT token with the admin role
+     When I access REST API endpoint "config" using HTTP GET method
+     Then The status code of the response is 200
+
+  Scenario: A user with the config role can access the config endpoint
+    Given I have a valid JWT token with the config role
+     When I access REST API endpoint "config" using HTTP GET method
+     Then The status code of the response is 200
+
+  Scenario: A user with the config role can access the info endpoint
+    Given I have a valid JWT token with the config role
+     When I access REST API endpoint "info" using HTTP GET method
+     Then The status code of the response is 200
+
+  Scenario: A user with the readonly role can access the info endpoint
+    Given I have a valid JWT token with the readonly role
+     When I access REST API endpoint "info" using HTTP GET method
+     Then The status code of the response is 200
+
+  Scenario: A user with the readonly role can't access the config endpoint
+    Given I have a valid JWT token with the readonly role
+     When I access REST API endpoint "config" using HTTP GET method
+     Then The status code of the response is 403
+      And The body of the response contains Insufficient permissions

tests/e2e/features/environment.py

@@ -7,9 +7,11 @@
 4. after_scenario
 """
 
+import os
 import requests
 import subprocess
 import time
+import json
 from behave.model import Scenario, Feature
 from behave.runner import Context
 
@@ -25,6 +27,8 @@
 except ImportError as e:
     print("Warning: unable to import module:", e)
 
+from tests.unit.authentication.test_jwk_token import create_jwks_keys, make_key
+
 
 def before_all(context: Context) -> None:
     """Run before and after the whole shooting match."""
@@ -55,7 +59,7 @@ def after_scenario(context: Context, scenario: Scenario) -> None:
         try:
             # Start the llama-stack container again
             subprocess.run(
-                ["docker", "start", "llama-stack"], check=True, capture_output=True
+                [os.getenv("CONTAINER_CMD", "docker"), "start", "llama-stack"],
             )
 
             # Wait for the service to be healthy
@@ -67,7 +71,7 @@ def after_scenario(context: Context, scenario: Scenario) -> None:
                 try:
                     result = subprocess.run(
                         [
-                            "docker",
+                            os.getenv("CONTAINER_CMD", "docker"),
                             "exec",
                             "llama-stack",
                             "curl",
@@ -108,13 +112,26 @@ def before_feature(context: Context, feature: Feature) -> None:
         switch_config(context.feature_config)
         restart_container("lightspeed-stack")
 
+    elif "JWKAuth" in feature.tags:
+        context.feature_config = (
+            "tests/e2e/configuration/lightspeed-stack-auth-jwk.yaml"
+        )
+        with open("tests/e2e/configuration/test_jwk/jwk.json", "w") as f:
+            context.test_key = make_key()
+            keys = create_jwks_keys([context.test_key], ["RS256"])
+            f.write(json.dumps(keys))
+
+        context.default_config_backup = create_config_backup("lightspeed-stack.yaml")
+        switch_config("tests/e2e/configuration/lightspeed-stack-auth-jwk.yaml")
+        restart_container("lightspeed-stack")
+
     if "Feedback" in feature.tags:
         context.feedback_conversations = []
 
 
 def after_feature(context: Context, feature: Feature) -> None:
     """Run after each feature file is exercised."""
-    if "Authorized" in feature.tags:
+    if "Authorized" in feature.tags or "JWKAuth" in feature.tags:
         switch_config(context.default_config_backup)
         restart_container("lightspeed-stack")
         remove_config_backup(context.default_config_backup)

tests/e2e/features/steps/health.py

@@ -1,5 +1,6 @@
 """Implementation of common test steps."""
 
+import os
 import subprocess
 import time
 from behave import given  # pyright: ignore[reportAttributeAccessIssue]
@@ -14,7 +15,13 @@ def llama_stack_connection_broken(context: Context) -> None:
 
     try:
         result = subprocess.run(
-            ["docker", "inspect", "-f", "{{.State.Running}}", "llama-stack"],
+            [
+                os.getenv("CONTAINER_CMD", "docker"),
+                "inspect",
+                "-f",
+                "{{.State.Running}}",
+                "llama-stack",
+            ],
             capture_output=True,
             text=True,
             check=True,
@@ -23,7 +30,9 @@ def llama_stack_connection_broken(context: Context) -> None:
         if result.stdout.strip():
             context.llama_stack_was_running = True
             subprocess.run(
-                ["docker", "stop", "llama-stack"], check=True, capture_output=True
+                [os.getenv("CONTAINER_CMD", "docker"), "stop", "llama-stack"],
+                check=True,
+                capture_output=True,
             )
 
             # Wait a moment for the connection to be fully disrupted

tests/e2e/features/steps/jwk_auth.py

@@ -0,0 +1,41 @@
+"""JWK auth steps - reusing unit test primitives."""
+
+from pathlib import Path
+import sys
+
+from behave import given  # pyright: ignore[reportAttributeAccessIssue]
+from behave.runner import Context
+
+
+from authlib.jose import JsonWebToken
+
+sys.path.append(str(Path(__file__).resolve().parents[4]))
+
+# Import at runtime to avoid module load issues
+from tests.unit.authentication.test_jwk_token import (
+    create_token_header,
+    create_token_payload,
+)
+
+
+@given("I have a valid JWT token with the {role} role")
+def create_role_token(context: Context, role: str) -> None:
+    """Create token with role using the shared test key."""
+    test_key = context.test_key
+
+    header = create_token_header(test_key["kid"])
+    payload = create_token_payload()
+
+    # This works thanks to the definitions in lightspeed-stack-auth-jwk.yaml
+    payload["roles"] = [role]  # Add role to existing payload
+
+    token = (
+        JsonWebToken(algorithms=["RS256"])
+        .encode(header, payload, test_key["private_key"])
+        .decode()
+    )
+
+    if not hasattr(context, "auth_headers"):
+        context.auth_headers = {}
+
+    context.auth_headers["Authorization"] = f"Bearer {token}"

tests/e2e/test_list.txt

@@ -8,3 +8,4 @@ features/info.feature
 features/query.feature
 features/streaming_query.feature
 features/rest_api.feature
+features/authorization_jwk.feature