Merge pull request #680 from radofuchs/LCORE_760_hardcoded_model_fix

LCORE-760: fix hardcoded model and provider in e2e tests

Author: tisnik

tests/e2e/features/conversations.feature

@@ -22,7 +22,7 @@ Feature: conversations endpoint API tests
      And The conversation with conversation_id from above is returned
      And The conversation details are following
      """
-     {"last_used_model": "gpt-4-turbo", "last_used_provider": "openai", "message_count": 1}
+     {"last_used_model": "{MODEL}", "last_used_provider": "{PROVIDER}", "message_count": 1}
      """
 
   Scenario: Check if conversations endpoint fails when the auth header is not present

tests/e2e/features/environment.py

@@ -20,14 +20,47 @@
     create_config_backup,
 )
 
-try:
-    import os  # noqa: F401
-except ImportError as e:
-    print("Warning: unable to import module:", e)
+
+def _fetch_models_from_service(hostname: str = "localhost", port: int = 8080) -> dict:
+    """Query /v1/models endpoint and return first LLM model.
+
+    Returns:
+        Dict with model_id and provider_id, or empty dict if unavailable
+    """
+    try:
+        url = f"http://{hostname}:{port}/v1/models"
+        response = requests.get(url, timeout=5)
+        response.raise_for_status()
+        data = response.json()
+
+        # Find first LLM model
+        for model in data.get("models", []):
+            if model.get("api_model_type") == "llm":
+                provider_id = model.get("provider_id")
+                model_id = model.get("provider_resource_id")
+                if provider_id and model_id:
+                    return {"model_id": model_id, "provider_id": provider_id}
+        return {}
+    except (requests.RequestException, ValueError, KeyError):
+        return {}
 
 
 def before_all(context: Context) -> None:
     """Run before and after the whole shooting match."""
+    # Get first LLM model from running service
+    llm_model = _fetch_models_from_service()
+
+    if llm_model:
+        context.default_model = llm_model["model_id"]
+        context.default_provider = llm_model["provider_id"]
+        print(
+            f"Detected LLM: {context.default_model} (provider: {context.default_provider})"
+        )
+    else:
+        # Fallback for development
+        context.default_model = "gpt-4-turbo"
+        context.default_provider = "openai"
+        print("⚠ Could not detect models, using fallback: gpt-4-turbo/openai")
 
 
 def before_scenario(context: Context, scenario: Scenario) -> None:

tests/e2e/features/info.feature

@@ -34,7 +34,7 @@ Feature: Info tests
     Given The system is in default state
      When I access REST API endpoint "models" using HTTP GET method
      Then The status code of the response is 200
-      And The body of the response for model gpt-4o-mini has proper structure
+      And The body of the response has proper model structure
 
 
   Scenario: Check if models endpoint is working

tests/e2e/features/query.feature

@@ -42,7 +42,7 @@ Feature: Query endpoint API tests
     And I store conversation details
     And I use "query" to ask question with same conversation_id
     """
-    {"query": "Write a simple code for reversing string", "system_prompt": "provide coding assistance", "model": "gpt-4-turbo", "provider": "openai"}
+    {"query": "Write a simple code for reversing string", "system_prompt": "provide coding assistance", "model": "{MODEL}", "provider": "{PROVIDER}"}
     """
     Then The status code of the response is 200
      And The response should contain following fragments
@@ -76,20 +76,20 @@ Scenario: Check if LLM responds for query request with error for missing query
      And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva
      When I use "query" to ask question with authorization header
      """
-     {"provider": "openai"}
+     {"provider": "{PROVIDER}"}
      """
      Then The status code of the response is 422
      And The body of the response is the following
         """
-        { "detail": [{"type": "missing", "loc": [ "body", "query" ], "msg": "Field required", "input": {"provider": "openai"}}] }
+        { "detail": [{"type": "missing", "loc": [ "body", "query" ], "msg": "Field required", "input": {"provider": "{PROVIDER}"}}] }
         """
 
   Scenario: Check if LLM responds for query request with error for missing model
     Given The system is in default state
      And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva
      When I use "query" to ask question with authorization header
      """
-     {"query": "Say hello", "provider": "openai"}
+     {"query": "Say hello", "provider": "{PROVIDER}"}
      """
      Then The status code of the response is 422
       And The body of the response contains Value error, Model must be specified if provider is specified
@@ -99,7 +99,7 @@ Scenario: Check if LLM responds for query request with error for missing query
     And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva
     When I use "query" to ask question with authorization header
     """
-    {"query": "Say hello", "model": "gpt-4-turbo"}
+    {"query": "Say hello", "model": "{MODEL}"}
     """
      Then The status code of the response is 422
       And The body of the response contains Value error, Provider must be specified if model is specified

tests/e2e/features/steps/common_http.py

@@ -7,6 +7,7 @@
 from behave.runner import Context
 from tests.e2e.utils.utils import (
     normalize_endpoint,
+    replace_placeholders,
     validate_json,
     validate_json_partially,
 )
@@ -170,7 +171,10 @@ def check_prediction_result(context: Context) -> None:
     assert context.response is not None, "Request needs to be performed first"
     assert context.text is not None, "Response does not contain any payload"
 
-    expected_body = json.loads(context.text)
+    # Replace {MODEL} and {PROVIDER} placeholders with actual values
+    json_str = replace_placeholders(context, context.text or "{}")
+
+    expected_body = json.loads(json_str)
     result = context.response.json()
 
     # compare both JSONs and print actual result in case of any difference

tests/e2e/features/steps/conversation.py

@@ -4,6 +4,7 @@
 from behave import step, when, then  # pyright: ignore[reportAttributeAccessIssue]
 from behave.runner import Context
 import requests
+from tests.e2e.utils.utils import replace_placeholders
 
 # default timeout for HTTP operations
 DEFAULT_TIMEOUT = 10
@@ -109,7 +110,10 @@ def check_returned_conversation_id(context: Context) -> None:
 @then("The conversation details are following")
 def check_returned_conversation_content(context: Context) -> None:
     """Check the conversation content in response."""
-    expected_data = json.loads(context.text)
+    # Replace {MODEL} and {PROVIDER} placeholders with actual values
+    json_str = replace_placeholders(context, context.text or "{}")
+
+    expected_data = json.loads(json_str)
     found_conversation = context.found_conversation
 
     assert (

tests/e2e/features/steps/info.py

@@ -27,31 +27,39 @@ def check_llama_version(context: Context, llama_version: str) -> None:
     ), f"llama-stack version is {response_json["llama_stack_version"]}"
 
 
-@then("The body of the response for model {model} has proper structure")
-def check_model_structure(context: Context, model: str) -> None:
-    """Check that the gpt-4o-mini model has the correct structure and required fields."""
+@then("The body of the response has proper model structure")
+def check_model_structure(context: Context) -> None:
+    """Check that the first LLM model has the correct structure and required fields."""
     response_json = context.response.json()
     assert response_json is not None, "Response is not valid JSON"
 
     assert "models" in response_json, "Response missing 'models' field"
     models = response_json["models"]
-    assert len(models) > 0, "Models list should not be empty"
+    assert len(models) > 0, "Response has empty list of models"
 
-    gpt_model = None
-    for model_id in models:
-        if "gpt-4o-mini" in model_id.get("identifier", ""):
-            gpt_model = model_id
+    # Find first LLM model (same logic as environment.py)
+    llm_model = None
+    for model in models:
+        if model.get("api_model_type") == "llm":
+            llm_model = model
             break
 
-    assert gpt_model is not None
+    assert llm_model is not None, "No LLM model found in response"
 
-    assert gpt_model["type"] == "model", "type should be 'model'"
-    assert gpt_model["api_model_type"] == "llm", "api_model_type should be 'llm'"
-    assert gpt_model["model_type"] == "llm", "model_type should be 'llm'"
-    assert gpt_model["provider_id"] == "openai", "provider_id should be 'openai'"
+    # Get expected values from context
+    expected_model = context.default_model
+    expected_provider = context.default_provider
+
+    # Validate structure and values
+    assert llm_model["type"] == "model", "type should be 'model'"
+    assert llm_model["api_model_type"] == "llm", "api_model_type should be 'llm'"
+    assert llm_model["model_type"] == "llm", "model_type should be 'llm'"
+    assert (
+        llm_model["provider_id"] == expected_provider
+    ), f"provider_id should be '{expected_provider}'"
     assert (
-        gpt_model["provider_resource_id"] == model
-    ), "provider_resource_id should be 'gpt-4o-mini'"
+        llm_model["provider_resource_id"] == expected_model
+    ), f"provider_resource_id should be '{expected_model}'"
     assert (
-        gpt_model["identifier"] == f"openai/{model}"
-    ), "identifier should be 'openai/gpt-4o-mini'"
+        llm_model["identifier"] == f"{expected_provider}/{expected_model}"
+    ), f"identifier should be '{expected_provider}/{expected_model}'"

tests/e2e/features/steps/llm_query_response.py

@@ -4,6 +4,7 @@
 import requests
 from behave import then, step  # pyright: ignore[reportAttributeAccessIssue]
 from behave.runner import Context
+from tests.e2e.utils.utils import replace_placeholders
 
 
 DEFAULT_LLM_TIMEOUT = 60
@@ -24,9 +25,11 @@ def ask_question(context: Context, endpoint: str) -> None:
     path = f"{context.api_prefix}/{endpoint}".replace("//", "/")
     url = base + path
 
-    # Use context.text if available, otherwise use empty query
-    data = json.loads(context.text or "{}")
-    print(data)
+    # Replace {MODEL} and {PROVIDER} placeholders with actual values
+    json_str = replace_placeholders(context, context.text or "{}")
+
+    data = json.loads(json_str)
+    print(f"Request data: {data}")
     context.response = requests.post(url, json=data, timeout=DEFAULT_LLM_TIMEOUT)
 
 
@@ -37,9 +40,11 @@ def ask_question_authorized(context: Context, endpoint: str) -> None:
     path = f"{context.api_prefix}/{endpoint}".replace("//", "/")
     url = base + path
 
-    # Use context.text if available, otherwise use empty query
-    data = json.loads(context.text or "{}")
-    print(data)
+    # Replace {MODEL} and {PROVIDER} placeholders with actual values
+    json_str = replace_placeholders(context, context.text or "{}")
+
+    data = json.loads(json_str)
+    print(f"Request data: {data}")
     context.response = requests.post(
         url, json=data, headers=context.auth_headers, timeout=DEFAULT_LLM_TIMEOUT
     )
@@ -58,12 +63,14 @@ def ask_question_in_same_conversation(context: Context, endpoint: str) -> None:
     path = f"{context.api_prefix}/{endpoint}".replace("//", "/")
     url = base + path
 
-    # Use context.text if available, otherwise use empty query
-    data = json.loads(context.text or "{}")
+    # Replace {MODEL} and {PROVIDER} placeholders with actual values
+    json_str = replace_placeholders(context, context.text or "{}")
+
+    data = json.loads(json_str)
     headers = context.auth_headers if hasattr(context, "auth_headers") else {}
     data["conversation_id"] = context.response_data["conversation_id"]
 
-    print(data)
+    print(f"Request data: {data}")
     context.response = requests.post(
         url, json=data, headers=headers, timeout=DEFAULT_LLM_TIMEOUT
     )

tests/e2e/features/streaming_query.feature

@@ -51,7 +51,7 @@ Feature: streaming_query endpoint API tests
     Then The status code of the response is 200
     And I use "streaming_query" to ask question with same conversation_id
     """
-    {"query": "Write a simple code for reversing string", "system_prompt": "provide coding assistance", "model": "gpt-4-turbo", "provider": "openai"}
+    {"query": "Write a simple code for reversing string", "system_prompt": "provide coding assistance", "model": "{MODEL}", "provider": "{PROVIDER}"}
     """
     Then The status code of the response is 200
     When I wait for the response to be completed
@@ -64,19 +64,19 @@ Feature: streaming_query endpoint API tests
     Given The system is in default state
     When I use "streaming_query" to ask question
     """
-    {"provider": "openai"}
+    {"provider": "{PROVIDER}"}
     """
      Then The status code of the response is 422
       And The body of the response is the following
           """
-          { "detail": [{"type": "missing", "loc": [ "body", "query" ], "msg": "Field required", "input": {"provider": "openai"}}] }
+          { "detail": [{"type": "missing", "loc": [ "body", "query" ], "msg": "Field required", "input": {"provider": "{PROVIDER}"}}] }
           """
 
   Scenario: Check if LLM responds for streaming_query request with error for missing model
     Given The system is in default state
     When I use "streaming_query" to ask question
     """
-    {"query": "Say hello", "provider": "openai"}
+    {"query": "Say hello", "provider": "{PROVIDER}"}
     """
      Then The status code of the response is 422
       And The body of the response contains Value error, Model must be specified if provider is specified
@@ -85,7 +85,7 @@ Feature: streaming_query endpoint API tests
     Given The system is in default state
     When I use "streaming_query" to ask question
     """
-    {"query": "Say hello", "model": "gpt-4-turbo"}
+    {"query": "Say hello", "model": "{MODEL}"}
     """
      Then The status code of the response is 422
       And The body of the response contains Value error, Provider must be specified if model is specified

tests/e2e/utils/utils.py

@@ -6,6 +6,7 @@
 import time
 import jsonschema
 from typing import Any
+from behave.runner import Context
 
 
 def normalize_endpoint(endpoint: str) -> str:
@@ -141,3 +142,19 @@ def restart_container(container_name: str) -> None:
 
     # Wait for container to be healthy
     wait_for_container_health(container_name)
+
+
+def replace_placeholders(context: Context, text: str) -> str:
+    """Replace {MODEL} and {PROVIDER} placeholders with actual values from context.
+
+    Args:
+        context: Behave context containing default_model and default_provider
+        text: String that may contain {MODEL} and {PROVIDER} placeholders
+
+    Returns:
+        String with placeholders replaced by actual values
+
+    """
+    result = text.replace("{MODEL}", context.default_model)
+    result = result.replace("{PROVIDER}", context.default_provider)
+    return result