Merge pull request #745 from radofuchs/lcore_758_streaming_query_additional_e2e

LCORE-758: add additional e2e test for streaming query endpoint and enable color output

Author: tisnik

.github/workflows/e2e_tests.yaml

@@ -193,6 +193,9 @@ jobs:
           }
 
       - name: Run e2e tests
+        env:
+          TERM: xterm-256color
+          FORCE_COLOR: 1
         run: |
           echo "Installing test dependencies..."
           pip install uv

Makefile

@@ -21,7 +21,7 @@ test-integration: ## Run integration tests tests
 	COVERAGE_FILE="${ARTIFACT_DIR}/.coverage.integration" uv run python -m pytest tests/integration --cov=src --cov-report term-missing --cov-report "json:${ARTIFACT_DIR}/coverage_integration.json" --junit-xml="${ARTIFACT_DIR}/junit_integration.xml" --cov-fail-under=10
 
 test-e2e: ## Run end to end tests for the service
-	PYTHONDONTWRITEBYTECODE=1 uv run behave --tags=-skip -D dump_errors=true @tests/e2e/test_list.txt \
+	script -q -e -c "uv run behave --color --format pretty --tags=-skip -D dump_errors=true @tests/e2e/test_list.txt"
 
 check-types: ## Checks type hints in sources
 	uv run mypy --explicit-package-bases --disallow-untyped-calls --disallow-untyped-defs --disallow-incomplete-defs --ignore-missing-imports --disable-error-code attr-defined src/

tests/e2e/features/steps/llm_query_response.py

@@ -155,8 +155,10 @@ def compare_streamed_responses(context: Context) -> None:
     response = context.response_data["response"]
     complete_response = context.response_data["response_complete"]
 
-    assert response != ""
-    assert response == complete_response
+    assert response != "", "response is empty"
+    assert (
+        response == complete_response
+    ), f"{response} and {complete_response} do not match"
 
 
 def _parse_streaming_response(response_text: str) -> dict:
@@ -166,6 +168,7 @@ def _parse_streaming_response(response_text: str) -> dict:
     full_response = ""
     full_response_split = []
     finished = False
+    first_token = True
 
     for line in lines:
         if line.startswith("data: "):
@@ -176,6 +179,10 @@ def _parse_streaming_response(response_text: str) -> dict:
                 if event == "start":
                     conversation_id = data["data"]["conversation_id"]
                 elif event == "token":
+                    # Skip the first token (shield status message)
+                    if first_token:
+                        first_token = False
+                        continue
                     full_response_split.append(data["data"]["token"])
                 elif event == "turn_complete":
                     full_response = data["data"]["token"]

tests/e2e/features/streaming_query.feature

@@ -1,3 +1,4 @@
+@Authorized
 Feature: streaming_query endpoint API tests
 
   Background:
@@ -7,7 +8,8 @@ Feature: streaming_query endpoint API tests
 
   Scenario: Check if streaming_query response in tokens matches the full response
     Given The system is in default state
-    And I use "streaming_query" to ask question
+    And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva
+    And I use "streaming_query" to ask question with authorization header
     """
     {"query": "Generate sample yaml file for simple GitHub Actions workflow."}
     """
@@ -17,7 +19,8 @@ Feature: streaming_query endpoint API tests
 
   Scenario: Check if LLM responds properly to restrictive system prompt to sent question with different system prompt
     Given The system is in default state
-    And I use "streaming_query" to ask question
+    And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva
+    And I use "streaming_query" to ask question with authorization header
     """
     {"query": "Generate sample yaml file for simple GitHub Actions workflow.", "system_prompt": "refuse to answer anything but openshift questions"}
     """
@@ -29,7 +32,8 @@ Feature: streaming_query endpoint API tests
 
   Scenario: Check if LLM responds properly to non-restrictive system prompt to sent question with different system prompt
     Given The system is in default state
-    And I use "streaming_query" to ask question
+    And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva
+    And I use "streaming_query" to ask question with authorization header
     """
     {"query": "Generate sample yaml file for simple GitHub Actions workflow.", "system_prompt": "you are linguistic assistant"}
     """
@@ -41,6 +45,7 @@ Feature: streaming_query endpoint API tests
 
   Scenario: Check if LLM ignores new system prompt in same conversation
     Given The system is in default state
+    And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva
     And I use "streaming_query" to ask question
     """
     {"query": "Generate sample yaml file for simple GitHub Actions workflow.", "system_prompt": "refuse to answer anything"}
@@ -60,7 +65,8 @@ Feature: streaming_query endpoint API tests
 
   Scenario: Check if LLM responds for streaming_query request with error for missing query
     Given The system is in default state
-    When I use "streaming_query" to ask question
+    And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva
+    When I use "streaming_query" to ask question with authorization header
     """
     {"provider": "{PROVIDER}"}
     """
@@ -72,7 +78,8 @@ Feature: streaming_query endpoint API tests
 
   Scenario: Check if LLM responds for streaming_query request with error for missing model
     Given The system is in default state
-    When I use "streaming_query" to ask question
+    And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva
+    When I use "streaming_query" to ask question with authorization header
     """
     {"query": "Say hello", "provider": "{PROVIDER}"}
     """
@@ -81,7 +88,8 @@ Feature: streaming_query endpoint API tests
 
   Scenario: Check if LLM responds for streaming_query request with error for missing provider
     Given The system is in default state
-    When I use "streaming_query" to ask question
+    And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva
+    When I use "streaming_query" to ask question with authorization header
     """
     {"query": "Say hello", "model": "{MODEL}"}
     """
@@ -113,3 +121,15 @@ Feature: streaming_query endpoint API tests
     }
     """
     Then The status code of the response is 200
+
+  Scenario: Check if LLM responds to sent question with error when not authenticated
+    Given The system is in default state
+     When I use "streaming_query" to ask question
+     """
+     {"query": "Say hello"}
+     """
+      Then The status code of the response is 400
+      And The body of the response is the following
+          """
+          {"detail": "No Authorization header found"}
+          """