From 1a91e8e775501ce6cd38aed20c6f21a95e653f74 Mon Sep 17 00:00:00 2001 From: Gilad Ravid Date: Thu, 21 Aug 2025 12:37:26 +0300 Subject: [PATCH 1/9] log previous assisted-chat-eval-test container's logs --- scripts/ci_test.sh | 6 ++++-- test/prow/entrypoint.sh | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/ci_test.sh b/scripts/ci_test.sh index 15bfac51..dacbe5b5 100755 --- a/scripts/ci_test.sh +++ b/scripts/ci_test.sh @@ -13,7 +13,7 @@ oc process -p IMAGE_NAME="$ASSISTED_CHAT_TEST" -p SSL_CLIENT_SECRET_NAME=assiste sleep 5 oc get pods -n "$NAMESPACE" -POD_NAME=$(oc get pods | tr -s ' ' | cut -d ' ' -f1 | grep assisted-chat-eval-tes) +POD_NAME=$(oc get pods | tr -s ' ' | cut -d ' ' -f1 | grep assisted-chat-eval-test) TIMEOUT=600 ELAPSED=0 @@ -24,7 +24,9 @@ while [ $ELAPSED -lt $TIMEOUT ]; do CURRENT_RESTARTS=$(oc get pod "$POD_NAME" -n "$NAMESPACE" -o=jsonpath='{.status.containerStatuses[0].restartCount}') if [[ $CURRENT_RESTARTS -gt 0 ]]; then echo "Pod ${POD_NAME} was restarted, so the tests should run at least once, exiting" - oc logs -n "$NAMESPACE" "$POD_NAME" + echo "########################## Start of logs ##########################" + oc logs -p -n "$NAMESPACE" "$POD_NAME" + echo "########################## End of logs ##########################" exit "$(oc get pod "$POD_NAME" -n "$NAMESPACE" -o=jsonpath='{.status.containerStatuses[0].lastState.terminated.exitCode}')" fi if [[ "$CURRENT_STATUS" == "Succeeded" ]]; then diff --git a/test/prow/entrypoint.sh b/test/prow/entrypoint.sh index 88edba8f..d1ce1f32 100644 --- a/test/prow/entrypoint.sh +++ b/test/prow/entrypoint.sh @@ -15,4 +15,4 @@ echo "$OCM_TOKEN" > test/evals/ocm_token.txt cd test/evals -#python eval.py --agent_endpoint "${AGENT_URL}:${AGENT_PORT}" +python eval.py --agent_endpoint "${AGENT_URL}:${AGENT_PORT}" From b09ceaa17990bd5548594c71b2f9b773c309da9c Mon Sep 17 00:00:00 2001 From: Gilad Ravid Date: Thu, 21 Aug 2025 14:32:49 +0300 Subject: [PATCH 2/9] add coderabbit suggestions --- scripts/ci_test.sh | 2 +- test/prow/entrypoint.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/ci_test.sh b/scripts/ci_test.sh index dacbe5b5..b02ec931 100755 --- a/scripts/ci_test.sh +++ b/scripts/ci_test.sh @@ -13,7 +13,7 @@ oc process -p IMAGE_NAME="$ASSISTED_CHAT_TEST" -p SSL_CLIENT_SECRET_NAME=assiste sleep 5 oc get pods -n "$NAMESPACE" -POD_NAME=$(oc get pods | tr -s ' ' | cut -d ' ' -f1 | grep assisted-chat-eval-test) +POD_NAME=$(oc get pods -n "$NAMESPACE" | tr -s ' ' | cut -d ' ' -f1 | grep assisted-chat-eval-test) TIMEOUT=600 ELAPSED=0 diff --git a/test/prow/entrypoint.sh b/test/prow/entrypoint.sh index d1ce1f32..798c3e07 100644 --- a/test/prow/entrypoint.sh +++ b/test/prow/entrypoint.sh @@ -9,7 +9,7 @@ OCM_TOKEN=$(curl -X POST https://sso.redhat.com/auth/realms/redhat-external/prot -H "Content-Type: application/x-www-form-urlencoded" \ -d "grant_type=client_credentials" \ -d "client_id=$CLIENT_ID" \ - -d "client_secret=$CLIENT_SECRET" | jq '.access_token') + -d "client_secret=$CLIENT_SECRET" | jq -r '.access_token') echo "$OCM_TOKEN" > test/evals/ocm_token.txt From 9b262b64769d6577c48b95c6351e38f245da721d Mon Sep 17 00:00:00 2001 From: Gilad Ravid Date: Mon, 25 Aug 2025 09:46:51 +0300 Subject: [PATCH 3/9] debug --- scripts/ci_test.sh | 4 +--- test/prow/entrypoint.sh | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/scripts/ci_test.sh b/scripts/ci_test.sh index b02ec931..4b3f8103 100755 --- a/scripts/ci_test.sh +++ b/scripts/ci_test.sh @@ -24,9 +24,7 @@ while [ $ELAPSED -lt $TIMEOUT ]; do CURRENT_RESTARTS=$(oc get pod "$POD_NAME" -n "$NAMESPACE" -o=jsonpath='{.status.containerStatuses[0].restartCount}') if [[ $CURRENT_RESTARTS -gt 0 ]]; then echo "Pod ${POD_NAME} was restarted, so the tests should run at least once, exiting" - echo "########################## Start of logs ##########################" - oc logs -p -n "$NAMESPACE" "$POD_NAME" - echo "########################## End of logs ##########################" + oc logs -p -n "$NAMESPACE" "$POD_NAME" || true exit "$(oc get pod "$POD_NAME" -n "$NAMESPACE" -o=jsonpath='{.status.containerStatuses[0].lastState.terminated.exitCode}')" fi if [[ "$CURRENT_STATUS" == "Succeeded" ]]; then diff --git a/test/prow/entrypoint.sh b/test/prow/entrypoint.sh index 798c3e07..3ae6c702 100644 --- a/test/prow/entrypoint.sh +++ b/test/prow/entrypoint.sh @@ -15,4 +15,4 @@ echo "$OCM_TOKEN" > test/evals/ocm_token.txt cd test/evals -python eval.py --agent_endpoint "${AGENT_URL}:${AGENT_PORT}" +python eval.py --agent_endpoint "${AGENT_URL}:${AGENT_PORT}" --endpoint_type query From 3ebf25660fed3089f4b82f8fbcf99fa6ef635e3e Mon Sep 17 00:00:00 2001 From: Gilad Ravid Date: Mon, 25 Aug 2025 11:07:24 +0300 Subject: [PATCH 4/9] debug --- test/evals/eval.py | 32 ++++++++++++++++++++++++++++++++ test/prow/entrypoint.sh | 2 +- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/test/evals/eval.py b/test/evals/eval.py index 7982caf2..5e9a304e 100644 --- a/test/evals/eval.py +++ b/test/evals/eval.py @@ -4,6 +4,38 @@ import logging import sys +# Monkey patch to fix httpx ResponseNotRead error in lsc_agent_eval +def patch_httpx_response_error(): + """Patch httpx Response to handle streaming response text access safely.""" + try: + import httpx + + # Store original text property + original_text = httpx.Response.text + + def safe_text(self): + """Safely access response text, handling streaming responses.""" + try: + return original_text.fget(self) + except httpx.ResponseNotRead: + # If it's a streaming response that hasn't been read, read it first + try: + self.read() + return original_text.fget(self) + except Exception: + # If we still can't read it, return a safe fallback + return f"" + + # Replace the text property with our safe version + httpx.Response.text = property(safe_text) + + except ImportError: + # httpx not available, skip patching + pass + +# Apply the patch before importing lsc_agent_eval +patch_httpx_response_error() + from lsc_agent_eval import AgentGoalEval # Configure logging to show all messages from agent_eval library diff --git a/test/prow/entrypoint.sh b/test/prow/entrypoint.sh index 3ae6c702..798c3e07 100644 --- a/test/prow/entrypoint.sh +++ b/test/prow/entrypoint.sh @@ -15,4 +15,4 @@ echo "$OCM_TOKEN" > test/evals/ocm_token.txt cd test/evals -python eval.py --agent_endpoint "${AGENT_URL}:${AGENT_PORT}" --endpoint_type query +python eval.py --agent_endpoint "${AGENT_URL}:${AGENT_PORT}" From 7fc3502b19982b741b5bdf98539795af4ebb145d Mon Sep 17 00:00:00 2001 From: Gilad Ravid Date: Tue, 26 Aug 2025 11:00:39 +0300 Subject: [PATCH 5/9] debug --- test/evals/eval.py | 32 -------------------------------- test/prow/entrypoint.sh | 2 +- 2 files changed, 1 insertion(+), 33 deletions(-) diff --git a/test/evals/eval.py b/test/evals/eval.py index 5e9a304e..7982caf2 100644 --- a/test/evals/eval.py +++ b/test/evals/eval.py @@ -4,38 +4,6 @@ import logging import sys -# Monkey patch to fix httpx ResponseNotRead error in lsc_agent_eval -def patch_httpx_response_error(): - """Patch httpx Response to handle streaming response text access safely.""" - try: - import httpx - - # Store original text property - original_text = httpx.Response.text - - def safe_text(self): - """Safely access response text, handling streaming responses.""" - try: - return original_text.fget(self) - except httpx.ResponseNotRead: - # If it's a streaming response that hasn't been read, read it first - try: - self.read() - return original_text.fget(self) - except Exception: - # If we still can't read it, return a safe fallback - return f"" - - # Replace the text property with our safe version - httpx.Response.text = property(safe_text) - - except ImportError: - # httpx not available, skip patching - pass - -# Apply the patch before importing lsc_agent_eval -patch_httpx_response_error() - from lsc_agent_eval import AgentGoalEval # Configure logging to show all messages from agent_eval library diff --git a/test/prow/entrypoint.sh b/test/prow/entrypoint.sh index 798c3e07..3ae6c702 100644 --- a/test/prow/entrypoint.sh +++ b/test/prow/entrypoint.sh @@ -15,4 +15,4 @@ echo "$OCM_TOKEN" > test/evals/ocm_token.txt cd test/evals -python eval.py --agent_endpoint "${AGENT_URL}:${AGENT_PORT}" +python eval.py --agent_endpoint "${AGENT_URL}:${AGENT_PORT}" --endpoint_type query From 8d89a8fb2b48159ad556cc5667ce50b785d6cd56 Mon Sep 17 00:00:00 2001 From: Gilad Ravid Date: Tue, 26 Aug 2025 11:50:13 +0300 Subject: [PATCH 6/9] update template --- template.yaml | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/template.yaml b/template.yaml index 2b4156c1..95167d6f 100644 --- a/template.yaml +++ b/template.yaml @@ -358,7 +358,22 @@ objects: password: ${env.ASSISTED_CHAT_POSTGRES_PASSWORD} ssl_mode: ${LLAMA_STACK_POSTGRES_SSL_MODE} ca_cert_path: /etc/tls/ca-bundle.pem - models: [] + models: + - metadata: {} + model_id: gemini/gemini-2.0-flash + provider_id: gemini + provider_model_id: gemini/gemini-2.0-flash + model_type: llm + - metadata: {} + model_id: gemini/gemini-2.5-pro + provider_id: gemini + provider_model_id: gemini/gemini-2.5-pro + model_type: llm + - metadata: {} + model_id: gemini/gemini-2.5-flash + provider_id: gemini + provider_model_id: gemini/gemini-2.5-flash + model_type: llm shields: [] vector_dbs: [] datasets: [] From 357bd93e52144560bf2ca9f9e29b0d494a4f7ab5 Mon Sep 17 00:00:00 2001 From: Gilad Ravid Date: Tue, 26 Aug 2025 12:29:02 +0300 Subject: [PATCH 7/9] debug --- test/prow/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/prow/entrypoint.sh b/test/prow/entrypoint.sh index 3ae6c702..798c3e07 100644 --- a/test/prow/entrypoint.sh +++ b/test/prow/entrypoint.sh @@ -15,4 +15,4 @@ echo "$OCM_TOKEN" > test/evals/ocm_token.txt cd test/evals -python eval.py --agent_endpoint "${AGENT_URL}:${AGENT_PORT}" --endpoint_type query +python eval.py --agent_endpoint "${AGENT_URL}:${AGENT_PORT}" From a3201ad6a047aa7cf36e1f872c14b5c31d9d24ee Mon Sep 17 00:00:00 2001 From: Gilad Ravid Date: Tue, 26 Aug 2025 13:41:31 +0300 Subject: [PATCH 8/9] change provider_model_id --- template.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/template.yaml b/template.yaml index 95167d6f..b5380cbe 100644 --- a/template.yaml +++ b/template.yaml @@ -362,17 +362,17 @@ objects: - metadata: {} model_id: gemini/gemini-2.0-flash provider_id: gemini - provider_model_id: gemini/gemini-2.0-flash + provider_model_id: gemini-2.0-flash model_type: llm - metadata: {} model_id: gemini/gemini-2.5-pro provider_id: gemini - provider_model_id: gemini/gemini-2.5-pro + provider_model_id: gemini-2.5-pro model_type: llm - metadata: {} model_id: gemini/gemini-2.5-flash provider_id: gemini - provider_model_id: gemini/gemini-2.5-flash + provider_model_id: gemini-2.5-flash model_type: llm shields: [] vector_dbs: [] From 4e31882f888510c63400ef610c9b9a2b0dd16b5d Mon Sep 17 00:00:00 2001 From: Gilad Ravid Date: Tue, 26 Aug 2025 16:05:51 +0300 Subject: [PATCH 9/9] change provider to vertex_ai --- template.yaml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/template.yaml b/template.yaml index b5380cbe..ea6b25e4 100644 --- a/template.yaml +++ b/template.yaml @@ -294,10 +294,11 @@ objects: - vector_io providers: inference: - - provider_id: gemini - provider_type: remote::gemini + - provider_id: vertex_ai + provider_type: remote::vertexai config: - api_key: dummy-to-stop-llama-stack-from-complaining-even-though-we-use-vertex-and-not-gemini-directly + project: "" + location: us-central1 vector_io: [] files: [] safety: [] @@ -360,18 +361,18 @@ objects: ca_cert_path: /etc/tls/ca-bundle.pem models: - metadata: {} - model_id: gemini/gemini-2.0-flash - provider_id: gemini + model_id: vertex_ai/gemini-2.0-flash + provider_id: vertex_ai provider_model_id: gemini-2.0-flash model_type: llm - metadata: {} - model_id: gemini/gemini-2.5-pro - provider_id: gemini + model_id: vertex_ai/gemini-2.5-pro + provider_id: vertex_ai provider_model_id: gemini-2.5-pro model_type: llm - metadata: {} - model_id: gemini/gemini-2.5-flash - provider_id: gemini + model_id: vertex_ai/gemini-2.5-flash + provider_id: vertex_ai provider_model_id: gemini-2.5-flash model_type: llm shields: []