creating a unique name for the clusters created by the evaluation tests

scripts/ci_test.sh

@@ -6,6 +6,9 @@ set -o pipefail
 
 SECRETS_BASE_PATH="${SECRETS_BASE_PATH:-/var/run/secrets}"
 JOB_NAME="assisted-chat-eval-test"
+UNIQUE_ID=$(head /dev/urandom | tr -dc 0-9a-z | head -c 8)
+
+echo "${UNIQUE_ID}" > ${SHARED_DIR}/eval_test_unique_id
 
 if [[ -n $ASSISTED_CHAT_TEST ]]; then
     echo "The variable ASSISTED_CHAT_TEST was proided with the value ${ASSISTED_CHAT_TEST}, using it to create the IMAGE and TAG variables for the template"
@@ -26,7 +29,10 @@ if ! oc get secret -n "$NAMESPACE" gemini &>/dev/null; then
     oc create secret generic -n $NAMESPACE gemini --from-file=api_key="${SECRETS_BASE_PATH}/gemini/api_key"
 fi
 
-oc process -p IMAGE_NAME="$ASSISTED_CHAT_TEST" -p SSL_CLIENT_SECRET_NAME=assisted-chat-ssl-ci -f test/prow/template.yaml --local | oc apply -n "$NAMESPACE" -f -
+oc process -p IMAGE_NAME="$ASSISTED_CHAT_TEST" \
+           -p SSL_CLIENT_SECRET_NAME=assisted-chat-ssl-ci \
+           -p JOB_ID=${UNIQUE_ID} \
+           -f test/prow/template.yaml --local | oc apply -n "$NAMESPACE" -f -
 
 sleep 5
 oc get pods -n "$NAMESPACE"

test/evals/eval_data.yaml

@@ -62,21 +62,21 @@
   description: Create SNO and then retrieve Discovery ISO in two steps with all the information provided
   conversation:
     - eval_id: create_eval_test_sno
-      eval_query: create a new single node cluster named eval-test-sno, running on version 4.19.7 with the x86_64 CPU architecture, configured under the base domain example.com, using the provided SSH key "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCmeaBFhSJ/MLECmqUaKweRgo10ABpwdvJ7v76qLYfP0pzfzYsF3hGP/fH5OQfHi9pTbWynjaEcPHVfaTaFWHvyMtv8PEMUIDgQPWlBSYzb+3AgQ5AsChhzTJCYnRdmCdzENlV+azgtb3mVfXiyCfjxhyy3QAV4hRrMaVtJGuUQfQ== [email protected]".
-      eval_types: [tool_eval, response_eval:accuracy, response_eval:sub-string]
+      eval_query: create a new single node cluster named eval-test-singlenode-ClustER-NAme, running on version 4.19.7 with the x86_64 CPU architecture, configured under the base domain example.com, using the provided SSH key "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCmeaBFhSJ/MLECmqUaKweRgo10ABpwdvJ7v76qLYfP0pzfzYsF3hGP/fH5OQfHi9pTbWynjaEcPHVfaTaFWHvyMtv8PEMUIDgQPWlBSYzb+3AgQ5AsChhzTJCYnRdmCdzENlV+azgtb3mVfXiyCfjxhyy3QAV4hRrMaVtJGuUQfQ== [email protected]".
+      eval_types: [tool_eval, response_eval:sub-string, response_eval:accuracy]
       expected_tool_calls:
         - - tool_name: create_cluster
             arguments:
-              name: "eval-test-sno"
+              name: "eval-test-singlenode-ClustER-NAme"
               version: "4\\.19\\.7"
               base_domain: "example\\.com"
               single_node: "(?i:true)"
               cpu_architecture: "x86_64"
               ssh_public_key: 'ssh-rsa\s+[A-Za-z0-9+/]+[=]{0,3}(\s+.+)?\s*'
-      expected_keywords: ["eval-test-sno", "4.19.7", "ID", "Discovery ISO", "download", "cluster"]
-      expected_response: I have created a cluster with name eval-test-sno, OpenShift version 4.19.7, base domain test.local. Next, you'll need to download the Discovery ISO, then boot your hosts with it. Do you want me to download the Discovery ISO?
+      expected_keywords: ["eval-test-singlenode-ClustER-NAme", "4.19.7", "ID", "Discovery ISO", "download", "cluster"]
+      expected_response: I have created a cluster with name eval-test-singlenode-ClustER-NAme, OpenShift version 4.19.7, base domain test.local. Next, you'll need to download the Discovery ISO, then boot your hosts with it. Would you like me to get the Discovery ISO download URL?
     - eval_id: get_iso_eval_test_sno
-      eval_query: Using the ID of the cluster you just created, get the Discovery ISO download URL for cluster 'eval-test-sno'
+      eval_query: Using the ID of the cluster you just created, get the Discovery ISO download URL for cluster 'eval-test-singlenode-ClustER-NAme'
       eval_types: [tool_eval, response_eval:sub-string]
       expected_tool_calls:
         - - tool_name: cluster_iso_download_url
@@ -88,30 +88,30 @@
   description: Create multi-node cluster and then retrieve ISO in two steps
   conversation:
     - eval_id: create_eval_test_multinode
-      eval_query: Create a multi-node cluster named 'eval-test-multinode' with OpenShift 4.18.22 and domain test.local
       eval_types: [tool_eval, response_eval:accuracy, response_eval:sub-string]
+      eval_query: Create a multi-node cluster named 'eval-test-multinode-ClustER-NAme' with OpenShift 4.18.22 and domain test.local
       expected_tool_calls:
         - - tool_name: create_cluster
             arguments:
-              name: "eval-test-multinode"
+              name: "eval-test-multinode-ClustER-NAme"
               version: "4\\.18\\.22"
               base_domain: "test\\.local"
               single_node: "(?i:false)"
               cpu_architecture: "x86_64"
               ssh_public_key: ""
-      expected_keywords: ["eval-test-multinode", "4.18.22", "ID", "Discovery ISO", "download", "cluster"]
-      expected_response: I have created a cluster with name eval-test-multinode, OpenShift version 4.18.22, base domain test.local. Next, you'll need to download the Discovery ISO, then boot your hosts with it. Do you want me to download the Discovery ISO?
+      expected_keywords: ["eval-test-multinode-ClustER-NAme", "4.18.22", "ID", "Discovery ISO", "cluster"]
+      expected_response: I have created a cluster with name eval-test-multinode-ClustER-NAme, OpenShift version 4.18.22, base domain test.local. Next, you'll need to download the Discovery ISO, then boot your hosts with it. Would you like me to get the Discovery ISO download URL?
     - eval_id: set_ssh_key_eval_test_ssh
       eval_query: Set the SSH key for the cluster you just created to "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCmeaBFhSJ/MLECmqUaKweRgo10ABpwdvJ7v76qLYfP0pzfzYsF3hGP/fH5OQfHi9pTbWynjaEcPHVfaTaFWHvyMtv8PEMUIDgQPWlBSYzb+3AgQ5AsChhzTJCYnRdmCdzENlV+azgtb3mVfXiyCfjxhyy3QAV4hRrMaVtJGuUQfQ== [email protected]"
-      eval_types: [tool_eval, response_eval:sub-string]
+      eval_types: [tool_eval, response_eval:accuracy]
       expected_tool_calls:
         - - tool_name: set_cluster_ssh_key
             arguments:
               cluster_id: "[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}"
               ssh_public_key: 'ssh-rsa\s+[A-Za-z0-9+/]+[=]{0,3}(\s+.+)?\s*'
-      expected_keywords: ["SSH", "key", "updated"]
+      expected_response: The SSH public key is set for the cluster for cluster
     - eval_id: get_iso_eval_test_multinode
-      eval_query: Using the ID of the cluster you just created, get the Discovery ISO for cluster 'eval-test-multinode'
+      eval_query: Using the ID of the cluster you just created, get the Discovery ISO for cluster 'eval-test-multinode-ClustER-NAme'
       eval_types: [tool_eval, response_eval:sub-string]
       expected_tool_calls:
         - - tool_name: cluster_iso_download_url
@@ -175,18 +175,18 @@
 - conversation_group: cluster_id_from_name
   conversation:
     - eval_id: create_single_node_cluser
-      eval_query: Create a multi-node cluster named 'eval-test-cluster-name' with OpenShift 4.18.22 and domain test.local. I do not have an SSH key to provide.
+      eval_query: Create a multi-node cluster named 'eval-test-ClustER-NAme' with OpenShift 4.18.22 and domain test.local. I do not have an SSH key to provide.
       eval_types: [response_eval:accuracy, response_eval:sub-string]
-      expected_keywords: ["eval-test-cluster-name", "4.18.22", "ID", "Discovery ISO", "download", "cluster"]
-      expected_response: I have created a cluster with name eval-test-cluster-name, OpenShift version 4.18.22, base domain test.local. Next, you'll need to download the Discovery ISO, then boot your hosts with it. Do you want me to download the Discovery ISO?
+      expected_keywords: ["eval-test-ClustER-NAme", "4.18.22", "ID", "Discovery ISO", "download", "cluster"]
+      expected_response: I have created a cluster with name eval-test-ClustER-NAme, OpenShift version 4.18.22, base domain test.local. Next, you'll need to download the Discovery ISO, then boot your hosts with it. Would you like me to get the Discovery ISO download URL?
     - eval_id: cluster_name_tool_call
-      eval_query: Show me information on cluster eval-test-cluster-name
+      eval_query: Show me information on cluster eval-test-ClustER-NAme
       eval_types: [tool_eval, response_eval:sub-string]
       expected_tool_calls:
         - - tool_name: list_clusters
             arguments: {}
         - - tool_name: cluster_info
             arguments:
               cluster_id: "[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}"
-      expected_keywords: ["cluster", "eval-test-cluster-name", "test.local", "4.18.22"]
+      expected_keywords: ["cluster", "eval-test-ClustER-NAme", "test.local", "4.18.22"]
       description: Test handling requesting a cluster by name

test/prow/entrypoint.sh

@@ -12,13 +12,14 @@ OCM_TOKEN=$(curl -X POST https://sso.redhat.com/auth/realms/redhat-external/prot
   -d "client_secret=$CLIENT_SECRET" | jq '.access_token' | sed "s/^['\"]*//; s/['\"]*$//")
 
 WORK_DIR=$(pwd)
+TEST_DIR="${WORK_DIR}/test/evals"
 TEMP_DIR=$(mktemp -d)
 
 cd $TEMP_DIR
 
 echo "$OCM_TOKEN" > ocm_token.txt
 echo "GEMINI_API_KEY=${GEMINI_API_KEY}" > .env
 
-TEST_DIR="${WORK_DIR}/test/evals"
+sed -i "s/ClustER-NAme/${UNIQUE_ID}/g" $TEST_DIR/eval_data.yaml
 
 python $TEST_DIR/eval.py --agent_endpoint "${AGENT_URL}:${AGENT_PORT}" --agent_auth_token_file $TEMP_DIR/ocm_token.txt --eval_data_yaml $TEST_DIR/eval_data.yaml

test/prow/template.yaml

@@ -41,6 +41,8 @@ objects:
             value: ${AGENT_URL}
           - name: AGENT_PORT
             value: ${AGENT_PORT}
+          - name: UNIQUE_ID
+            value: ${JOB_ID}
 
 parameters:
 - name: JOB_ID
@@ -62,4 +64,3 @@ parameters:
   value: gemini
 - name: GEMINI_API_SECRET_KEY_NAME
   value: api_key
-