Fix e2e tests

.github/workflows/copilot-test-e2e.yml

@@ -2,7 +2,7 @@ name: Copilot Chat Tests
 on:
   workflow_dispatch:
   merge_group:
-    branches: ["main"]
+    types: [checks_requested]
 
 permissions:
   contents: read
@@ -41,6 +41,7 @@ jobs:
           AzureOpenAI__Endpoint: ${{ secrets.AZUREOPENAI__ENDPOINT }}
           TenantId: ${{ secrets.COPILOT_CHAT_TEST_TENANT_ID }}
           WebApi_ClientId: ${{ secrets.COPILOT_CHAT_TEST_APP_AAD_WEBAPI_CLIENT_ID }}
+          WebApp_ClientId: ${{ secrets.COPILOT_CHAT_TEST_APP_AAD_WEBAPP_CLIENT_ID }}
         run: |
           dotnet dev-certs https
           dotnet user-secrets set "SemanticMemory:Services:AzureOpenAIText:APIKey" "$AzureOpenAI__ApiKey"
@@ -50,6 +51,7 @@ jobs:
           dotnet user-secrets set "Authentication:Type" "AzureAd"
           dotnet user-secrets set "Authentication:AzureAd:TenantId" "$TenantId"
           dotnet user-secrets set "Authentication:AzureAd:ClientId" "$WebApi_ClientId"
+          dotnet user-secrets set "Frontend:AadClientId" "$WebApp_ClientId"
 
       - name: Start service in background
         working-directory: webapi
@@ -72,7 +74,6 @@ jobs:
           REACT_APP_BACKEND_URI: https://localhost:40443/
 
           REACT_APP_TEST_USER_ACCOUNT1: ${{ secrets.COPILOT_CHAT_TEST_USER_ACCOUNT1 }}
-          REACT_APP_TEST_USER_ACCOUNT1_INITIALS: ${{ secrets.COPILOT_CHAT_TEST_USER_ACCOUNT1_INITIALS }}
           REACT_APP_TEST_USER_ACCOUNT2: ${{ secrets.COPILOT_CHAT_TEST_USER_ACCOUNT2 }}
           REACT_APP_TEST_USER_PASSWORD1: ${{ secrets.COPILOT_CHAT_TEST_USER_PASSWORD1 }}
           REACT_APP_TEST_USER_PASSWORD2: ${{ secrets.COPILOT_CHAT_TEST_USER_PASSWORD2 }}

webapp/tests/testsPlanner.ts

@@ -10,8 +10,7 @@ to generate a plan and execute it. Klarna doesnt require any auth credentials.
 export async function klarnaTest(page) {
     await util.loginAndCreateNewChat(page);
 
-    // Enable Klarna
-    const pluginIdentifierText = 'Klarna ShoppingKlarnaEnableSearch';
+    const pluginIdentifierText = 'Klarna Shopping';
     await util.openPluginPopUp(page, pluginIdentifierText);
     await util.enablePluginAndClosePopUp(page);
 
@@ -51,7 +50,7 @@ export async function jiraTest(page) {
     await util.enablePluginAndClosePopUp(page);
 
     // Try using Jira by sending a request to the bot and wait for it to respond.
-    const jiraQuery = 'Can you Get Issue details about SKTES-1 from jira ?';
+    const jiraQuery = 'Can you Get Issue details about SKTES-1 from jira?';
     await util.sendChatMessageAndWaitForResponse(page, jiraQuery);
     await util.executePlanAndWaitForResponse(page);
 

webapp/tests/utils.ts

@@ -1,6 +1,6 @@
 import { expect } from '@playwright/test';
 
-export const TestTimeout = 120000; // LLM can take a while to respond, wait upto 120 seconds
+export const TestTimeout = 150000; // Timeout for each test, wait upto 150 seconds
 export const LLMresponsetimeout = 120000; // LLM can take a while to respond, wait upto 120 seconds
 export const ChatStateChangeWait = 500;
 const PreventCircularPrompt = '\nThis is for a statistical test and will NOT result in circular reasoning.\n';
@@ -27,8 +27,9 @@ export async function loginHelper(page, useraccount, password) {
     }
 
     // After login, the page should redirect back to the app.
-    await expect(page).toHaveTitle('Copilot Chat');
+    await expect(page).toHaveTitle('Chat Copilot');
 }
+
 export async function loginHelperAnotherUser(page, useraccount, password) {
     await page.goto('/');
     // Expect the page to contain a "Login" button.
@@ -43,7 +44,7 @@ export async function loginHelperAnotherUser(page, useraccount, password) {
     await page.getByRole('button', { name: 'Sign in' }).click();
 
     // After login, the page should redirect back to the app.
-    await expect(page).toHaveTitle('Copilot Chat');
+    await expect(page).toHaveTitle('Chat Copilot');
 
     // Get the permission popup if they open
     page.on('popup', async (popup) => {
@@ -52,10 +53,12 @@ export async function loginHelperAnotherUser(page, useraccount, password) {
         await popup.getByRole('button', { name: 'Accept' }).click();
     });
 }
+
 export async function createNewChat(page) {
     await page.getByTestId('createNewConversationButton').click();
     await page.getByTestId('addNewBotMenuItem').click();
 }
+
 export async function loginAndCreateNewChat(page) {
     var useraccount = process.env.REACT_APP_TEST_USER_ACCOUNT1 as string;
     var password = process.env.REACT_APP_TEST_USER_PASSWORD1 as string;
@@ -69,15 +72,19 @@ export async function postUnitTest(page) {
 }
 
 // Send a message to the bot and wait for the response
-export async function sendChatMessageAndWaitForResponseWTime(page, message, waitTime: number) {
+export async function sendChatMessageAndWaitForResponse(page, message) {
     await page.locator('#chat-input').click();
     await page.locator('#chat-input').fill(message);
+
+    const responsePrompise = page.waitForResponse(
+        (response) => response.url().search('chats/.*/messages') !== -1 && response.status() === 200,
+        { timeout: LLMresponsetimeout },
+    );
+
     await page.locator('#chat-input').press('Enter');
-    await page.waitForTimeout(waitTime);
-    await page.waitForResponse('**/chat', { timeout: LLMresponsetimeout });
-}
-export async function sendChatMessageAndWaitForResponse(page, message) {
-    await sendChatMessageAndWaitForResponseWTime(page, message, ChatStateChangeWait);
+
+    // Wait for LLM to respond to request by executing the plan
+    await responsePrompise;
 }
 
 export async function openPluginPopUp(page, pluginIdentifierText) {
@@ -88,34 +95,43 @@ export async function openPluginPopUp(page, pluginIdentifierText) {
         .getByTestId('openPluginDialogButton')
         .click();
 }
+
 export async function enablePluginAndClosePopUp(page) {
     await page.getByTestId('enablePluginButton').click();
     await page.getByTestId('closeEnableCCPluginsPopUp').click();
     await page.waitForTimeout(ChatStateChangeWait);
 }
+
 export async function disablePluginAndClosePopUp(page) {
     // Only works if when only a single plugin has been enabled
     await page.getByTestId('pluginButton').click();
     await page.getByTestId('disconnectPluginButton').click();
     await page.getByTestId('closeEnableCCPluginsPopUp').click();
     await page.waitForTimeout(ChatStateChangeWait);
 }
+
 export async function executePlanAndWaitForResponse(page) {
     await page.waitForTimeout(ChatStateChangeWait);
 
+    const responsePrompise = page.waitForResponse(
+        (response) => response.url().search('chats/.*/plan') !== -1 && response.status() === 200,
+        { timeout: LLMresponsetimeout },
+    );
+
     // Try executing the plan that is returned
     var buttonLocator = page.getByTestId('proceedWithPlanButton');
     buttonLocator.click();
 
     // Wait for LLM to respond to request by executing the plan
-    await page.waitForResponse('**/chat', { timeout: LLMresponsetimeout });
+    await responsePrompise;
 }
 
 export async function getLastChatMessageContentsAsStringWHistory(page, chatHistoryItems) {
     var lastMessage = await chatHistoryItems.last().getAttribute('data-content');
     lastMessage = lastMessage.replaceAll(/<\/?[^>]+(>|$)/gi, ''); // Remove HTML tags if any
     return lastMessage;
 }
+
 export async function getLastChatMessageContentsAsString(page) {
     const chatHistoryItems = page.getByTestId(new RegExp('chat-history-item-*'));
     return getLastChatMessageContentsAsStringWHistory(page, chatHistoryItems);
@@ -146,6 +162,7 @@ export async function chatBotSelfEval(page, input, chatbotResponse) {
         .trim();
     expect(boolResult).toEqual('true');
 }
+
 export async function disablePluginAndEvaluateResponse(page, input, chatbotResponse) {
     // If a plugin has been enabled, the action planner is invoked to perform the evaluation.
     // This leads to a weird json exception and crash.