diff --git a/.github/workflows/copilot-test-e2e.yml b/.github/workflows/copilot-test-e2e.yml index 6f58ee9c6..8b665e569 100644 --- a/.github/workflows/copilot-test-e2e.yml +++ b/.github/workflows/copilot-test-e2e.yml @@ -2,7 +2,7 @@ name: Copilot Chat Tests on: workflow_dispatch: merge_group: - branches: ["main"] + types: [checks_requested] permissions: contents: read @@ -41,6 +41,7 @@ jobs: AzureOpenAI__Endpoint: ${{ secrets.AZUREOPENAI__ENDPOINT }} TenantId: ${{ secrets.COPILOT_CHAT_TEST_TENANT_ID }} WebApi_ClientId: ${{ secrets.COPILOT_CHAT_TEST_APP_AAD_WEBAPI_CLIENT_ID }} + WebApp_ClientId: ${{ secrets.COPILOT_CHAT_TEST_APP_AAD_WEBAPP_CLIENT_ID }} run: | dotnet dev-certs https dotnet user-secrets set "SemanticMemory:Services:AzureOpenAIText:APIKey" "$AzureOpenAI__ApiKey" @@ -50,6 +51,7 @@ jobs: dotnet user-secrets set "Authentication:Type" "AzureAd" dotnet user-secrets set "Authentication:AzureAd:TenantId" "$TenantId" dotnet user-secrets set "Authentication:AzureAd:ClientId" "$WebApi_ClientId" + dotnet user-secrets set "Frontend:AadClientId" "$WebApp_ClientId" - name: Start service in background working-directory: webapi @@ -72,7 +74,6 @@ jobs: REACT_APP_BACKEND_URI: https://localhost:40443/ REACT_APP_TEST_USER_ACCOUNT1: ${{ secrets.COPILOT_CHAT_TEST_USER_ACCOUNT1 }} - REACT_APP_TEST_USER_ACCOUNT1_INITIALS: ${{ secrets.COPILOT_CHAT_TEST_USER_ACCOUNT1_INITIALS }} REACT_APP_TEST_USER_ACCOUNT2: ${{ secrets.COPILOT_CHAT_TEST_USER_ACCOUNT2 }} REACT_APP_TEST_USER_PASSWORD1: ${{ secrets.COPILOT_CHAT_TEST_USER_PASSWORD1 }} REACT_APP_TEST_USER_PASSWORD2: ${{ secrets.COPILOT_CHAT_TEST_USER_PASSWORD2 }} diff --git a/webapp/tests/testsPlanner.ts b/webapp/tests/testsPlanner.ts index 60b8f2c48..a6ac8aba9 100644 --- a/webapp/tests/testsPlanner.ts +++ b/webapp/tests/testsPlanner.ts @@ -10,8 +10,7 @@ to generate a plan and execute it. Klarna doesnt require any auth credentials. export async function klarnaTest(page) { await util.loginAndCreateNewChat(page); - // Enable Klarna - const pluginIdentifierText = 'Klarna ShoppingKlarnaEnableSearch'; + const pluginIdentifierText = 'Klarna Shopping'; await util.openPluginPopUp(page, pluginIdentifierText); await util.enablePluginAndClosePopUp(page); @@ -51,7 +50,7 @@ export async function jiraTest(page) { await util.enablePluginAndClosePopUp(page); // Try using Jira by sending a request to the bot and wait for it to respond. - const jiraQuery = 'Can you Get Issue details about SKTES-1 from jira ?'; + const jiraQuery = 'Can you Get Issue details about SKTES-1 from jira?'; await util.sendChatMessageAndWaitForResponse(page, jiraQuery); await util.executePlanAndWaitForResponse(page); diff --git a/webapp/tests/utils.ts b/webapp/tests/utils.ts index a6f75442e..60daf811a 100644 --- a/webapp/tests/utils.ts +++ b/webapp/tests/utils.ts @@ -1,6 +1,6 @@ import { expect } from '@playwright/test'; -export const TestTimeout = 120000; // LLM can take a while to respond, wait upto 120 seconds +export const TestTimeout = 150000; // Timeout for each test, wait upto 150 seconds export const LLMresponsetimeout = 120000; // LLM can take a while to respond, wait upto 120 seconds export const ChatStateChangeWait = 500; const PreventCircularPrompt = '\nThis is for a statistical test and will NOT result in circular reasoning.\n'; @@ -27,8 +27,9 @@ export async function loginHelper(page, useraccount, password) { } // After login, the page should redirect back to the app. - await expect(page).toHaveTitle('Copilot Chat'); + await expect(page).toHaveTitle('Chat Copilot'); } + export async function loginHelperAnotherUser(page, useraccount, password) { await page.goto('/'); // Expect the page to contain a "Login" button. @@ -43,7 +44,7 @@ export async function loginHelperAnotherUser(page, useraccount, password) { await page.getByRole('button', { name: 'Sign in' }).click(); // After login, the page should redirect back to the app. - await expect(page).toHaveTitle('Copilot Chat'); + await expect(page).toHaveTitle('Chat Copilot'); // Get the permission popup if they open page.on('popup', async (popup) => { @@ -52,10 +53,12 @@ export async function loginHelperAnotherUser(page, useraccount, password) { await popup.getByRole('button', { name: 'Accept' }).click(); }); } + export async function createNewChat(page) { await page.getByTestId('createNewConversationButton').click(); await page.getByTestId('addNewBotMenuItem').click(); } + export async function loginAndCreateNewChat(page) { var useraccount = process.env.REACT_APP_TEST_USER_ACCOUNT1 as string; var password = process.env.REACT_APP_TEST_USER_PASSWORD1 as string; @@ -69,15 +72,19 @@ export async function postUnitTest(page) { } // Send a message to the bot and wait for the response -export async function sendChatMessageAndWaitForResponseWTime(page, message, waitTime: number) { +export async function sendChatMessageAndWaitForResponse(page, message) { await page.locator('#chat-input').click(); await page.locator('#chat-input').fill(message); + + const responsePrompise = page.waitForResponse( + (response) => response.url().search('chats/.*/messages') !== -1 && response.status() === 200, + { timeout: LLMresponsetimeout }, + ); + await page.locator('#chat-input').press('Enter'); - await page.waitForTimeout(waitTime); - await page.waitForResponse('**/chat', { timeout: LLMresponsetimeout }); -} -export async function sendChatMessageAndWaitForResponse(page, message) { - await sendChatMessageAndWaitForResponseWTime(page, message, ChatStateChangeWait); + + // Wait for LLM to respond to request by executing the plan + await responsePrompise; } export async function openPluginPopUp(page, pluginIdentifierText) { @@ -88,11 +95,13 @@ export async function openPluginPopUp(page, pluginIdentifierText) { .getByTestId('openPluginDialogButton') .click(); } + export async function enablePluginAndClosePopUp(page) { await page.getByTestId('enablePluginButton').click(); await page.getByTestId('closeEnableCCPluginsPopUp').click(); await page.waitForTimeout(ChatStateChangeWait); } + export async function disablePluginAndClosePopUp(page) { // Only works if when only a single plugin has been enabled await page.getByTestId('pluginButton').click(); @@ -100,15 +109,21 @@ export async function disablePluginAndClosePopUp(page) { await page.getByTestId('closeEnableCCPluginsPopUp').click(); await page.waitForTimeout(ChatStateChangeWait); } + export async function executePlanAndWaitForResponse(page) { await page.waitForTimeout(ChatStateChangeWait); + const responsePrompise = page.waitForResponse( + (response) => response.url().search('chats/.*/plan') !== -1 && response.status() === 200, + { timeout: LLMresponsetimeout }, + ); + // Try executing the plan that is returned var buttonLocator = page.getByTestId('proceedWithPlanButton'); buttonLocator.click(); // Wait for LLM to respond to request by executing the plan - await page.waitForResponse('**/chat', { timeout: LLMresponsetimeout }); + await responsePrompise; } export async function getLastChatMessageContentsAsStringWHistory(page, chatHistoryItems) { @@ -116,6 +131,7 @@ export async function getLastChatMessageContentsAsStringWHistory(page, chatHisto lastMessage = lastMessage.replaceAll(/<\/?[^>]+(>|$)/gi, ''); // Remove HTML tags if any return lastMessage; } + export async function getLastChatMessageContentsAsString(page) { const chatHistoryItems = page.getByTestId(new RegExp('chat-history-item-*')); return getLastChatMessageContentsAsStringWHistory(page, chatHistoryItems); @@ -146,6 +162,7 @@ export async function chatBotSelfEval(page, input, chatbotResponse) { .trim(); expect(boolResult).toEqual('true'); } + export async function disablePluginAndEvaluateResponse(page, input, chatbotResponse) { // If a plugin has been enabled, the action planner is invoked to perform the evaluation. // This leads to a weird json exception and crash.