From d3b19f1619d1761c6b3d69ba3f3f3146f206f2ed Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Wed, 30 Aug 2023 08:35:35 -0700 Subject: [PATCH] Add scenario test readme (#272) ### Motivation and Context We fix the scenario tests. It' time to document how to run the tests locally. ### Description 1. Create a README with setup instructions to run the tests locally. 2. Add a timeout of 30 seconds for each test. 3. Rename some secrets so that they make more sense. ### Contribution Checklist - [ ] The code builds clean without any errors or warnings - [ ] The PR follows the [Contribution Guidelines](https://github.com/microsoft/copilot-chat/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/copilot-chat/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [ ] All unit tests pass, and I have added new tests where possible - [ ] I didn't break anyone :smile: --- .github/workflows/copilot-test-e2e.yml | 6 +- webapp/playwright.config.ts | 87 +++++++++++++------------- webapp/tests/README.md | 25 ++++++++ 3 files changed, 73 insertions(+), 45 deletions(-) create mode 100644 webapp/tests/README.md diff --git a/.github/workflows/copilot-test-e2e.yml b/.github/workflows/copilot-test-e2e.yml index 7b250c0f..a103fcb2 100644 --- a/.github/workflows/copilot-test-e2e.yml +++ b/.github/workflows/copilot-test-e2e.yml @@ -39,7 +39,7 @@ jobs: env: AzureOpenAI__ApiKey: ${{ secrets.AZUREOPENAI__APIKEY }} AzureOpenAI__Endpoint: ${{ secrets.AZUREOPENAI__ENDPOINT }} - TenantId: ${{ secrets.COPILOT_CHAT_TEST_APP_AAD_TENANT_ID }} + TenantId: ${{ secrets.COPILOT_CHAT_TEST_TENANT_ID }} WebApi_ClientId: ${{ secrets.COPILOT_CHAT_TEST_APP_AAD_WEBAPI_CLIENT_ID }} run: | dotnet dev-certs https @@ -70,8 +70,8 @@ jobs: REACT_APP_BACKEND_URI: https://localhost:40443/ REACT_APP_AUTH_TYPE: AzureAd - REACT_APP_AAD_AUTHORITY: https://login.microsoftonline.com/${{ secrets.COPILOT_CHAT_TEST_APP_AAD_TENANT_ID }} - REACT_APP_AAD_CLIENT_ID: ${{ secrets.COPILOT_CHAT_TEST_APP_AAD_CLIENT_ID }} + REACT_APP_AAD_AUTHORITY: https://login.microsoftonline.com/${{ secrets.COPILOT_CHAT_TEST_TENANT_ID }} + REACT_APP_AAD_CLIENT_ID: ${{ secrets.COPILOT_CHAT_TEST_APP_AAD_WEBAPP_CLIENT_ID }} REACT_APP_AAD_API_SCOPE: api://${{ secrets.COPILOT_CHAT_TEST_APP_AAD_WEBAPI_CLIENT_ID }}/access_as_user REACT_APP_TEST_USER_ACCOUNT1: ${{ secrets.COPILOT_CHAT_TEST_USER_ACCOUNT1 }} diff --git a/webapp/playwright.config.ts b/webapp/playwright.config.ts index f9eca30b..152cede4 100644 --- a/webapp/playwright.config.ts +++ b/webapp/playwright.config.ts @@ -10,53 +10,56 @@ const filename = fileURLToPath(import.meta.url); const dirname = path.dirname(filename); const envPath = path.resolve(dirname, '.env'); if (fs.existsSync(envPath)) { - dotenv.config({ path: envPath }); + dotenv.config({ path: envPath }); } /** * See https://playwright.dev/docs/test-configuration. */ export default defineConfig({ - testDir: './tests', - /* Run tests in files in parallel */ - fullyParallel: true, - /* Fail the build on CI if you accidentally left test.only in the source code. */ - forbidOnly: !!process.env.CI, - /* Retry on CI only */ - retries: process.env.CI ? 3 : 0, - /* Opt out of parallel tests on CI. */ - workers: process.env.CI ? 1 : undefined, - /* Reporter to use. See https://playwright.dev/docs/test-reporters */ - reporter: 'html', - /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ - use: { - /* Base URL to use in actions like `await page.goto('/')`. */ - baseURL: 'http://localhost:3000', - - /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ - trace: 'on-first-retry', - - /* Ignore certificate errors. */ - ignoreHTTPSErrors: true, - - /* Add permissions to copy text to clipboard */ - permissions: ['clipboard-write'], - }, - - /* Configure projects for major browsers */ - projects: [ - { - name: 'chromium', - use: { ...devices['Desktop Chrome'] }, + testDir: './tests', + /* Run tests in files in parallel */ + fullyParallel: true, + /* Fail the build on CI if you accidentally left test.only in the source code. */ + forbidOnly: !!process.env.CI, + /* Retry on CI only */ + retries: process.env.CI ? 3 : 0, + /* Opt out of parallel tests on CI. */ + workers: process.env.CI ? 1 : undefined, + /* Reporter to use. See https://playwright.dev/docs/test-reporters */ + reporter: 'html', + /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ + use: { + /* Base URL to use in actions like `await page.goto('/')`. */ + baseURL: 'http://localhost:3000', + + /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ + trace: 'on-first-retry', + + /* Ignore certificate errors. */ + ignoreHTTPSErrors: true, + + /* Add permissions to copy text to clipboard */ + permissions: ['clipboard-write'], + }, + + /* Each test is given 30 seconds to complete by default. */ + timeout: 30000, + + /* Configure projects for major browsers */ + projects: [ + { + name: 'chromium', + use: { ...devices['Desktop Chrome'] }, + }, + // Add more browsers here. + ], + + /* Run your local dev server before starting the tests */ + webServer: { + command: 'yarn start', + url: 'http://localhost:3000', + reuseExistingServer: !process.env.CI, + timeout: 120000, }, - // Add more browsers here. - ], - - /* Run your local dev server before starting the tests */ - webServer: { - command: 'yarn start', - url: 'http://localhost:3000', - reuseExistingServer: !process.env.CI, - timeout: 120000, - }, }); diff --git a/webapp/tests/README.md b/webapp/tests/README.md new file mode 100644 index 00000000..93675224 --- /dev/null +++ b/webapp/tests/README.md @@ -0,0 +1,25 @@ +# Copilot Chat Web App Scenario Tests + +## How to set up the tests to run locally + +### Install Playwright + +Playwright is a dependency included in package.json. You just need to run `yarn install` followed by `yarn playwright install --with-deps` at the webapp/ root to install Playwright. + +> (Optional) Install the [VS Code Extension](https://marketplace.visualstudio.com/items?itemName=ms-playwright.playwright). + +### Set up App registrations + +Follow the [instructions](https://github.com/microsoft/chat-copilot#optional-enable-backend-authentication-via-azure-ad) to create two app registrations. This is needed for the multi-user chat test. + +### Configure the environment + +- Follow the [instructions](https://github.com/microsoft/chat-copilot#optional-enable-backend-authentication-via-azure-ad) to configure the /webapp/.env file and the /webapi/appsettings.josn file. + +- You need two test accounts to run the multi-user chat. Make sure the two accounts are under the correct tenant. Enter the account credentials in the .env file. + +### Running the tests + +- Open a terminal window to start the webapi. + +- Once the webapi is ready, run `yarn playwright test` in another terminal or use the [VS Code Extension](https://marketplace.visualstudio.com/items?itemName=ms-playwright.playwright).