diff --git a/.circleci/config.yml b/.circleci/config.yml index e171759f1c4..ddd224eca65 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3933,6 +3933,9 @@ jobs: image: ubuntu-2204:2023.10.1 resource_class: xlarge working_directory: ~/project + parameters: + browser: + type: string steps: - checkout - setup_google_dns @@ -3962,7 +3965,7 @@ jobs: echo "Expires at: $EXPIRES_AT" neon branches create \ --project-id $NEON_PROJECT_ID \ - --name preview/commit-${CIRCLE_SHA1:0:7} \ + --name preview/commit-${CIRCLE_SHA1:0:7}-<< parameters.browser >> \ --expires-at $EXPIRES_AT \ --parent br-fancy-paper-ad1olsb3 \ --api-key $NEON_API_KEY || true @@ -3972,7 +3975,7 @@ jobs: E2E_UI_TEST_DATABASE_URL=$(neon connection-string \ --project-id $NEON_PROJECT_ID \ --api-key $NEON_API_KEY \ - --branch preview/commit-${CIRCLE_SHA1:0:7} \ + --branch preview/commit-${CIRCLE_SHA1:0:7}-<< parameters.browser >> \ --database-name yuneng-trial-db \ --role neondb_owner) echo $E2E_UI_TEST_DATABASE_URL @@ -3984,7 +3987,7 @@ jobs: -e UI_USERNAME="admin" \ -e UI_PASSWORD="gm" \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ - --name litellm-docker-database \ + --name litellm-docker-database-<< parameters.browser >> \ -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ @@ -4000,7 +4003,7 @@ jobs: sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - run: name: Start outputting logs - command: docker logs -f litellm-docker-database + command: docker logs -f litellm-docker-database-<< parameters.browser >> background: true - run: name: Wait for app to be ready @@ -4009,6 +4012,7 @@ jobs: name: Run Playwright Tests command: | npx playwright test \ + --project << parameters.browser >> \ --config ui/litellm-dashboard/e2e_tests/playwright.config.ts \ --reporter=html \ --output=test-results @@ -4214,6 +4218,20 @@ workflows: - main - /litellm_.*/ - e2e_ui_testing: + name: e2e_ui_testing_chromium + browser: chromium + context: e2e_ui_tests + requires: + - ui_build + - build_docker_database_image + filters: + branches: + only: + - main + - /litellm_.*/ + - e2e_ui_testing: + name: e2e_ui_testing_firefox + browser: firefox context: e2e_ui_tests requires: - ui_build @@ -4525,7 +4543,8 @@ workflows: - litellm_assistants_api_testing - auth_ui_unit_tests - db_migration_disable_update_check - - e2e_ui_testing + - e2e_ui_testing_chromium + - e2e_ui_testing_firefox - litellm_proxy_unit_testing_key_generation - litellm_proxy_unit_testing_part1 - litellm_proxy_unit_testing_part2 diff --git a/ui/litellm-dashboard/e2e_tests/constants.ts b/ui/litellm-dashboard/e2e_tests/constants.ts index b07bd68fcf1..58b56af0a2b 100644 --- a/ui/litellm-dashboard/e2e_tests/constants.ts +++ b/ui/litellm-dashboard/e2e_tests/constants.ts @@ -1 +1,6 @@ export const ADMIN_STORAGE_PATH = "admin.storageState.json"; + +export const E2E_UPDATE_LIMITS_KEY_ID_PREFIX = "102c"; +export const E2E_DELETE_KEY_ID_PREFIX = "94a5"; +export const E2E_DELETE_KEY_NAME = "e2eDeleteKey"; +export const E2E_REGENERATE_KEY_ID_PREFIX = "593a"; diff --git a/ui/litellm-dashboard/e2e_tests/tests/keys/deleteKey.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/keys/deleteKey.spec.ts new file mode 100644 index 00000000000..a5841316251 --- /dev/null +++ b/ui/litellm-dashboard/e2e_tests/tests/keys/deleteKey.spec.ts @@ -0,0 +1,25 @@ +import { test, expect } from "@playwright/test"; +import { ADMIN_STORAGE_PATH, E2E_DELETE_KEY_ID_PREFIX, E2E_DELETE_KEY_NAME } from "../../constants"; +import { Page } from "../../fixtures/pages"; +import { navigateToPage } from "../../helpers/navigation"; + +test.describe("Delete Key", () => { + test.use({ storageState: ADMIN_STORAGE_PATH }); + + test("Able to delete a key", async ({ page }) => { + await navigateToPage(page, Page.ApiKeys); + await expect(page.getByRole("button", { name: "Next" })).toBeVisible(); + await page + .locator("button", { + hasText: E2E_DELETE_KEY_ID_PREFIX, + }) + .click(); + await page.getByRole("button", { name: "Delete Key" }).click(); + await page.getByRole("textbox", { name: E2E_DELETE_KEY_NAME }).click(); + await page.getByRole("textbox", { name: E2E_DELETE_KEY_NAME }).fill(E2E_DELETE_KEY_NAME); + const deleteButton = page.getByRole("button", { name: "Delete", exact: true }); + await expect(deleteButton).toBeEnabled(); + await deleteButton.click(); + await expect(page.getByText("Key deleted successfully")).toBeVisible(); + }); +}); diff --git a/ui/litellm-dashboard/e2e_tests/tests/keys/regenerateKey.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/keys/regenerateKey.spec.ts new file mode 100644 index 00000000000..0188a4f81ce --- /dev/null +++ b/ui/litellm-dashboard/e2e_tests/tests/keys/regenerateKey.spec.ts @@ -0,0 +1,21 @@ +import { test, expect } from "@playwright/test"; +import { ADMIN_STORAGE_PATH, E2E_REGENERATE_KEY_ID_PREFIX } from "../../constants"; +import { Page } from "../../fixtures/pages"; +import { navigateToPage } from "../../helpers/navigation"; + +test.describe("Regenerate Key", () => { + test.use({ storageState: ADMIN_STORAGE_PATH }); + + test("Able to regenerate a key", async ({ page }) => { + await navigateToPage(page, Page.ApiKeys); + await expect(page.getByRole("button", { name: "Next" })).toBeVisible(); + await page + .locator("button", { + hasText: E2E_REGENERATE_KEY_ID_PREFIX, + }) + .click(); + await page.getByRole("button", { name: "Regenerate Key" }).click(); + await page.getByRole("button", { name: "Regenerate", exact: true }).click(); + await expect(page.getByText("Virtual Key regenerated")).toBeVisible(); + }); +}); diff --git a/ui/litellm-dashboard/e2e_tests/tests/keys/updateKeyLimits.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/keys/updateKeyLimits.spec.ts new file mode 100644 index 00000000000..6cae36272ab --- /dev/null +++ b/ui/litellm-dashboard/e2e_tests/tests/keys/updateKeyLimits.spec.ts @@ -0,0 +1,27 @@ +import { test, expect } from "@playwright/test"; +import { ADMIN_STORAGE_PATH, E2E_UPDATE_LIMITS_KEY_ID_PREFIX } from "../../constants"; +import { Page } from "../../fixtures/pages"; +import { navigateToPage } from "../../helpers/navigation"; + +test.describe("Update Key TPM and RPM Limits", () => { + test.use({ storageState: ADMIN_STORAGE_PATH }); + + test("Able to update a key's TPM and RPM limits", async ({ page }) => { + await navigateToPage(page, Page.ApiKeys); + await expect(page.getByRole("button", { name: "Next" })).toBeVisible(); + await page + .locator("button", { + hasText: E2E_UPDATE_LIMITS_KEY_ID_PREFIX, + }) + .click(); + await page.getByRole("tab", { name: "Settings" }).click(); + await page.getByRole("button", { name: "Edit Settings" }).click(); + await page.getByRole("spinbutton", { name: "TPM Limit" }).click(); + await page.getByRole("spinbutton", { name: "TPM Limit" }).fill("123"); + await page.getByRole("spinbutton", { name: "RPM Limit" }).click(); + await page.getByRole("spinbutton", { name: "RPM Limit" }).fill("456"); + await page.getByRole("button", { name: "Save Changes" }).click(); + await expect(page.getByRole("paragraph").filter({ hasText: "TPM: 123" })).toBeVisible(); + await expect(page.getByRole("paragraph").filter({ hasText: "RPM: 456" })).toBeVisible(); + }); +}); diff --git a/ui/litellm-dashboard/scripts/e2e_tests/neonHelperScripts.ts b/ui/litellm-dashboard/scripts/e2e_tests/neonHelperScripts.ts index 3078a0d90d2..089ad4e7926 100644 --- a/ui/litellm-dashboard/scripts/e2e_tests/neonHelperScripts.ts +++ b/ui/litellm-dashboard/scripts/e2e_tests/neonHelperScripts.ts @@ -1,4 +1,4 @@ -import { createApiClient } from "@neondatabase/api-client"; +import { createApiClient, EndpointType } from "@neondatabase/api-client"; import { config } from "dotenv"; import { resolve } from "path"; @@ -27,6 +27,13 @@ export async function createNeonE2ETestingBranch(projectId: string, parentBranch parent_id: parentBranchId, expires_at: expireAt ?? new Date(Date.now() + 1000 * 60 * 30).toISOString(), }, + endpoints: [ + { + type: EndpointType.ReadWrite, + autoscaling_limit_min_cu: 0.25, + autoscaling_limit_max_cu: 1, + }, + ], }); return response; } catch (error) { @@ -35,13 +42,15 @@ export async function createNeonE2ETestingBranch(projectId: string, parentBranch } export async function getNeonE2ETestingBranchConnectionString() { - await createNeonE2ETestingBranch(PROJECT_ID, PARENT_BRANCH); - + const createBranchResponse = await createNeonE2ETestingBranch(PROJECT_ID, PARENT_BRANCH); + const projectId = createBranchResponse.data.branch.project_id; const response = await apiClient.getConnectionUri({ database_name: NEON_E2E_UI_TEST_DB_NAME, role_name: "neondb_owner", - projectId: PROJECT_ID, + projectId: projectId, }); console.log("connection string:", response.data.uri); return response.data.uri; } + +getNeonE2ETestingBranchConnectionString();