WIP: Run E2E against Grafana Cloud

.github/workflows/cloud-e2e.yml

@@ -0,0 +1,128 @@
+name: Grafana Cloud E2E tests
+
+on:
+  schedule:
+    - cron: '0 9 * * *' # Daily at 09:00 UTC
+
+  # TODO: test, remove later
+  push:
+    branches:
+      - main
+  pull_request:
+
+defaults:
+  run:
+    shell: bash
+
+permissions:
+  contents: read
+
+jobs:
+  # @TODO: Remove the `build` job later
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-go@v6
+        with:
+          go-version: 'stable'
+
+      - name: Build backend
+        uses: magefile/mage-action@6f50bbb8ea47d56e62dee92392788acbc8192d0b
+        with:
+          args: buildAll
+          version: latest
+
+      - name: Install frontend dependencies
+        run: npm ci --no-audit --fund=false
+
+      - name: Build frontend
+        run: npm run build
+        env:
+          NODE_OPTIONS: '--max_old_space_size=4096'
+
+  e2e:
+    name: Bench Tests
+    needs: build
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+
+    permissions:
+      contents: read
+      id-token: write
+
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          persist-credentials: false
+
+      #- name: Setup Node.js environment
+      #  uses: actions/setup-node@v6
+      #  with:
+      #    node-version-file: .nvmrc
+      #    cache: 'npm'
+
+      - name: Get secrets from Vault
+        id: get-secrets
+        # Only execute this action in a trusted context
+        if: |
+          github.event_name != 'pull_request' || 
+          github.event.pull_request.head.repo.full_name == github.repository
+        uses: grafana/shared-workflows/actions/get-vault-secrets@get-vault-secrets/v1.3.0
+        with:
+          common_secrets: |
+            PLAYWRIGHT_GRAFANA_PASSWORD=data-sources/e2e:grafana-pw
+            PLAYWRIGHT_GRAFANA_USERNAME=data-sources/e2e:grafana-username
+          repo_secrets: |
+            DS_INSTANCE_HOST=ds-instance:host
+            DS_INSTANCE_PASSWORD=ds-instance:password
+            DS_INSTANCE_PORT=ds-instance:port
+            DS_INSTANCE_USERNAME=ds-instance:username
+          export_env: false
+
+      - name: report instance
+        run: |
+          echo "${{ fromJSON(steps.get-secrets.outputs.secrets).DS_INSTANCE_HOST }}" "${{ fromJSON(steps.get-secrets.outputs.secrets).DS_INSTANCE_PORT }}"
+
+      #- name: Install npm dependencies
+      #  run: |
+      #    if [ -f yarn.lock ]; then
+      #      yarn install --frozen-lockfile --non-interactive
+      #    else
+      #      npm ci --no-audit --fund=false
+      #    fi
+
+      #- name: Install Playwright Browsers
+      #  run: npx playwright install --with-deps chromium
+
+      - name: Run Grafana Bench tests
+        run: |
+          set -euo pipefail 
+          docker run \
+            --rm \
+            --network=host \
+            --volume "$PWD:/tests" \
+            us-docker.pkg.dev/grafanalabs-dev/docker-grafana-bench-dev/grafana-bench-playwright:dev-latest test \
+            --grafana-version "rrc-instant" \
+            --grafana-admin-password "${{ fromJSON(steps.get-secrets.outputs.secrets).PLAYWRIGHT_GRAFANA_PASSWORD }}" \
+            --grafana-admin-user "${{ fromJSON(steps.get-secrets.outputs.secrets).PLAYWRIGHT_GRAFANA_USERNAME }}" \
+            --grafana-url "https://datasourcese2e.grafana-dev.net" \
+            --log-level debug \
+            --pw-execute "npm run e2e" \
+            --pw-prepare "npm ci --no-audit --fund=false; npx playwright install" \
+            --test-env 'CI=true,DS_INSTANCE_HOST=${{ fromJSON(steps.get-secrets.outputs.secrets).DS_INSTANCE_HOST }},DS_INSTANCE_PASSWORD=${{ fromJSON(steps.get-secrets.outputs.secrets).DS_INSTANCE_PASSWORD }},DS_INSTANCE_PORT=${{ fromJSON(steps.get-secrets.outputs.secrets).DS_INSTANCE_PORT }},DS_INSTANCE_USERNAME=${{ fromJSON(steps.get-secrets.outputs.secrets).DS_INSTANCE_USERNAME }},DS_PDC_NETWORK_NAME=datasources-pdc-network-aws-datasource' \
+            --test-runner playwright \
+            --test-verbose
+
+
+      #- name: Archive Playwright artifacts
+      #  uses: actions/upload-artifact@v4
+      #  if: ${{ !cancelled() }}
+      #  with:
+      #    name: playwright-report
+      #    path: playwright-report/
+      #    retention-days: 1

.github/workflows/push.yml

@@ -16,6 +16,6 @@ jobs:
       contents: read
       id-token: write
     with:
-      plugin-version-suffix: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || '' }}
       go-version: '1.24'
       golangci-lint-version: '2.1.6'
+      plugin-version-suffix: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || '' }}

cspell.config.json

@@ -18,6 +18,7 @@
     "CHSQL",
     "ClickHouse",
     "closedate",
+    "combobox",
     "commitish",
     "concats",
     "createddate",
@@ -78,9 +79,9 @@
     "timerange",
     "timeseries",
     "timespan",
-    "toggleTip",
     "TLSCA",
     "TLSSSL",
+    "toggleTip",
     "Toggletip",
     "totime",
     "traceid",

playwright.config.ts

@@ -1,3 +1,5 @@
+import { dirname } from 'path';
+
 import { defineConfig, devices } from '@playwright/test';
 import type { PluginOptions } from '@grafana/plugin-e2e';
 
@@ -42,14 +44,14 @@ export default defineConfig<PluginOptions>({
   projects: [
     {
       name: 'auth',
-      testDir: 'node_modules/@grafana/plugin-e2e/dist/auth',
+      testDir: `${dirname(require.resolve('@grafana/plugin-e2e'))}/auth`,
       testMatch: [/.*\.js/],
     },
     {
       name: 'run-tests',
       use: {
         ...devices['Desktop Chrome'],
-        storageState: 'playwright/.auth/admin.json',
+        storageState: `playwright/.auth/${process.env.GRAFANA_ADMIN_USER || 'admin'}.json`,
       },
       dependencies: ['auth'],
     },

tests/e2e/configEditor.spec.ts

@@ -1,26 +1,44 @@
 import { expect, test } from '@grafana/plugin-e2e';
+import { Page } from '@playwright/test';
 
 const PLUGIN_UID = 'grafana-clickhouse-datasource';
-const CLICKHOUSE_DB_URL = Boolean(process.env.CI) ? 'clickhouse-server' : 'localhost';
+
+// Determine ClickHouse DB URL based on environment
+function resolveClickhouseUrl(env = process.env) {
+  const { CI, DS_INSTANCE_HOST } = env;
+  return CI ? (DS_INSTANCE_HOST || 'clickhouse-server') : 'localhost';
+}
+
+// Configure Private Data Source Connect if network name is provided
+async function configurePDC(page: Page, networkName: string) {
+  await page.getByRole('combobox', { name: 'Private data source connect' }).click();
+  await page.getByText(networkName).click();
+}
 
 test.describe('Config Editor', () => {
   test('invalid credentials should return an error', async ({ createDataSourceConfigPage, page }) => {
     const configPage = await createDataSourceConfigPage({ type: PLUGIN_UID });
-    await page.getByPlaceholder('Server address').fill(CLICKHOUSE_DB_URL);
+    await page.getByPlaceholder('Server address').fill(resolveClickhouseUrl());
     await expect(configPage.saveAndTest()).not.toBeOK();
+    await configPage.deleteDataSource();
   });
 
   test('valid credentials should display a success alert on the page', async ({ createDataSourceConfigPage, page }) => {
     const configPage = await createDataSourceConfigPage({ type: PLUGIN_UID });
 
-    await page.getByPlaceholder('Server address').fill(CLICKHOUSE_DB_URL);
-    await page.getByPlaceholder('9000').fill('9000');
-    await page.getByPlaceholder('default').fill('default');
+    await page.getByPlaceholder('Server address').fill(resolveClickhouseUrl());
+    await page.getByPlaceholder('9000').fill(process.env.DS_INSTANCE_PORT ?? '9000');
+    await page.getByPlaceholder('default').fill(process.env.DS_INSTANCE_USERNAME ?? 'default');
+    await page.getByPlaceholder('password').fill(process.env.DS_INSTANCE_PASSWORD ?? '');
+
+    if (process.env.DS_PDC_NETWORK_NAME) {
+      await configurePDC(page, process.env.DS_PDC_NETWORK_NAME);
+    }
 
     await configPage.saveAndTest();
     await expect(configPage).toHaveAlert('success', { hasNotText: 'Datasource updated' });
 
-    await page.pause();
+    await configPage.deleteDataSource();
   });
 
   test('mandatory fields should show error if left empty', async ({ createDataSourceConfigPage, page }) => {
@@ -35,5 +53,7 @@ test.describe('Config Editor', () => {
     await expect(page.getByText('Port is required')).toBeVisible();
 
     await expect(configPage.saveAndTest()).not.toBeOK();
+
+    await configPage.deleteDataSource();
   });
 });