test(cloud-agent-next): run the shared e2e scenarios against a deployed stack

.github/workflows/e2e-deployed.yml

@@ -0,0 +1,102 @@
+name: E2E Deployed
+
+on:
+  workflow_dispatch:
+    inputs:
+      worker_url:
+        description: 'HTTPS URL of the deployed cloud-agent-next Worker'
+        required: true
+        type: string
+      fake_llm_url:
+        description: 'HTTPS URL of the deployed fake-LLM Worker'
+        required: true
+        type: string
+      backend_url:
+        description: 'HTTPS URL of the Kilo backend that issues stream tickets'
+        required: true
+        type: string
+        default: 'https://api.kilo.ai'
+
+permissions:
+  contents: read
+
+concurrency:
+  group: e2e-deployed
+  cancel-in-progress: false
+
+jobs:
+  e2e-deployed:
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    env:
+      WORKER_URL: ${{ inputs.worker_url }}
+      FAKE_LLM_URL: ${{ inputs.fake_llm_url }}
+      E2E_BACKEND_URL: ${{ inputs.backend_url }}
+      # Secrets are exposed as environment variables only; never echoed.
+      E2E_USER_TOKEN: ${{ secrets.E2E_USER_TOKEN }}
+      FAKE_LLM_ADMIN_TOKEN: ${{ secrets.FAKE_LLM_ADMIN_TOKEN }}
+      E2E_INTERNAL_API_SECRET: ${{ secrets.E2E_INTERNAL_API_SECRET }}
+    steps:
+      - uses: useblacksmith/checkout@41cdeedae8edb2e684ba22896a5fd2a3cb85db6b # v1
+        with:
+          lfs: true
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4.4.0
+
+      - name: Setup Node
+        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
+        with:
+          node-version-file: '.nvmrc'
+          cache: 'pnpm'
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Run deployed e2e matrix
+        run: |
+          set -o pipefail
+          pnpm --filter cloud-agent-next run e2e:deployed 2>&1 | tee e2e-deployed.log
+
+      - name: Summarize deployed e2e results
+        if: always()
+        run: |
+          log='e2e-deployed.log'
+
+          {
+            echo '## Deployed E2E matrix'
+            echo
+          } >> "$GITHUB_STEP_SUMMARY"
+
+          if [[ ! -f "$log" ]]; then
+            echo 'The runner produced no log.' >> "$GITHUB_STEP_SUMMARY"
+            exit 0
+          fi
+
+          counts=$(grep -E '^Summary: [0-9]+ passed, [0-9]+ failed, [0-9]+ unsupported$' "$log" | tail -1 || true)
+          unsupported=$(grep -E '^unsupported: ' "$log" || true)
+
+          if [[ -n "$counts" ]]; then
+            echo "$counts" >> "$GITHUB_STEP_SUMMARY"
+          else
+            echo 'The runner did not reach its summary line.' >> "$GITHUB_STEP_SUMMARY"
+          fi
+
+          if [[ -n "$unsupported" ]]; then
+            {
+              echo
+              echo '### Unsupported scenarios'
+              echo
+              echo '```'
+              echo "$unsupported"
+              echo '```'
+            } >> "$GITHUB_STEP_SUMMARY"
+          fi
+
+      - name: Upload deployed e2e log
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: e2e-deployed-log
+          path: e2e-deployed.log
+          retention-days: 7

dev/local/cli.ts

@@ -19,6 +19,7 @@ import {
   resolveSessionNextAuthUrl,
   resolveDeletionMockSessionEnv,
   resolveFakeLlmSessionEnv,
+  resolveE2eInternalSecretSessionEnv,
   services,
 } from './services';
 import { acquireProcessLock, withProcessLockAsync } from './process-lock';
@@ -65,6 +66,7 @@ import {
   buildFollowLogPipeCommand,
   buildLogPipeCommand,
   probePort,
+  waitForPort,
   restartServiceInTmux,
   buildStartCommand,
   shellQuote,
@@ -578,7 +580,7 @@ async function cmdUp(args: string[], repoRoot: string): Promise<string | undefin
     for (const name of serviceNames) {
       const service = getService(name);
       if (service.type !== 'worker' || !findServicePane(sessionName, name)) continue;
-      const outcome = await restartServiceInTmux(sessionName, name);
+      const outcome = await restartServiceInTmux(sessionName, name, undefined, serviceNames);
       if (outcome === 'gave-up') throw new Error(`${name} did not restart with refreshed secrets`);
     }
     if (Object.keys(mobileEnv).length > 0) {
@@ -601,7 +603,7 @@ async function cmdUp(args: string[], repoRoot: string): Promise<string | undefin
       !findServicePane(sessionName, 'cloud-agent-public-tunnels')
     ) {
       const previous = snapshotCloudAgentPublicTunnelEnv(repoRoot);
-      startServiceInTmux(sessionName, 'cloud-agent-public-tunnels');
+      startServiceInTmux(sessionName, 'cloud-agent-public-tunnels', undefined, serviceNames);
       const captured = await waitForCloudAgentPublicTunnelCapture(
         repoRoot,
         previous,
@@ -733,18 +735,24 @@ async function cmdUp(args: string[], repoRoot: string): Promise<string | undefin
   if (deletionMockEnv) Object.assign(sessionEnv, deletionMockEnv);
   const fakeLlmEnv = resolveFakeLlmSessionEnv({ serviceNames });
   if (fakeLlmEnv) Object.assign(sessionEnv, fakeLlmEnv);
+  const e2eInternalSecretEnv = resolveE2eInternalSecretSessionEnv({ serviceNames });
+  if (e2eInternalSecretEnv) Object.assign(sessionEnv, e2eInternalSecretEnv);
   createSession(sessionName, sessionEnv);
 
   // --- Start each service in its own tmux window ---
   const SIDEBAR_WIDTH = 40;
 
   // --- Start capture services first (tunnel, stripe) and wait for output ---
+  // The fake-LLM Worker is a capture service too: the public-tunnel service
+  // probes it before publishing and exits if it is not answering, so it must be
+  // started and ready before the tunnel capture runs.
   const captureServiceSet = new Set([
     'kiloclaw-tunnel',
     'stripe',
     'app-builder-tunnel',
     'bitbucket-webhook-tunnel',
     'cloud-agent-public-tunnels',
+    'fake-llm-worker',
   ]);
   const captureServices = serviceNames.filter(n => captureServiceSet.has(n));
   const otherServices = serviceNames.filter(n => !captureServiceSet.has(n));
@@ -785,8 +793,18 @@ async function cmdUp(args: string[], repoRoot: string): Promise<string | undefin
       ? snapshotCloudAgentPublicTunnelEnv(repoRoot)
       : undefined;
 
+    // Start the fake-LLM Worker and wait for it to answer before any tunnel
+    // capture: the tunnels refuse to publish the fake when the guard probe
+    // cannot authenticate against it.
+    if (captureServices.includes('fake-llm-worker')) {
+      startServiceInTmux(sessionName, 'fake-llm-worker', sessionEnv, serviceNames);
+      startedServices.push('fake-llm-worker');
+      await waitForPort(getService('fake-llm-worker').port, 'fake-llm-worker', CAPTURE_TIMEOUT_MS);
+    }
+
     for (const name of captureServices) {
-      startServiceInTmux(sessionName, name, sessionEnv);
+      if (name === 'fake-llm-worker') continue;
+      startServiceInTmux(sessionName, name, sessionEnv, serviceNames);
       startedServices.push(name);
       await sleep(300);
     }
@@ -943,7 +961,7 @@ async function cmdUp(args: string[], repoRoot: string): Promise<string | undefin
       continue;
     }
 
-    startServiceInTmux(sessionName, name, sessionEnv);
+    startServiceInTmux(sessionName, name, sessionEnv, serviceNames);
     startedServices.push(name);
     await sleep(300);
   }

dev/local/dashboard.tsx

@@ -9,6 +9,7 @@ import {
   getAlwaysOnGroupIds,
   resolveGroups,
   resolveGroupTransitiveDeps,
+  planTunnelRestart,
 } from './services';
 import type { ServiceGroup } from './services';
 import {
@@ -443,12 +444,13 @@ function Dashboard({
         }
         const publicTunnelSnapshot = snapshotCloudAgentPublicTunnelEnv(repoRoot);
         const previousPublicTunnelSnapshot = lastPublicTunnelSnapshot.current;
+        const { reloadTarget } = planTunnelRestart([...runningServices]);
         if (
           previousPublicTunnelSnapshot?.values.WORKER_URL !== undefined &&
           publicTunnelSnapshot.values.WORKER_URL !== undefined &&
           publicTunnelSnapshot.values.WORKER_URL !==
             previousPublicTunnelSnapshot.values.WORKER_URL &&
-          runningServices.has('cloud-agent-next') &&
+          reloadTarget !== undefined &&
           !reloadingWorker.current
         ) {
           reloadingWorker.current = true;
@@ -459,7 +461,7 @@ function Dashboard({
             CAPTURE_TIMEOUT_MS
           )
             .then(captured =>
-              captured ? restartServiceInTmux(sessionName, 'cloud-agent-next') : undefined
+              captured ? restartServiceInTmux(sessionName, reloadTarget) : undefined
             )
             .finally(() => {
               reloadingWorker.current = false;
@@ -788,10 +790,15 @@ function Dashboard({
       }
       if (reloadingWorker.current) return;
       const previous = snapshotCloudAgentPublicTunnelEnv(repoRoot);
+      const { selection, reloadTarget } = planTunnelRestart([...runningServices]);
       reloadingWorker.current = true;
       void (async () => {
         try {
-          const outcome = await restartServiceInTmux(sessionName, item.name);
+          // The active selection must travel with the restart: without it a
+          // tunnels relaunch (or a recreate after the pane vanishes) falls back
+          // to the three-port form and the fake-LLM tunnel URL is never
+          // published.
+          const outcome = await restartServiceInTmux(sessionName, item.name, undefined, selection);
           if (outcome === 'gave-up') {
             setStackIssues(['tunnels: restart did not finish']);
             return;
@@ -806,13 +813,13 @@ function Dashboard({
             return;
           }
           if (
-            !runningServices.has('cloud-agent-next') &&
-            !findServicePane(sessionName, 'cloud-agent-next')
+            reloadTarget === undefined ||
+            (!runningServices.has(reloadTarget) && !findServicePane(sessionName, reloadTarget))
           ) {
             return;
           }
           setStackIssues(['reloading worker for new tunnel URL']);
-          await restartServiceInTmux(sessionName, 'cloud-agent-next');
+          await restartServiceInTmux(sessionName, reloadTarget);
         } finally {
           reloadingWorker.current = false;
         }

dev/local/runner.ts

@@ -2,7 +2,7 @@ import { execFileSync } from 'node:child_process';
 import * as fs from 'node:fs';
 import * as net from 'node:net';
 import * as path from 'node:path';
-import { getService, getInfraProfile, getAllInfraProfiles } from './services';
+import { getService, getInfraProfile, getAllInfraProfiles, serviceCommand } from './services';
 import {
   createWindow,
   sendKeys,
@@ -87,20 +87,21 @@ export async function waitForPort(port: number, name: string, maxWaitMs: number)
 // Command building
 // ---------------------------------------------------------------------------
 
-export function buildStartCommand(serviceName: string): string {
+export function buildStartCommand(serviceName: string, serviceNames?: readonly string[]): string {
   const svc = getService(serviceName);
+  const command = serviceNames ? serviceCommand(serviceName, serviceNames) : svc.command;
 
   // Use pnpm --filter instead of cd to avoid cwd issues on restart —
   // after ctrl-c the shell stays in the service dir, so a relative cd
   // would try to descend into <dir>/<dir> which doesn't exist.
-  if (svc.dir !== '.' && svc.command[0] === 'pnpm') {
-    const [, ...rest] = svc.command;
+  if (svc.dir !== '.' && command[0] === 'pnpm') {
+    const [, ...rest] = command;
     return `pnpm --filter {./${svc.dir}} ${rest.join(' ')}`;
   }
 
   const parts: string[] = [];
   if (svc.dir !== '.') parts.push(`cd ${shellQuote(path.join(findRepoRoot(), svc.dir))}`);
-  parts.push(svc.command.join(' '));
+  parts.push(command.join(' '));
 
   return parts.join(' && ');
 }
@@ -112,11 +113,14 @@ export function buildStartCommand(serviceName: string): string {
 export function startServiceInTmux(
   sessionName: string,
   serviceName: string,
-  env?: Record<string, string>
+  env?: Record<string, string>,
+  serviceNames?: readonly string[]
 ): void {
   const svc = getService(serviceName);
   const startupCommand =
-    svc.type === 'infra' ? buildInfraLogCommand(serviceName) : buildStartCommand(serviceName);
+    svc.type === 'infra'
+      ? buildInfraLogCommand(serviceName)
+      : buildStartCommand(serviceName, serviceNames);
   const winIndex = createWindow(sessionName, serviceName, env, startupCommand);
   setPaneServiceIdentity(sessionName, winIndex, 0, serviceName);
   const logPath = path.join(findRepoRoot(), 'dev', 'logs', `${serviceName}.log`);
@@ -395,11 +399,12 @@ const inFlightRestarts = new Map<string, () => void>();
 export function restartServiceInTmux(
   sessionName: string,
   serviceName: string,
-  env?: Record<string, string>
+  env?: Record<string, string>,
+  serviceNames?: readonly string[]
 ): Promise<RestartOutcome> {
   const svc = getService(serviceName);
   if (svc.type === 'infra') return Promise.resolve('not-running');
-  const cmd = buildStartCommand(serviceName);
+  const cmd = buildStartCommand(serviceName, serviceNames);
   // Find the service wherever it lives (own window or joined into window 0).
   // Bail before touching the in-flight map: if the pane is already gone, a
   // previous restart's poll may be mid-recreate and must not be cancelled.
@@ -441,7 +446,7 @@ export function restartServiceInTmux(
         // window instead of leaving the service stopped after reporting
         // "Restarted"; the new window inherits the tmux session environment.
         try {
-          startServiceInTmux(sessionName, serviceName, env);
+          startServiceInTmux(sessionName, serviceName, env, serviceNames);
           settle('recreated');
         } catch {
           // Same policy as below: keep the TUI alive; the next explicit