ci: quarantine platform-stalled E2E suites from push lanes (#10316)

.github/workflows/e2e.yml

@@ -203,10 +203,20 @@ jobs:
           # The docker leg runs vitest directly instead of through
           # test:integration:sandbox:docker: that script would rebuild the image
           # the step above just built.
+          # external-context-mem0-write, external-context-auto-recall,
+          # context-compress-interactive and qwen-serve-channel-workers are
+          # quarantined to the nightly isolated matrix below as a precaution.
+          # Their #10272 startup stalls were the goal-runtime wait bug fixed
+          # by #10290, not a platform defect, and they have not stalled on
+          # any platform since. The nightly canaries run on
+          # ubuntu-latest — which never reproduced the stall — so they keep
+          # functional coverage on hosted Linux but give no signal about the
+          # former stall platforms; restoring these suites to the push lanes
+          # requires verifying them on those runners first.
           if [[ "${{ matrix.sandbox }}" == "sandbox:docker" ]]; then
-            npx cross-env QWEN_SANDBOX=docker vitest run --root ./integration-tests --exclude '**/interactive/cron-interactive.test.ts' --exclude '**/channel-plugin.test.ts' --shard='${{ matrix.shard }}'
+            npx cross-env QWEN_SANDBOX=docker vitest run --root ./integration-tests --exclude '**/interactive/cron-interactive.test.ts' --exclude '**/channel-plugin.test.ts' --exclude '**/interactive/external-context-mem0-write.test.ts' --exclude '**/interactive/external-context-auto-recall.test.ts' --exclude '**/interactive/context-compress-interactive.test.ts' --exclude '**/cli/qwen-serve-channel-workers.test.ts' --shard='${{ matrix.shard }}'
           else
-            npm run test:integration:sandbox:none -- --exclude '**/interactive/cron-interactive.test.ts' --exclude '**/channel-plugin.test.ts' --shard='${{ matrix.shard }}'
+            npm run test:integration:sandbox:none -- --exclude '**/interactive/cron-interactive.test.ts' --exclude '**/channel-plugin.test.ts' --exclude '**/interactive/external-context-mem0-write.test.ts' --exclude '**/interactive/external-context-auto-recall.test.ts' --exclude '**/interactive/context-compress-interactive.test.ts' --exclude '**/cli/qwen-serve-channel-workers.test.ts' --shard='${{ matrix.shard }}'
           fi
 
       # The sandbox build retags the same image name every run, so on a
@@ -275,7 +285,7 @@ jobs:
           OPENAI_API_KEY: '${{ secrets.OPENAI_API_KEY }}'
           OPENAI_BASE_URL: '${{ secrets.OPENAI_BASE_URL }}'
           OPENAI_MODEL: '${{ secrets.OPENAI_MODEL }}'
-        run: 'npx cross-env VERBOSE=true KEEP_OUTPUT=true QWEN_SANDBOX=false vitest run --root ./integration-tests --exclude "**/interactive/cron-interactive.test.ts" --exclude "**/channel-plugin.test.ts" --shard="${{ matrix.shard }}"'
+        run: 'npx cross-env VERBOSE=true KEEP_OUTPUT=true QWEN_SANDBOX=false vitest run --root ./integration-tests --exclude "**/interactive/cron-interactive.test.ts" --exclude "**/channel-plugin.test.ts" --exclude "**/interactive/external-context-mem0-write.test.ts" --exclude "**/interactive/external-context-auto-recall.test.ts" --exclude "**/interactive/context-compress-interactive.test.ts" --exclude "**/cli/qwen-serve-channel-workers.test.ts" --shard="${{ matrix.shard }}"'
 
   isolated-nightly:
     name: '${{ matrix.label }} (nightly)'
@@ -291,6 +301,16 @@ jobs:
             test_file: 'interactive/cron-interactive.test.ts'
           - label: 'channel-plugin E2E'
             test_file: 'channel-plugin.test.ts'
+          # Quarantined from the push lanes as a precaution; see the
+          # excludes and rationale in the push jobs above.
+          - label: 'external-context-mem0-write E2E'
+            test_file: 'interactive/external-context-mem0-write.test.ts'
+          - label: 'external-context-auto-recall E2E'
+            test_file: 'interactive/external-context-auto-recall.test.ts'
+          - label: 'context-compress-interactive E2E'
+            test_file: 'interactive/context-compress-interactive.test.ts'
+          - label: 'qwen-serve-channel-workers E2E'
+            test_file: 'cli/qwen-serve-channel-workers.test.ts'
     steps:
       - name: 'Checkout'
         uses: 'actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10' # v6.0.3

scripts/tests/e2e-workflow.test.js

@@ -4,7 +4,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { readFileSync } from 'node:fs';
+import { existsSync, readFileSync } from 'node:fs';
 import { describe, expect, it } from 'vitest';
 import { parse } from 'yaml';
 
@@ -32,4 +32,70 @@ describe('e2e workflow', () => {
     expect(group).toContain('github.event_name');
     expect(group).toContain('github.head_ref || github.ref_name');
   });
+
+  it('quarantines the nightly-isolated suites from every push lane', () => {
+    // cron-interactive is timing-flaky, and external-context-mem0-write,
+    // external-context-auto-recall, context-compress-interactive and
+    // qwen-serve-channel-workers are quarantined to the nightly isolated
+    // matrix as a precaution after the #10272 startup stalls (since traced
+    // to the goal-runtime wait bug fixed by #10290). Every nightly canary
+    // must be excluded from every push lane, and every push-lane exclusion
+    // must keep its nightly canary — dropping either side silently loses
+    // coverage.
+    const nightlyFiles = yml.jobs['isolated-nightly'].strategy.matrix.include
+      .map((entry) => entry.test_file)
+      .sort();
+    // Anchor the oracle to the filesystem: a renamed suite would otherwise
+    // match no exclude (vitest ignores a non-matching --exclude silently)
+    // and no canary file while this test compared e2e.yml strings only.
+    for (const file of nightlyFiles) {
+      expect(existsSync(`integration-tests/${file}`), file).toBe(true);
+    }
+    const countExcludes = (run) => {
+      const counts = new Map();
+      for (const match of run.matchAll(/--exclude ['"]([^'"]+)['"]/g)) {
+        const file = match[1].replace(/^\*\*\//, '');
+        counts.set(file, (counts.get(file) ?? 0) + 1);
+      }
+      return counts;
+    };
+    // Discover push lanes instead of listing job names: every job with a
+    // `Run E2E tests` step is one, so a future lane that forgets the
+    // excludes fails here instead of stalling on the quarantined suites.
+    let lanes = 0;
+    for (const [jobName, job] of Object.entries(yml.jobs)) {
+      const step = job.steps.find((s) => s.name === 'Run E2E tests');
+      if (!step) continue;
+      lanes += 1;
+      // One vitest invocation per matrix leg; counting them separately
+      // catches an exclude moved between legs, which a whole-step count
+      // cancels out.
+      const invocations = step.run
+        .split('\n')
+        .filter((line) => !line.trim().startsWith('#'))
+        .filter((line) => /vitest run|test:integration/.test(line));
+      expect(invocations.length, jobName).toBeGreaterThan(0);
+      for (const invocation of invocations) {
+        const counts = countExcludes(invocation);
+        expect([...counts.keys()].sort(), `${jobName}: ${invocation}`).toEqual(
+          nightlyFiles,
+        );
+        for (const file of nightlyFiles) {
+          expect(counts.get(file), `${jobName}: ${file}`).toBe(1);
+        }
+      }
+    }
+    expect(lanes).toBeGreaterThan(0);
+  });
+
+  it('runs the nightly canary step on its matrix file', () => {
+    // The quarantine test above pins the canary matrix against the push
+    // lanes but never reads the canary run step: if that step stopped
+    // consuming matrix.test_file, every canary would silently run nothing
+    // while continue-on-error painted the jobs green.
+    const canaryStep = yml.jobs['isolated-nightly'].steps.find(
+      (step) => step.name === 'Run ${{ matrix.label }} tests',
+    );
+    expect(canaryStep.run).toContain('${{ matrix.test_file }}');
+  });
 });