fix(ci): stop shared-pool hosts exiting all-green E2E runs red (#10325)

.github/workflows/e2e.yml

@@ -196,6 +196,9 @@ jobs:
           OPENAI_MODEL: '${{ secrets.OPENAI_MODEL }}'
           KEEP_OUTPUT: 'true'
           VERBOSE: 'true'
+          # Mapped for integration-tests/vitest.config.ts, which exempts
+          # self-hosted runners from pressure-flake unhandled errors.
+          RUNNER_ENVIRONMENT: '${{ runner.environment }}'
         run: |-
           # The docker leg runs vitest directly instead of through
           # test:integration:sandbox:docker: that script would rebuild the image

integration-tests/globalSetup.test.ts

@@ -0,0 +1,85 @@
+/**
+ * @license
+ * Copyright 2026 Qwen Team
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+// The env keys globalSetup's setup() writes, saved so a case can restore the
+// suite-wide values after re-importing the module and running its lifecycle.
+const SETUP_ENV_KEYS = [
+  'INTEGRATION_TEST_FILE_DIR',
+  'QWEN_CODE_INTEGRATION_TEST',
+  'TELEMETRY_LOG_FILE',
+  'E2E_TEST_FILE_DIR',
+  'TEST_CLI_PATH',
+  'VERBOSE',
+  'KEEP_OUTPUT',
+] as const;
+
+describe('globalSetup memory-file save/restore', () => {
+  let qwenHome: string;
+  let savedEnv: Map<string, string | undefined>;
+
+  beforeEach(async () => {
+    qwenHome = await mkdtemp(join(tmpdir(), 'qwen-globalsetup-test-'));
+    savedEnv = new Map(
+      [...SETUP_ENV_KEYS, 'QWEN_HOME'].map((key) => [key, process.env[key]]),
+    );
+    process.env['QWEN_HOME'] = qwenHome;
+    // Let teardown remove the run directories this case creates.
+    process.env['KEEP_OUTPUT'] = 'false';
+  });
+
+  afterEach(async () => {
+    for (const [key, value] of savedEnv) {
+      if (value === undefined) {
+        delete process.env[key];
+      } else {
+        process.env[key] = value;
+      }
+    }
+    vi.resetModules();
+    await rm(qwenHome, { recursive: true, force: true });
+  });
+
+  // memoryFilePath is captured at module import time, so point QWEN_HOME at
+  // the scratch dir BEFORE a fresh import of the module.
+  async function loadGlobalSetup() {
+    vi.resetModules();
+    return import('./globalSetup.js');
+  }
+
+  it('restores the saved memory file after the run', async () => {
+    await writeFile(join(qwenHome, 'QWEN.md'), 'original content', 'utf-8');
+    const { setup, teardown } = await loadGlobalSetup();
+    await setup();
+    await writeFile(join(qwenHome, 'QWEN.md'), 'mutated by tests', 'utf-8');
+
+    await expect(teardown()).resolves.toBeUndefined();
+
+    await expect(readFile(join(qwenHome, 'QWEN.md'), 'utf-8')).resolves.toBe(
+      'original content',
+    );
+  });
+
+  it('does not exit an all-green run red when the restore cannot write', async () => {
+    // The persistent pool runners can carry a readable-but-unwritable
+    // QWEN.md left behind by a privileged job; before #10325 the teardown
+    // restore threw on it and exited every all-green E2E run on that host
+    // red with no failing test. Swap the file for a directory after setup()
+    // read it — the write then fails regardless of privilege, since root
+    // bypasses permission bits.
+    await writeFile(join(qwenHome, 'QWEN.md'), 'original content', 'utf-8');
+    const { setup, teardown } = await loadGlobalSetup();
+    await setup();
+    await rm(join(qwenHome, 'QWEN.md'), { force: true });
+    await mkdir(join(qwenHome, 'QWEN.md'));
+
+    await expect(teardown()).resolves.toBeUndefined();
+  });
+});

integration-tests/globalSetup.ts

@@ -119,8 +119,17 @@ export async function teardown() {
   }
 
   if (originalMemoryContent !== null) {
-    await mkdir(dirname(memoryFilePath), { recursive: true });
-    await writeFile(memoryFilePath, originalMemoryContent, 'utf-8');
+    try {
+      await mkdir(dirname(memoryFilePath), { recursive: true });
+      await writeFile(memoryFilePath, originalMemoryContent, 'utf-8');
+    } catch (e) {
+      // Best-effort restore: on the persistent pool runners a privileged job
+      // can leave a readable-but-unwritable QWEN.md behind, and the throw
+      // turned every all-green E2E run on that host red with no failing test
+      // ('Startup Error: EACCES'; #10325). Keep the warning visible so the
+      // poisoned host is still diagnosable.
+      console.error(`Warning: could not restore ${memoryFilePath}:`, e);
+    }
   } else {
     try {
       await unlink(memoryFilePath);

integration-tests/vitest.config.ts

@@ -34,14 +34,18 @@ export default defineConfig({
         maxForks: 4,
       },
     },
-    // The worker->main `onTaskUpdate` RPC runs on a 60s budget; under the
-    // resource pressure of the macOS E2E lane a stall longer than that
-    // surfaces as an unhandled error and exits an all-green run red (the
-    // same failure class the core, cli, and scripts suites hit on these
-    // lanes). Test failures still fail the run; only unhandled errors stop
-    // being fatal, and only off Linux — the ubuntu shards and Linux local
-    // runs keep the unhandled-error signal.
-    dangerouslyIgnoreUnhandledErrors: process.platform !== 'linux',
+    // The worker->main `onTaskUpdate` RPC runs on a 60s budget; under
+    // resource pressure a stall longer than that surfaces as an unhandled
+    // error and exits an all-green run red (the same failure class the
+    // core, cli, and scripts suites hit on the macOS lane). Since #10085
+    // the Linux shards run on the shared self-hosted pool instead of
+    // ubuntu-hosted VMs and hit the same pressure class there (#10325), so
+    // self-hosted runners are exempted as well. Test failures still fail
+    // the run; only unhandled errors stop being fatal — github-hosted Linux
+    // (the nightly isolated legs) and local Linux runs keep the signal.
+    dangerouslyIgnoreUnhandledErrors:
+      process.platform !== 'linux' ||
+      process.env['RUNNER_ENVIRONMENT'] === 'self-hosted',
   },
   resolve: {
     alias: {

scripts/tests/integration-vitest-config.test.ts

@@ -4,7 +4,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { describe, expect, it } from 'vitest';
+import { afterEach, describe, expect, it, vi } from 'vitest';
 import integrationConfig from '../../integration-tests/vitest.config.js';
 
 describe('integration Vitest config', () => {
@@ -17,11 +17,50 @@ describe('integration Vitest config', () => {
     expect(integrationConfig.test?.poolOptions?.threads).toBeUndefined();
   });
 
-  it('keeps unhandled errors fatal only on Linux', () => {
-    // toBe, not toBeFalsy: a deleted flag is `undefined` and must fail
-    // this pin on every platform, including Linux where the value is false.
-    expect(integrationConfig.test?.dangerouslyIgnoreUnhandledErrors).toBe(
-      process.platform !== 'linux',
-    );
+  describe('unhandled-error exemption', () => {
+    const savedRunnerEnvironment = process.env['RUNNER_ENVIRONMENT'];
+
+    afterEach(() => {
+      if (savedRunnerEnvironment === undefined) {
+        delete process.env['RUNNER_ENVIRONMENT'];
+      } else {
+        process.env['RUNNER_ENVIRONMENT'] = savedRunnerEnvironment;
+      }
+      vi.resetModules();
+    });
+
+    // The flag reads RUNNER_ENVIRONMENT at config import time, so each case
+    // re-imports the config under a controlled value instead of trusting the
+    // ambient one.
+    async function configFor(runnerEnvironment: string | undefined) {
+      vi.resetModules();
+      if (runnerEnvironment === undefined) {
+        delete process.env['RUNNER_ENVIRONMENT'];
+      } else {
+        process.env['RUNNER_ENVIRONMENT'] = runnerEnvironment;
+      }
+      const { default: config } = await import(
+        '../../integration-tests/vitest.config.js'
+      );
+      return config;
+    }
+
+    it('exempts self-hosted pool runners on every platform', async () => {
+      // Dropping the self-hosted clause makes the shared pool's pressure
+      // flakes exit all-green E2E runs red again (#10325).
+      const config = await configFor('self-hosted');
+      expect(config.test?.dangerouslyIgnoreUnhandledErrors).toBe(true);
+    });
+
+    it('keeps unhandled errors fatal on github-hosted Linux and local runs', async () => {
+      // toBe, not toBeFalsy: a deleted flag is `undefined` and must fail
+      // this pin on every platform, including Linux where the value is false.
+      for (const environment of ['github-hosted', undefined]) {
+        const config = await configFor(environment);
+        expect(config.test?.dangerouslyIgnoreUnhandledErrors).toBe(
+          process.platform !== 'linux',
+        );
+      }
+    });
   });
 });