test(infra): retry recursive temp-dir removal on main (main twin of #11968)

scripts/ad-hoc/codemod-rm-maxretries.mjs

@@ -0,0 +1,101 @@
+#!/usr/bin/env node
+/**
+ * One-shot codemod (#11966): give every recursive temp-dir removal in tests the retry
+ * options Node already supports, so a WAL/backup/worker still writing into the directory
+ * turns into a retried delete instead of a red shard:
+ *
+ *   rmSync(dir, { recursive: true, force: true })
+ *   → rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 })
+ *
+ * Applies to `rmSync(`, `fs.rmSync(`, `rm(` / `fs.rm(` / `fs.promises.rm(` (async) and
+ * `rmdirSync(` calls whose option object literal contains `recursive: true` and no
+ * `maxRetries`. Only the option object is touched — call sites, assertions and imports are
+ * left as they are. Usage: node scripts/ad-hoc/codemod-rm-maxretries.mjs [dir=tests]
+ */
+import fs from "node:fs";
+import path from "node:path";
+
+const root = process.argv[2] || "tests";
+const CALL = /\b(?:fs\.promises\.|fsp\.|fs\.|promises\.)?(?:rmSync|rmdirSync|rm)\(/g;
+let files = 0;
+let sites = 0;
+
+function walk(dir, out = []) {
+  for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
+    const p = path.join(dir, e.name);
+    if (e.isDirectory()) {
+      if (e.name === "node_modules" || e.name === "fixtures") continue;
+      walk(p, out);
+    } else if (/\.(ts|tsx|mts|cts|js|mjs|cjs)$/.test(e.name)) out.push(p);
+  }
+  return out;
+}
+
+// Find the closing brace of the option object literal that starts at `open`.
+function objectEnd(src, open) {
+  let depth = 0;
+  for (let i = open; i < src.length; i++) {
+    const c = src[i];
+    if (c === "{") depth++;
+    else if (c === "}") {
+      depth--;
+      if (depth === 0) return i;
+    } else if (c === '"' || c === "'" || c === "`") {
+      const q = c;
+      i++;
+      while (i < src.length && src[i] !== q) {
+        if (src[i] === "\\") i++;
+        i++;
+      }
+    }
+  }
+  return -1;
+}
+
+for (const file of walk(root)) {
+  const src = fs.readFileSync(file, "utf8");
+  let out = "";
+  let last = 0;
+  let touched = 0;
+  for (const m of src.matchAll(CALL)) {
+    const callStart = m.index + m[0].length;
+    // Locate the option object: the first `{` before the call's closing paren at depth 0.
+    let depth = 0;
+    let objOpen = -1;
+    for (let i = callStart; i < src.length; i++) {
+      const c = src[i];
+      if (c === "(" || c === "[") depth++;
+      else if (c === ")" || c === "]") {
+        if (depth === 0) break;
+        depth--;
+      } else if (c === "{" && depth === 0) {
+        objOpen = i;
+        break;
+      }
+    }
+    if (objOpen === -1) continue;
+    const objClose = objectEnd(src, objOpen);
+    if (objClose === -1) continue;
+    const obj = src.slice(objOpen, objClose + 1);
+    if (!/\brecursive:\s*true\b/.test(obj) || /\bmaxRetries\b/.test(obj)) continue;
+    // Insert before the closing brace, respecting an existing trailing comma / newline.
+    const inner = obj.slice(1, -1);
+    const trimmed = inner.replace(/\s+$/, "");
+    const trailing = inner.slice(trimmed.length);
+    const sep = trimmed.endsWith(",") ? " " : ", ";
+    const multiline = /\n/.test(trailing);
+    const insert = multiline
+      ? `${trimmed}${trimmed.endsWith(",") ? "" : ","}\n${trailing.replace(/\n$/, "")}  maxRetries: 5,\n  retryDelay: 100,${trailing}`
+      : `${trimmed}${sep}maxRetries: 5, retryDelay: 100${trailing}`;
+    out += src.slice(last, objOpen + 1) + insert;
+    last = objClose;
+    touched++;
+  }
+  if (touched) {
+    out += src.slice(last);
+    fs.writeFileSync(file, out);
+    files++;
+    sites += touched;
+  }
+}
+console.log(`[codemod-rm-maxretries] ${sites} call site(s) in ${files} file(s) under ${root}`);

tests/_setup/isolateDataDir.ts

@@ -33,7 +33,7 @@ if (!process.env.DATA_DIR) {
   // Best-effort cleanup so a long suite run does not leak hundreds of temp DBs.
   process.on("exit", () => {
     try {
-      fs.rmSync(dir, { recursive: true, force: true });
+      fs.rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
     } catch {
       // ignore — the OS reaps its temp dir eventually.
     }

tests/e2e/system-failover.test.ts

@@ -366,7 +366,7 @@ test.after(async () => {
   await serverA.stop();
   await serverB.stop();
   core.closeDbInstance();
-  await fsp.rm(TEST_DATA_DIR, { recursive: true, force: true });
+  await fsp.rm(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
 });
 
 test("primary healthy: request routes to Server A only", async () => {

tests/integration/_chatPipelineHarness.ts

@@ -286,7 +286,7 @@ export async function createChatPipelineHarness(prefix) {
     clearSkillState();
     await new Promise((resolve) => setTimeout(resolve, 20));
     core.resetDbInstance();
-    fs.rmSync(testDataDir, { recursive: true, force: true });
+    fs.rmSync(testDataDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
     fs.mkdirSync(testDataDir, { recursive: true });
     initTranslators();
   }
@@ -300,7 +300,7 @@ export async function createChatPipelineHarness(prefix) {
     clearSkillState();
     resetAllCircuitBreakers();
     core.resetDbInstance();
-    fs.rmSync(testDataDir, { recursive: true, force: true });
+    fs.rmSync(testDataDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
   }
 
   async function seedConnection(provider: string, overrides: SeedConnectionOverrides = {}) {

tests/integration/agent-bridge-bypass-flow.test.ts

@@ -24,7 +24,7 @@ const DEFAULT_PATTERNS = [".bank.", ".gov.", "okta.com", "auth0.com"];
 
 function resetDb() {
   core.resetDbInstance();
-  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
   fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
 }
 
@@ -34,7 +34,7 @@ test.beforeEach(() => {
 });
 
 test.after(() => {
-  try { fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); } catch { /* noop */ }
+  try { fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } catch { /* noop */ }
 });
 
 // ── POST patterns ──────────────────────────────────────────────────────────

tests/integration/agent-bridge-cert-flow.test.ts

@@ -30,7 +30,7 @@ function certFilePath() {
 }
 
 function resetCertDir() {
-  fs.rmSync(certDir(), { recursive: true, force: true });
+  fs.rmSync(certDir(), { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
   fs.mkdirSync(certDir(), { recursive: true });
 }
 
@@ -39,7 +39,7 @@ test.beforeEach(() => {
 });
 
 test.after(() => {
-  try { fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); } catch { /* noop */ }
+  try { fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } catch { /* noop */ }
 });
 
 // ── GET /cert ─────────────────────────────────────────────────────────────

tests/integration/agent-bridge-mappings.test.ts

@@ -24,7 +24,7 @@ const mappingsRoute = await import(
 
 function resetDb() {
   core.resetDbInstance();
-  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
   fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
 }
 
@@ -33,7 +33,7 @@ test.beforeEach(() => {
 });
 
 test.after(() => {
-  try { fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); } catch { /* noop */ }
+  try { fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } catch { /* noop */ }
 });
 
 // ── GET (empty) ────────────────────────────────────────────────────────────

tests/integration/agent-bridge-routes.test.ts

@@ -39,7 +39,7 @@ const routeGuard = await import("../../src/server/authz/routeGuard.ts");
 
 function resetDb() {
   core.resetDbInstance();
-  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
   fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
 }
 
@@ -49,7 +49,7 @@ test.beforeEach(() => {
 
 test.after(() => {
   try {
-    fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+    fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
   } catch {
     /* noop */
   }

tests/integration/all-statuses-route.test.ts

@@ -39,7 +39,7 @@ async function resetStorage() {
   delete process.env.INITIAL_PASSWORD;
   core.resetDbInstance();
   apiKeysDb.resetApiKeyState();
-  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
   fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
 }
 
@@ -55,7 +55,7 @@ test.beforeEach(async () => {
 
 test.after(async () => {
   await resetStorage();
-  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
 });
 
 // ── Auth tests ────────────────────────────────────────────────────────────────
@@ -288,6 +288,6 @@ test("grok-build status uses GROK_HOME and returns its managed endpoint", async
   } finally {
     if (original === undefined) delete process.env.GROK_HOME;
     else process.env.GROK_HOME = original;
-    fs.rmSync(grokHome, { recursive: true, force: true });
+    fs.rmSync(grokHome, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
   }
 });

tests/integration/antigravity-projectid-discovery-persist-8491.test.ts

@@ -32,7 +32,7 @@ const { clearAntigravityProjectCache } = await import(
 test.after(() => {
   core.resetDbInstance();
   if (fs.existsSync(TEST_DATA_DIR)) {
-    fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+    fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
   }
 });
 

tests/integration/api-keys.test.ts

@@ -25,7 +25,7 @@ async function resetStorage() {
   delete process.env.INITIAL_PASSWORD;
   core.resetDbInstance();
   apiKeysDb.resetApiKeyState();
-  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
   fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
 }
 
@@ -63,7 +63,7 @@ test.beforeEach(async () => {
 
 test.after(async () => {
   await resetStorage();
-  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
 });
 
 test("API keys routes require management auth when login protection is enabled", async () => {

tests/integration/api-routes-critical.test.ts

@@ -24,7 +24,7 @@ async function resetStorage() {
   delete process.env.ENABLE_SOCKS5_PROXY;
   core.resetDbInstance();
   apiKeysDb.resetApiKeyState();
-  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
   fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
 }
 
@@ -59,7 +59,7 @@ test.beforeEach(async () => {
 
 test.after(async () => {
   await resetStorage();
-  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
 });
 
 test("critical routes: v1 management proxies covers auth, lookup, where-used, patch, and delete branches", async () => {

tests/integration/audit-log-level-filter.test.ts

@@ -20,7 +20,7 @@ const auditRoute = await import("../../src/app/api/compliance/audit-log/route.ts
 
 function resetDb() {
   core.resetDbInstance();
-  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
   fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
 }
 
@@ -38,7 +38,7 @@ test.beforeEach(() => {
 
 test.after(() => {
   core.resetDbInstance();
-  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
 });
 
 /**

tests/integration/batch-e2e-rate-limit.test.ts

@@ -268,7 +268,7 @@ async function stopProcess(child: ReturnType<typeof spawn>) {
 async function removeDirWithRetry(dir: string) {
   for (let attempt = 0; attempt < 5; attempt++) {
     try {
-      fs.rmSync(dir, { recursive: true, force: true });
+      fs.rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
       return;
     } catch (error) {
       if (attempt === 4) throw error;

tests/integration/chat-pipeline.test.ts

@@ -373,7 +373,7 @@ async function resetStorage() {
   invalidateMemorySettingsCache();
   await new Promise((resolve) => setTimeout(resolve, 20));
   core.resetDbInstance();
-  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
   fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
   initTranslators();
 }
@@ -512,7 +512,7 @@ test.after(async () => {
   clearInflight();
   resetAllCircuitBreakers();
   core.resetDbInstance();
-  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
 });
 
 test("chat pipeline handles OpenAI passthrough with valid API key auth", async () => {

tests/integration/chatcore-compression-integration.test.ts

@@ -28,7 +28,7 @@ async function resetStorage() {
   readCacheDb.invalidateDbCache();
   await new Promise((resolve) => setTimeout(resolve, 20));
   core.resetDbInstance();
-  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
   fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
 }
 
@@ -40,7 +40,7 @@ test.after(async () => {
   globalThis.fetch = originalFetch;
   core.closeDbInstance();
   try {
-    fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+    fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
   } catch {}
 });
 

tests/integration/chatcore-context-window-boundary.test.ts

@@ -21,7 +21,7 @@ test.after(async () => {
   globalThis.fetch = originalFetch;
   core.closeDbInstance();
   try {
-    fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+    fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
   } catch {}
 });