test(infra): retry recursive temp-dir removal instead of failing a shard on ENOTEMPTY (#11966)

scripts/ad-hoc/codemod-rm-maxretries.mjs

@@ -0,0 +1,101 @@
+#!/usr/bin/env node
+/**
+ * One-shot codemod (#11966): give every recursive temp-dir removal in tests the retry
+ * options Node already supports, so a WAL/backup/worker still writing into the directory
+ * turns into a retried delete instead of a red shard:
+ *
+ *   rmSync(dir, { recursive: true, force: true })
+ *   → rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 })
+ *
+ * Applies to `rmSync(`, `fs.rmSync(`, `rm(` / `fs.rm(` / `fs.promises.rm(` (async) and
+ * `rmdirSync(` calls whose option object literal contains `recursive: true` and no
+ * `maxRetries`. Only the option object is touched — call sites, assertions and imports are
+ * left as they are. Usage: node scripts/ad-hoc/codemod-rm-maxretries.mjs [dir=tests]
+ */
+import fs from "node:fs";
+import path from "node:path";
+
+const root = process.argv[2] || "tests";
+const CALL = /\b(?:fs\.promises\.|fsp\.|fs\.|promises\.)?(?:rmSync|rmdirSync|rm)\(/g;
+let files = 0;
+let sites = 0;
+
+function walk(dir, out = []) {
+  for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
+    const p = path.join(dir, e.name);
+    if (e.isDirectory()) {
+      if (e.name === "node_modules" || e.name === "fixtures") continue;
+      walk(p, out);
+    } else if (/\.(ts|tsx|mts|cts|js|mjs|cjs)$/.test(e.name)) out.push(p);
+  }
+  return out;
+}
+
+// Find the closing brace of the option object literal that starts at `open`.
+function objectEnd(src, open) {
+  let depth = 0;
+  for (let i = open; i < src.length; i++) {
+    const c = src[i];
+    if (c === "{") depth++;
+    else if (c === "}") {
+      depth--;
+      if (depth === 0) return i;
+    } else if (c === '"' || c === "'" || c === "`") {
+      const q = c;
+      i++;
+      while (i < src.length && src[i] !== q) {
+        if (src[i] === "\\") i++;
+        i++;
+      }
+    }
+  }
+  return -1;
+}
+
+for (const file of walk(root)) {
+  const src = fs.readFileSync(file, "utf8");
+  let out = "";
+  let last = 0;
+  let touched = 0;
+  for (const m of src.matchAll(CALL)) {
+    const callStart = m.index + m[0].length;
+    // Locate the option object: the first `{` before the call's closing paren at depth 0.
+    let depth = 0;
+    let objOpen = -1;
+    for (let i = callStart; i < src.length; i++) {
+      const c = src[i];
+      if (c === "(" || c === "[") depth++;
+      else if (c === ")" || c === "]") {
+        if (depth === 0) break;
+        depth--;
+      } else if (c === "{" && depth === 0) {
+        objOpen = i;
+        break;
+      }
+    }
+    if (objOpen === -1) continue;
+    const objClose = objectEnd(src, objOpen);
+    if (objClose === -1) continue;
+    const obj = src.slice(objOpen, objClose + 1);
+    if (!/\brecursive:\s*true\b/.test(obj) || /\bmaxRetries\b/.test(obj)) continue;
+    // Insert before the closing brace, respecting an existing trailing comma / newline.
+    const inner = obj.slice(1, -1);
+    const trimmed = inner.replace(/\s+$/, "");
+    const trailing = inner.slice(trimmed.length);
+    const sep = trimmed.endsWith(",") ? " " : ", ";
+    const multiline = /\n/.test(trailing);
+    const insert = multiline
+      ? `${trimmed}${trimmed.endsWith(",") ? "" : ","}\n${trailing.replace(/\n$/, "")}  maxRetries: 5,\n  retryDelay: 100,${trailing}`
+      : `${trimmed}${sep}maxRetries: 5, retryDelay: 100${trailing}`;
+    out += src.slice(last, objOpen + 1) + insert;
+    last = objClose;
+    touched++;
+  }
+  if (touched) {
+    out += src.slice(last);
+    fs.writeFileSync(file, out);
+    files++;
+    sites += touched;
+  }
+}
+console.log(`[codemod-rm-maxretries] ${sites} call site(s) in ${files} file(s) under ${root}`);

scripts/check/check-forgotten-sibling-tests.mjs

@@ -173,7 +173,7 @@ function arg(name, fallback = "") {
 }
 
 function git(root, args) {
-  return execFileSync("git", args, { cwd: root, encoding: "utf8" });
+  return execFileSync("git", args, { cwd: root, encoding: "utf8", maxBuffer: 64 * 1024 * 1024 });
 }
 
 function changedEntries(root, base) {

tests/_setup/isolateDataDir.ts

@@ -33,7 +33,7 @@ if (!process.env.DATA_DIR) {
   // Best-effort cleanup so a long suite run does not leak hundreds of temp DBs.
   process.on("exit", () => {
     try {
-      fs.rmSync(dir, { recursive: true, force: true });
+      fs.rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
     } catch {
       // ignore — the OS reaps its temp dir eventually.
     }

tests/e2e/system-failover.test.ts

@@ -366,7 +366,7 @@ test.after(async () => {
   await serverA.stop();
   await serverB.stop();
   core.closeDbInstance();
-  await fsp.rm(TEST_DATA_DIR, { recursive: true, force: true });
+  await fsp.rm(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
 });
 
 test("primary healthy: request routes to Server A only", async () => {

tests/integration/_chatPipelineHarness.ts

@@ -286,7 +286,7 @@ export async function createChatPipelineHarness(prefix) {
     clearSkillState();
     await new Promise((resolve) => setTimeout(resolve, 20));
     core.resetDbInstance();
-    fs.rmSync(testDataDir, { recursive: true, force: true });
+    fs.rmSync(testDataDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
     fs.mkdirSync(testDataDir, { recursive: true });
     initTranslators();
   }
@@ -300,7 +300,7 @@ export async function createChatPipelineHarness(prefix) {
     clearSkillState();
     resetAllCircuitBreakers();
     core.resetDbInstance();
-    fs.rmSync(testDataDir, { recursive: true, force: true });
+    fs.rmSync(testDataDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
   }
 
   async function seedConnection(provider: string, overrides: SeedConnectionOverrides = {}) {

tests/integration/agent-bridge-bypass-flow.test.ts

@@ -24,7 +24,7 @@ const DEFAULT_PATTERNS = [".bank.", ".gov.", "okta.com", "auth0.com"];
 
 function resetDb() {
   core.resetDbInstance();
-  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
   fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
 }
 
@@ -34,7 +34,11 @@ test.beforeEach(() => {
 });
 
 test.after(() => {
-  try { fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); } catch { /* noop */ }
+  try {
+    fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
+  } catch {
+    /* noop */
+  }
 });
 
 // ── POST patterns ──────────────────────────────────────────────────────────
@@ -48,7 +52,10 @@ test("POST /bypass: stores user patterns", async () => {
     })
   );
   assert.equal(res.status, 200);
-  const body = await res.json() as { ok: boolean; patterns: Array<{ pattern: string; source: string }> };
+  const body = (await res.json()) as {
+    ok: boolean;
+    patterns: Array<{ pattern: string; source: string }>;
+  };
   assert.equal(body.ok, true);
   assert.ok(Array.isArray(body.patterns));
   const userPatterns = body.patterns.filter((p) => p.source === "user");
@@ -64,7 +71,7 @@ test("POST /bypass: invalid body returns 400", async () => {
     })
   );
   assert.equal(res.status, 400);
-  const body = await res.json() as Record<string, unknown>;
+  const body = (await res.json()) as Record<string, unknown>;
   const errMsg = (body.error as Record<string, unknown>)?.message as string;
   assert.ok(!errMsg.includes("at /"), "stack trace leaked in 400 error");
 });
@@ -83,7 +90,7 @@ test("GET /bypass: shows default + user patterns", async () => {
 
   const res = await bypassRoute.GET();
   assert.equal(res.status, 200);
-  const body = await res.json() as { patterns: Array<{ pattern: string; source: string }> };
+  const body = (await res.json()) as { patterns: Array<{ pattern: string; source: string }> };
   assert.ok(Array.isArray(body.patterns));
 
   const sources = new Set(body.patterns.map((p) => p.source));
@@ -119,7 +126,10 @@ test("DELETE /bypass?pattern=X: removes a user pattern", async () => {
     })
   );
   assert.equal(deleteRes.status, 200);
-  const deleteBody = await deleteRes.json() as { ok: boolean; patterns: Array<{ pattern: string; source: string }> };
+  const deleteBody = (await deleteRes.json()) as {
+    ok: boolean;
+    patterns: Array<{ pattern: string; source: string }>;
+  };
   assert.equal(deleteBody.ok, true);
 
   // Verify it's gone
@@ -142,19 +152,18 @@ test("DELETE /bypass: missing pattern param returns 400", async () => {
     })
   );
   assert.equal(res.status, 400);
-  const body = await res.json() as Record<string, unknown>;
+  const body = (await res.json()) as Record<string, unknown>;
   const errMsg = (body.error as Record<string, unknown>)?.message as string;
   assert.ok(!errMsg.includes("at /"), "stack trace leaked in DELETE 400");
 });
 
 test("DELETE /bypass?pattern=X: no-op when pattern not in user list", async () => {
   const res = await bypassRoute.DELETE(
-    new Request(
-      "http://localhost/api/tools/agent-bridge/bypass?pattern=not-in-list.com",
-      { method: "DELETE" }
-    )
+    new Request("http://localhost/api/tools/agent-bridge/bypass?pattern=not-in-list.com", {
+      method: "DELETE",
+    })
   );
   assert.equal(res.status, 200);
-  const body = await res.json() as { ok: boolean };
+  const body = (await res.json()) as { ok: boolean };
   assert.equal(body.ok, true);
 });

tests/integration/agent-bridge-cert-flow.test.ts

@@ -19,7 +19,8 @@ process.env.DISABLE_SQLITE_AUTO_BACKUP = "true";
 
 const certRoute = await import("../../src/app/api/tools/agent-bridge/cert/route.ts");
 const downloadRoute = await import("../../src/app/api/tools/agent-bridge/cert/download/route.ts");
-const regenerateRoute = await import("../../src/app/api/tools/agent-bridge/cert/regenerate/route.ts");
+const regenerateRoute =
+  await import("../../src/app/api/tools/agent-bridge/cert/regenerate/route.ts");
 
 function certDir() {
   return path.join(TEST_DATA_DIR, "mitm");
@@ -30,7 +31,7 @@ function certFilePath() {
 }
 
 function resetCertDir() {
-  fs.rmSync(certDir(), { recursive: true, force: true });
+  fs.rmSync(certDir(), { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
   fs.mkdirSync(certDir(), { recursive: true });
 }
 
@@ -39,15 +40,19 @@ test.beforeEach(() => {
 });
 
 test.after(() => {
-  try { fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); } catch { /* noop */ }
+  try {
+    fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
+  } catch {
+    /* noop */
+  }
 });
 
 // ── GET /cert ─────────────────────────────────────────────────────────────
 
 test("GET /cert: returns exists:false when no cert file", async () => {
   const res = await certRoute.GET();
   assert.equal(res.status, 200);
-  const body = await res.json() as Record<string, unknown>;
+  const body = (await res.json()) as Record<string, unknown>;
   assert.equal(body.exists, false);
   assert.equal(body.trusted, false);
   assert.equal(body.path, null);
@@ -59,7 +64,7 @@ test("GET /cert: returns exists:true when cert file present", async () => {
 
   const res = await certRoute.GET();
   assert.equal(res.status, 200);
-  const body = await res.json() as Record<string, unknown>;
+  const body = (await res.json()) as Record<string, unknown>;
   assert.equal(body.exists, true);
   // trusted may be false in test env (no system store)
   assert.ok(typeof body.trusted === "boolean");
@@ -83,7 +88,7 @@ test("POST /cert: returns 404 when no cert file", async () => {
     })
   );
   assert.equal(res.status, 404);
-  const body = await res.json() as Record<string, unknown>;
+  const body = (await res.json()) as Record<string, unknown>;
   const errMsg = (body.error as Record<string, unknown>)?.message as string;
   assert.ok(!errMsg.includes("at /"), "stack trace leaked in 404 error message");
 });
@@ -106,7 +111,7 @@ MIIBpDCCAQ2gAwIBAgIUFakeMITMCertForTestingOnlyXX==
 
   // In test env: installCert may throw because the PEM is fake; we accept
   // either 200 (mocked) or 500 (real OS failure) — NOT a 500 with stack trace
-  const body = await res.json() as Record<string, unknown>;
+  const body = (await res.json()) as Record<string, unknown>;
   const errMsg = (body.error as Record<string, unknown>)?.message as string | undefined;
   if (errMsg) {
     assert.ok(!errMsg.includes("at /"), "stack trace leaked in POST /cert error");
@@ -118,7 +123,7 @@ MIIBpDCCAQ2gAwIBAgIUFakeMITMCertForTestingOnlyXX==
 test("GET /cert/download: 404 when no cert file", async () => {
   const res = await downloadRoute.GET();
   assert.equal(res.status, 404);
-  const body = await res.json() as Record<string, unknown>;
+  const body = (await res.json()) as Record<string, unknown>;
   const errMsg = (body.error as Record<string, unknown>)?.message as string;
   assert.ok(!errMsg.includes("at /"), "stack trace leaked in download 404");
 });