fix(widgets): re-arm a dev session on a changed folder id and bound persistent stat errors

apps/runtime/src/application/widget-dev-sessions.ts

@@ -185,6 +185,8 @@ export function createWidgetDevSessions(
     watch?: boolean;
     /** How often a session waiting on an answer looks for it. */
     answerPollMs?: number;
+    /** How long a folder may keep failing to be looked at before its session stops (`DEV_ENGINE_ROOT_UNREADABLE_MS`). */
+    rootUnreadableMs?: number;
   } = {},
 ): WidgetDevSessions {
   const live = new Map<string, LiveSession>();
@@ -261,13 +263,24 @@ export function createWidgetDevSessions(
       return refusal(400, "ROOT_NOT_LOCAL", "give a folder on a drive of this machine; a network share or device path is not developed from");
     }
     const resolved = resolve(given);
+    // Only "not found" means not there: a folder an antivirus or indexer holds (`EPERM`, `EBUSY`) is there, unreadable.
+    const unreadable = (cause: unknown) => {
+      const code = (cause as NodeJS.ErrnoException).code;
+      return code !== undefined && code !== "ENOENT" && code !== "ENOTDIR"
+        ? refusal(403, "ROOT_UNREADABLE", `${resolved} cannot be read on this node (${code})`)
+        : undefined;
+    };
     try {
       if (!statSync(resolved).isDirectory()) return refusal(400, "ROOT_NOT_A_FOLDER", `${resolved} is not a folder`);
-    } catch {
-      return refusal(404, "ROOT_NOT_FOUND", `${resolved} does not exist on this node`);
+    } catch (cause) {
+      return unreadable(cause) ?? refusal(404, "ROOT_NOT_FOUND", `${resolved} does not exist on this node`);
+    }
+    let root: string;
+    try {
+      root = realpathSync.native(resolved);
+    } catch (cause) {
+      return unreadable(cause) ?? refusal(404, "ROOT_NOT_FOUND", `${resolved} could not be resolved on this node`);
     }
-    const root = realOrUndefined(resolved);
-    if (root === undefined) return refusal(404, "ROOT_NOT_FOUND", `${resolved} could not be resolved on this node`);
     if (isRemoteOrDevicePath(root)) {
       return refusal(400, "ROOT_NOT_LOCAL", "the folder resolves to a network share or device path, which is not developed from");
     }
@@ -742,8 +755,9 @@ export function createWidgetDevSessions(
   };
 
   /**
-   * Whether a live session's folder has gone (deleted, renamed, or replaced by another folder at the same path while it
-   * was watched); when it has, the session is stopped as `folder-gone` before anything is built or installed from it. A
+   * Whether a live session's folder has gone (deleted, renamed, or no longer a folder; another folder made at the same
+   * path is watched in its place by the engine); when it has, the session is stopped as `folder-gone` before anything is
+   * built or installed from it. A
    * platform watcher does not always report this (Windows reports nothing), so the check is made before each build is
    * followed, not only on a watcher error. The session's engine answers, since it knows which folder it watches; a folder
    * that could not be looked at this time (a busy or locked folder on Windows) is not taken as gone.
@@ -779,6 +793,7 @@ export function createWidgetDevSessions(
         root: stored.root,
         cacheRoot: cacheRoot(),
         watch: options.watch !== false,
+        ...(options.rootUnreadableMs === undefined ? {} : { rootUnreadableMs: options.rootUnreadableMs }),
         generationsBefore: before,
         allowedIsolations: WIDGET_DEV_ALLOWED_ISOLATIONS,
         baseline: () => live.get(sessionId)?.baseline,
@@ -793,7 +808,7 @@ export function createWidgetDevSessions(
           });
         },
         onWatchError: (error) => {
-          process.stderr.write(`widget dev: ${sessionId} stopped watching ${stored.root}: ${error.message}\n`);
+          process.stderr.write(`widget dev: ${sessionId} stopped watching ${stored.root}: ${error.message}; what it ran keeps running\n`);
           // Said as it is: a session whose folder is no longer watched is stopped, not live.
           void serial(sessionId, async () => {
             if (live.get(sessionId) === session) markStopped(sessionId, "watch-failed");
@@ -1033,8 +1048,9 @@ export function createWidgetDevSessions(
         const checked = checkRoot(stored.root, { kind: stored.initiative?.kind ?? "person" });
         if (!checked.ok) {
           const gone = checked.code === "ROOT_NOT_FOUND" || checked.code === "ROOT_NOT_A_FOLDER";
-          markStopped(stored.sessionId, gone ? "folder-gone" : "root-refused");
-          process.stderr.write(`widget dev: ${stored.root} could not be watched again (${checked.code}), so its session was stopped; what it ran keeps running\n`);
+          // A folder that is there but cannot be read is not gone, nor refused: watching it failed.
+          markStopped(stored.sessionId, gone ? "folder-gone" : checked.code === "ROOT_UNREADABLE" ? "watch-failed" : "root-refused");
+          process.stderr.write(`widget dev: ${stored.root} could not be watched again (${checked.code}: ${checked.message}), so its session was stopped; what it ran keeps running\n`);
           continue;
         }
         if (live.size >= WIDGET_DEV_LIVE_MAX) {

apps/runtime/src/develop-widget-tool.ts

@@ -48,7 +48,7 @@ export function describeDevSession(view: WidgetDevSessionView): string {
   if (activation.state === "none") lines.push("No generation runs yet.");
   if (view.status === "stopped" && view.stopReason !== undefined && view.stopReason !== "requested") {
     const why = {
-      "watch-failed": "watching its folder failed",
+      "watch-failed": "watching its folder failed (the platform stopped reporting changes, or the folder could not be read for 30 s; the node's log names the error)",
       "folder-gone": "its folder is gone",
       capacity: "the node already watches as many folders as it can",
       "root-refused": "its folder is no longer one it may watch",

apps/runtime/test/widget-dev-sessions.spec.ts

@@ -1,3 +1,4 @@
+import { spawn } from "node:child_process";
 import type * as fs from "node:fs";
 import type * as fsPromises from "node:fs/promises";
 import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
@@ -1021,7 +1022,7 @@ describe("a widget dev session", () => {
     expect(stopped).toMatchObject({ status: "stopped", stopReason: "folder-gone", activation: { state: "active", generation: 1 } });
   });
 
-  it("stops a watched session as folder-gone when its folder is deleted and made again before anything looks", async () => {
+  it("keeps a watched session live, and builds the new folder, when its folder is deleted and made again before anything looks", async () => {
     services.widgetDev?.close();
     services.widgetDev = createWidgetDevSessions(() => services, { watch: true, answerPollMs: 20 });
     const started = session(await call("POST", "/widget-dev/sessions", { root }));
@@ -1031,8 +1032,78 @@ describe("a widget dev session", () => {
     // so nothing looks between the delete and the new folder; no retries are asked for, since `rmSync` would not run them.
     rmSync(root, { recursive: true, force: true });
     writePackage("<!doctype html><p>a new folder</p>\n");
+    const rebuilt = await eventually(started.sessionId, (view) => view.activation.state === "active" && view.activation.generation === 2);
+    expect(rebuilt).toMatchObject({ status: "live", activation: { state: "active", generation: 2 } });
+
+    // The new folder is the one watched: a save in it builds and runs.
+    writePackage("<!doctype html><p>saved in the new folder</p>\n");
+    const saved = await eventually(started.sessionId, (view) => view.activation.state === "active" && view.activation.generation === 3);
+    expect(saved).toMatchObject({ status: "live", activation: { state: "active", generation: 3 } });
+  });
+
+  it("keeps a watched session live when another process deletes its folder and makes it again a moment later", async () => {
+    services.widgetDev?.close();
+    services.widgetDev = createWidgetDevSessions(() => services, { watch: true, answerPollMs: 20 });
+    const started = session(await call("POST", "/widget-dev/sessions", { root }));
+    expect(started.status).toBe("live");
+
+    // As `rmdir /s /q out && xcopy source out` in a terminal: another process, and a moment with no folder at the path.
+    const source = join(dir, "source");
+    writePackage("<!doctype html><p>made again by another process</p>\n", [], source);
+    const script = [
+      "const fs = require('node:fs');",
+      "const [out, from] = process.argv.slice(1);",
+      "fs.rmSync(out, { recursive: true, force: true });",
+      "setTimeout(() => fs.cpSync(from, out, { recursive: true }), 400);",
+    ].join("\n");
+    const child = spawn(process.execPath, ["-e", script, root, source], { stdio: "ignore" });
+    expect(await new Promise((done) => child.on("exit", done))).toBe(0);
+
+    const rebuilt = await eventually(started.sessionId, (view) => view.status === "stopped" || (view.activation.state === "active" && view.activation.generation === 2));
+    expect(rebuilt).toMatchObject({ status: "live", activation: { state: "active", generation: 2 } });
+  });
+
+  it("keeps a watched session live when a build runs while another process makes its folder again, and builds it once back", async () => {
+    services.widgetDev?.close();
+    services.widgetDev = createWidgetDevSessions(() => services, { watch: true, answerPollMs: 20 });
+    const started = session(await call("POST", "/widget-dev/sessions", { root }));
+    expect(started.status).toBe("live");
+
+    const source = join(dir, "source");
+    writePackage("<!doctype html><p>made again by another process</p>\n", [], source);
+    const script = [
+      "const fs = require('node:fs');",
+      "const [out, from] = process.argv.slice(1);",
+      "fs.rmSync(out, { recursive: true, force: true });",
+      "setTimeout(() => fs.cpSync(from, out, { recursive: true }), 800);",
+    ].join("\n");
+    const child = spawn(process.execPath, ["-e", script, root, source], { stdio: "ignore" });
+    const exited = new Promise((done) => child.on("exit", done));
+
+    // A build that runs while no folder is at the path: it fails on the missing files, and the session stays live.
+    const deadline = Date.now() + 5_000;
+    while (existsSync(root) && Date.now() < deadline) await new Promise((done) => setTimeout(done, 5));
+    expect(existsSync(root)).toBe(false);
+    const during = await call("POST", `/widget-dev/sessions/${started.sessionId}/rebuild`);
+    expect(during.status).toBe(200);
+    expect(session(during)).toMatchObject({ status: "live", lastBuild: { ok: false, trigger: "rebuild" }, activation: { state: "active", generation: 1 } });
+
+    expect(await exited).toBe(0);
+    const rebuilt = await eventually(started.sessionId, (view) => view.status === "stopped" || (view.activation.state === "active" && view.activation.generation === 2));
+    expect(rebuilt).toMatchObject({ status: "live", activation: { state: "active", generation: 2 } });
+  });
+
+  it("stops a watched session as watch-failed when its folder cannot be looked at for the time bound, and keeps what runs", async () => {
+    services.widgetDev?.close();
+    services.widgetDev = createWidgetDevSessions(() => services, { watch: true, answerPollMs: 20, rootUnreadableMs: 300 });
+    const started = session(await call("POST", "/widget-dev/sessions", { root }));
+    expect(started.status).toBe("live");
+
+    // The engine looks at the canonical path a start stores (on macOS, `/private/var/...` for a `/var/...` temp folder).
+    statFailure.path = realpathSync.native(root);
+    // Bounded: the node looks at the folder once a second, so the bound is passed on the second look at the latest.
     const stopped = await eventually(started.sessionId, (view) => view.status === "stopped");
-    expect(stopped).toMatchObject({ status: "stopped", stopReason: "folder-gone", activation: { state: "active", generation: 1 } });
+    expect(stopped).toMatchObject({ status: "stopped", stopReason: "watch-failed", activation: { state: "active", generation: 1 } });
   });
 
   it("keeps a watched session live through a folder it cannot look at for a moment", async () => {
@@ -1041,7 +1112,8 @@ describe("a widget dev session", () => {
     const started = session(await call("POST", "/widget-dev/sessions", { root }));
     expect(started.status).toBe("live");
 
-    statFailure.path = resolve(root);
+    // The engine looks at the canonical path a start stores (on macOS, `/private/var/...` for a `/var/...` temp folder).
+    statFailure.path = realpathSync.native(root);
     const rebuilt = await call("POST", `/widget-dev/sessions/${started.sessionId}/rebuild`);
     expect(rebuilt.status).toBe(200);
     expect(session(rebuilt).status).toBe("live");
@@ -1052,6 +1124,39 @@ describe("a widget dev session", () => {
     const next = await eventually(started.sessionId, (view) => view.activation.state === "active" && view.activation.generation === 2);
     expect(next).toMatchObject({ status: "live", activation: { state: "active", generation: 2 } });
   });
+
+  it("stops a watched session as folder-gone when a folder above it is swapped for a link to another tree, and builds nothing there", async () => {
+    services.widgetDev?.close();
+    services.widgetDev = createWidgetDevSessions(() => services, { watch: true, answerPollMs: 20 });
+    const started = session(await call("POST", "/widget-dev/sessions", { root }));
+    expect(started.status).toBe("live");
+
+    // `projects/timer` now leads, through `projects`, to a tree the person never chose for this session.
+    const elsewhere = join(dir, "elsewhere");
+    writePackage("<!doctype html><p>elsewhere</p>\n", [], join(elsewhere, "timer"), { id: "com.example.elsewhere" });
+    const projects = join(dir, "projects");
+    rmSync(projects, { recursive: true, force: true });
+    symlinkSync(elsewhere, projects, process.platform === "win32" ? "junction" : "dir");
+
+    const stopped = await eventually(started.sessionId, (view) => view.status === "stopped");
+    expect(stopped).toMatchObject({ status: "stopped", stopReason: "folder-gone", packageId: PACKAGE, activation: { state: "active", generation: 1 } });
+  });
+
+  it("says a folder that is there but cannot be read cannot be read, at a start and at a resume", async () => {
+    // A start and a resume look at the path as given, before it is made canonical.
+    statFailure.path = resolve(root);
+    const refused = await call("POST", "/widget-dev/sessions", { root });
+    expect(refused.status).toBe(403);
+    expect(refused.body).toMatchObject({ code: "ROOT_UNREADABLE" });
+    expect((refused.body as { message: string }).message).toContain("cannot be read on this node (EPERM)");
+
+    writeDevSessions(join(dir, "node"), [{ sessionId: "wdev_unreadable", root, status: "live" as const, startedAt: new Date().toISOString() }]);
+    services.widgetDev?.close();
+    services.widgetDev = createWidgetDevSessions(() => services, { watch: false });
+    await services.widgetDev.resume();
+    const views = ((await call("GET", "/widget-dev/sessions")).body as { sessions: WidgetDevSessionView[] }).sessions;
+    expect(views.find((view) => view.sessionId === "wdev_unreadable")).toMatchObject({ status: "stopped", stopReason: "watch-failed" });
+  });
 });
 
 describe("the directory a widget dev session lists its builds in", () => {