Core: Recover from a docgen worker death instead of going dark for the session

code/core/src/shared/open-service/services/docgen/worker/docgen-worker-client.test.ts

@@ -215,4 +215,52 @@ describe('DocgenWorkerClient.extract', () => {
 
     await expect(promise).rejects.toThrow(/exited unexpectedly/);
   });
+
+  it('names the death in every extract that follows it', async () => {
+    const { createDocgenWorkerClient } = await loadModule();
+    const client = createDocgenWorkerClient(DESCRIPTORS)!;
+
+    const first = client.extract({ id: 'x' } as any);
+    ackInit(fakeWorkers[0]);
+    await Promise.resolve();
+    fakeWorkers[0].emit('error', new Error('worker ran out of memory'));
+    await expect(first).rejects.toThrow('worker ran out of memory');
+
+    // Never served a request, so the next extract reuses the corpse rather than respawning into
+    // the same failure - and says what that failure was.
+    await expect(client.extract({ id: 'y' } as any)).rejects.toThrow(
+      'docgen worker is no longer running: worker ran out of memory'
+    );
+    expect(fakeWorkers).toHaveLength(1);
+  });
+
+  it('spawns a replacement once a dead worker had served requests', async () => {
+    const { createDocgenWorkerClient } = await loadModule();
+    const client = createDocgenWorkerClient(DESCRIPTORS)!;
+
+    const first = client.extract({ id: 'x' } as any);
+    ackInit(fakeWorkers[0]);
+    await Promise.resolve();
+    const extractMsg = fakeWorkers[0].posted.find((m) => m.type === 'extract') as { id: number };
+    fakeWorkers[0].emit('message', {
+      type: 'extract',
+      id: extractMsg.id,
+      payload: { id: 'x', name: 'X', path: './x.stories.ts', jsDocTags: {} },
+    } satisfies DocgenWorkerResponse);
+    await expect(first).resolves.toMatchObject({ id: 'x' });
+
+    fakeWorkers[0].emit('error', new Error('worker ran out of memory'));
+
+    const second = client.extract({ id: 'y' } as any);
+    expect(fakeWorkers).toHaveLength(2);
+    ackInit(fakeWorkers[1]);
+    await Promise.resolve();
+    const retryMsg = fakeWorkers[1].posted.find((m) => m.type === 'extract') as { id: number };
+    fakeWorkers[1].emit('message', {
+      type: 'extract',
+      id: retryMsg.id,
+      payload: { id: 'y', name: 'Y', path: './y.stories.ts', jsDocTags: {} },
+    } satisfies DocgenWorkerResponse);
+    await expect(second).resolves.toMatchObject({ id: 'y' });
+  });
 });

code/core/src/shared/open-service/services/docgen/worker/docgen-worker-client.ts

@@ -44,7 +44,8 @@ class DocgenWorker implements DocgenWorkerClient {
   private readonly ready: Promise<void>;
   private rejectReady: (error: unknown) => void = () => undefined;
   private nextId = 0;
-  private dead = false;
+  private death: Error | undefined;
+  private served = false;
 
   constructor(
     scriptPath: string,
@@ -55,7 +56,7 @@ class DocgenWorker implements DocgenWorkerClient {
     this.worker.on('message', (msg: DocgenWorkerResponse) => this.handleMessage(msg));
     this.worker.on('error', (error) => this.fail(error));
     this.worker.on('exit', (code) => {
-      if (!this.dead) {
+      if (!this.death) {
         this.fail(new Error(`docgen worker exited unexpectedly with code ${code}`));
       }
     });
@@ -84,9 +85,24 @@ class DocgenWorker implements DocgenWorkerClient {
     this.post({ type: 'init', descriptors, logLevel: logger.getLogLevel() });
   }
 
+  /**
+   * Whether a fresh worker would be worth spawning in this one's place.
+   *
+   * Only once it has served an extract: a worker that died without ever answering died of something
+   * a replacement would hit again, and respawning would rebuild a multi-second TypeScript program
+   * per request to fail the same way.
+   */
+  get replaceable(): boolean {
+    return this.death !== undefined && this.served;
+  }
+
   async extract(entry: IndexEntry): Promise<DocgenPayload | undefined> {
-    if (this.dead) {
-      throw new Error('docgen worker is no longer running');
+    if (this.death) {
+      // Carries the death forward: the crash itself is one event, but every later extract fails on
+      // it, so a bare "no longer running" is the only thing anyone downstream ever sees.
+      throw new Error(`docgen worker is no longer running: ${this.death.message}`, {
+        cause: this.death,
+      });
     }
     await this.ready;
     return new Promise<DocgenPayload | undefined>((resolve, reject) => {
@@ -130,6 +146,7 @@ class DocgenWorker implements DocgenWorkerClient {
       clearTimeout(pending.timer);
     }
     this.pending.delete(msg.id);
+    this.served = true;
     this.keepProcessAliveWhileBusy();
     if (msg.error) {
       pending.reject(errorLikeToError(msg.error));
@@ -139,11 +156,11 @@ class DocgenWorker implements DocgenWorkerClient {
   }
 
   private fail(error: Error): void {
-    if (this.dead) {
+    if (this.death) {
       return;
     }
-    logger.debug(`docgen worker failure: ${error.message}`);
-    this.dead = true;
+    logger.warn(`docgen worker died, so component docs are unavailable: ${error.message}`);
+    this.death = error;
     // Without this, an `extract` awaiting `ready` hangs when the worker dies before `init`.
     this.rejectReady(error);
     this.rejectAllPending(error);
@@ -188,6 +205,10 @@ export function createDocgenWorkerClient(
 
   return {
     async extract(entry) {
+      if (worker?.replaceable) {
+        logger.debug('docgen worker died after serving requests; spawning a replacement');
+        worker = undefined;
+      }
       worker ??= new DocgenWorker(scriptPath, descriptors);
       return worker.extract(entry);
     },