From c1d5e8738a05992e3d0939caf8af8b1633e98d35 Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Tue, 21 Jul 2026 13:37:33 +0400 Subject: [PATCH 1/4] test: make automatic restart recovery deterministic Fixes #123 --- tests/fixtures/fake-upstream.mjs | 4 ++++ tests/upstream-manager.test.ts | 11 ++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/fixtures/fake-upstream.mjs b/tests/fixtures/fake-upstream.mjs index ae30b47b..27134351 100644 --- a/tests/fixtures/fake-upstream.mjs +++ b/tests/fixtures/fake-upstream.mjs @@ -68,6 +68,7 @@ const failOnRestartPath = process.env.TEST_FAIL_ON_RESTART_PATH; const failListResourcesPath = process.env.TEST_FAIL_LIST_RESOURCES_PATH; const failListPromptsPath = process.env.TEST_FAIL_LIST_PROMPTS_PATH; const crashOnCallToolPath = process.env.TEST_CRASH_ON_CALL_TOOL_PATH; +const crashOnCallToolObservedPath = process.env.TEST_CRASH_ON_CALL_TOOL_OBSERVED_PATH; const crashAfterInitializedPath = process.env.TEST_CRASH_AFTER_INITIALIZED_PATH; const startCountPath = process.env.TEST_START_COUNT_PATH; const initializedPath = process.env.TEST_INITIALIZED_PATH; @@ -176,6 +177,9 @@ if (stderrMessage) { } } if (crashOnCallToolPath && existsSync(crashOnCallToolPath)) { + if (crashOnCallToolObservedPath) { + writeFileSync(crashOnCallToolObservedPath, "observed"); + } throw new Error("test upstream configured to stay unavailable after an abrupt exit"); } if (failOnRestartPath) { diff --git a/tests/upstream-manager.test.ts b/tests/upstream-manager.test.ts index d657aeba..0f4f2075 100644 --- a/tests/upstream-manager.test.ts +++ b/tests/upstream-manager.test.ts @@ -493,6 +493,9 @@ describe("upstream process manager", () => { it("automatically recovers a crashed profile after a bounded backoff", async () => { const directory = await mkdtemp(join(tmpdir(), "miftah-auto-restart-")); const crashPath = join(directory, "crash"); + const crashObservedPath = join(directory, "crash-observed"); + const restartGatePath = join(directory, "restart-gate"); + const restartReadyPath = join(directory, "restart-ready"); const startCountPath = join(directory, "starts"); await writeFile(startCountPath, ""); const manager = new UpstreamProcessManager( @@ -502,6 +505,9 @@ describe("upstream process manager", () => { args: [fixture], env: { TEST_CRASH_ON_CALL_TOOL_PATH: crashPath, + TEST_CRASH_ON_CALL_TOOL_OBSERVED_PATH: crashObservedPath, + TEST_HANG_ON_START_PATH: restartGatePath, + TEST_HANG_ON_START_READY_PATH: restartReadyPath, TEST_START_COUNT_PATH: startCountPath } }, @@ -511,9 +517,11 @@ describe("upstream process manager", () => { try { const session = await manager.get("work"); - await writeFile(crashPath, "crash"); + await Promise.all([writeFile(crashPath, "crash"), writeFile(restartGatePath, "restart")]); await expect(session.callTool({ name: "whoami", arguments: {} })).rejects.toThrow(); + await waitFor(() => existsSync(restartReadyPath), Boolean); await unlink(crashPath); + await unlink(restartGatePath); await waitFor(() => countStarts(startCountPath), (count) => count === 2); const recovered = await waitFor( @@ -522,6 +530,7 @@ describe("upstream process manager", () => { ); if (!recovered) throw new Error("Expected recovered health"); expect(recovered.restartCount).toBe(1); + expect(existsSync(crashObservedPath)).toBe(false); await expect((await manager.get("work")).callTool({ name: "whoami", arguments: {} })).resolves.toMatchObject({ content: [{ type: "text", text: "unknown" }] }); From cbf6939a74ec14724bdaa1703cd2da7f2a2b8f1b Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Tue, 21 Jul 2026 21:19:27 +0400 Subject: [PATCH 2/4] test: harden automatic recovery control --- tests/upstream-manager.test.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/upstream-manager.test.ts b/tests/upstream-manager.test.ts index 0f4f2075..158ddc65 100644 --- a/tests/upstream-manager.test.ts +++ b/tests/upstream-manager.test.ts @@ -512,7 +512,7 @@ describe("upstream process manager", () => { } }, { work: {} }, - { startupTimeoutMs: 1_000, restartOnCrash: true, maxRestarts: 2 } + { startupTimeoutMs: 5_000, restartOnCrash: true, maxRestarts: 2 } ); try { @@ -520,6 +520,7 @@ describe("upstream process manager", () => { await Promise.all([writeFile(crashPath, "crash"), writeFile(restartGatePath, "restart")]); await expect(session.callTool({ name: "whoami", arguments: {} })).rejects.toThrow(); await waitFor(() => existsSync(restartReadyPath), Boolean); + await delay(1_100); await unlink(crashPath); await unlink(restartGatePath); @@ -610,6 +611,7 @@ describe("upstream process manager", () => { it("releases a profile capacity reservation after a failed startup", async () => { const directory = await mkdtemp(join(tmpdir(), "miftah-failed-start-capacity-")); const failurePath = join(directory, "fail"); + const crashObservedPath = join(directory, "crash-observed"); await writeFile(failurePath, "fail"); const manager = new UpstreamProcessManager( { @@ -618,7 +620,12 @@ describe("upstream process manager", () => { args: [fixture] }, { - work: { env: { TEST_CRASH_ON_CALL_TOOL_PATH: failurePath } }, + work: { + env: { + TEST_CRASH_ON_CALL_TOOL_PATH: failurePath, + TEST_CRASH_ON_CALL_TOOL_OBSERVED_PATH: crashObservedPath + } + }, personal: {} }, { startupTimeoutMs: 1_000, maxConcurrentProfiles: 1 } @@ -626,6 +633,7 @@ describe("upstream process manager", () => { try { await expect(manager.get("work")).rejects.toMatchObject({ code: "UPSTREAM_INIT_FAILED" }); + expect(existsSync(crashObservedPath)).toBe(true); await expect((await manager.get("personal")).listTools()).resolves.toMatchObject({ tools: expect.any(Array) }); } finally { await manager.close(); From fe7394b971aa76c834f6c0c0b63ace3f6266af24 Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Tue, 21 Jul 2026 21:29:49 +0400 Subject: [PATCH 3/4] test: remove timing race from recovery check --- tests/upstream-manager.test.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/upstream-manager.test.ts b/tests/upstream-manager.test.ts index 158ddc65..c9607326 100644 --- a/tests/upstream-manager.test.ts +++ b/tests/upstream-manager.test.ts @@ -493,7 +493,7 @@ describe("upstream process manager", () => { it("automatically recovers a crashed profile after a bounded backoff", async () => { const directory = await mkdtemp(join(tmpdir(), "miftah-auto-restart-")); const crashPath = join(directory, "crash"); - const crashObservedPath = join(directory, "crash-observed"); + const recoveryCrashObservedPath = join(directory, "recovery-crash-observed"); const restartGatePath = join(directory, "restart-gate"); const restartReadyPath = join(directory, "restart-ready"); const startCountPath = join(directory, "starts"); @@ -505,7 +505,7 @@ describe("upstream process manager", () => { args: [fixture], env: { TEST_CRASH_ON_CALL_TOOL_PATH: crashPath, - TEST_CRASH_ON_CALL_TOOL_OBSERVED_PATH: crashObservedPath, + TEST_CRASH_ON_CALL_TOOL_OBSERVED_PATH: recoveryCrashObservedPath, TEST_HANG_ON_START_PATH: restartGatePath, TEST_HANG_ON_START_READY_PATH: restartReadyPath, TEST_START_COUNT_PATH: startCountPath @@ -519,8 +519,14 @@ describe("upstream process manager", () => { const session = await manager.get("work"); await Promise.all([writeFile(crashPath, "crash"), writeFile(restartGatePath, "restart")]); await expect(session.callTool({ name: "whoami", arguments: {} })).rejects.toThrow(); + expect(existsSync(recoveryCrashObservedPath)).toBe(false); await waitFor(() => existsSync(restartReadyPath), Boolean); - await delay(1_100); + const restarting = await waitFor( + () => manager.listHealth().find((health) => health.profile === "work"), + (health) => health?.processState === "starting" && health.restartCount === 1 + ); + expect(restarting).toMatchObject({ processState: "starting", restartCount: 1 }); + expect(await countStarts(startCountPath)).toBe(2); await unlink(crashPath); await unlink(restartGatePath); @@ -531,7 +537,7 @@ describe("upstream process manager", () => { ); if (!recovered) throw new Error("Expected recovered health"); expect(recovered.restartCount).toBe(1); - expect(existsSync(crashObservedPath)).toBe(false); + expect(existsSync(recoveryCrashObservedPath)).toBe(false); await expect((await manager.get("work")).callTool({ name: "whoami", arguments: {} })).resolves.toMatchObject({ content: [{ type: "text", text: "unknown" }] }); From 6958b971001f1bd67d44dc18737e49570dd308b1 Mon Sep 17 00:00:00 2001 From: mohammed naji Date: Tue, 21 Jul 2026 21:32:22 +0400 Subject: [PATCH 4/4] test: wait for restart entry before releasing gate --- tests/upstream-manager.test.ts | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tests/upstream-manager.test.ts b/tests/upstream-manager.test.ts index c9607326..39691806 100644 --- a/tests/upstream-manager.test.ts +++ b/tests/upstream-manager.test.ts @@ -520,17 +520,11 @@ describe("upstream process manager", () => { await Promise.all([writeFile(crashPath, "crash"), writeFile(restartGatePath, "restart")]); await expect(session.callTool({ name: "whoami", arguments: {} })).rejects.toThrow(); expect(existsSync(recoveryCrashObservedPath)).toBe(false); + await waitFor(() => countStarts(startCountPath), (count) => count === 2); await waitFor(() => existsSync(restartReadyPath), Boolean); - const restarting = await waitFor( - () => manager.listHealth().find((health) => health.profile === "work"), - (health) => health?.processState === "starting" && health.restartCount === 1 - ); - expect(restarting).toMatchObject({ processState: "starting", restartCount: 1 }); - expect(await countStarts(startCountPath)).toBe(2); await unlink(crashPath); await unlink(restartGatePath); - await waitFor(() => countStarts(startCountPath), (count) => count === 2); const recovered = await waitFor( () => manager.listHealth().find((health) => health.profile === "work"), (health) => health?.processState === "running"