Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 25 additions & 18 deletions apps/server/src/diagnostics/ProcessDiagnostics.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,11 @@ import * as ResourceAttribution from "../resourceTelemetry/ResourceAttribution.t
import * as ResourceTelemetry from "../resourceTelemetry/ResourceTelemetry.ts";
import * as ProcessDiagnostics from "./ProcessDiagnostics.ts";

// ProcessDiagnostics treats process.pid as the server, so the fixture PID is an
// offset from it. A fixed PID collides whenever the test worker is assigned it.
const SERVER_PID = process.pid;
const CHILD_PID = SERVER_PID + 1;

function makeNativeSnapshot(
processes: ResourceMonitorSnapshotEvent["processes"],
): ResourceMonitorSnapshotEvent {
Expand Down Expand Up @@ -66,7 +71,7 @@ describe("ProcessDiagnostics", () => {
Effect.gen(function* () {
const snapshot = makeNativeSnapshot([
{
pid: process.pid,
pid: SERVER_PID,
ppid: 1,
startTimeMs: 1_000,
runTimeMs: 60_000,
Expand All @@ -82,8 +87,8 @@ describe("ProcessDiagnostics", () => {
ioSemantics: "storage",
},
{
pid: 4_242,
ppid: process.pid,
pid: CHILD_PID,
ppid: SERVER_PID,
startTimeMs: 2_000,
runTimeMs: 4_000,
name: "agent",
Expand All @@ -106,7 +111,7 @@ describe("ProcessDiagnostics", () => {
return yield* processDiagnostics.read;
}).pipe(Effect.provide(layer));

expect(diagnostics.processes.map((process) => process.pid)).toEqual([4242]);
expect(diagnostics.processes.map((process) => process.pid)).toEqual([CHILD_PID]);
expect(diagnostics.processes[0]?.startTimeMs).toBe(2_000);
expect(diagnostics.processes[0]?.cpuPercent).toBe(1.5);
expect(diagnostics.processes[0]?.rssBytes).toBe(2_048);
Expand All @@ -122,7 +127,7 @@ describe("ProcessDiagnostics", () => {
const result = yield* Effect.service(ProcessDiagnostics.ProcessDiagnostics).pipe(
Effect.flatMap((processDiagnostics) =>
processDiagnostics.signal({
pid: 4_242,
pid: CHILD_PID,
startTimeMs: 2_000,
signal: "SIGINT",
}),
Expand All @@ -131,10 +136,12 @@ describe("ProcessDiagnostics", () => {
);

expect(result).toEqual({
pid: 4242,
pid: CHILD_PID,
signal: "SIGINT",
signaled: false,
message: Option.some("Process 4242 no longer matches the selected process identity."),
message: Option.some(
`Process ${CHILD_PID} no longer matches the selected process identity.`,
),
});
}),
);
Expand All @@ -143,8 +150,8 @@ describe("ProcessDiagnostics", () => {
Effect.gen(function* () {
const snapshot = makeNativeSnapshot([
{
pid: 4_242,
ppid: process.pid,
pid: CHILD_PID,
ppid: SERVER_PID,
startTimeMs: 2_000,
runTimeMs: 4_000,
name: "agent",
Expand Down Expand Up @@ -185,7 +192,7 @@ describe("ProcessDiagnostics", () => {
const result = yield* Effect.service(ProcessDiagnostics.ProcessDiagnostics).pipe(
Effect.flatMap((processDiagnostics) =>
processDiagnostics.signal({
pid: 4_242,
pid: CHILD_PID,
startTimeMs: 2_000,
signal: "SIGINT",
}),
Expand All @@ -194,11 +201,11 @@ describe("ProcessDiagnostics", () => {
);

expect(result).toEqual({
pid: 4_242,
pid: CHILD_PID,
signal: "SIGINT",
signaled: false,
message: Option.some(
"Could not refresh process 4242; refusing to signal a stale identity.",
`Could not refresh process ${CHILD_PID}; refusing to signal a stale identity.`,
),
});
}),
Expand All @@ -211,7 +218,7 @@ describe("ProcessDiagnostics", () => {
);
const snapshot = makeNativeSnapshot([
{
pid: 4_242,
pid: CHILD_PID,
ppid: 1,
startTimeMs: 2_000,
runTimeMs: 4_000,
Expand All @@ -233,7 +240,7 @@ describe("ProcessDiagnostics", () => {
type: "desktopTelemetry",
sequence: 1,
sampledAtUnixMs,
electronPid: 4_242,
electronPid: CHILD_PID,
power: {
source: "electron-main",
idle: "false",
Expand All @@ -249,7 +256,7 @@ describe("ProcessDiagnostics", () => {
speedLimitPercent: Option.none(),
electronProcesses: [
{
pid: 4_242,
pid: CHILD_PID,
creationTimeMs: 2_000,
type: "Browser",
name: "electron",
Expand All @@ -265,7 +272,7 @@ describe("ProcessDiagnostics", () => {
const result = yield* Effect.service(ProcessDiagnostics.ProcessDiagnostics).pipe(
Effect.flatMap((processDiagnostics) =>
processDiagnostics.signal({
pid: 4_242,
pid: CHILD_PID,
startTimeMs: 2_000,
signal: "SIGKILL",
}),
Expand All @@ -274,10 +281,10 @@ describe("ProcessDiagnostics", () => {
);

expect(result).toEqual({
pid: 4_242,
pid: CHILD_PID,
signal: "SIGKILL",
signaled: false,
message: Option.some("Process 4242 is not a signalable T3 backend descendant."),
message: Option.some(`Process ${CHILD_PID} is not a signalable T3 backend descendant.`),
});

const diagnostics = yield* Effect.service(ProcessDiagnostics.ProcessDiagnostics).pipe(
Expand Down
71 changes: 42 additions & 29 deletions apps/server/src/resourceTelemetry/ResourceTelemetry.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,15 @@ import * as NativeTelemetryClient from "./NativeTelemetryClient.ts";
import * as ResourceAttribution from "./ResourceAttribution.ts";
import * as ResourceTelemetry from "./ResourceTelemetry.ts";

// ResourceTelemetry classifies process.pid as the server, so fixture PIDs are
// offsets from it. Fixed PIDs collide whenever the test worker is assigned one.
const SERVER_PID = process.pid;
const CODEX_PID = SERVER_PID + 1;
const SHORT_LIVED_CHILD_PID = SERVER_PID + 2;
const ELECTRON_PID = SERVER_PID + 3;
const MONITOR_PID = SERVER_PID + 4;
const RESTARTED_MONITOR_PID = SERVER_PID + 5;

function processSample(
input: Partial<ResourceMonitorProcessSample> &
Pick<ResourceMonitorProcessSample, "pid" | "ppid" | "startTimeMs">,
Expand Down Expand Up @@ -50,31 +59,31 @@ function nativeSnapshot(input: {
}): ResourceMonitorSnapshotEvent {
const processes = [
processSample({
pid: process.pid,
pid: SERVER_PID,
ppid: 1,
startTimeMs: 100,
cpuTimeMs: input.sequence * 10,
}),
processSample({
pid: 4_242,
ppid: process.pid,
pid: CODEX_PID,
ppid: SERVER_PID,
startTimeMs: 200,
name: "codex",
command: "codex app-server",
cpuTimeMs: input.childCpuTimeMs,
ioWriteBytes: input.childWriteBytes,
}),
processSample({
pid: 5_000,
pid: ELECTRON_PID,
ppid: 1,
startTimeMs: 300,
name: "electron",
command: "electron",
cpuTimeMs: input.sequence * 20,
}),
processSample({
pid: 9_000,
ppid: process.pid,
pid: MONITOR_PID,
ppid: SERVER_PID,
startTimeMs: 400,
name: "t3-resource-monitor",
command: "t3-resource-monitor",
Expand Down Expand Up @@ -111,7 +120,7 @@ function desktopSnapshot(sampledAtUnixMs: number): DesktopHostTelemetrySnapshot
type: "desktopTelemetry",
sequence: 1,
sampledAtUnixMs,
electronPid: 5_000,
electronPid: ELECTRON_PID,
power: {
source: "electron-main",
idle: "false",
Expand All @@ -127,7 +136,7 @@ function desktopSnapshot(sampledAtUnixMs: number): DesktopHostTelemetrySnapshot
speedLimitPercent: Option.some(90),
electronProcesses: [
{
pid: 5_000,
pid: ELECTRON_PID,
creationTimeMs: 300,
type: "Browser",
name: "electron",
Expand All @@ -154,17 +163,17 @@ describe("ResourceTelemetry", () => {
childWriteBytes: index * 100,
});
const processes = sample.processes
.filter((entry) => entry.pid === process.pid || entry.pid === 4_242)
.filter((entry) => entry.pid === SERVER_PID || entry.pid === CODEX_PID)
.map((entry) =>
entry.pid === 4_242
entry.pid === CODEX_PID
? { ...entry, startTimeMs: index }
: { ...entry, ioWriteBytes: index * 100 },
);
if (index === 0 || index === 2) {
processes.push(
processSample({
pid: 4_243,
ppid: process.pid,
pid: SHORT_LIVED_CHILD_PID,
ppid: SERVER_PID,
startTimeMs: 500,
cpuTimeMs: index === 0 ? 100 : 350,
ioWriteBytes: index === 0 ? 1_000 : 5_000,
Expand Down Expand Up @@ -192,18 +201,20 @@ describe("ResourceTelemetry", () => {

expect(history.topProcesses).toHaveLength(sampleCount + 2);
expect(
history.topProcesses.find((entry) => entry.identity.pid === process.pid),
history.topProcesses.find((entry) => entry.identity.pid === SERVER_PID),
).toMatchObject({
sampleCount,
cpuTimeMs: (sampleCount - 1) * 10,
ioWriteBytes: (sampleCount - 1) * 100,
});
const children = history.topProcesses.filter((entry) => entry.identity.pid === 4_242);
const children = history.topProcesses.filter((entry) => entry.identity.pid === CODEX_PID);
expect(children).toHaveLength(sampleCount);
for (const child of children) {
expect(child).toMatchObject({ sampleCount: 1, cpuTimeMs: 0, ioWriteBytes: 0 });
}
expect(history.topProcesses.find((entry) => entry.identity.pid === 4_243)).toMatchObject({
expect(
history.topProcesses.find((entry) => entry.identity.pid === SHORT_LIVED_CHILD_PID),
).toMatchObject({
sampleCount: 2,
cpuTimeMs: 250,
ioWriteBytes: 4_000,
Expand Down Expand Up @@ -307,7 +318,7 @@ describe("ResourceTelemetry", () => {
sampledAtUnixMs,
childCpuTimeMs: 100,
childWriteBytes: 1_000,
externalProcesses: [{ pid: 5_000, startTimeMs: 300 }],
externalProcesses: [{ pid: ELECTRON_PID, startTimeMs: 300 }],
});
const desktop = {
...desktopSnapshot(sampledAtUnixMs),
Expand Down Expand Up @@ -336,9 +347,9 @@ describe("ResourceTelemetry", () => {
return yield* telemetry.refresh;
}).pipe(Effect.provide(layerTelemetry));

expect(snapshot.processes.find((entry) => entry.identity.pid === 5_000)?.category).toBe(
"electron-main",
);
expect(
snapshot.processes.find((entry) => entry.identity.pid === ELECTRON_PID)?.category,
).toBe("electron-main");
expect(snapshot.groups.electron.processStarts).toBe(1);
expect(snapshot.groups.backend.processStarts).toBe(3);
}),
Expand Down Expand Up @@ -574,7 +585,7 @@ describe("ResourceTelemetry", () => {
version: 3,
type: "hello",
sidecarVersion: "0.1.0",
sidecarPid: 9_000,
sidecarPid: MONITOR_PID,
platform: "test",
arch: "test",
capabilities: {
Expand Down Expand Up @@ -629,7 +640,9 @@ describe("ResourceTelemetry", () => {
const telemetry = yield* ResourceTelemetry.ResourceTelemetry;
const attribution = yield* ResourceAttribution.ResourceAttribution;

expect(yield* Ref.get(externalProcesses)).toEqual([{ pid: 5_000, startTimeMs: 300 }]);
expect(yield* Ref.get(externalProcesses)).toEqual([
{ pid: ELECTRON_PID, startTimeMs: 300 },
]);

yield* attribution.record({
component: "provider-event-log",
Expand Down Expand Up @@ -657,7 +670,7 @@ describe("ResourceTelemetry", () => {

yield* TestClock.adjust(Duration.seconds(1));
const second = yield* telemetry.refresh;
const codex = second.processes.find((entry) => entry.identity.pid === 4_242);
const codex = second.processes.find((entry) => entry.identity.pid === CODEX_PID);
expect(codex?.cpuPercent).toBe(25);
expect(codex?.ioWriteBytesPerSecond).toBe(4_000);
expect(second.groups.backend.ioWriteBytes).toBe(4_000);
Expand All @@ -671,13 +684,13 @@ describe("ResourceTelemetry", () => {
});
expect(history.retainedSampleCount).toBeGreaterThan(0);
expect(
history.topProcesses.find((entry) => entry.identity.pid === 4_242)?.sampleCount,
history.topProcesses.find((entry) => entry.identity.pid === CODEX_PID)?.sampleCount,
).toBe(2);
expect(history.topProcesses.find((entry) => entry.identity.pid === 4_242)?.cpuTimeMs).toBe(
250,
);
expect(
history.topProcesses.find((entry) => entry.identity.pid === 4_242)?.ioWriteBytes,
history.topProcesses.find((entry) => entry.identity.pid === CODEX_PID)?.cpuTimeMs,
).toBe(250);
expect(
history.topProcesses.find((entry) => entry.identity.pid === CODEX_PID)?.ioWriteBytes,
).toBe(4_000);
expect(history.buckets.reduce((total, bucket) => total + bucket.ioWriteBytes, 0)).toBe(
4_000,
Expand All @@ -691,14 +704,14 @@ describe("ResourceTelemetry", () => {
...current,
hello: Option.map(current.hello, (hello) => ({
...hello,
sidecarPid: 9_001,
sidecarPid: RESTARTED_MONITOR_PID,
})),
restartCount: 3,
}));
yield* TestClock.adjust(Duration.seconds(1));
const restarted = yield* telemetry.refresh;
expect(DateTime.toEpochMillis(restarted.readAt)).toBe(startedAt + 2_000);
expect(Option.getOrNull(restarted.health.sidecarPid)).toBe(9_001);
expect(Option.getOrNull(restarted.health.sidecarPid)).toBe(RESTARTED_MONITOR_PID);

yield* Ref.update(nativeHealth, (current) => ({
...current,
Expand Down
Loading