Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions apps/desktop/src/ipc/methods/preview.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,24 @@ describe("preview IPC methods", () => {
}),
);

effectIt.effect("returns typed click outcomes across the preview IPC handler", () =>
Effect.gen(function* () {
const result = {
_tag: "NotSent",
reason: "target-disabled",
} as const;
const manager = PreviewManager.PreviewManager.of({
automationClick: () => Effect.succeed(result),
} as unknown as PreviewManager.PreviewManager["Service"]);

expect(
yield* PreviewIpc.automationClick
.handler({ tabId: "tab-1", input: { locator: "role=button[name='Send']" } })
.pipe(Effect.provideService(PreviewManager.PreviewManager, manager)),
).toEqual(result);
}),
);

it("keeps the public automation status tab id limit", () => {
const encode = Schema.encodeUnknownSync(PreviewAutomationStatus);
const tabId = "t".repeat(129);
Expand Down
5 changes: 3 additions & 2 deletions apps/desktop/src/ipc/methods/preview.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import {
DesktopPreviewAnnotationThemeInputSchema,
DesktopPreviewArtifactInputSchema,
DesktopPreviewAutomationClickInputSchema,
DesktopPreviewAutomationClickResultSchema,
DesktopPreviewAutomationEvaluateInputSchema,
DesktopPreviewAutomationPressInputSchema,
DesktopPreviewAutomationScrollInputSchema,
Expand Down Expand Up @@ -302,10 +303,10 @@ export const automationSnapshot = DesktopIpc.makeIpcMethod({
export const automationClick = DesktopIpc.makeIpcMethod({
channel: IpcChannels.PREVIEW_AUTOMATION_CLICK_CHANNEL,
payload: DesktopPreviewAutomationClickInputSchema,
result: Schema.Void,
result: DesktopPreviewAutomationClickResultSchema,
handler: Effect.fn("desktop.ipc.preview.automationClick")(function* ({ tabId, input }) {
const manager = yield* PreviewManager.PreviewManager;
yield* manager.automationClick(tabId, input);
return yield* manager.automationClick(tabId, input);
}),
});

Expand Down
129 changes: 129 additions & 0 deletions apps/desktop/src/preview/Manager.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { it as effectIt } from "@effect/vitest";
import { DESKTOP_PREVIEW_RECORDING_CAPTURE_TRIGGER } from "@t3tools/contracts";
import type { DesktopPreviewRecordingFrame } from "@t3tools/contracts";
import { HostProcessPlatform } from "@t3tools/shared/hostProcess";
import * as NodeVM from "node:vm";
import * as Cause from "effect/Cause";
import * as Deferred from "effect/Deferred";
import * as Effect from "effect/Effect";
Expand Down Expand Up @@ -3747,4 +3748,132 @@ describe("Preview automation diagnostics", () => {
expect(JSON.stringify(error)).not.toContain(selector);
expect("locator" in error).toBe(false);
});

it("does not invent an ambiguous target match count", () => {
const error = new PreviewManager.PreviewAutomationTargetLookupError({
operation: "click",
tabId: "tab_1",
selectorKind: "locator",
selectorLength: 12,
failureKind: "ambiguous",
});

expect(error.message).toBe(
"Preview automation click matched multiple elements for locator (12 characters)",
);
});

effectIt.effect("returns typed click lookup failures without dispatching input", () =>
withManager((manager) =>
Effect.gen(function* () {
const selector = "role=button[name='target-secret']";
let lookupResult: unknown = { notFound: true, failureKind: "missing" };
let lookupExpression = "";
const sendCommand = vi.fn(async (method: string, params?: Record<string, unknown>) => {
if (method !== "Runtime.evaluate") return undefined;
const expression = String(params?.["expression"] ?? "");
if (expression.includes("const parsed = injected.parseSelector")) {
lookupExpression = expression;
return { result: { value: lookupResult } };
}
return { result: { value: true } };
});
fromId.mockReturnValue({
...makeTestPreviewWebContents(
vi.fn(async () => ({
toJPEG: () => Buffer.from("unused-click-frame"),
toPNG: () => Buffer.from("unused-click-frame"),
getSize: () => ({ width: 1280, height: 720 }),
})),
),
isDevToolsOpened: () => false,
debugger: {
isAttached: () => false,
attach: vi.fn(),
sendCommand,
on: vi.fn(),
off: vi.fn(),
},
} as never);

yield* manager.createTab("tab_lookup");
yield* manager.registerWebview("tab_lookup", 42);

const missing = yield* manager.automationClick("tab_lookup", { locator: selector });
lookupResult = { notFound: true, failureKind: "hidden" };
const hidden = yield* manager.automationClick("tab_lookup", { locator: selector });
lookupResult = { notFound: true, failureKind: "disabled" };
const disabled = yield* manager.automationClick("tab_lookup", { locator: selector });
lookupResult = { notFound: true, failureKind: "ambiguous", matchCount: 3 };
const ambiguous = yield* manager.automationClick("tab_lookup", { locator: selector });
const snapshot = yield* manager.automationSnapshot("tab_lookup");

expect([missing, hidden, disabled, ambiguous]).toEqual([
{ _tag: "NotSent", reason: "target-missing" },
{ _tag: "NotSent", reason: "target-hidden" },
{ _tag: "NotSent", reason: "target-disabled" },
{ _tag: "NotSent", reason: "target-ambiguous", matchCount: 3 },
]);
expect(snapshot.actionTimeline.filter((action) => action.action === "click")).toEqual([
expect.objectContaining({ status: "failed" }),
expect.objectContaining({ status: "failed" }),
expect.objectContaining({ status: "failed" }),
expect.objectContaining({ status: "failed" }),
]);
expect(sendCommand).not.toHaveBeenCalledWith("Input.dispatchMouseEvent", expect.anything());

const element = {
scrollIntoView: vi.fn(),
getBoundingClientRect: () => ({ left: 20, top: 10, width: 40, height: 20 }),
};
const runLookup = (
matches: ReadonlyArray<typeof element>,
state: { readonly visible: boolean; readonly enabled: boolean },
) => {
const querySelectorAll = vi.fn(() => matches);
const elementState = vi.fn((_element: typeof element, name: keyof typeof state) => ({
matches: state[name],
}));
const result = NodeVM.runInNewContext(lookupExpression, {
document: {},
globalThis: {
__t3PlaywrightInjected: {
parseSelector: vi.fn(() => ({ parts: [] })),
querySelectorAll,
elementState,
},
},
});
return { result, querySelectorAll, elementState };
};

const missingLookup = runLookup([], { visible: true, enabled: true });
expect(missingLookup.result).toEqual({ notFound: true, failureKind: "missing" });
expect(missingLookup.querySelectorAll).toHaveBeenCalledOnce();

const ambiguousLookup = runLookup([element, element, element], {
visible: true,
enabled: true,
});
expect(ambiguousLookup.result).toEqual({
notFound: true,
failureKind: "ambiguous",
matchCount: 3,
});
expect(ambiguousLookup.querySelectorAll).toHaveBeenCalledOnce();

const hiddenLookup = runLookup([element], { visible: false, enabled: true });
expect(hiddenLookup.result).toEqual({ notFound: true, failureKind: "hidden" });
expect(hiddenLookup.elementState).toHaveBeenCalledWith(element, "visible");

const disabledLookup = runLookup([element], { visible: true, enabled: false });
expect(disabledLookup.result).toEqual({ notFound: true, failureKind: "disabled" });
expect(disabledLookup.elementState).toHaveBeenCalledWith(element, "enabled");

const visibleLookup = runLookup([element], { visible: true, enabled: true });
expect(visibleLookup.result).toEqual({ x: 40, y: 20 });
expect(element.scrollIntoView).toHaveBeenCalledWith({ block: "center", inline: "center" });
}),
),
);
});
88 changes: 80 additions & 8 deletions apps/desktop/src/preview/Manager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import { DESKTOP_PREVIEW_RECORDING_CAPTURE_TRIGGER } from "@t3tools/contracts";
import type {
DesktopPreviewAnnotationTheme,
DesktopPreviewAutomationClickResult,
DesktopPreviewAutomationStatus,
DesktopPreviewColorScheme,
DesktopPreviewFavicon,
Expand Down Expand Up @@ -3541,19 +3542,34 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
locator,
);
const point = yield* evaluateWithDebugger<
{ x: number; y: number } | { invalidSelector: true; message: string } | { notFound: true }
| { x: number; y: number }
| { invalidSelector: true; message: string }
| {
notFound: true;
failureKind: "missing" | "hidden" | "disabled";
}
| {
notFound: true;
failureKind: "ambiguous";
matchCount: number;
}
>(
tabId,
send,
`(() => {
try {
const injected = globalThis.__t3PlaywrightInjected;
const parsed = injected.parseSelector(${locatorJson});
const element = injected.querySelector(parsed, document, true);
if (!element) return { notFound: true };
const matches = injected.querySelectorAll(parsed, document);
if (matches.length === 0) return { notFound: true, failureKind: "missing" };
if (matches.length > 1) {
return { notFound: true, failureKind: "ambiguous", matchCount: matches.length };
}
const element = matches[0];
const visible = injected.elementState(element, "visible");
const enabled = injected.elementState(element, "enabled");
if (!visible.matches || !enabled.matches) return { notFound: true };
if (!visible.matches) return { notFound: true, failureKind: "hidden" };
if (!enabled.matches) return { notFound: true, failureKind: "disabled" };
element.scrollIntoView({ block: "center", inline: "center" });
const rect = element.getBoundingClientRect();
return { x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 };
Expand All @@ -3573,10 +3589,12 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
});
}
if ("notFound" in point) {
return yield* new PreviewAutomationTargetNotFoundError({
return yield* new PreviewAutomationTargetLookupError({
operation: "click",
tabId,
...automationSelectorDiagnostics(input),
failureKind: point.failureKind,
...(point.failureKind === "ambiguous" ? { matchCount: point.matchCount } : {}),
});
}
return point;
Expand Down Expand Up @@ -3655,8 +3673,33 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
input: PreviewAutomationClickInput,
) {
const wc = yield* requireWebContents(tabId);
yield* withControlSession(tabId, wc, "click", (send) =>
performAutomationClick(tabId, input, send),
return yield* Effect.gen(function* () {
yield* withControlSession(tabId, wc, "click", (send) =>
performAutomationClick(tabId, input, send),
);
return { _tag: "Dispatched" } satisfies DesktopPreviewAutomationClickResult;
}).pipe(
Effect.catchTags({
PreviewAutomationTargetLookupError: (
error,
): Effect.Effect<
DesktopPreviewAutomationClickResult,
PreviewAutomationTargetLookupError
> => {
if (error.failureKind === "ambiguous") {
if (error.matchCount === undefined) return Effect.fail(error);
return Effect.succeed<DesktopPreviewAutomationClickResult>({
_tag: "NotSent",
reason: "target-ambiguous",
matchCount: error.matchCount,
});
}
return Effect.succeed<DesktopPreviewAutomationClickResult>({
_tag: "NotSent",
reason: `target-${error.failureKind}`,
});
},
}),
);
});

Expand Down Expand Up @@ -4269,6 +4312,34 @@ export class PreviewAutomationTargetNotFoundError extends Schema.TaggedErrorClas
}
}

export class PreviewAutomationTargetLookupError extends Schema.TaggedErrorClass<PreviewAutomationTargetLookupError>()(
"PreviewAutomationTargetLookupError",
{
operation: Schema.String,
tabId: Schema.String,
selectorKind: PreviewAutomationSelectorKind,
selectorLength: Schema.optionalKey(Schema.Number),
failureKind: Schema.Literals(["missing", "hidden", "disabled", "ambiguous"]),
matchCount: Schema.optionalKey(Schema.Int.check(Schema.isGreaterThan(0))),
},
) {
override get message(): string {
const target = previewAutomationTargetLabel(this.selectorKind, this.selectorLength);
if (this.failureKind === "hidden") {
return `Preview automation ${this.operation} found ${target}, but it is not visible`;
}
if (this.failureKind === "disabled") {
return `Preview automation ${this.operation} found ${target}, but it is disabled`;
}
if (this.failureKind === "ambiguous") {
return this.matchCount === undefined
? `Preview automation ${this.operation} matched multiple elements for ${target}`
: `Preview automation ${this.operation} matched ${this.matchCount} elements for ${target}`;
}
return `Preview automation ${this.operation} could not find ${target}`;
}
}

export class PreviewAutomationTargetNotEditableError extends Schema.TaggedErrorClass<PreviewAutomationTargetNotEditableError>()(
"PreviewAutomationTargetNotEditableError",
{
Expand Down Expand Up @@ -4387,6 +4458,7 @@ export const PreviewManagerError = Schema.Union([
PreviewAutomationDebuggerAttachedError,
PreviewAutomationEvaluationError,
PreviewAutomationTargetNotFoundError,
PreviewAutomationTargetLookupError,
PreviewAutomationTargetNotEditableError,
PreviewAutomationCoordinatesOutsideViewportError,
PreviewAutomationInvalidSelectorError,
Expand Down Expand Up @@ -4473,7 +4545,7 @@ export class PreviewManager extends Context.Service<
readonly automationClick: (
tabId: string,
input: PreviewAutomationClickInput,
) => Effect.Effect<void, PreviewManagerError>;
) => Effect.Effect<DesktopPreviewAutomationClickResult, PreviewManagerError>;
readonly automationType: (
tabId: string,
input: PreviewAutomationTypeInput,
Expand Down
45 changes: 45 additions & 0 deletions apps/server/src/mcp/McpHttpServer.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,51 @@ it.effect("returns bounded structural preview snapshot failures", () =>
).pipe(Effect.provide(TestLayer)),
);

it.effect("returns a typed click lookup reason through the MCP tool", () =>
Effect.scoped(
Effect.gen(function* () {
const locator = "role=button[name='request-secret']";
const server = yield* McpServer.McpServer;
const broker = yield* PreviewAutomationBroker.PreviewAutomationBroker;
const events = yield* broker.connect({
clientId: "mcp-click-failure-client",
environmentId,
});
yield* Stream.runForEach(events, (event) =>
event.type === "connected"
? Effect.void
: broker.respond({
clientId: "mcp-click-failure-client",
connectionId: event.connectionId,
requestId: event.request.requestId,
ok: false,
error: {
_tag: "PreviewAutomationTargetLookupError",
message: "The preview click target is disabled.",
detail: { failureKind: "disabled" },
},
}),
).pipe(Effect.forkScoped);
yield* Effect.yieldNow;

const result = yield* server
.callTool({ name: "preview_click", arguments: { locator } })
.pipe(
Effect.provideService(McpInvocationContext.McpInvocationContext, invocation),
Effect.provideService(McpSchema.McpServerClient, client),
);

expect(result.isError).toBe(true);
expect(result.content).toEqual([
{
type: "text",
text: `Preview automation click found locator (${locator.length} characters), but it is disabled.`,
},
]);
}),
).pipe(Effect.provide(TestLayer)),
);

it.effect("terminates HTTP MCP sessions with DELETE", () =>
Effect.scoped(
Effect.gen(function* () {
Expand Down
Loading
Loading