From 05cc4ac4bdf893825a9c9f6d6ca68024e1f2dba1 Mon Sep 17 00:00:00 2001 From: Vellum Assistant Date: Wed, 25 Feb 2026 01:54:39 -0500 Subject: [PATCH 1/2] feat: add recording intent detection and standalone routing with feature flag Co-Authored-By: Claude --- assistant/src/config/core-schema.ts | 3 + assistant/src/config/defaults.ts | 1 + assistant/src/config/schema.ts | 1 + assistant/src/daemon/handlers/misc.ts | 41 +++++++++++ assistant/src/daemon/handlers/sessions.ts | 23 ++++++ assistant/src/daemon/recording-intent.ts | 86 +++++++++++++++++++++++ 6 files changed, 155 insertions(+) create mode 100644 assistant/src/daemon/recording-intent.ts diff --git a/assistant/src/config/core-schema.ts b/assistant/src/config/core-schema.ts index bde0f7d8738..8fabca34a10 100644 --- a/assistant/src/config/core-schema.ts +++ b/assistant/src/config/core-schema.ts @@ -291,6 +291,9 @@ export const DaemonConfigSchema = z.object({ .int('daemon.titleGenerationMaxTokens must be an integer') .positive('daemon.titleGenerationMaxTokens must be a positive integer') .default(30), + standaloneRecording: z + .boolean({ error: 'daemon.standaloneRecording must be a boolean' }) + .default(true), }); export type TimeoutConfig = z.infer; diff --git a/assistant/src/config/defaults.ts b/assistant/src/config/defaults.ts index a1890f20695..b26d24941f0 100644 --- a/assistant/src/config/defaults.ts +++ b/assistant/src/config/defaults.ts @@ -286,6 +286,7 @@ export const DEFAULT_CONFIG: AssistantConfig = { stopTimeoutMs: 5000, sigkillGracePeriodMs: 2000, titleGenerationMaxTokens: 30, + standaloneRecording: true, }, notifications: { enabled: false, diff --git a/assistant/src/config/schema.ts b/assistant/src/config/schema.ts index 5be97b642ce..ccdf09eb59d 100644 --- a/assistant/src/config/schema.ts +++ b/assistant/src/config/schema.ts @@ -445,6 +445,7 @@ export const AssistantConfigSchema = z.object({ stopTimeoutMs: 5000, sigkillGracePeriodMs: 2000, titleGenerationMaxTokens: 30, + standaloneRecording: true, }), notifications: NotificationsConfigSchema.default({ enabled: false, diff --git a/assistant/src/daemon/handlers/misc.ts b/assistant/src/daemon/handlers/misc.ts index 53640c79922..d9d4e6ad197 100644 --- a/assistant/src/daemon/handlers/misc.ts +++ b/assistant/src/daemon/handlers/misc.ts @@ -10,6 +10,9 @@ import { checkIngressForSecrets } from '../../security/secret-ingress.js'; import { parseSlashCandidate } from '../../skills/slash-commands.js'; import { classifySessionError, buildSessionErrorMessage } from '../session-error.js'; import { resolveBlobPath, deleteBlob, isValidBlobId } from '../ipc-blob-store.js'; +import { getConfig } from '../../config/loader.js'; +import { isRecordingOnly, detectStopRecordingIntent } from '../recording-intent.js'; +import { handleRecordingStart, handleRecordingStop } from './recording.js'; import type { TaskSubmit, SuggestionRequest, @@ -63,6 +66,44 @@ export async function handleTaskSubmit( return; } + // ── Standalone recording intent interception ────────────────────────── + // Intercept recording-only and stop-recording prompts before they reach + // the classifier. This prevents "record my screen" from creating a CU + // session and routes it to the standalone recording flow instead. + const config = getConfig(); + if (config.daemon.standaloneRecording) { + if (detectStopRecordingIntent(msg.task)) { + // Find the active session for this socket so we can resolve the + // conversation that owns the recording. + const activeSessionId = ctx.socketToSession.get(socket); + if (activeSessionId) { + handleRecordingStop(activeSessionId, ctx); + } + rlog.info('Recording stop intent intercepted'); + ctx.send(socket, { type: 'assistant_text_delta', text: 'Stopping the recording.' }); + ctx.send(socket, { type: 'message_complete' }); + return; + } + + if (isRecordingOnly(msg.task)) { + // Create a conversation so the recording can be attached later (M4). + const conversation = conversationStore.createConversation(msg.task); + ctx.socketToSession.set(socket, conversation.id); + + handleRecordingStart(conversation.id, { promptForSource: true }, socket, ctx); + + ctx.send(socket, { + type: 'task_routed', + sessionId: conversation.id, + interactionType: 'text_qa', + }); + rlog.info({ sessionId: conversation.id }, 'Recording-only intent intercepted — routed to standalone recording'); + ctx.send(socket, { type: 'assistant_text_delta', text: 'Starting screen recording.' }); + ctx.send(socket, { type: 'message_complete' }); + return; + } + } + // Slash candidates always route to text_qa — bypass classifier const slashCandidate = parseSlashCandidate(msg.task); const interactionType = slashCandidate.kind === 'candidate' diff --git a/assistant/src/daemon/handlers/sessions.ts b/assistant/src/daemon/handlers/sessions.ts index e29f4410c81..8a8d37ae1ad 100644 --- a/assistant/src/daemon/handlers/sessions.ts +++ b/assistant/src/daemon/handlers/sessions.ts @@ -10,6 +10,8 @@ import { getAttachmentsForMessage, setAttachmentThumbnail } from '../../memory/a import { generateVideoThumbnail } from '../video-thumbnail.js'; import type { UserMessageAttachment } from '../ipc-contract.js'; import { normalizeThreadType } from '../ipc-protocol.js'; +import { isRecordingOnly, detectStopRecordingIntent } from '../recording-intent.js'; +import { handleRecordingStart, handleRecordingStop } from './recording.js'; import type { UserMessage, ConfirmationResponse, @@ -85,6 +87,27 @@ export async function handleUserMessage( attributes: { source: 'user_message' }, }); + // ── Standalone recording intent interception ────────────────────────── + const config = getConfig(); + const messageText = msg.content ?? ''; + if (config.daemon.standaloneRecording && messageText) { + if (detectStopRecordingIntent(messageText)) { + handleRecordingStop(msg.sessionId, ctx); + rlog.info('Recording stop intent intercepted in user_message'); + ctx.send(socket, { type: 'assistant_text_delta', text: 'Stopping the recording.' }); + ctx.send(socket, { type: 'message_complete', sessionId: msg.sessionId }); + return; + } + + if (isRecordingOnly(messageText)) { + handleRecordingStart(msg.sessionId, { promptForSource: true }, socket, ctx); + rlog.info('Recording-only intent intercepted in user_message'); + ctx.send(socket, { type: 'assistant_text_delta', text: 'Starting screen recording.' }); + ctx.send(socket, { type: 'message_complete', sessionId: msg.sessionId }); + return; + } + } + const ipcChannel = parseChannelId(msg.channel) ?? 'vellum'; const queuedChannelMetadata = { userMessageChannel: ipcChannel, diff --git a/assistant/src/daemon/recording-intent.ts b/assistant/src/daemon/recording-intent.ts new file mode 100644 index 00000000000..91e6c7d746e --- /dev/null +++ b/assistant/src/daemon/recording-intent.ts @@ -0,0 +1,86 @@ +// Recording intent detection for standalone screen recording routing. +// Used by task/message handlers to intercept recording-related prompts +// before they reach the classifier or create a CU session. + +// ─── Start recording patterns ──────────────────────────────────────────────── + +const START_RECORDING_PATTERNS: RegExp[] = [ + /\brecord\s+(my\s+)?screen\b/i, + /\brecord\s+the\s+screen\b/i, + /\bscreen\s+record(ing)?\b/i, + /\bstart\s+recording\b/i, + /\bbegin\s+recording\b/i, + /\bcapture\s+(my\s+)?(screen|display)\b/i, + /\bmake\s+a\s+(screen\s+)?recording\b/i, +]; + +// ─── Stop recording patterns ──────────────────────────────────────────────── + +const STOP_RECORDING_PATTERNS: RegExp[] = [ + /\bstop\s+(the\s+)?recording\b/i, + /\bend\s+(the\s+)?recording\b/i, + /\bfinish\s+(the\s+)?recording\b/i, + /\bhalt\s+(the\s+)?recording\b/i, +]; + +// ─── Clause removal for mixed-intent prompts ───────────────────────────────── + +const RECORDING_CLAUSE_PATTERNS: RegExp[] = [ + /\b(and\s+)?(also\s+)?record\s+(my\s+|the\s+)?screen\b/i, + /\b(and\s+)?(also\s+)?screen\s+record(ing)?\b/i, + /\b(and\s+)?(also\s+)?start\s+recording\b/i, + /\b(and\s+)?(also\s+)?begin\s+recording\b/i, + /\b(and\s+)?(also\s+)?capture\s+(my\s+)?(screen|display)\b/i, + /\b(and\s+)?(also\s+)?make\s+a\s+(screen\s+)?recording\b/i, + /\bwhile\s+(you\s+)?record(ing)?\s+(my\s+|the\s+)?screen\b/i, + /\brecord\s+(my\s+|the\s+)?screen\s+while\b/i, +]; + +// ─── Public API ────────────────────────────────────────────────────────────── + +/** + * Returns true if the user's message includes any recording-related phrases. + * Does not distinguish between recording-only and mixed-intent prompts. + */ +export function detectRecordingIntent(taskText: string): boolean { + return START_RECORDING_PATTERNS.some((p) => p.test(taskText)); +} + +/** + * Returns true if the prompt is purely about recording with no additional task. + * "record my screen" -> true + * "record my screen while I work" -> false (has CU task component) + * "open Chrome and record my screen" -> false (has CU task component) + */ +export function isRecordingOnly(taskText: string): boolean { + if (!detectRecordingIntent(taskText)) return false; + + // Strip the recording clause and check if anything substantive remains + const stripped = stripRecordingIntent(taskText); + // If after removing the recording clause, only whitespace/punctuation remains, + // this is a recording-only prompt. + return stripped.replace(/[.,;!?\s]+/g, '').length === 0; +} + +/** + * Returns true if the user wants to stop recording. + * Requires explicit "stop/end/finish/halt recording" phrasing -- + * bare "stop", "end it", or "quit" are too ambiguous and will not match. + */ +export function detectStopRecordingIntent(taskText: string): boolean { + return STOP_RECORDING_PATTERNS.some((p) => p.test(taskText)); +} + +/** + * Removes recording-related clauses from a task, returning the cleaned text. + * Used when a recording intent is embedded in a broader CU task so the + * recording portion can be handled separately while the task continues. + */ +export function stripRecordingIntent(taskText: string): string { + let result = taskText; + for (const pattern of RECORDING_CLAUSE_PATTERNS) { + result = result.replace(pattern, ''); + } + // Clean up any leftover double spaces or leading/trailing whitespace + return result.replace(/\s{2,}/g, ' ').trim(); +} From bf77c29be420084f3bc63b4b06a21239bdae6715 Mon Sep 17 00:00:00 2001 From: Jason Zhou Date: Wed, 25 Feb 2026 02:12:13 -0500 Subject: [PATCH 2/2] fix: address M3 PR #8699 review feedback (#8700) * fix: add isStopRecordingOnly guard, sessionId in message_complete, check stop return value Co-Authored-By: Claude * fix: handle polite filler words in isStopRecordingOnly and isRecordingOnly (#8701) Co-authored-by: Vellum Assistant Co-authored-by: Claude --------- Co-authored-by: Vellum Assistant Co-authored-by: Claude --- assistant/src/daemon/handlers/misc.ts | 16 +++++--- assistant/src/daemon/handlers/sessions.ts | 11 +++-- assistant/src/daemon/recording-intent.ts | 49 +++++++++++++++++++++-- 3 files changed, 63 insertions(+), 13 deletions(-) diff --git a/assistant/src/daemon/handlers/misc.ts b/assistant/src/daemon/handlers/misc.ts index d9d4e6ad197..7eb50c00552 100644 --- a/assistant/src/daemon/handlers/misc.ts +++ b/assistant/src/daemon/handlers/misc.ts @@ -11,7 +11,7 @@ import { parseSlashCandidate } from '../../skills/slash-commands.js'; import { classifySessionError, buildSessionErrorMessage } from '../session-error.js'; import { resolveBlobPath, deleteBlob, isValidBlobId } from '../ipc-blob-store.js'; import { getConfig } from '../../config/loader.js'; -import { isRecordingOnly, detectStopRecordingIntent } from '../recording-intent.js'; +import { isRecordingOnly, isStopRecordingOnly } from '../recording-intent.js'; import { handleRecordingStart, handleRecordingStop } from './recording.js'; import type { TaskSubmit, @@ -72,16 +72,20 @@ export async function handleTaskSubmit( // session and routes it to the standalone recording flow instead. const config = getConfig(); if (config.daemon.standaloneRecording) { - if (detectStopRecordingIntent(msg.task)) { + if (isStopRecordingOnly(msg.task)) { // Find the active session for this socket so we can resolve the // conversation that owns the recording. const activeSessionId = ctx.socketToSession.get(socket); + let stopped = false; if (activeSessionId) { - handleRecordingStop(activeSessionId, ctx); + stopped = handleRecordingStop(activeSessionId, ctx) !== undefined; } rlog.info('Recording stop intent intercepted'); - ctx.send(socket, { type: 'assistant_text_delta', text: 'Stopping the recording.' }); - ctx.send(socket, { type: 'message_complete' }); + ctx.send(socket, { + type: 'assistant_text_delta', + text: stopped ? 'Stopping the recording.' : 'No active recording to stop.', + }); + ctx.send(socket, { type: 'message_complete', sessionId: activeSessionId }); return; } @@ -99,7 +103,7 @@ export async function handleTaskSubmit( }); rlog.info({ sessionId: conversation.id }, 'Recording-only intent intercepted — routed to standalone recording'); ctx.send(socket, { type: 'assistant_text_delta', text: 'Starting screen recording.' }); - ctx.send(socket, { type: 'message_complete' }); + ctx.send(socket, { type: 'message_complete', sessionId: conversation.id }); return; } } diff --git a/assistant/src/daemon/handlers/sessions.ts b/assistant/src/daemon/handlers/sessions.ts index 8a8d37ae1ad..5457f6ee820 100644 --- a/assistant/src/daemon/handlers/sessions.ts +++ b/assistant/src/daemon/handlers/sessions.ts @@ -10,7 +10,7 @@ import { getAttachmentsForMessage, setAttachmentThumbnail } from '../../memory/a import { generateVideoThumbnail } from '../video-thumbnail.js'; import type { UserMessageAttachment } from '../ipc-contract.js'; import { normalizeThreadType } from '../ipc-protocol.js'; -import { isRecordingOnly, detectStopRecordingIntent } from '../recording-intent.js'; +import { isRecordingOnly, isStopRecordingOnly } from '../recording-intent.js'; import { handleRecordingStart, handleRecordingStop } from './recording.js'; import type { UserMessage, @@ -91,10 +91,13 @@ export async function handleUserMessage( const config = getConfig(); const messageText = msg.content ?? ''; if (config.daemon.standaloneRecording && messageText) { - if (detectStopRecordingIntent(messageText)) { - handleRecordingStop(msg.sessionId, ctx); + if (isStopRecordingOnly(messageText)) { + const stopped = handleRecordingStop(msg.sessionId, ctx) !== undefined; rlog.info('Recording stop intent intercepted in user_message'); - ctx.send(socket, { type: 'assistant_text_delta', text: 'Stopping the recording.' }); + ctx.send(socket, { + type: 'assistant_text_delta', + text: stopped ? 'Stopping the recording.' : 'No active recording to stop.', + }); ctx.send(socket, { type: 'message_complete', sessionId: msg.sessionId }); return; } diff --git a/assistant/src/daemon/recording-intent.ts b/assistant/src/daemon/recording-intent.ts index 91e6c7d746e..2ca7d525d92 100644 --- a/assistant/src/daemon/recording-intent.ts +++ b/assistant/src/daemon/recording-intent.ts @@ -23,6 +23,15 @@ const STOP_RECORDING_PATTERNS: RegExp[] = [ /\bhalt\s+(the\s+)?recording\b/i, ]; +// ─── Stop-recording clause removal for mixed-intent prompts ───────────────── + +const STOP_RECORDING_CLAUSE_PATTERNS: RegExp[] = [ + /\b(and\s+)?(also\s+)?stop\s+(the\s+)?recording\b/i, + /\b(and\s+)?(also\s+)?end\s+(the\s+)?recording\b/i, + /\b(and\s+)?(also\s+)?finish\s+(the\s+)?recording\b/i, + /\b(and\s+)?(also\s+)?halt\s+(the\s+)?recording\b/i, +]; + // ─── Clause removal for mixed-intent prompts ───────────────────────────────── const RECORDING_CLAUSE_PATTERNS: RegExp[] = [ @@ -36,6 +45,10 @@ const RECORDING_CLAUSE_PATTERNS: RegExp[] = [ /\brecord\s+(my\s+|the\s+)?screen\s+while\b/i, ]; +/** Common polite/filler words stripped before checking intent-only status. */ +const FILLER_PATTERN = + /\b(please|pls|plz|can\s+you|could\s+you|would\s+you|now|right\s+now|thanks|thank\s+you|thx|ty|for\s+me|ok(ay)?|hey|hi|just)\b/gi; + // ─── Public API ────────────────────────────────────────────────────────────── /** @@ -57,9 +70,11 @@ export function isRecordingOnly(taskText: string): boolean { // Strip the recording clause and check if anything substantive remains const stripped = stripRecordingIntent(taskText); - // If after removing the recording clause, only whitespace/punctuation remains, - // this is a recording-only prompt. - return stripped.replace(/[.,;!?\s]+/g, '').length === 0; + // Also remove common polite/filler words that don't change the intent + const withoutFillers = stripped.replace(FILLER_PATTERN, ''); + // If after removing the recording clause and fillers, only whitespace/punctuation + // remains, this is a recording-only prompt. + return withoutFillers.replace(/[.,;!?\s]+/g, '').length === 0; } /** @@ -84,3 +99,31 @@ export function stripRecordingIntent(taskText: string): string { // Clean up any leftover double spaces or leading/trailing whitespace return result.replace(/\s{2,}/g, ' ').trim(); } + +/** + * Removes stop-recording clauses from a message, returning the cleaned text. + * Analogous to stripRecordingIntent but for stop-recording phrases. + */ +export function stripStopRecordingIntent(taskText: string): string { + let result = taskText; + for (const pattern of STOP_RECORDING_CLAUSE_PATTERNS) { + result = result.replace(pattern, ''); + } + return result.replace(/\s{2,}/g, ' ').trim(); +} + +/** + * Returns true if the prompt is purely about stopping recording with no + * additional task. Analogous to isRecordingOnly but for stop-recording. + * "stop recording" -> true + * "how do I stop recording?" -> false (has additional context) + * "stop recording and close the browser" -> false (has CU task component) + */ +export function isStopRecordingOnly(taskText: string): boolean { + if (!detectStopRecordingIntent(taskText)) return false; + + const stripped = stripStopRecordingIntent(taskText); + // Also remove common polite/filler words that don't change the intent + const withoutFillers = stripped.replace(FILLER_PATTERN, ''); + return withoutFillers.replace(/[.,;!?\s]+/g, '').length === 0; +}