diff --git a/apps/desktop/src/settings/DesktopClientSettings.test.ts b/apps/desktop/src/settings/DesktopClientSettings.test.ts index 6bf0e87c9324..20e814ce4449 100644 --- a/apps/desktop/src/settings/DesktopClientSettings.test.ts +++ b/apps/desktop/src/settings/DesktopClientSettings.test.ts @@ -32,6 +32,7 @@ const clientSettings: ClientSettings = { diffIgnoreWhitespace: true, diffLayout: "stacked", environmentIdentificationMode: "artwork", + enableCompletionSounds: false, favorites: [], fontFamilyCode: "", fontFamilyComposer: "", diff --git a/apps/mobile/assets/interaction-sounds/bloom.wav b/apps/mobile/assets/interaction-sounds/bloom.wav new file mode 100644 index 000000000000..251a32d096f5 Binary files /dev/null and b/apps/mobile/assets/interaction-sounds/bloom.wav differ diff --git a/apps/mobile/assets/interaction-sounds/success.wav b/apps/mobile/assets/interaction-sounds/success.wav new file mode 100644 index 000000000000..72d214295c3b Binary files /dev/null and b/apps/mobile/assets/interaction-sounds/success.wav differ diff --git a/apps/mobile/src/App.tsx b/apps/mobile/src/App.tsx index c268056f0322..6785ca73d70d 100644 --- a/apps/mobile/src/App.tsx +++ b/apps/mobile/src/App.tsx @@ -13,6 +13,7 @@ import { ConfirmDialogHost } from "./components/ConfirmDialogHost"; import { CloudAuthProvider } from "./features/cloud/CloudAuthProvider"; import { prepareNativeShowcaseCapture } from "./features/showcase/nativeShowcaseScene"; import { IncomingShareProvider } from "./features/sharing/IncomingShareProvider"; +import { InteractionSoundCoordinator } from "./features/interaction-sounds/InteractionSoundCoordinator"; import { AppearancePreferencesProvider, useAppearancePreferences, @@ -76,6 +77,7 @@ function AppContent() { return ( <> + diff --git a/apps/mobile/src/features/interaction-sounds/InteractionSoundCoordinator.tsx b/apps/mobile/src/features/interaction-sounds/InteractionSoundCoordinator.tsx new file mode 100644 index 000000000000..bc5711f96526 --- /dev/null +++ b/apps/mobile/src/features/interaction-sounds/InteractionSoundCoordinator.tsx @@ -0,0 +1,114 @@ +import { useAtomValue } from "@effect/atom-react"; +import { scopedThreadKey } from "@t3tools/client-runtime/environment"; +import { + observeThreadSoundState, + shouldPlayInteractionSound, + type InteractionSoundCue, + type ThreadSoundStateByKey, +} from "@t3tools/client-runtime/interaction-sounds"; +import type { ScopedThreadRef } from "@t3tools/contracts"; +import { useAudioPlayer } from "expo-audio"; +import { AsyncResult } from "effect/unstable/reactivity"; +import { useEffect, useMemo, useRef } from "react"; + +import { mobilePreferencesAtom } from "../../state/preferences"; +import { liveEnvironmentIdsAtom } from "../../state/shell"; +import { useThreadRefs, useThreadShell } from "../../state/entities"; +import { replayInteractionSound } from "./interactionSoundPlayback"; + +const SUCCESS_SOUND = require("../../../assets/interaction-sounds/success.wav"); +const BLOOM_SOUND = require("../../../assets/interaction-sounds/bloom.wav"); + +type InteractionSoundPlayers = Readonly< + Record> +>; + +export function InteractionSoundCoordinator() { + const threadRefs = useThreadRefs(); + const liveEnvironmentIds = useAtomValue(liveEnvironmentIdsAtom); + const preferences = useAtomValue(mobilePreferencesAtom); + const successPlayer = useAudioPlayer(SUCCESS_SOUND); + const bloomPlayer = useAudioPlayer(BLOOM_SOUND); + const previouslyLiveEnvironmentIdsRef = useRef(new Set()); + const players = useMemo( + () => ({ bloom: bloomPlayer, success: successPlayer }), + [bloomPlayer, successPlayer], + ); + const settingsHydrated = AsyncResult.isSuccess(preferences); + const completionSoundEnabled = settingsHydrated + ? preferences.value.completionSoundEnabled !== false + : true; + + useEffect(() => { + for (const environmentId of liveEnvironmentIds) { + previouslyLiveEnvironmentIdsRef.current.add(environmentId); + } + }, [liveEnvironmentIds]); + + return threadRefs.map((threadRef) => ( + + )); +} + +function InteractionSoundThreadCoordinator({ + threadRef, + environmentLive, + environmentPreviouslyLive, + completionSoundEnabled, + settingsHydrated, + players, +}: { + readonly threadRef: ScopedThreadRef; + readonly environmentLive: boolean; + readonly environmentPreviouslyLive: boolean; + readonly completionSoundEnabled: boolean; + readonly settingsHydrated: boolean; + readonly players: InteractionSoundPlayers; +}) { + const thread = useThreadShell(threadRef); + const previousStateRef = useRef(null); + const environmentObservedLiveRef = useRef(environmentPreviouslyLive); + + useEffect(() => { + if (thread === null) { + return; + } + const environmentWasLive = environmentObservedLiveRef.current || environmentPreviouslyLive; + const observation = observeThreadSoundState(previousStateRef.current, thread, { + environmentLive, + environmentPreviouslyLive: environmentWasLive, + settingsHydrated, + }); + if (environmentLive || environmentPreviouslyLive) { + environmentObservedLiveRef.current = true; + } + previousStateRef.current = observation.state; + for (const cue of observation.cues) { + if (shouldPlayInteractionSound(cue, completionSoundEnabled)) { + void replayInteractionSound(players[cue]).catch((error: unknown) => { + console.warn(`[interaction-sounds] Could not play ${cue} cue.`, error); + }); + } + } + }, [ + completionSoundEnabled, + environmentLive, + environmentPreviouslyLive, + players, + settingsHydrated, + thread, + ]); + + return null; +} diff --git a/apps/mobile/src/features/interaction-sounds/interactionSoundPlayback.test.ts b/apps/mobile/src/features/interaction-sounds/interactionSoundPlayback.test.ts new file mode 100644 index 000000000000..16cc41ef065a --- /dev/null +++ b/apps/mobile/src/features/interaction-sounds/interactionSoundPlayback.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it, vi } from "vite-plus/test"; + +import { replayInteractionSound } from "./interactionSoundPlayback"; + +describe("mobile interaction sounds", () => { + it("starts a fresh player without seeking", async () => { + const player = { + currentTime: 0, + play: vi.fn(), + seekTo: vi.fn(() => Promise.resolve()), + }; + + await replayInteractionSound(player); + + expect(player.seekTo).not.toHaveBeenCalled(); + expect(player.play).toHaveBeenCalledTimes(1); + }); + + it("rewinds a previously played cue before replaying it", async () => { + const calls: string[] = []; + const player = { + currentTime: 0.4, + play: vi.fn(() => { + calls.push("play"); + }), + seekTo: vi.fn(async () => { + calls.push("seek"); + }), + }; + + await replayInteractionSound(player); + + expect(player.seekTo).toHaveBeenCalledWith(0); + expect(calls).toEqual(["seek", "play"]); + }); +}); diff --git a/apps/mobile/src/features/interaction-sounds/interactionSoundPlayback.ts b/apps/mobile/src/features/interaction-sounds/interactionSoundPlayback.ts new file mode 100644 index 000000000000..205ae67e3f4c --- /dev/null +++ b/apps/mobile/src/features/interaction-sounds/interactionSoundPlayback.ts @@ -0,0 +1,10 @@ +import type { AudioPlayer } from "expo-audio"; + +type InteractionSoundPlayer = Pick; + +export async function replayInteractionSound(player: InteractionSoundPlayer): Promise { + if (player.currentTime > 0) { + await player.seekTo(0); + } + player.play(); +} diff --git a/apps/mobile/src/features/settings/SettingsRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsRouteScreen.tsx index 3bd25a20c8da..6542573a7a65 100644 --- a/apps/mobile/src/features/settings/SettingsRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsRouteScreen.tsx @@ -542,11 +542,23 @@ function ConfiguredSettingsRouteScreen() { } function GeneralSettingsSection() { + const preferencesResult = useAtomValue(mobilePreferencesAtom); + const savePreferences = useAtomSet(updateMobilePreferencesAtom); + const completionSoundEnabled = AsyncResult.isSuccess(preferencesResult) + ? preferencesResult.value.completionSoundEnabled !== false + : true; + return ( + savePreferences({ completionSoundEnabled: value })} + /> ); } diff --git a/apps/mobile/src/persistence/mobile-preferences.ts b/apps/mobile/src/persistence/mobile-preferences.ts index f14a73f15cb2..1d3b6946c233 100644 --- a/apps/mobile/src/persistence/mobile-preferences.ts +++ b/apps/mobile/src/persistence/mobile-preferences.ts @@ -21,6 +21,7 @@ export interface Preferences { readonly lightThemeId?: MobileThemeId; readonly darkThemeId?: MobileThemeId; readonly themeMode?: MobileThemeMode; + readonly completionSoundEnabled?: boolean; readonly baseFontSize?: number; readonly terminalFontSize?: number | null; readonly markdownFontSize?: number; @@ -90,6 +91,7 @@ function sanitizePreferences(parsed: Preferences): Preferences { lightThemeId?: MobileThemeId; darkThemeId?: MobileThemeId; themeMode?: MobileThemeMode; + completionSoundEnabled?: boolean; baseFontSize?: number; terminalFontSize?: number | null; markdownFontSize?: number; @@ -133,6 +135,9 @@ function sanitizePreferences(parsed: Preferences): Preferences { ) { preferences.themeMode = parsed.themeMode; } + if (typeof parsed.completionSoundEnabled === "boolean") { + preferences.completionSoundEnabled = parsed.completionSoundEnabled; + } if (typeof parsed.baseFontSize === "number") preferences.baseFontSize = parsed.baseFontSize; if (typeof parsed.terminalFontSize === "number" || parsed.terminalFontSize === null) { preferences.terminalFontSize = parsed.terminalFontSize; diff --git a/apps/mobile/src/state/entities.ts b/apps/mobile/src/state/entities.ts index 8199dee34866..52200bb5ec3d 100644 --- a/apps/mobile/src/state/entities.ts +++ b/apps/mobile/src/state/entities.ts @@ -33,6 +33,10 @@ export function useThreadShells(): ReadonlyArray { return useAtomValue(environmentThreadShells.threadShellsAtom); } +export function useThreadRefs(): ReadonlyArray { + return useAtomValue(environmentThreadShells.threadRefsAtom); +} + export function useProject(ref: ScopedProjectRef | null): EnvironmentProject | null { return useAtomValue(ref === null ? EMPTY_PROJECT_ATOM : environmentProjects.projectAtom(ref)); } diff --git a/apps/mobile/src/state/shell.ts b/apps/mobile/src/state/shell.ts index e879dd25e292..01b4f04463bc 100644 --- a/apps/mobile/src/state/shell.ts +++ b/apps/mobile/src/state/shell.ts @@ -2,6 +2,7 @@ import { createEnvironmentShellAtoms, createEnvironmentShellSummaryAtom, createEnvironmentSnapshotAtom, + createLiveEnvironmentIdsAtom, createShellEnvironmentAtoms, } from "@t3tools/client-runtime/state/shell"; @@ -15,3 +16,9 @@ export const environmentShellSummaryAtom = createEnvironmentShellSummaryAtom({ catalogValueAtom: environmentCatalog.catalogValueAtom, shellStateValueAtom: environmentShell.stateValueAtom, }); + +export const liveEnvironmentIdsAtom = createLiveEnvironmentIdsAtom({ + catalogValueAtom: environmentCatalog.catalogValueAtom, + shellStateValueAtom: environmentShell.stateValueAtom, + label: "mobile-live-environment-ids", +}); diff --git a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts index e262bce34aaf..7d30cc2cef80 100644 --- a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts +++ b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts @@ -341,6 +341,7 @@ projectionSnapshotLayer("ProjectionSnapshotQuery", (it) => { latestTurn: { turnId: asTurnId("turn-1"), state: "completed", + initiatingUserMessageId: null, requestedAt: "2026-02-24T00:00:08.000Z", startedAt: "2026-02-24T00:00:08.000Z", completedAt: "2026-02-24T00:00:08.000Z", @@ -471,6 +472,7 @@ projectionSnapshotLayer("ProjectionSnapshotQuery", (it) => { latestTurn: { turnId: asTurnId("turn-1"), state: "completed", + initiatingUserMessageId: null, requestedAt: "2026-02-24T00:00:08.000Z", startedAt: "2026-02-24T00:00:08.000Z", completedAt: "2026-02-24T00:00:08.000Z", @@ -1649,6 +1651,10 @@ projectionSnapshotLayer("ProjectionSnapshotQuery", (it) => { assert.equal(threadShell._tag, "Some"); if (threadShell._tag === "Some") { assert.equal(threadShell.value.latestTurn?.turnId, asTurnId("turn-running")); + assert.equal( + threadShell.value.latestTurn?.initiatingUserMessageId, + asMessageId("message-user-2"), + ); assert.equal(threadShell.value.latestTurn?.state, "running"); assert.equal(threadShell.value.latestTurn?.startedAt, "2026-04-02T00:00:30.000Z"); } @@ -1657,6 +1663,10 @@ projectionSnapshotLayer("ProjectionSnapshotQuery", (it) => { assert.equal(threadDetail._tag, "Some"); if (threadDetail._tag === "Some") { assert.equal(threadDetail.value.latestTurn?.turnId, asTurnId("turn-running")); + assert.equal( + threadDetail.value.latestTurn?.initiatingUserMessageId, + asMessageId("message-user-2"), + ); assert.equal(threadDetail.value.latestTurn?.state, "running"); assert.equal(threadDetail.value.latestTurn?.startedAt, "2026-04-02T00:00:30.000Z"); } diff --git a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts index 5f82a26e2a36..1d6bf5d38738 100644 --- a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts +++ b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts @@ -142,6 +142,7 @@ const ProjectionCheckpointDbRowSchema = ProjectionCheckpoint.mapFields( const ProjectionLatestTurnDbRowSchema = Schema.Struct({ threadId: ProjectionThread.fields.threadId, turnId: TurnId, + pendingMessageId: Schema.NullOr(MessageId), state: Schema.String, requestedAt: IsoDateTime, startedAt: Schema.NullOr(IsoDateTime), @@ -322,6 +323,7 @@ function mapLatestTurn( ): OrchestrationLatestTurn { return { turnId: row.turnId, + initiatingUserMessageId: row.pendingMessageId, state: row.state === "error" ? "error" @@ -772,6 +774,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { SELECT turns.thread_id AS "threadId", turns.turn_id AS "turnId", + turns.pending_message_id AS "pendingMessageId", turns.state, turns.requested_at AS "requestedAt", turns.started_at AS "startedAt", @@ -796,6 +799,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { SELECT turns.thread_id AS "threadId", turns.turn_id AS "turnId", + turns.pending_message_id AS "pendingMessageId", turns.state, turns.requested_at AS "requestedAt", turns.started_at AS "startedAt", @@ -822,6 +826,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { SELECT turns.thread_id AS "threadId", turns.turn_id AS "turnId", + turns.pending_message_id AS "pendingMessageId", turns.state, turns.requested_at AS "requestedAt", turns.started_at AS "startedAt", @@ -1391,6 +1396,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { SELECT turns.thread_id AS "threadId", turns.turn_id AS "turnId", + turns.pending_message_id AS "pendingMessageId", turns.state, turns.requested_at AS "requestedAt", turns.started_at AS "startedAt", @@ -2002,6 +2008,7 @@ pending_approval_requests AS ( } latestTurnByThread.set(row.threadId, { turnId: row.turnId, + initiatingUserMessageId: row.pendingMessageId, state: row.state === "error" ? "error" diff --git a/apps/web/package.json b/apps/web/package.json index 7fbcd6defb27..7dff9f9b3fcc 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -32,6 +32,7 @@ "@tanstack/react-pacer": "^0.19.4", "@tanstack/react-router": "^1.160.2", "class-variance-authority": "^0.7.1", + "cuelume": "^0.2.1", "culori": "^4.0.2", "effect": "catalog:", "heic-to": "^1.5.2", diff --git a/apps/web/src/components/settings/SettingsPanels.tsx b/apps/web/src/components/settings/SettingsPanels.tsx index bac8f2937328..4f3f7bf49f63 100644 --- a/apps/web/src/components/settings/SettingsPanels.tsx +++ b/apps/web/src/components/settings/SettingsPanels.tsx @@ -548,6 +548,9 @@ export function useSettingsRestore(onRestored?: () => void) { ...(settings.contextWindowMeterEnabled !== DEFAULT_UNIFIED_SETTINGS.contextWindowMeterEnabled ? ["Context window indicator"] : []), + ...(settings.enableCompletionSounds !== DEFAULT_UNIFIED_SETTINGS.enableCompletionSounds + ? ["Completion sound"] + : []), ...(settings.enableLegacyTokenStreaming !== DEFAULT_UNIFIED_SETTINGS.enableLegacyTokenStreaming ? ["Stream token by token"] @@ -621,6 +624,7 @@ export function useSettingsRestore(onRestored?: () => void) { settings.fontSizeTerminal, settings.glassOpacity, settings.panelAnimationDurationMs, + settings.enableCompletionSounds, settings.enableLegacyTokenStreaming, settings.enableProviderUpdateChecks, settings.continueThreadsAfterServerUpdate, @@ -712,6 +716,7 @@ export function useSettingsRestore(onRestored?: () => void) { environmentIdentificationMode: DEFAULT_UNIFIED_SETTINGS.environmentIdentificationMode, glassOpacity: DEFAULT_UNIFIED_SETTINGS.glassOpacity, panelAnimationDurationMs: DEFAULT_UNIFIED_SETTINGS.panelAnimationDurationMs, + enableCompletionSounds: DEFAULT_UNIFIED_SETTINGS.enableCompletionSounds, sidebarThreadPreviewCount: DEFAULT_UNIFIED_SETTINGS.sidebarThreadPreviewCount, sidebarProjectGroupingMode: DEFAULT_UNIFIED_SETTINGS.sidebarProjectGroupingMode, sidebarAutoSettleAfterDays: DEFAULT_UNIFIED_SETTINGS.sidebarAutoSettleAfterDays, @@ -2231,6 +2236,32 @@ export function GeneralSettingsPanel() { } /> + + updateSettings({ + enableCompletionSounds: DEFAULT_UNIFIED_SETTINGS.enableCompletionSounds, + }) + } + /> + ) : null + } + control={ + + updateSettings({ enableCompletionSounds: Boolean(checked) }) + } + aria-label="Play a sound when a turn completes" + /> + } + /> + ) : null} {primaryEnvironmentAuthenticated ? : null} + {primaryEnvironmentAuthenticated ? : null} {appShell} {/* Above the router: a theme draft is judged by walking the app, so the @@ -262,6 +280,113 @@ function FontAppearanceSync() { return null; } +function InteractionSoundCoordinator() { + const threadRefs = useThreadRefs(); + const liveEnvironmentIds = useAtomValue(liveEnvironmentIdsAtom); + const completionSoundEnabled = useClientSettings((settings) => settings.enableCompletionSounds); + const settingsHydrated = useClientSettingsHydrated(); + const previouslyLiveEnvironmentIdsRef = useRef(new Set()); + + useEffect(() => { + const cleanup = () => { + document.removeEventListener("pointerdown", prime, true); + document.removeEventListener("keydown", prime, true); + }; + const prime = (event: Event) => { + if (!event.isTrusted) { + return; + } + // Cuelume owns a lazy AudioContext. Touch it from the first real user + // gesture at an effectively inaudible level so later background cues + // are not rejected by browser autoplay policy. + playInteractionSound("press", { volume: 0.0001 }); + cleanup(); + }; + + document.addEventListener("pointerdown", prime, true); + document.addEventListener("keydown", prime, true); + return cleanup; + }, []); + + useEffect(() => { + for (const environmentId of liveEnvironmentIds) { + previouslyLiveEnvironmentIdsRef.current.add(environmentId); + } + }, [liveEnvironmentIds]); + + return threadRefs.map((threadRef) => ( + + )); +} + +function InteractionSoundThreadCoordinator({ + threadRef, + environmentLive, + environmentPreviouslyLive, + completionSoundEnabled, + settingsHydrated, +}: { + readonly threadRef: ScopedThreadRef; + readonly environmentLive: boolean; + readonly environmentPreviouslyLive: boolean; + readonly completionSoundEnabled: boolean; + readonly settingsHydrated: boolean; +}) { + const thread = useThreadShell(threadRef); + const previousStateRef = useRef(null); + const environmentObservedLiveRef = useRef(environmentPreviouslyLive); + + useEffect(() => { + if (thread === null) { + return; + } + const environmentWasLive = environmentObservedLiveRef.current || environmentPreviouslyLive; + const observation = observeThreadSoundState(previousStateRef.current, thread, { + environmentLive, + environmentPreviouslyLive: environmentWasLive, + settingsHydrated, + }); + if (environmentLive || environmentPreviouslyLive) { + environmentObservedLiveRef.current = true; + } + previousStateRef.current = observation.state; + for (const cue of observation.cues) { + if (shouldPlayInteractionSound(cue, completionSoundEnabled)) { + playInteractionSound(cue); + } + } + }, [ + completionSoundEnabled, + environmentLive, + environmentPreviouslyLive, + settingsHydrated, + thread, + ]); + + return null; +} + +function playInteractionSound( + cue: InteractionSoundCue | "press", + options?: { readonly volume?: number }, +) { + try { + play(cue, options); + } catch (error) { + console.warn(`[interaction-sounds] Could not play ${cue} cue.`, error); + } +} + function DocumentTitleSync() { const primaryServerVersion = useAtomValue(primaryServerConfigAtom)?.environment.serverVersion ?? null; diff --git a/apps/web/src/state/shell.ts b/apps/web/src/state/shell.ts index b1719819da9d..bba0c0406282 100644 --- a/apps/web/src/state/shell.ts +++ b/apps/web/src/state/shell.ts @@ -5,6 +5,7 @@ import { import { createEnvironmentShellAtoms, createEnvironmentSnapshotAtom, + createLiveEnvironmentIdsAtom, createShellEnvironmentAtoms, type EnvironmentShellState, } from "@t3tools/client-runtime/state/shell"; @@ -21,6 +22,12 @@ export const shellEnvironment = createShellEnvironmentAtoms(connectionAtomRuntim export const environmentShell = createEnvironmentShellAtoms(connectionAtomRuntime); export const environmentSnapshotAtom = createEnvironmentSnapshotAtom(environmentShell.stateAtom); +export const liveEnvironmentIdsAtom = createLiveEnvironmentIdsAtom({ + catalogValueAtom: environmentCatalog.catalogValueAtom, + shellStateValueAtom: environmentShell.stateValueAtom, + label: "web-live-environment-ids", +}); + export const allEnvironmentShellsBootstrappedAtom = Atom.make((get) => { const catalog = AsyncResult.value(get(environmentCatalog.catalogAtom)); if (Option.isNone(catalog)) { diff --git a/docs/README.md b/docs/README.md index 4e6f82bfb826..67d58cbf3f37 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,6 +10,7 @@ - [Source control](./user/source-control.md) - [Project settings](./user/project-settings.md) - [Appearance and themes](./user/appearance.md) +- [Interaction sounds](./user/interaction-sounds.md) - [Keyboard shortcuts](./user/keybindings.md) - [Import browser sessions](./user/browser-import.md) - [Usage and limits](./user/usage.md) diff --git a/docs/user/interaction-sounds.md b/docs/user/interaction-sounds.md new file mode 100644 index 000000000000..be809650856f --- /dev/null +++ b/docs/user/interaction-sounds.md @@ -0,0 +1,24 @@ +# Interaction Sounds + +T3 Code plays short interaction sounds for important thread transitions on web, desktop, and +mobile: + +- **Completion:** a success cue plays when a turn started by you completes, even if another thread + is open. +- **Input required:** a bloom cue plays when a thread begins waiting for your input or approval. + +Completion sounds do not play for cached startup state, unchanged state, or background provider +work that was not started by a user message. Input-required sounds can still play for background +work so you know when an agent is blocked on your response. + +## Completion Sound Setting + +Completion sounds are enabled by default. + +- On web and desktop, open **Settings** → **General** → **Completion sound**. +- On mobile, open **Settings** → **General** → **Completion Sound**. + +Turning this setting off disables only the completion cue. The input-required cue remains enabled so +T3 Code can still alert you when an agent is blocked on your response. + +The mobile setting is stored on that device. Changing it does not change the web or desktop setting. diff --git a/package.json b/package.json index 4e5aca36d135..bc4ec0582221 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "start:marketing": "vp run --filter @t3tools/marketing preview", "start:mock-update-server": "node scripts/mock-update-server.ts", "screenshots:mobile": "node scripts/mobile-showcase.ts", + "sounds:generate": "node scripts/generate-interaction-sound-assets.mjs", "icons:export": "node scripts/export-brand-icons.ts", "icons:check": "node scripts/export-brand-icons.ts --check", "build": "vp run --filter './apps/*' build", diff --git a/packages/client-runtime/package.json b/packages/client-runtime/package.json index fdab38c9cc62..3cc3357963f2 100644 --- a/packages/client-runtime/package.json +++ b/packages/client-runtime/package.json @@ -75,6 +75,10 @@ "types": "./src/operations/projects.ts", "default": "./src/operations/projects.ts" }, + "./interaction-sounds": { + "types": "./src/interactionSounds.ts", + "default": "./src/interactionSounds.ts" + }, "./platform": { "types": "./src/platform/index.ts", "default": "./src/platform/index.ts" diff --git a/packages/client-runtime/src/interactionSounds.test.ts b/packages/client-runtime/src/interactionSounds.test.ts new file mode 100644 index 000000000000..61b791bc76a4 --- /dev/null +++ b/packages/client-runtime/src/interactionSounds.test.ts @@ -0,0 +1,468 @@ +import { EnvironmentId, MessageId, ThreadId, TurnId } from "@t3tools/contracts"; +import { describe, expect, it } from "vite-plus/test"; +import type { EnvironmentThreadShell } from "./state/shell.ts"; +import { + captureThreadSoundState, + deriveInteractionSoundCues, + observeThreadSoundState, + shouldPlayInteractionSound, +} from "./interactionSounds.ts"; + +function makeThread(overrides: Partial = {}): EnvironmentThreadShell { + return { + environmentId: "environment-1", + id: "thread-1", + projectId: "project-1", + title: "Thread", + modelSelection: null, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + latestTurn: null, + createdAt: "2026-07-11T12:00:00.000Z", + updatedAt: "2026-07-11T12:00:00.000Z", + archivedAt: null, + session: null, + latestUserMessageAt: null, + hasPendingApprovals: false, + hasPendingUserInput: false, + hasActionableProposedPlan: false, + ...overrides, + } as EnvironmentThreadShell; +} + +describe("interaction sounds", () => { + it("plays success when a turn is associated with its initiating user message", () => { + const running = makeThread({ + latestTurn: { + turnId: TurnId.make("turn-1"), + initiatingUserMessageId: MessageId.make("message-1"), + state: "running", + requestedAt: "2026-07-11T12:00:02.000Z", + startedAt: "2026-07-11T12:00:03.000Z", + completedAt: null, + assistantMessageId: null, + }, + }); + const completed = makeThread({ + latestTurn: { + ...running.latestTurn!, + state: "completed", + completedAt: "2026-07-11T12:00:05.000Z", + }, + }); + + expect(deriveInteractionSoundCues(captureThreadSoundState([running]), [completed])).toEqual([ + "success", + ]); + }); + + it("supports older shells where a normal user message precedes provider startup", () => { + const running = makeThread({ + latestUserMessageAt: "2026-07-11T12:00:00.000Z", + latestTurn: { + turnId: TurnId.make("legacy-turn"), + state: "running", + requestedAt: "2026-07-11T12:00:02.000Z", + startedAt: "2026-07-11T12:00:03.000Z", + completedAt: null, + assistantMessageId: null, + }, + }); + const completed = makeThread({ + latestUserMessageAt: running.latestUserMessageAt, + latestTurn: { + ...running.latestTurn!, + state: "completed", + completedAt: "2026-07-11T12:00:05.000Z", + }, + }); + + expect(deriveInteractionSoundCues(captureThreadSoundState([running]), [completed])).toEqual([ + "success", + ]); + }); + + it("does not let a later steering message associate a background subagent turn", () => { + const backgroundRunning = makeThread({ + latestUserMessageAt: "2026-07-11T12:00:00.000Z", + latestTurn: { + turnId: TurnId.make("subagent-turn"), + initiatingUserMessageId: null, + state: "running", + requestedAt: "2026-07-11T12:05:00.000Z", + startedAt: "2026-07-11T12:05:00.000Z", + completedAt: null, + assistantMessageId: null, + }, + }); + const completedAfterSteering = makeThread({ + latestUserMessageAt: "2026-07-11T12:06:00.000Z", + latestTurn: { + ...backgroundRunning.latestTurn!, + state: "completed", + completedAt: "2026-07-11T12:06:05.000Z", + }, + }); + + expect( + deriveInteractionSoundCues(captureThreadSoundState([backgroundRunning]), [ + completedAfterSteering, + ]), + ).toEqual([]); + }); + + it("does not associate an old user message with later background work", () => { + const beforeBackgroundWork = makeThread({ + latestUserMessageAt: "2026-07-11T12:00:00.000Z", + }); + const completedBackgroundTurn = makeThread({ + latestUserMessageAt: beforeBackgroundWork.latestUserMessageAt, + latestTurn: { + turnId: TurnId.make("background-turn"), + state: "completed", + requestedAt: "2026-07-11T12:05:00.000Z", + startedAt: "2026-07-11T12:05:00.000Z", + completedAt: "2026-07-11T12:05:05.000Z", + assistantMessageId: null, + }, + }); + + expect( + deriveInteractionSoundCues(captureThreadSoundState([beforeBackgroundWork]), [ + completedBackgroundTurn, + ]), + ).toEqual([]); + }); + + it("plays bloom when a thread starts requesting user input", () => { + const thread = makeThread(); + + expect( + deriveInteractionSoundCues(captureThreadSoundState([thread]), [ + makeThread({ hasPendingUserInput: true }), + ]), + ).toEqual(["bloom"]); + }); + + it("plays bloom when a thread starts requesting approval", () => { + const thread = makeThread(); + + expect( + deriveInteractionSoundCues(captureThreadSoundState([thread]), [ + makeThread({ hasPendingApprovals: true }), + ]), + ).toEqual(["bloom"]); + }); + + it("plays bloom when pending input changes directly to pending approval", () => { + const pendingInput = makeThread({ hasPendingUserInput: true }); + const pendingApproval = makeThread({ hasPendingApprovals: true }); + + expect( + deriveInteractionSoundCues(captureThreadSoundState([pendingInput]), [pendingApproval]), + ).toEqual(["bloom"]); + }); + + it("plays bloom when pending approval changes directly to pending input", () => { + const pendingApproval = makeThread({ hasPendingApprovals: true }); + const pendingInput = makeThread({ hasPendingUserInput: true }); + + expect( + deriveInteractionSoundCues(captureThreadSoundState([pendingApproval]), [pendingInput]), + ).toEqual(["bloom"]); + }); + + it("keeps thread state distinct when scoped IDs contain colons", () => { + const first = makeThread({ + environmentId: EnvironmentId.make("a:b"), + id: ThreadId.make("c"), + }); + const second = makeThread({ + environmentId: EnvironmentId.make("a"), + id: ThreadId.make("b:c"), + hasPendingUserInput: true, + }); + const previous = captureThreadSoundState([first, second]); + + expect(previous.size).toBe(2); + expect( + deriveInteractionSoundCues(previous, [{ ...first, hasPendingUserInput: true }, second]), + ).toEqual(["bloom"]); + }); + + it("does not replay cues for unchanged state", () => { + const thread = makeThread({ + latestUserMessageAt: "2026-07-11T12:00:00.000Z", + hasPendingUserInput: true, + hasPendingApprovals: true, + latestTurn: { + turnId: TurnId.make("turn-1"), + state: "completed", + requestedAt: "2026-07-11T12:00:00.000Z", + startedAt: "2026-07-11T12:00:01.000Z", + completedAt: "2026-07-11T12:00:05.000Z", + assistantMessageId: null, + }, + }); + + expect(deriveInteractionSoundCues(captureThreadSoundState([thread]), [thread])).toEqual([]); + }); + + it("does not replay success when a completed turn timestamp is corrected", () => { + const completed = makeThread({ + latestUserMessageAt: "2026-07-11T12:00:00.000Z", + latestTurn: { + turnId: TurnId.make("turn-1"), + state: "completed", + requestedAt: "2026-07-11T12:00:01.000Z", + startedAt: "2026-07-11T12:00:02.000Z", + completedAt: "2026-07-11T12:00:05.000Z", + assistantMessageId: null, + }, + }); + const corrected = makeThread({ + latestUserMessageAt: completed.latestUserMessageAt, + latestTurn: { + ...completed.latestTurn!, + completedAt: "2026-07-11T12:00:06.000Z", + }, + }); + + expect(deriveInteractionSoundCues(captureThreadSoundState([completed]), [corrected])).toEqual( + [], + ); + }); + + it("does not play cues while existing threads are first hydrated", () => { + const thread = makeThread({ + hasPendingUserInput: true, + latestTurn: { + turnId: TurnId.make("turn-1"), + state: "completed", + requestedAt: "2026-07-11T12:00:00.000Z", + startedAt: "2026-07-11T12:00:01.000Z", + completedAt: "2026-07-11T12:00:05.000Z", + assistantMessageId: null, + }, + }); + + expect(deriveInteractionSoundCues(new Map(), [thread])).toEqual([]); + }); + + it("preserves pre-hydration thread state so cues can play after settings hydrate", () => { + const running = makeThread({ + latestUserMessageAt: "2026-07-11T12:00:00.000Z", + latestTurn: { + turnId: TurnId.make("turn-1"), + state: "running", + requestedAt: "2026-07-11T12:00:01.000Z", + startedAt: "2026-07-11T12:00:01.000Z", + completedAt: null, + assistantMessageId: null, + }, + }); + const completed = makeThread({ + latestUserMessageAt: running.latestUserMessageAt, + latestTurn: { + ...running.latestTurn!, + state: "completed", + completedAt: "2026-07-11T12:00:05.000Z", + }, + }); + + const seeded = observeThreadSoundState(null, running, { + environmentLive: true, + environmentPreviouslyLive: false, + settingsHydrated: false, + }); + const frozen = observeThreadSoundState(seeded.state, completed, { + environmentLive: true, + environmentPreviouslyLive: true, + settingsHydrated: false, + }); + const hydrated = observeThreadSoundState(frozen.state, completed, { + environmentLive: true, + environmentPreviouslyLive: true, + settingsHydrated: true, + }); + + expect(hydrated.cues).toEqual(["success"]); + }); + + it("preserves a thread baseline while its environment is synchronizing", () => { + const running = makeThread({ + latestTurn: { + turnId: TurnId.make("turn-1"), + initiatingUserMessageId: MessageId.make("message-1"), + state: "running", + requestedAt: "2026-07-11T12:00:01.000Z", + startedAt: "2026-07-11T12:00:01.000Z", + completedAt: null, + assistantMessageId: null, + }, + }); + const completedDuringSync = makeThread({ + latestTurn: { + ...running.latestTurn!, + state: "completed", + completedAt: "2026-07-11T12:00:05.000Z", + }, + }); + const beforeSync = captureThreadSoundState([running]); + const whileSynchronizing = observeThreadSoundState(beforeSync, completedDuringSync, { + environmentLive: false, + environmentPreviouslyLive: true, + settingsHydrated: true, + }); + const reconnected = observeThreadSoundState(whileSynchronizing.state, completedDuringSync, { + environmentLive: true, + environmentPreviouslyLive: true, + settingsHydrated: true, + }); + + expect(reconnected.cues).toEqual(["success"]); + }); + + it("refreshes cached startup state until the environment first becomes live", () => { + const staleRunning = makeThread({ + latestTurn: { + turnId: TurnId.make("turn-1"), + initiatingUserMessageId: MessageId.make("message-1"), + state: "running", + requestedAt: "2026-07-11T12:00:00.000Z", + startedAt: "2026-07-11T12:00:01.000Z", + completedAt: null, + assistantMessageId: null, + }, + }); + const refreshedCompleted = makeThread({ + latestTurn: { + ...staleRunning.latestTurn!, + state: "completed", + completedAt: "2026-07-11T12:00:05.000Z", + }, + }); + const seeded = observeThreadSoundState(null, staleRunning, { + environmentLive: false, + environmentPreviouslyLive: false, + settingsHydrated: true, + }); + const refreshed = observeThreadSoundState(seeded.state, refreshedCompleted, { + environmentLive: false, + environmentPreviouslyLive: false, + settingsHydrated: true, + }); + const firstLive = observeThreadSoundState(refreshed.state, refreshedCompleted, { + environmentLive: true, + environmentPreviouslyLive: false, + settingsHydrated: true, + }); + + expect(firstLive.cues).toEqual([]); + }); + + it("plays later cues after the first live snapshot seeds the baseline", () => { + const running = makeThread({ + latestTurn: { + turnId: TurnId.make("turn-1"), + initiatingUserMessageId: MessageId.make("message-1"), + state: "running", + requestedAt: "2026-07-11T12:00:00.000Z", + startedAt: "2026-07-11T12:00:01.000Z", + completedAt: null, + assistantMessageId: null, + }, + }); + const completed = makeThread({ + latestTurn: { + ...running.latestTurn!, + state: "completed", + completedAt: "2026-07-11T12:00:05.000Z", + }, + }); + let environmentObservedLive = false; + const firstLive = observeThreadSoundState(null, running, { + environmentLive: true, + environmentPreviouslyLive: environmentObservedLive, + settingsHydrated: true, + }); + environmentObservedLive = true; + const laterUpdate = observeThreadSoundState(firstLive.state, completed, { + environmentLive: true, + environmentPreviouslyLive: environmentObservedLive, + settingsHydrated: true, + }); + + expect(firstLive.cues).toEqual([]); + expect(laterUpdate.cues).toEqual(["success"]); + }); + + it("detects a user-input request received while its environment is synchronizing", () => { + const idle = makeThread(); + const pendingInputDuringSync = makeThread({ hasPendingUserInput: true }); + const beforeSync = captureThreadSoundState([idle]); + const whileSynchronizing = observeThreadSoundState(beforeSync, pendingInputDuringSync, { + environmentLive: false, + environmentPreviouslyLive: true, + settingsHydrated: true, + }); + const reconnected = observeThreadSoundState(whileSynchronizing.state, pendingInputDuringSync, { + environmentLive: true, + environmentPreviouslyLive: true, + settingsHydrated: true, + }); + + expect(reconnected.cues).toEqual(["bloom"]); + }); + + it("does not replay pending input when a historical thread reappears", () => { + const discovered = observeThreadSoundState(null, makeThread({ hasPendingUserInput: true }), { + environmentLive: true, + environmentPreviouslyLive: true, + settingsHydrated: true, + }); + + expect(discovered.cues).toEqual([]); + }); + + it("does not replay completion when a historical thread reappears", () => { + const discovered = observeThreadSoundState( + null, + makeThread({ + latestTurn: { + turnId: TurnId.make("remote-turn"), + initiatingUserMessageId: MessageId.make("remote-message"), + state: "completed", + requestedAt: "2026-07-11T12:00:01.000Z", + startedAt: "2026-07-11T12:00:02.000Z", + completedAt: "2026-07-11T12:00:05.000Z", + assistantMessageId: null, + }, + }), + { + environmentLive: true, + environmentPreviouslyLive: true, + settingsHydrated: true, + }, + ); + + expect(discovered.cues).toEqual([]); + }); + + it("seeds a thread from the first live hydration without playing a cue", () => { + const discovered = observeThreadSoundState(null, makeThread({ hasPendingUserInput: true }), { + environmentLive: true, + environmentPreviouslyLive: false, + settingsHydrated: true, + }); + + expect(discovered.cues).toEqual([]); + }); + + it("keeps input-request cues enabled when completion sounds are disabled", () => { + expect(shouldPlayInteractionSound("success", false)).toBe(false); + expect(shouldPlayInteractionSound("bloom", false)).toBe(true); + }); +}); diff --git a/packages/client-runtime/src/interactionSounds.ts b/packages/client-runtime/src/interactionSounds.ts new file mode 100644 index 000000000000..1282f4c6bb54 --- /dev/null +++ b/packages/client-runtime/src/interactionSounds.ts @@ -0,0 +1,149 @@ +import type { EnvironmentThreadShell } from "./state/shell.ts"; + +export type InteractionSoundCue = "bloom" | "success"; + +interface ThreadSoundState { + readonly completedTurn: string | null; + readonly userInitiatedTurn: string | null; + readonly hasPendingUserInput: boolean; + readonly hasPendingApprovals: boolean; +} + +export type ThreadSoundStateByKey = ReadonlyMap; + +export function shouldPlayInteractionSound( + cue: InteractionSoundCue, + completionSoundEnabled: boolean, +): boolean { + return cue !== "success" || completionSoundEnabled; +} + +function threadKey(thread: EnvironmentThreadShell): string { + return JSON.stringify([thread.environmentId, thread.id]); +} + +function completedTurn(thread: EnvironmentThreadShell): string | null { + const latestTurn = thread.latestTurn; + if (latestTurn?.state !== "completed" || latestTurn.completedAt === null) { + return null; + } + return latestTurn.turnId; +} + +const USER_TURN_START_WINDOW_MS = 2 * 60 * 1_000; + +function userInitiatedTurn(thread: EnvironmentThreadShell): string | null { + const latestTurn = thread.latestTurn; + if (latestTurn === null) { + return null; + } + + // Current servers expose the exact message/turn association from the + // projection. A null association explicitly identifies synthetic provider + // work, while undefined means the shell came from an older server. + if (latestTurn.initiatingUserMessageId !== undefined) { + return latestTurn.initiatingUserMessageId === null ? null : latestTurn.turnId; + } + + if (thread.latestUserMessageAt === null) { + return null; + } + + const requestedAt = Date.parse(latestTurn.requestedAt); + const latestUserMessageAt = Date.parse(thread.latestUserMessageAt); + if (!Number.isFinite(requestedAt) || !Number.isFinite(latestUserMessageAt)) { + return null; + } + + // A normal prompt is recorded before provider startup, while synthetic + // background turns have no nearby initiating message. Keep the same bounded + // adoption window used for queued turn starts so an old prompt cannot claim + // unrelated background work. A later steering message is also excluded + // because it falls after requestedAt. + const startupDelay = requestedAt - latestUserMessageAt; + if (startupDelay < 0 || startupDelay > USER_TURN_START_WINDOW_MS) { + return null; + } + + return latestTurn.turnId; +} + +export function captureThreadSoundState( + threads: ReadonlyArray, +): ThreadSoundStateByKey { + return new Map( + threads.map((thread) => [ + threadKey(thread), + { + completedTurn: completedTurn(thread), + userInitiatedTurn: userInitiatedTurn(thread), + hasPendingUserInput: thread.hasPendingUserInput, + hasPendingApprovals: thread.hasPendingApprovals, + }, + ]), + ); +} + +export function deriveInteractionSoundCues( + previous: ThreadSoundStateByKey, + threads: ReadonlyArray, +): InteractionSoundCue[] { + const cues: InteractionSoundCue[] = []; + + for (const thread of threads) { + const prior = previous.get(threadKey(thread)); + const nextCompletedTurn = completedTurn(thread); + const nextUserInitiatedTurn = userInitiatedTurn(thread); + + if ( + prior && + nextCompletedTurn !== null && + prior.completedTurn !== nextCompletedTurn && + nextUserInitiatedTurn === nextCompletedTurn + ) { + cues.push("success"); + } + if ( + prior && + ((thread.hasPendingUserInput && !prior.hasPendingUserInput) || + (thread.hasPendingApprovals && !prior.hasPendingApprovals)) + ) { + cues.push("bloom"); + } + } + + return cues; +} + +export interface ThreadSoundObservation { + readonly state: ThreadSoundStateByKey; + readonly cues: ReadonlyArray; +} + +/** + * Advance one thread's sound state. Coordinators subscribe to individual + * thread atoms so streaming updates only revisit the thread that changed. + */ +export function observeThreadSoundState( + previous: ThreadSoundStateByKey | null, + thread: EnvironmentThreadShell, + options: { + readonly environmentLive: boolean; + readonly environmentPreviouslyLive: boolean; + readonly settingsHydrated: boolean; + }, +): ThreadSoundObservation { + const current = [thread]; + if (previous === null || !options.environmentPreviouslyLive) { + return { state: captureThreadSoundState(current), cues: [] }; + } + + if (!options.environmentLive || !options.settingsHydrated) { + return { state: previous, cues: [] }; + } + + return { + state: captureThreadSoundState(current), + cues: deriveInteractionSoundCues(previous, current), + }; +} diff --git a/packages/client-runtime/src/state/shell.test.ts b/packages/client-runtime/src/state/shell.test.ts index f1326e0a5cbe..e339b7056855 100644 --- a/packages/client-runtime/src/state/shell.test.ts +++ b/packages/client-runtime/src/state/shell.test.ts @@ -6,7 +6,11 @@ import { Atom, AtomRegistry } from "effect/unstable/reactivity"; import { PrimaryConnectionTarget } from "../connection/model.ts"; import type { EnvironmentShellState } from "./shell.ts"; -import { createEnvironmentServerConfigsAtom, createEnvironmentShellSummaryAtom } from "./shell.ts"; +import { + createEnvironmentServerConfigsAtom, + createEnvironmentShellSummaryAtom, + createLiveEnvironmentIdsAtom, +} from "./shell.ts"; const ENVIRONMENT_ID = EnvironmentId.make("environment-1"); const OTHER_ENVIRONMENT_ID = EnvironmentId.make("environment-2"); @@ -77,6 +81,11 @@ function makeHarness() { catalogValueAtom, serverConfigValueAtom: configAtoms, }); + const liveEnvironmentIdsAtom = createLiveEnvironmentIdsAtom({ + catalogValueAtom, + shellStateValueAtom: shellStateAtoms, + label: "test-live-environment-ids", + }); return { registry: AtomRegistry.make(), @@ -84,6 +93,7 @@ function makeHarness() { configAtom: configAtoms, summaryAtom, serverConfigsAtom, + liveEnvironmentIdsAtom, }; } @@ -127,4 +137,30 @@ describe("environment shell projections", () => { harness.registry.set(harness.configAtom(ENVIRONMENT_ID), config); expect(harness.registry.get(harness.serverConfigsAtom)).toBe(withConfig); }); + + it("projects live environment IDs and preserves identity while membership is unchanged", () => { + const harness = makeHarness(); + const empty = harness.registry.get(harness.liveEnvironmentIdsAtom); + + harness.registry.set( + harness.shellStateAtom(OTHER_ENVIRONMENT_ID), + shellState({ + status: "live", + updatedAt: "2026-06-02T00:00:00.000Z", + }), + ); + const live = harness.registry.get(harness.liveEnvironmentIdsAtom); + + expect(live).not.toBe(empty); + expect(live).toEqual(new Set([OTHER_ENVIRONMENT_ID])); + + harness.registry.set( + harness.shellStateAtom(OTHER_ENVIRONMENT_ID), + shellState({ + status: "live", + updatedAt: "2026-06-03T00:00:00.000Z", + }), + ); + expect(harness.registry.get(harness.liveEnvironmentIdsAtom)).toBe(live); + }); }); diff --git a/packages/client-runtime/src/state/shell.ts b/packages/client-runtime/src/state/shell.ts index 95d90f9b36f2..4e05a3834c24 100644 --- a/packages/client-runtime/src/state/shell.ts +++ b/packages/client-runtime/src/state/shell.ts @@ -376,6 +376,30 @@ export function createEnvironmentShellSummaryAtom(input: { }).pipe(Atom.withLabel("environment-shell-summary")); } +export function createLiveEnvironmentIdsAtom(input: { + readonly catalogValueAtom: Atom.Atom; + readonly shellStateValueAtom: (environmentId: EnvironmentId) => Atom.Atom; + readonly label: string; +}) { + let previousLiveEnvironmentIds: ReadonlySet = new Set(); + return Atom.make((get): ReadonlySet => { + const next = new Set(); + for (const environmentId of get(input.catalogValueAtom).entries.keys()) { + if (get(input.shellStateValueAtom(environmentId)).status === "live") { + next.add(environmentId); + } + } + if ( + next.size === previousLiveEnvironmentIds.size && + [...next].every((environmentId) => previousLiveEnvironmentIds.has(environmentId)) + ) { + return previousLiveEnvironmentIds; + } + previousLiveEnvironmentIds = next; + return previousLiveEnvironmentIds; + }).pipe(Atom.withLabel(input.label)); +} + export function createEnvironmentServerConfigsAtom(input: { readonly catalogValueAtom: Atom.Atom; readonly serverConfigValueAtom: (environmentId: EnvironmentId) => Atom.Atom; diff --git a/packages/contracts/src/orchestration.ts b/packages/contracts/src/orchestration.ts index da4eac53d9e6..5a89703ee261 100644 --- a/packages/contracts/src/orchestration.ts +++ b/packages/contracts/src/orchestration.ts @@ -459,6 +459,7 @@ export type OrchestrationLatestTurnState = typeof OrchestrationLatestTurnState.T export const OrchestrationLatestTurn = Schema.Struct({ turnId: TurnId, + initiatingUserMessageId: Schema.optional(Schema.NullOr(MessageId)), state: OrchestrationLatestTurnState, requestedAt: IsoDateTime, startedAt: Schema.NullOr(IsoDateTime), diff --git a/packages/contracts/src/settings.test.ts b/packages/contracts/src/settings.test.ts index f01d24c3ac8c..5df4df364062 100644 --- a/packages/contracts/src/settings.test.ts +++ b/packages/contracts/src/settings.test.ts @@ -371,6 +371,18 @@ describe("ServerSettings thread settlement", () => { }); }); +describe("ClientSettings completion sound", () => { + it("defaults the completion sound on", () => { + expect(decodeClientSettings({}).enableCompletionSounds).toBe(true); + }); + + it("preserves an explicit disabled preference", () => { + expect(decodeClientSettings({ enableCompletionSounds: false }).enableCompletionSounds).toBe( + false, + ); + }); +}); + describe("ServerSettings.providerInstances (slice-2 invariant)", () => { it("defaults text generation to Luna at low reasoning effort", () => { expect(DEFAULT_SERVER_SETTINGS.textGenerationModelSelection).toEqual({ diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index 7e25c8444dbb..727ed373fd77 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -279,6 +279,7 @@ export const ClientSettingsSchema = Schema.Struct({ glassOpacity: GlassOpacity.pipe( Schema.withDecodingDefault(Effect.succeed(DEFAULT_GLASS_OPACITY)), ), + enableCompletionSounds: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))), fontSizeInterface: InterfaceFontSize.pipe( Schema.withDecodingDefault(Effect.succeed(DEFAULT_INTERFACE_FONT_SIZE)), ), @@ -1237,6 +1238,7 @@ export const ClientSettingsPatch = Schema.Struct({ environmentIdentificationMode: Schema.optionalKey(EnvironmentIdentificationMode), glassOpacity: Schema.optionalKey(GlassOpacity), onboardingCompletedAt: Schema.optionalKey(Schema.NullOr(Schema.String)), + enableCompletionSounds: Schema.optionalKey(Schema.Boolean), fontSizeInterface: Schema.optionalKey(InterfaceFontSize), fontSizePrompt: Schema.optionalKey(PromptFontSize), fontSizeCode: Schema.optionalKey(CodeFontSize), diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 95403e871a9b..12799bcc31c2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -614,6 +614,9 @@ importers: culori: specifier: ^4.0.2 version: 4.0.2 + cuelume: + specifier: ^0.2.1 + version: 0.2.1 effect: specifier: 4.0.0-beta.103 version: 4.0.0-beta.103(patch_hash=af36b7948b6f9c56623074662b51dade5699880c1a7c71245de73e13c3185fb6) @@ -6338,6 +6341,9 @@ packages: csstype@3.2.3: resolution: {integrity: sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==} + cuelume@0.2.1: + resolution: {integrity: sha512-UFN5GtPRhnkL55r5rz7oalgy72XTtV50QqOec3+wHer0O1hlwOteej99dlRxeFX2ViLUD/CpCu47v5XCTa2tEQ==} + culori@4.0.2: resolution: {integrity: sha512-1+BhOB8ahCn4O0cep0Sh2l9KCOfOdY+BXJnKMHFFzDEouSr/el18QwXEMRlOj9UY5nCeA8UN3a/82rUWRBeyBw==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} @@ -16487,6 +16493,8 @@ snapshots: csstype@3.2.3: {} + cuelume@0.2.1: {} + culori@4.0.2: {} debounce-fn@4.0.0: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index ee1bd25547f6..94f1159150a9 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -79,6 +79,7 @@ minimumReleaseAgeExclude: - "@effect/sql-sqlite-bun@4.0.0-beta.103" - "@effect/vitest@4.0.0-beta.103" - alchemy@2.0.0-beta.65 + - cuelume@0.2.1 - effect@4.0.0-beta.103 - "@legendapp/list@3.3.5" - "@expo/cli@57.0.20" diff --git a/scripts/generate-interaction-sound-assets.mjs b/scripts/generate-interaction-sound-assets.mjs new file mode 100644 index 000000000000..0ca3b0ff2e2c --- /dev/null +++ b/scripts/generate-interaction-sound-assets.mjs @@ -0,0 +1,111 @@ +import * as NodeFS from "node:fs"; +import * as NodePath from "node:path"; +import * as NodeURL from "node:url"; + +const SAMPLE_RATE = 44_100; +const TARGET_PEAK = 0.42; +const scriptDirectory = NodePath.dirname(NodeURL.fileURLToPath(import.meta.url)); +const outputDirectory = NodePath.resolve( + scriptDirectory, + "../apps/mobile/assets/interaction-sounds", +); + +const recipes = { + bloom: { + duration: 0.85, + masterGain: 0.5, + layers: [ + { frequency: 528, attack: 0.06, decay: 0.32, peak: 0.06 }, + { frequency: 528, detune: 12, attack: 0.06, decay: 0.34, peak: 0.05 }, + ], + shimmer: { delay: 0.15, feedback: 0.2, wet: 0.12 }, + }, + success: { + duration: 0.75, + masterGain: 0.5, + layers: [ + { frequency: 880, attack: 0.004, decay: 0.09, peak: 0.06 }, + { frequency: 1108.73, offset: 0.06, attack: 0.004, decay: 0.1, peak: 0.06 }, + { frequency: 1318.51, offset: 0.12, attack: 0.004, decay: 0.18, peak: 0.07 }, + ], + }, +}; + +function envelope(layer, elapsed) { + if (elapsed < 0) return 0; + if (elapsed < layer.attack) return elapsed / layer.attack; + const decayElapsed = elapsed - layer.attack; + if (decayElapsed >= layer.decay) return 0; + return Math.exp((-7 * decayElapsed) / layer.decay); +} + +function renderRecipe(recipe) { + const sampleCount = Math.ceil(recipe.duration * SAMPLE_RATE); + const dry = new Float64Array(sampleCount); + + for (const layer of recipe.layers) { + const offset = layer.offset ?? 0; + const frequency = layer.frequency * 2 ** ((layer.detune ?? 0) / 1200); + for (let index = 0; index < sampleCount; index += 1) { + const time = index / SAMPLE_RATE; + const elapsed = time - offset; + const amplitude = envelope(layer, elapsed); + if (amplitude === 0) continue; + dry[index] += + Math.sin(2 * Math.PI * frequency * elapsed) * amplitude * layer.peak * recipe.masterGain; + } + } + + const output = Float64Array.from(dry); + if (recipe.shimmer) { + const delaySamples = Math.round(recipe.shimmer.delay * SAMPLE_RATE); + let repeat = 1; + let repeatGain = recipe.shimmer.wet; + while (repeatGain >= 0.001) { + const offset = delaySamples * repeat; + for (let index = 0; index + offset < sampleCount; index += 1) { + output[index + offset] += dry[index] * repeatGain; + } + repeat += 1; + repeatGain *= recipe.shimmer.feedback; + } + } + + let peak = 0; + for (const sample of output) peak = Math.max(peak, Math.abs(sample)); + const scale = peak === 0 ? 1 : TARGET_PEAK / peak; + return Int16Array.from(output, (sample) => + Math.round(Math.max(-1, Math.min(1, sample * scale)) * 0x7fff), + ); +} + +function encodeWav(samples) { + const bytesPerSample = 2; + const dataSize = samples.length * bytesPerSample; + const buffer = Buffer.alloc(44 + dataSize); + buffer.write("RIFF", 0); + buffer.writeUInt32LE(36 + dataSize, 4); + buffer.write("WAVE", 8); + buffer.write("fmt ", 12); + buffer.writeUInt32LE(16, 16); + buffer.writeUInt16LE(1, 20); + buffer.writeUInt16LE(1, 22); + buffer.writeUInt32LE(SAMPLE_RATE, 24); + buffer.writeUInt32LE(SAMPLE_RATE * bytesPerSample, 28); + buffer.writeUInt16LE(bytesPerSample, 32); + buffer.writeUInt16LE(16, 34); + buffer.write("data", 36); + buffer.writeUInt32LE(dataSize, 40); + for (let index = 0; index < samples.length; index += 1) { + buffer.writeInt16LE(samples[index], 44 + index * bytesPerSample); + } + return buffer; +} + +NodeFS.mkdirSync(outputDirectory, { recursive: true }); +for (const [name, recipe] of Object.entries(recipes)) { + NodeFS.writeFileSync( + NodePath.resolve(outputDirectory, `${name}.wav`), + encodeWav(renderRecipe(recipe)), + ); +}