diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.json index 1e7a33773ac..ec85659baaf 100644 --- a/.github/workspace-dep-closures.json +++ b/.github/workspace-dep-closures.json @@ -3147,6 +3147,26 @@ "crates/workspace-hack", "services/deleted_item_poller" ], + "dictation": [ + "crates/ai_usage", + "crates/bot_id", + "crates/bot_token", + "crates/cowlike", + "crates/dictation", + "crates/harness_id", + "crates/harness_token", + "crates/macro_auth", + "crates/macro_authorization", + "crates/macro_env", + "crates/macro_env_var", + "crates/macro_user_id", + "crates/macro_uuid", + "crates/model-error-response", + "crates/model_user", + "crates/rate_limit", + "crates/remote_env_var", + "crates/workspace-hack" + ], "document_cognition_service": [ "crates/activity", "crates/agent", @@ -3330,6 +3350,7 @@ "crates/cowlike", "crates/crm", "crates/database_env_vars", + "crates/dictation", "crates/document_sub_type", "crates/documents", "crates/dynamodb_client", diff --git a/.sqlx/query-16a038db0df182a666e0717ac7f9243efeeb51622f9bea833501850b099cc291.json b/.sqlx/query-16a038db0df182a666e0717ac7f9243efeeb51622f9bea833501850b099cc291.json new file mode 100644 index 00000000000..d34422565a0 --- /dev/null +++ b/.sqlx/query-16a038db0df182a666e0717ac7f9243efeeb51622f9bea833501850b099cc291.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO ai_pricing (model, price_per_million_in, price_per_million_out, price_per_audio_minute)\n VALUES ($1, $2, $3, $4)\n ON CONFLICT (model) DO UPDATE\n SET price_per_million_in = EXCLUDED.price_per_million_in,\n price_per_million_out = EXCLUDED.price_per_million_out,\n price_per_audio_minute = EXCLUDED.price_per_audio_minute,\n updated_at = NOW()\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Float4", + "Float4", + "Float4" + ] + }, + "nullable": [] + }, + "hash": "16a038db0df182a666e0717ac7f9243efeeb51622f9bea833501850b099cc291" +} diff --git a/.sqlx/query-cc6d835ae3a35891b84f625bc774d487f4907b7f4ac77a6b870113b7ea2581fb.json b/.sqlx/query-40ce78006bc37d7088397407fae17e249fa9f8524426c950480b3df116d6729e.json similarity index 63% rename from .sqlx/query-cc6d835ae3a35891b84f625bc774d487f4907b7f4ac77a6b870113b7ea2581fb.json rename to .sqlx/query-40ce78006bc37d7088397407fae17e249fa9f8524426c950480b3df116d6729e.json index dce905120c9..f2bbc0d1671 100644 --- a/.sqlx/query-cc6d835ae3a35891b84f625bc774d487f4907b7f4ac77a6b870113b7ea2581fb.json +++ b/.sqlx/query-40ce78006bc37d7088397407fae17e249fa9f8524426c950480b3df116d6729e.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n feature,\n user_id,\n entity,\n model,\n input_tokens,\n output_tokens,\n price_per_million_in,\n price_per_million_out,\n total,\n created_at\n FROM ai_usage\n WHERE ($1::timestamptz IS NULL OR created_at >= $1)\n AND ($2::timestamptz IS NULL OR created_at < $2)\n AND (cardinality($3::text[]) = 0 OR user_id = ANY($3))\n AND (cardinality($4::text[]) = 0 OR feature = ANY($4))\n ORDER BY created_at DESC\n ", + "query": "\n SELECT\n feature,\n user_id,\n entity,\n model,\n input_tokens,\n output_tokens,\n audio_seconds,\n price_per_million_in,\n price_per_million_out,\n price_per_audio_minute,\n total,\n created_at\n FROM ai_usage\n WHERE ($1::timestamptz IS NULL OR created_at >= $1)\n AND ($2::timestamptz IS NULL OR created_at < $2)\n AND (cardinality($3::text[]) = 0 OR user_id = ANY($3))\n AND (cardinality($4::text[]) = 0 OR feature = ANY($4))\n ORDER BY created_at DESC\n ", "describe": { "columns": [ { @@ -35,21 +35,31 @@ }, { "ordinal": 6, + "name": "audio_seconds", + "type_info": "Float8" + }, + { + "ordinal": 7, "name": "price_per_million_in", "type_info": "Float4" }, { - "ordinal": 7, + "ordinal": 8, "name": "price_per_million_out", "type_info": "Float4" }, { - "ordinal": 8, + "ordinal": 9, + "name": "price_per_audio_minute", + "type_info": "Float4" + }, + { + "ordinal": 10, "name": "total", "type_info": "Float4" }, { - "ordinal": 9, + "ordinal": 11, "name": "created_at", "type_info": "Timestamptz" } @@ -72,8 +82,10 @@ true, true, true, + true, + true, false ] }, - "hash": "cc6d835ae3a35891b84f625bc774d487f4907b7f4ac77a6b870113b7ea2581fb" + "hash": "40ce78006bc37d7088397407fae17e249fa9f8524426c950480b3df116d6729e" } diff --git a/.sqlx/query-645d4533a5ece189935ffc7859067772f61584ab24b98f4b6a66ba30dbf09ae8.json b/.sqlx/query-645d4533a5ece189935ffc7859067772f61584ab24b98f4b6a66ba30dbf09ae8.json new file mode 100644 index 00000000000..71b33dff7ba --- /dev/null +++ b/.sqlx/query-645d4533a5ece189935ffc7859067772f61584ab24b98f4b6a66ba30dbf09ae8.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE ai_usage\n SET price_per_million_in = $2::real,\n price_per_million_out = $3::real,\n price_per_audio_minute = $4::real,\n total = CASE WHEN (audio_seconds IS NULL) <> ($4::real IS NULL) THEN NULL\n ELSE (input_tokens::real / 1000000.0::real) * $2::real\n + (output_tokens::real / 1000000.0::real) * $3::real\n + (COALESCE(audio_seconds, 0) / 60.0 * COALESCE($4::real, 0))::real END\n WHERE model = $1\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Float4", + "Float4", + "Float4" + ] + }, + "nullable": [] + }, + "hash": "645d4533a5ece189935ffc7859067772f61584ab24b98f4b6a66ba30dbf09ae8" +} diff --git a/.sqlx/query-8dc9ec20b47e16110d9ec18da057b14461c616516d03ca31d7e17749938b829a.json b/.sqlx/query-68dcb987626d91eaa4526ab79ba646da0d94e452286665dcc1e136468a7b2df6.json similarity index 58% rename from .sqlx/query-8dc9ec20b47e16110d9ec18da057b14461c616516d03ca31d7e17749938b829a.json rename to .sqlx/query-68dcb987626d91eaa4526ab79ba646da0d94e452286665dcc1e136468a7b2df6.json index 1082325c79c..1510f834bdc 100644 --- a/.sqlx/query-8dc9ec20b47e16110d9ec18da057b14461c616516d03ca31d7e17749938b829a.json +++ b/.sqlx/query-68dcb987626d91eaa4526ab79ba646da0d94e452286665dcc1e136468a7b2df6.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT price_per_million_in, price_per_million_out\n FROM ai_pricing\n WHERE model = $1\n ", + "query": "\n SELECT price_per_million_in, price_per_million_out, price_per_audio_minute\n FROM ai_pricing\n WHERE model = $1\n ", "describe": { "columns": [ { @@ -12,6 +12,11 @@ "ordinal": 1, "name": "price_per_million_out", "type_info": "Float4" + }, + { + "ordinal": 2, + "name": "price_per_audio_minute", + "type_info": "Float4" } ], "parameters": { @@ -21,8 +26,9 @@ }, "nullable": [ false, - false + false, + true ] }, - "hash": "8dc9ec20b47e16110d9ec18da057b14461c616516d03ca31d7e17749938b829a" + "hash": "68dcb987626d91eaa4526ab79ba646da0d94e452286665dcc1e136468a7b2df6" } diff --git a/.sqlx/query-81bed360250642bbb9f8f46d705bfe31be1a467c32bb2c8a16cf61cc668fb6c1.json b/.sqlx/query-81bed360250642bbb9f8f46d705bfe31be1a467c32bb2c8a16cf61cc668fb6c1.json deleted file mode 100644 index bd17d0b32c5..00000000000 --- a/.sqlx/query-81bed360250642bbb9f8f46d705bfe31be1a467c32bb2c8a16cf61cc668fb6c1.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO ai_pricing (model, price_per_million_in, price_per_million_out)\n VALUES ($1, $2, $3)\n ON CONFLICT (model) DO UPDATE\n SET price_per_million_in = EXCLUDED.price_per_million_in,\n price_per_million_out = EXCLUDED.price_per_million_out,\n updated_at = NOW()\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Float4", - "Float4" - ] - }, - "nullable": [] - }, - "hash": "81bed360250642bbb9f8f46d705bfe31be1a467c32bb2c8a16cf61cc668fb6c1" -} diff --git a/.sqlx/query-8f6b8f408336fd56bf212f1b17f32f36dc1fdbe6d45e4ccfe3ab938974d828c3.json b/.sqlx/query-8f6b8f408336fd56bf212f1b17f32f36dc1fdbe6d45e4ccfe3ab938974d828c3.json deleted file mode 100644 index 5390457b2c0..00000000000 --- a/.sqlx/query-8f6b8f408336fd56bf212f1b17f32f36dc1fdbe6d45e4ccfe3ab938974d828c3.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE ai_usage\n SET price_per_million_in = $2::real,\n price_per_million_out = $3::real,\n total = (input_tokens::real / 1000000.0::real) * $2::real\n + (output_tokens::real / 1000000.0::real) * $3::real\n WHERE model = $1\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Float4", - "Float4" - ] - }, - "nullable": [] - }, - "hash": "8f6b8f408336fd56bf212f1b17f32f36dc1fdbe6d45e4ccfe3ab938974d828c3" -} diff --git a/.sqlx/query-006d6742d6218213f59326abc8c5e0df14ddf5e7c04a2cf05d8bff6ea4279add.json b/.sqlx/query-c552084079684d8689a7330053ba60061cd45ce5071f59c394858f3fb2c8ea09.json similarity index 55% rename from .sqlx/query-006d6742d6218213f59326abc8c5e0df14ddf5e7c04a2cf05d8bff6ea4279add.json rename to .sqlx/query-c552084079684d8689a7330053ba60061cd45ce5071f59c394858f3fb2c8ea09.json index 1f1640067f9..52bef1d396d 100644 --- a/.sqlx/query-006d6742d6218213f59326abc8c5e0df14ddf5e7c04a2cf05d8bff6ea4279add.json +++ b/.sqlx/query-c552084079684d8689a7330053ba60061cd45ce5071f59c394858f3fb2c8ea09.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO ai_usage (\n id, feature, user_id, entity, model,\n input_tokens, output_tokens,\n price_per_million_in, price_per_million_out, total\n )\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)\n ", + "query": "\n INSERT INTO ai_usage (\n id, feature, user_id, entity, model,\n input_tokens, output_tokens, audio_seconds,\n price_per_million_in, price_per_million_out, price_per_audio_minute, total\n )\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)\n ", "describe": { "columns": [], "parameters": { @@ -12,6 +12,8 @@ "Text", "Int8", "Int8", + "Float8", + "Float4", "Float4", "Float4", "Float4" @@ -19,5 +21,5 @@ }, "nullable": [] }, - "hash": "006d6742d6218213f59326abc8c5e0df14ddf5e7c04a2cf05d8bff6ea4279add" + "hash": "c552084079684d8689a7330053ba60061cd45ce5071f59c394858f3fb2c8ea09" } diff --git a/Cargo.lock b/Cargo.lock index 5161bd3b27d..f4d35161cf5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5288,6 +5288,41 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "dictation" +version = "0.1.0" +dependencies = [ + "ai_usage", + "async-openai", + "axum", + "axum-extra", + "backoff", + "bytes", + "http-body-util", + "infer 0.19.0", + "macro_authorization", + "macro_env_var", + "macro_user_id", + "model-error-response", + "model_user", + "opus-pure", + "rate_limit", + "reqwest 0.13.4", + "rootcause", + "serde", + "serde_json", + "symphonia", + "thiserror 2.0.18", + "tokio", + "tower 0.5.3", + "tracing", + "tracing-subscriber", + "utoipa", + "uuid", + "wiremock", + "workspace-hack", +] + [[package]] name = "diff" version = "0.1.13" @@ -5523,6 +5558,7 @@ dependencies = [ "cool_asserts", "cowlike", "crm", + "dictation", "document_sub_type", "documents", "doppler-config", @@ -6718,6 +6754,12 @@ dependencies = [ "zune-inflate", ] +[[package]] +name = "extended" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365" + [[package]] name = "fallible-iterator" version = "0.2.0" @@ -12390,6 +12432,12 @@ dependencies = [ "tokio-stream", ] +[[package]] +name = "opus-pure" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de6a3cef053e1fdea38a5e50edb0789590a31184c5a59f6cd5e462fe844793e7" + [[package]] name = "ordered-float" version = "5.3.0" @@ -16631,6 +16679,107 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "symphonia" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7edef6a96b696d4e0cab5ee9ebb7ca155ed95f30a6b45bbb8b97d2727f02424" +dependencies = [ + "lazy_static", + "symphonia-core", + "symphonia-format-isomp4", + "symphonia-format-mkv", + "symphonia-format-ogg", + "symphonia-format-riff", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-common" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acc3fcc18ec9b8cdd48614e259c4cf0d27b71d41e5d9b120b42c5adab12d7c4" +dependencies = [ + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-core" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01c412864d599d4750d0c3d684d7e093ec05e5309681ef5252cc1096a437f6e0" +dependencies = [ + "bitflags 2.11.1", + "bytemuck", + "lazy_static", + "log", + "num-complex", + "smallvec", +] + +[[package]] +name = "symphonia-format-isomp4" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e681a70e1870d34e02abf1dbc51e4267c3f1827801474e8870be8c689fc4dc3" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-format-mkv" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d015c5c0558864665894b3f4cbd95e10abb01b9c868e751c72670f326a56360e" +dependencies = [ + "lazy_static", + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-format-ogg" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5495e7f7e3c7035328d82b6d6e377eef289bb0c4105bdeb557fc93a833f994" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-format-riff" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ff70929083a8c1a5f6cd7c904b6071c7914ad04739b510c2f7239dfc9b7dabe" +dependencies = [ + "extended", + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-metadata" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83713a97705d77bdef7cdbc0768fd6e5a54e4cd7e48d60a806ae85639e2c87c6" +dependencies = [ + "lazy_static", + "log", + "regex-lite", + "smallvec", + "symphonia-core", +] + [[package]] name = "syn" version = "1.0.109" @@ -19298,6 +19447,7 @@ dependencies = [ "ahash", "aho-corasick", "anyhow", + "async-openai", "aws-config", "aws-credential-types", "aws-runtime", @@ -19314,6 +19464,7 @@ dependencies = [ "base64 0.22.1", "bitflags 2.11.1", "bstr", + "bytemuck", "byteorder", "bytes", "bzip2", diff --git a/Cargo.toml b/Cargo.toml index dfcb78beb04..ac0ec63ed52 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,7 @@ members = [ "crates/cursor_cloud_agents", "crates/claude_cloud_agents", "crates/embedding", + "crates/dictation", "crates/email_api_client", "crates/entity_mutation", "crates/entity_registry/entity_registry", @@ -185,6 +186,7 @@ aws-types = "1" aws_lambda_events = { version = "0.16.0", default-features = false } axum = { version = "0.8", features = ["tower-log"] } axum-extra = { version = "0.10" } +backoff = { version = "0.4.0", features = ["tokio"] } base64 = "0.22.1" bebop = "3.1.3" cached = { version = "0.54.0", features = ["async_tokio_rt_multi_thread"] } @@ -220,6 +222,7 @@ http-body-util = "0.1.2" httpdate = "1" image = "0.25" indexmap = "2" +infer = "0.19.0" indoc = "2" inquire = "0.7" itertools = "0.14" @@ -241,6 +244,7 @@ lol_html = "2.4.0" lru = "0.16" mockall = "0.13" nom = "8" +opus-pure = "0.2.1" opentelemetry = { version = "0.31.0", default-features = false } opentelemetry-appender-tracing = { version = "0.31.1", default-features = false } opentelemetry-http = { version = "0.31.0", default-features = false } @@ -309,6 +313,12 @@ sqlx = { version = "0.8.6", features = [ sqlx-core = { version = "0.8.6" } sse-core = { version = "0.2.3", default-features = false, features = ["std"] } strum = { version = "0.27.1", features = ["derive"] } +symphonia = { version = "0.6.1", default-features = false, features = [ + "isomp4", + "mkv", + "ogg", + "wav", +] } tempfile = "3" thiserror = "2.0.0" time = "0.3.37" diff --git a/apps/web/package.json b/apps/web/package.json index b5982fbab59..a7352d8df6f 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -102,6 +102,7 @@ "@solid-primitives/rootless": "^1.5.2", "@solid-primitives/scheduled": "^1.5.2", "@solid-primitives/storage": "^4.3.3", + "@solid-primitives/stream": "^0.7.4", "@solidjs/router": "^0.15.3", "@tailwindcss/vite": "^4.2.4", "@tanstack/pacer": "^0.17.3", diff --git a/apps/web/src/components/ui/components/CollapsedInput.tsx b/apps/web/src/components/ui/components/CollapsedInput.tsx index feed0364051..664e0d3a112 100644 --- a/apps/web/src/components/ui/components/CollapsedInput.tsx +++ b/apps/web/src/components/ui/components/CollapsedInput.tsx @@ -26,6 +26,8 @@ export type CollapsedInputProps = { */ disabled?: boolean; class?: string; + /** Extra composer action placed immediately before Send. */ + trailingAction?: JSX.Element; /** * Target of the real input this trigger stands in for. Focused via the * `focusInput` directive when the trigger is clicked, so the iOS virtual @@ -112,6 +114,7 @@ export function CollapsedInput(props: CollapsedInputProps) { {attachmentCount()} + {props.trailingAction} ({ + createComposerDictation: () => { + const [active, setActive] = createSignal(false); + return { + active, + phase: () => (active() ? 'listening' : 'idle'), + volumeHistory: () => [], + message: () => '', + label: () => 'Start dictation', + disabled: () => false, + start: async () => { + setActive(true); + }, + confirm: async () => { + setActive(false); + }, + cancel: () => setActive(false), + }; + }, +})); + const editor = vi.hoisted(() => ({ lexical: undefined as LexicalEditor | undefined, clear: vi.fn(), diff --git a/apps/web/src/features/block-agent/ui/AgentInput.tsx b/apps/web/src/features/block-agent/ui/AgentInput.tsx index 3889cad07e3..3242ec37d73 100644 --- a/apps/web/src/features/block-agent/ui/AgentInput.tsx +++ b/apps/web/src/features/block-agent/ui/AgentInput.tsx @@ -10,6 +10,12 @@ import { createResizeObserver } from '@solid-primitives/resize-observer'; * `@core/component/AI/component/input/ChatInput.tsx`. */ +import { + DictationButton, + DictationFeedback, + DictationPanel, +} from '@app/features/dictation/components/dictation-controls'; +import { createComposerDictation } from '@app/features/dictation/composer-dictation'; import { InputProvider } from '@channel/Input/context'; import { Input } from '@channel/Input/Input'; import type { InputAttachmentData, InputCommands } from '@channel/Input/types'; @@ -116,6 +122,7 @@ export function AgentInput(props: AgentInputProps) { setHeight(element.getBoundingClientRect().height); }); useTouchOutsideToDismissKeyboard(() => containerRef); + const dictation = createComposerDictation(() => editor.lexical); const attachments = () => props.attachments ?? []; const canAttach = () => props.onAttachFiles !== undefined && !props.disabled; @@ -132,7 +139,8 @@ export function AgentInput(props: AgentInputProps) { const canSend = () => (markdown().trim().length > 0 || attachments().length > 0) && !hasPendingAttachments() && - !props.disabled; + !props.disabled && + !dictation.active(); // The channel composer's chips, drop zone, and overlay read their state // from `Input.Root`'s context; this is that context, over this composer's @@ -161,6 +169,7 @@ export function AgentInput(props: AgentInputProps) { // in flight, or a previous advance the log has not confirmed yet. Attached // files are something to send in their own right, so they hold it back too. const canSendNext = () => + !dictation.active() && markdown().trim().length === 0 && attachments().length === 0 && props.hasQueuedMessages === true && @@ -277,7 +286,7 @@ export function AgentInput(props: AgentInputProps) { {/* h-auto beats Surface's size-full so the in-flow controls are not clipped over the editor (that was Auto sitting on the placeholder). */} -
+
{/* Chips above the text, media and documents in their own rows, exactly as the channel composer lays them out. */} @@ -348,7 +362,11 @@ export function AgentInput(props: AgentInputProps) { disabled={props.disabled} /> -
+
+
+ +
); diff --git a/apps/web/src/features/channel/Input/ChannelInput.tsx b/apps/web/src/features/channel/Input/ChannelInput.tsx index c53550cb621..04ead08ab4d 100644 --- a/apps/web/src/features/channel/Input/ChannelInput.tsx +++ b/apps/web/src/features/channel/Input/ChannelInput.tsx @@ -35,6 +35,13 @@ import { Switch, } from 'solid-js'; import { Dynamic } from 'solid-js/web'; +import { + DictationButton, + DictationFeedback, + DictationPanel, +} from '../../dictation/components/dictation-controls'; +import { createComposerDictation } from '../../dictation/composer-dictation'; +import type { DictationController } from '../../dictation/core/types'; import { useAgentMentionUsers } from '../use-agent-mention-users'; import { useMessageBotMentionUsers } from '../use-channel-bot-mention-users'; import { CHANNEL_FILE_PICKER_ACCEPT } from './accepted-file-types'; @@ -87,7 +94,10 @@ export type ChannelInputProps = InputCallbacks & { flat?: boolean; }; -function WebDefaultActions(props: { input: InputData }) { +function WebDefaultActions(props: { + input: InputData; + dictation: DictationController; +}) { return ( <> @@ -97,13 +107,17 @@ function WebDefaultActions(props: { input: InputData }) { + ); } -function IosDefaultActions(props: { input: InputData }) { +function IosDefaultActions(props: { + input: InputData; + dictation: DictationController; +}) { return ( <> @@ -113,19 +127,25 @@ function IosDefaultActions(props: { input: InputData }) { + ); } -function DefaultActions(props: { input: InputData }) { +function DefaultActions(props: { + input: InputData; + dictation: DictationController; +}) { return ( } + fallback={ + + } > - + ); } @@ -166,7 +186,8 @@ export function ChannelInput(props: ChannelInputProps) { attachFiles: (files) => inputState.commands.attachFiles(files), }); - const isCollapsed = () => !!props.collapsible && collapsedInput.isCollapsed(); + const isCollapsed = () => + !!props.collapsible && collapsedInput.isCollapsed() && !dictation.active(); let isEditorConnected = false; let acceptTyping = false; @@ -252,7 +273,7 @@ export function ChannelInput(props: ChannelInputProps) { onEnter: () => { if (isTouchDevice()) return false; typingTracker.stop(); - inputState.commands.send(); + void commands.send(); return true; }, onPasteFilesAndDirs: (files, directories) => { @@ -264,6 +285,19 @@ export function ChannelInput(props: ChannelInputProps) { }); const markdownHandle = markdownEditor.buildHandle(); const lexicalEditor = () => markdownHandle.lexical; + const composerDictation = createComposerDictation(lexicalEditor); + const dictation: DictationController = { + ...composerDictation, + start: () => { + collapsedInput.expand(); + return composerDictation.start(); + }, + }; + const commands = { + ...inputState.commands, + // Keyboard, toolbar, and external handles all share this guard. + send: async () => (dictation.active() ? false : inputState.commands.send()), + }; const { isCompact: oneLineInput } = createComposerLayout(lexicalEditor(), { container: layout, mode: () => @@ -353,7 +387,7 @@ export function ChannelInput(props: ChannelInputProps) { collapsedInput.expand(); focusEditor(); }, - send: () => inputState.commands.send(), + send: commands.send, attachFiles: (files) => inputState.commands.attachFiles(files), insertEntityMention, previewEntityMentionInsertion, @@ -375,6 +409,10 @@ export function ChannelInput(props: ChannelInputProps) { runWithInputFocused: true, hide: true, keyDownHandler: () => { + if (dictation.active()) { + dictation.cancel(); + return true; + } // Block upstream escape handlers when ESC should close inline menus. return markdownEditor.controls.isInlineMenuOpen(); }, @@ -386,7 +424,12 @@ export function ChannelInput(props: ChannelInputProps) { onDragStart={(valid) => inputState.setIsDraggedOver(valid)} onDragEnd={() => inputState.setIsDraggedOver(false)} > - + @@ -439,7 +482,7 @@ export function ChannelInput(props: ChannelInputProps) { {props.children} - + @@ -448,7 +491,7 @@ export function ChannelInput(props: ChannelInputProps) { }; return ( - + {/* File picker opened from the CollapsedInput attach button. */} lexicalEditor().getRootElement()} onAttach={collapsedInput.attach} onOpen={collapsedInput.expand} - onSend={() => void inputState.commands.send()} + trailingAction={} + onSend={() => void commands.send()} /> {renderSurfaceContent()} + + ); } diff --git a/apps/web/src/features/channel/Input/tests/input-slots.test.tsx b/apps/web/src/features/channel/Input/tests/input-slots.test.tsx index 30dff3411ed..b045d94d11b 100644 --- a/apps/web/src/features/channel/Input/tests/input-slots.test.tsx +++ b/apps/web/src/features/channel/Input/tests/input-slots.test.tsx @@ -2,11 +2,12 @@ * @vitest-environment jsdom */ +import { isMobile } from '@core/mobile/isMobile'; import type { IUser } from '@core/user/types'; import { render as renderBare, screen } from '@solidjs/testing-library'; import { QueryClient, QueryClientProvider } from '@tanstack/solid-query'; import userEvent from '@testing-library/user-event'; -import { type JSX, onMount } from 'solid-js'; +import { createSignal, type JSX, onMount } from 'solid-js'; import { Portal } from 'solid-js/web'; import { beforeEach, describe, expect, it, vi } from 'vitest'; @@ -16,9 +17,39 @@ const editorMocks = vi.hoisted(() => ({ focus: vi.fn(), mentionUsers: undefined as (() => IUser[]) | undefined, emitChange: undefined as ((markdown: string) => void) | undefined, + onEnter: undefined as (() => boolean) | undefined, +})); + +vi.mock('../../../dictation/composer-dictation', () => ({ + createComposerDictation: () => { + const [active, setActive] = createSignal(false); + return { + active, + phase: () => (active() ? 'listening' : 'idle'), + volumeHistory: () => [], + message: () => '', + label: () => 'Start dictation', + disabled: () => false, + start: async () => { + setActive(true); + }, + confirm: async () => { + setActive(false); + }, + cancel: () => setActive(false), + }; + }, })); vi.hoisted(() => { + vi.stubGlobal( + 'ResizeObserver', + class { + observe() {} + unobserve() {} + disconnect() {} + } + ); Object.defineProperty(window, 'matchMedia', { writable: true, value: (query: string) => ({ @@ -34,6 +65,8 @@ vi.hoisted(() => { }); }); +vi.mock('@core/mobile/isMobile', () => ({ isMobile: vi.fn(() => false) })); + vi.mock('@core/component/LexicalMarkdown/utils/create-composer-layout', () => ({ createComposerLayout: ( _editor: unknown, @@ -206,7 +239,10 @@ vi.mock( editorMocks.emitChange = handler; return builder; }, - onEnter: () => builder, + onEnter: (handler: () => boolean) => { + editorMocks.onEnter = handler; + return builder; + }, buildHandle: () => handle, controls, lexical, @@ -277,7 +313,59 @@ describe('Input slots', () => { editorMocks.clear.mockClear(); editorMocks.focus.mockClear(); editorMocks.emitChange = undefined; + editorMocks.onEnter = undefined; editorMocks.mentionUsers = undefined; + vi.mocked(isMobile).mockReturnValue(false); + }); + + it('blocks handle and keyboard sends during dictation without clearing the draft', async () => { + const user = userEvent.setup(); + const onSend = vi.fn(); + let handle: InputHandle | undefined; + const { container } = render(() => ( + { + handle = value; + }} + onSend={onSend} + /> + )); + + await user.click(screen.getByRole('button', { name: 'Start dictation' })); + expect(container.querySelector('[data-input-layout]')).toHaveProperty( + 'inert', + true + ); + await handle?.send(); + editorMocks.onEnter?.(); + expect(onSend).not.toHaveBeenCalled(); + expect(editorMocks.clear).not.toHaveBeenCalled(); + + await user.keyboard('{Escape}'); + expect(screen.queryByRole('group', { name: 'Dictation' })).toBeNull(); + await handle?.send(); + expect(onSend).toHaveBeenCalledOnce(); + expect(onSend.mock.calls[0]?.[0]?.value).toBe('existing draft'); + }); + + it('starts dictation from the collapsed channel composer', async () => { + vi.mocked(isMobile).mockReturnValue(true); + const user = userEvent.setup(); + const { container } = render(() => ( + + )); + const collapsed = container.querySelector('[data-composer-collapsed]'); + expect(collapsed).toBeTruthy(); + const microphone = collapsed?.querySelector( + 'button[aria-label="Start dictation"]' + ); + expect(microphone).toBeTruthy(); + await user.click(microphone!); + expect(container.querySelector('[data-composer-collapsed]')).toBeNull(); + expect(screen.getByRole('group', { name: 'Dictation' })).toBeTruthy(); + await user.click(screen.getByRole('button', { name: 'Cancel dictation' })); + expect(screen.queryByRole('group', { name: 'Dictation' })).toBeNull(); }); it('offers Cursor within its rollout before account setup', () => { diff --git a/apps/web/src/features/dictation/browser/audio-recorder.test.ts b/apps/web/src/features/dictation/browser/audio-recorder.test.ts new file mode 100644 index 00000000000..6a3ee0cc63a --- /dev/null +++ b/apps/web/src/features/dictation/browser/audio-recorder.test.ts @@ -0,0 +1,308 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { + MAX_RECORDING_BYTES, + MAX_RECORDING_MS, + RECORDING_CHUNK_MS, +} from '../core/recording'; +import { + AudioRecorder, + AudioRecorderManager, + type RecordingClock, +} from './audio-recorder'; + +let now = 0; +let deadline: (() => void) | undefined; +const cancelDeadline = vi.fn(); +const clock: RecordingClock = { + now: () => now, + schedule: (callback, delay) => { + expect(delay).toBe(MAX_RECORDING_MS); + deadline = callback; + return cancelDeadline; + }, +}; +let manager: AudioRecorderManager; + +const stopTrack = vi.fn(); +const stream = { + getTracks: () => [{ stop: stopTrack }], +} as unknown as MediaStream; +const getUserMedia = vi.fn(async () => stream); + +/** Constant analyser output; 0 is silence and 255 saturates the meter. */ +let frequencyByte = 0; +const frames: FrameRequestCallback[] = []; + +class FakeAudioContext { + state = 'running'; + resume = vi.fn(async () => {}); + close = vi.fn(async () => { + this.state = 'closed'; + }); + createAnalyser = () => ({ + fftSize: 0, + minDecibels: 0, + maxDecibels: 0, + smoothingTimeConstant: 0, + frequencyBinCount: 64, + getByteFrequencyData: (buffer: Uint8Array) => buffer.fill(frequencyByte), + }); + createMediaStreamSource = () => ({ connect: vi.fn(), disconnect: vi.fn() }); +} + +class FakeMediaRecorder { + static latest: FakeMediaRecorder | undefined; + static isTypeSupported = vi.fn((type: string) => type === 'audio/mp4'); + mimeType: string; + state: RecordingState = 'inactive'; + timeslice = 0; + ondataavailable: ((event: { data: Blob }) => void) | null = null; + onerror: (() => void) | null = null; + onstop: (() => void) | null = null; + start = vi.fn((timeslice: number) => { + this.state = 'recording'; + this.timeslice = timeslice; + }); + stop = vi.fn(() => { + this.state = 'inactive'; + }); + finish() { + this.state = 'inactive'; + this.ondataavailable?.({ data: new Blob(['final']) }); + this.onstop?.(); + } + constructor(_stream: MediaStream, options: MediaRecorderOptions) { + this.mimeType = options.mimeType ?? ''; + FakeMediaRecorder.latest = this; + } + chunk(text: string) { + this.ondataavailable?.({ data: new Blob([text]) }); + } +} + +const latest = () => { + const media = FakeMediaRecorder.latest; + if (!media) throw new Error('no MediaRecorder was constructed'); + return media; +}; + +function callbacks() { + return { + onLevel: vi.fn<(level: number) => void>(), + onRecording: vi.fn<(audio: Blob) => void>(), + onLimit: vi.fn<() => void>(), + onError: vi.fn<(error: Error) => void>(), + onInterrupted: vi.fn<() => void>(), + }; +} + +beforeEach(() => { + vi.clearAllMocks(); + frequencyByte = 0; + now = 0; + deadline = undefined; + manager = new AudioRecorderManager(clock); + frames.length = 0; + FakeMediaRecorder.latest = undefined; + FakeMediaRecorder.isTypeSupported.mockImplementation( + (type: string) => type === 'audio/mp4' + ); + getUserMedia.mockResolvedValue(stream); + vi.stubGlobal('isSecureContext', true); + vi.stubGlobal('MediaRecorder', FakeMediaRecorder); + vi.stubGlobal('AudioContext', FakeAudioContext); + vi.stubGlobal('navigator', { mediaDevices: { getUserMedia } }); + vi.stubGlobal('requestAnimationFrame', (frame: FrameRequestCallback) => { + frames.push(frame); + return frames.length; + }); + vi.stubGlobal('cancelAnimationFrame', vi.fn()); +}); +afterEach(() => vi.unstubAllGlobals()); + +describe('AudioRecorder', () => { + it('reports support only with a secure context and a recordable format', () => { + expect(AudioRecorder.isSupported()).toBe(true); + FakeMediaRecorder.isTypeSupported.mockReturnValue(false); + expect(AudioRecorder.isSupported()).toBe(false); + FakeMediaRecorder.isTypeSupported.mockReturnValue(true); + vi.stubGlobal('isSecureContext', false); + expect(AudioRecorder.isSupported()).toBe(false); + }); + + it('collects chunks, meters each one, and delivers the recording on stop', async () => { + frequencyByte = 255; + const sinks = callbacks(); + const recorder = manager.createSession(sinks); + await recorder.start(); + const media = latest(); + expect(media.timeslice).toBe(RECORDING_CHUNK_MS); + expect(media.mimeType).toBe('audio/mp4'); + + media.chunk('one'); + frequencyByte = 0; + for (const frame of frames.splice(0)) frame(0); + media.chunk('two'); + expect(sinks.onLevel.mock.calls.map(([level]) => level)).toEqual([1, 0]); + + recorder.stop(); + expect(sinks.onRecording).not.toHaveBeenCalled(); + media.finish(); + expect(sinks.onRecording).toHaveBeenCalledOnce(); + const audio = sinks.onRecording.mock.calls[0][0]; + expect(audio.type).toBe('audio/mp4'); + expect(audio.size).toBe('one'.length + 'two'.length + 'final'.length); + expect(stopTrack).toHaveBeenCalledOnce(); + expect(recorder.recording).toBe(false); + expect(sinks.onError).not.toHaveBeenCalled(); + expect(sinks.onLimit).not.toHaveBeenCalled(); + expect(cancelDeadline).toHaveBeenCalledOnce(); + }); + + it('discards audio and releases the microphone on cancel', async () => { + const sinks = callbacks(); + const recorder = manager.createSession(sinks); + await recorder.start(); + latest().chunk('secret'); + recorder.cancel(); + expect(latest().stop).toHaveBeenCalledOnce(); + expect(sinks.onRecording).not.toHaveBeenCalled(); + expect(stopTrack).toHaveBeenCalledOnce(); + expect(recorder.recording).toBe(false); + }); + + it('releases a microphone granted after cancellation', async () => { + let grant!: (value: MediaStream) => void; + getUserMedia.mockImplementationOnce( + () => + new Promise((resolve) => { + grant = resolve; + }) + ); + const recorder = manager.createSession(callbacks()); + const starting = recorder.start(); + recorder.cancel(); + grant(stream); + await starting; + expect(stopTrack).toHaveBeenCalledOnce(); + expect(recorder.recording).toBe(false); + expect(FakeMediaRecorder.latest).toBeUndefined(); + }); + + it('rejects start when the microphone is unavailable', async () => { + getUserMedia.mockRejectedValueOnce(new Error('Permission denied')); + const sinks = callbacks(); + const recorder = manager.createSession(sinks); + await expect(recorder.start()).rejects.toThrow('Permission denied'); + expect(recorder.recording).toBe(false); + expect(sinks.onError).not.toHaveBeenCalled(); + }); + + it('uses elapsed timestamps when chunk delivery is delayed', async () => { + const sinks = callbacks(); + const recorder = manager.createSession(sinks); + await recorder.start(); + const media = latest(); + now = MAX_RECORDING_MS - 1; + media.chunk('before'); + expect(media.stop).not.toHaveBeenCalled(); + now = MAX_RECORDING_MS + 10_000; + media.chunk('delayed'); + expect(sinks.onLimit).toHaveBeenCalledOnce(); + expect(media.stop).toHaveBeenCalledOnce(); + expect(sinks.onRecording).not.toHaveBeenCalled(); + media.finish(); + expect(sinks.onRecording).toHaveBeenCalledOnce(); + expect(sinks.onLevel).toHaveBeenCalledTimes(2); + media.chunk('late'); + expect(sinks.onLevel).toHaveBeenCalledTimes(2); + }); + + it('stops at its deadline even when no chunks arrive', async () => { + const sinks = callbacks(); + const recorder = manager.createSession(sinks); + await recorder.start(); + deadline?.(); + deadline?.(); + expect(latest().stop).toHaveBeenCalledOnce(); + expect(sinks.onLimit).toHaveBeenCalledOnce(); + latest().finish(); + expect(sinks.onRecording).toHaveBeenCalledOnce(); + }); + + it('reports capture failures once and releases the microphone', async () => { + const sinks = callbacks(); + const recorder = manager.createSession(sinks); + await recorder.start(); + const media = latest(); + media.onerror?.(); + media.onerror?.(); + expect(sinks.onError).toHaveBeenCalledOnce(); + expect(sinks.onError.mock.calls[0][0].message).toMatch(/recording failed/i); + expect(stopTrack).toHaveBeenCalledOnce(); + expect(sinks.onRecording).not.toHaveBeenCalled(); + }); + + it('lets only one recorder hold the microphone', async () => { + const first = callbacks(); + const second = callbacks(); + const one = manager.createSession(first); + await one.start(); + const two = manager.createSession(second); + await two.start(); + expect(one.recording).toBe(false); + expect(two.recording).toBe(true); + expect(first.onRecording).not.toHaveBeenCalled(); + expect(first.onInterrupted).toHaveBeenCalledOnce(); + two.stop(); + latest().finish(); + expect(second.onRecording).toHaveBeenCalledOnce(); + expect(stopTrack).toHaveBeenCalledTimes(2); + }); + + it('delivers the final recording when the capture device stops itself', async () => { + const sinks = callbacks(); + const recorder = manager.createSession(sinks); + await recorder.start(); + latest().chunk('before device disconnected'); + latest().finish(); + expect(sinks.onRecording).toHaveBeenCalledOnce(); + expect(stopTrack).toHaveBeenCalledOnce(); + expect(recorder.recording).toBe(false); + }); + + it('rejects an oversized final chunk without delivering partial audio', async () => { + const sinks = callbacks(); + const recorder = manager.createSession(sinks); + await recorder.start(); + recorder.stop(); + latest().ondataavailable?.({ + data: new Blob([new Uint8Array(MAX_RECORDING_BYTES + 1)]), + }); + latest().finish(); + expect(sinks.onError).toHaveBeenCalledOnce(); + expect(sinks.onRecording).not.toHaveBeenCalled(); + expect(stopTrack).toHaveBeenCalledOnce(); + }); + + it('does not let a late permission grant steal the next session', async () => { + let grant!: (stream: MediaStream) => void; + getUserMedia.mockImplementationOnce( + () => + new Promise((resolve) => { + grant = resolve; + }) + ); + const first = callbacks(); + const one = manager.createSession(first); + const starting = one.start(); + const two = manager.createSession(callbacks()); + await two.start(); + expect(first.onInterrupted).toHaveBeenCalledOnce(); + grant(stream); + await starting; + expect(one.recording).toBe(false); + expect(two.recording).toBe(true); + two.cancel(); + }); +}); diff --git a/apps/web/src/features/dictation/browser/audio-recorder.ts b/apps/web/src/features/dictation/browser/audio-recorder.ts new file mode 100644 index 00000000000..2c92c814983 --- /dev/null +++ b/apps/web/src/features/dictation/browser/audio-recorder.ts @@ -0,0 +1,234 @@ +import { createAmplitudeFromStream } from '@solid-primitives/stream'; +import { createRoot } from 'solid-js'; +import { + type AudioRecorderCallbacks, + MAX_RECORDING_BYTES, + MAX_RECORDING_MS, + RECORDING_CHUNK_MS, + type RecorderHandle, +} from '../core/recording'; + +const MIME_TYPES = [ + 'audio/webm;codecs=opus', + 'audio/mp4', + 'audio/ogg;codecs=opus', +]; +const AUDIO_BITS_PER_SECOND = 64_000; +/** Stop early enough that the final chunk still fits under the server limit. */ +const SIZE_HEADROOM_BYTES = 64 * 1024; + +/** A monotonic clock and cancellable deadline, supplied explicitly in tests. */ +export interface RecordingClock { + now(): number; + schedule(callback: () => void, delay: number): () => void; +} + +const browserClock: RecordingClock = { + now: () => performance.now(), + schedule: (callback, delay) => { + const timeout = window.setTimeout(callback, delay); + return () => window.clearTimeout(timeout); + }, +}; + +type State = 'idle' | 'starting' | 'recording' | 'stopping' | 'done'; + +const stopTracks = (stream: MediaStream) => + stream.getTracks().forEach((track) => track.stop()); + +/** + * One capture session, initialized with the callbacks receiving its audio. + * Owns microphone tracks, the encoder, analyser, and deadline until released. + */ +export class AudioRecorder implements RecorderHandle { + private state: State = 'idle'; + private stream: MediaStream | undefined; + private recorder: MediaRecorder | undefined; + private releaseMeter: (() => void) | undefined; + private cancelDeadline: (() => void) | undefined; + private chunks: Blob[] = []; + private size = 0; + private startedAt = 0; + + constructor( + private readonly callbacks: AudioRecorderCallbacks, + private readonly owner: AudioRecorderManager = audioRecorder, + private readonly clock: RecordingClock = browserClock + ) {} + + static isSupported(): boolean { + return !!( + typeof window !== 'undefined' && + window.isSecureContext && + typeof navigator.mediaDevices?.getUserMedia === 'function' && + typeof MediaRecorder !== 'undefined' && + MIME_TYPES.some((type) => MediaRecorder.isTypeSupported(type)) + ); + } + + /** Whether this instance currently owns the microphone. */ + get recording(): boolean { + return this.state === 'recording' || this.state === 'stopping'; + } + + async start(): Promise { + if (this.state !== 'idle') return; + this.owner.claim(this); + this.state = 'starting'; + let stream: MediaStream; + try { + stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + } catch (error) { + this.release(); + throw error; + } + // Cancelled while the permission prompt was open: the grant arrives late. + if (this.state !== 'starting') { + stopTracks(stream); + return; + } + this.stream = stream; + try { + const mimeType = MIME_TYPES.find((type) => + MediaRecorder.isTypeSupported(type) + ); + if (!mimeType) throw new Error('No supported recording format'); + const recorder = new MediaRecorder(stream, { + mimeType, + audioBitsPerSecond: AUDIO_BITS_PER_SECOND, + }); + const readLevel = this.startMeter(stream); + recorder.ondataavailable = ({ data }) => this.onChunk(data, readLevel()); + recorder.onerror = () => + this.fail(new Error('Audio recording failed. Please try again.')); + recorder.onstop = () => this.onStop(); + this.recorder = recorder; + this.startedAt = this.clock.now(); + this.state = 'recording'; + recorder.start(RECORDING_CHUNK_MS); + this.cancelDeadline = this.clock.schedule( + () => this.stopAtLimit(), + MAX_RECORDING_MS + ); + } catch (error) { + this.release(); + throw error; + } + } + + stop(): void { + if (this.state !== 'recording' || !this.recorder) return; + this.state = 'stopping'; + if (this.recorder.state !== 'inactive') this.recorder.stop(); + } + + cancel(): void { + if (this.state === 'done') return; + const recorder = this.recorder; + this.release(); + // A stop issued after release is ignored by onStop: state is already done. + if (recorder && recorder.state !== 'inactive') recorder.stop(); + } + + /** Release ownership without moving focus away from the next composer. */ + interrupt(): void { + if (this.state === 'done') return; + this.cancel(); + this.callbacks.onInterrupted(); + } + + private startMeter(stream: MediaStream): () => number { + let amplitude = () => 0; + this.releaseMeter = createRoot((dispose) => { + // Library level is 0..100; the analyser is never routed to speakers. + const [level] = createAmplitudeFromStream(stream); + amplitude = level; + return dispose; + }); + return () => amplitude() / 100; + } + + private stopAtLimit() { + if (this.state !== 'recording') return; + this.callbacks.onLimit?.(); + this.stop(); + } + + private onChunk(data: Blob, level: number) { + if (!this.recording) return; + // Delayed/final chunks can exceed the headroom; never retain or upload them. + if (this.size + data.size > MAX_RECORDING_BYTES) { + this.fail( + new Error('Recording is too large. Please record a shorter message.') + ); + return; + } + if (data.size) this.chunks.push(data); + this.size += data.size; + if (this.state !== 'recording') return; + this.callbacks.onLevel?.(level); + if ( + this.clock.now() - this.startedAt >= MAX_RECORDING_MS || + this.size >= MAX_RECORDING_BYTES - SIZE_HEADROOM_BYTES + ) { + this.stopAtLimit(); + } + } + + private onStop() { + // Device disconnection also delivers final data followed by a stop event. + if (!this.recording) return; + const audio = new Blob(this.chunks, { type: this.recorder?.mimeType }); + const { onRecording } = this.callbacks; + this.release(); + onRecording(audio); + } + + private fail(error: Error) { + if (!this.recording) return; + this.cancel(); + this.callbacks.onError(error); + } + + private release() { + this.state = 'done'; + this.chunks = []; + this.size = 0; + this.cancelDeadline?.(); + this.cancelDeadline = undefined; + if (this.recorder) { + this.recorder.ondataavailable = null; + this.recorder.onerror = null; + this.recorder.onstop = null; + this.recorder = undefined; + } + this.releaseMeter?.(); + this.releaseMeter = undefined; + if (this.stream) stopTracks(this.stream); + this.stream = undefined; + this.owner.release(this); + } +} + +/** Shared microphone ownership; callbacks and captured bytes stay session-local. */ +export class AudioRecorderManager { + private active: AudioRecorder | undefined; + + constructor(private readonly clock: RecordingClock = browserClock) {} + + createSession(callbacks: AudioRecorderCallbacks): AudioRecorder { + return new AudioRecorder(callbacks, this, this.clock); + } + + claim(session: AudioRecorder) { + this.active?.interrupt(); + this.active = session; + } + + release(session: AudioRecorder) { + if (this.active === session) this.active = undefined; + } +} + +/** App singleton. Browser resources are acquired only when a session starts. */ +export const audioRecorder = new AudioRecorderManager(); diff --git a/apps/web/src/features/dictation/components/dictation-controls.tsx b/apps/web/src/features/dictation/components/dictation-controls.tsx new file mode 100644 index 00000000000..ddba6e7fedc --- /dev/null +++ b/apps/web/src/features/dictation/components/dictation-controls.tsx @@ -0,0 +1,99 @@ +import CheckIcon from '@phosphor-icons/core/regular/check.svg?component-solid'; +import MicrophoneIcon from '@phosphor-icons/core/regular/microphone.svg?component-solid'; +import XIcon from '@phosphor-icons/core/regular/x.svg?component-solid'; +import { Button } from '@ui'; +import { Show } from 'solid-js'; +import { match } from 'ts-pattern'; +import type { DictationController } from '../core/types'; +import { VolumeTimeline } from './volume-timeline'; + +export function DictationButton(props: { + dictation: DictationController; + disabled?: boolean; +}) { + return ( + + ); +} + +export function DictationPanel(props: { dictation: DictationController }) { + return ( + +
{ + if (event.key === 'Escape') { + event.preventDefault(); + event.stopPropagation(); + props.dictation.cancel(); + } + }} + > +
+ + + {match(props.dictation.phase()) + .with('starting', () => 'Starting…') + .with('finishing', () => 'Finishing…') + .with('review', () => 'Ready') + .otherwise(() => 'Listening…')} + +
+ + +
+ + ); +} + +export function DictationFeedback(props: { dictation: DictationController }) { + const message = () => props.dictation.message(); + return ( + +

+ {message()} +

+ + ); +} diff --git a/apps/web/src/features/dictation/components/volume-timeline.tsx b/apps/web/src/features/dictation/components/volume-timeline.tsx new file mode 100644 index 00000000000..cf5573ab817 --- /dev/null +++ b/apps/web/src/features/dictation/components/volume-timeline.tsx @@ -0,0 +1,62 @@ +import { createElementSize } from '@solid-primitives/resize-observer'; +import { createEffect, createSignal, Index, on, onCleanup } from 'solid-js'; +import { VOLUME_INTERVAL_MS } from '../core/volume'; + +export function VolumeTimeline(props: { levels: readonly number[] }) { + const [container, setContainer] = createSignal(); + const size = createElementSize(container); + let strip!: HTMLDivElement; + // One extra bar enters from beyond the right edge while the strip slides. + const visibleLevels = () => { + const count = Math.ceil((size.width ?? 0) / 6) + 1; + const levels = props.levels.slice(-count); + return [ + ...Array(Math.max(0, count - levels.length)).fill(0), + ...levels, + ]; + }; + + createEffect( + on( + () => props.levels, + (levels) => { + if ( + !levels.length || + window.matchMedia('(prefers-reduced-motion: reduce)').matches + ) + return; + const animation = strip.animate?.( + [{ transform: 'translateX(6px)' }, { transform: 'translateX(0)' }], + { duration: VOLUME_INTERVAL_MS, easing: 'linear' } + ); + onCleanup(() => animation?.cancel()); + }, + { defer: true } + ) + ); + + return ( + + ); +} diff --git a/apps/web/src/features/dictation/composer-dictation.ts b/apps/web/src/features/dictation/composer-dictation.ts new file mode 100644 index 00000000000..27bdf53dbcd --- /dev/null +++ b/apps/web/src/features/dictation/composer-dictation.ts @@ -0,0 +1,34 @@ +import { Telemetry } from '@macro-inc/observability'; +import { transcribeAudio } from '@queries/dictation/transcribe'; +import type { LexicalEditor } from 'lexical'; +import { $getRoot } from 'lexical'; +import { AudioRecorder, audioRecorder } from './browser/audio-recorder'; +import { createRecordedDictation } from './primitives/create-recorded-dictation'; + +/** Append plain speech without reparsing or replacing the existing rich draft. */ +export function createComposerDictation(editor: () => LexicalEditor) { + const language = navigator.language || 'en-US'; + return createRecordedDictation({ + supported: AudioRecorder.isSupported(), + startTrace: () => Telemetry.span('dictation.session'), + createRecorder: (callbacks) => audioRecorder.createSession(callbacks), + transcribe: (audio, signal, trace) => + transcribeAudio(audio, language, signal, trace), + onConfirm: (text: string) => { + editor().update( + () => { + const root = $getRoot(); + const existing = root.getTextContent(); + root + .selectEnd() + .insertText( + `${existing && !/\s$/.test(existing) ? ' ' : ''}${text}` + ); + }, + { discrete: true } + ); + editor().focus(); + }, + onCancel: () => editor().focus(), + }); +} diff --git a/apps/web/src/features/dictation/core/recording.ts b/apps/web/src/features/dictation/core/recording.ts new file mode 100644 index 00000000000..1f950f58479 --- /dev/null +++ b/apps/web/src/features/dictation/core/recording.ts @@ -0,0 +1,42 @@ +import type { Span } from '@macro-inc/observability'; +import { VOLUME_INTERVAL_MS, type VolumeLevel } from './volume'; + +/** Mirrors the server body limit for `/dictation/transcribe`. */ +export const MAX_RECORDING_BYTES = 8 * 1024 * 1024; +/** Leave room for codec padding below the server's five-minute limit. */ +export const MAX_RECORDING_MS = 5 * 60 * 1000 - 1_000; +/** Requested chunk cadence; browsers may deliver chunks late. */ +export const RECORDING_CHUNK_MS = VOLUME_INTERVAL_MS; + +export type AudioRecorderCallbacks = { + /** Microphone level for the chunk that just finished, while recording. */ + onLevel?: (level: VolumeLevel) => void; + /** Final encoded audio, including the browser's last data event. */ + onRecording: (audio: Blob) => void; + /** Another composer acquired the microphone; this session was discarded. */ + onInterrupted: () => void; + /** The size or duration cap was reached; the recorder stopped itself. */ + onLimit?: () => void; + /** Capture failed after it had started. The microphone is already released. */ + onError: (error: Error) => void; +}; + +/** What the primitives need from a recorder; `AudioRecorder` is the browser one. */ +export interface RecorderHandle { + /** Acquire the microphone and begin capturing. Rejects on permission failure. */ + start(): Promise; + /** Finish capturing; `onRecording` receives the audio. */ + stop(): void; + /** Discard everything and release the microphone. */ + cancel(): void; +} + +export type CreateRecorder = ( + callbacks: AudioRecorderCallbacks +) => RecorderHandle; + +export type TranscribeAudio = ( + audio: Blob, + signal: AbortSignal, + trace?: Pick +) => Promise; diff --git a/apps/web/src/features/dictation/core/types.ts b/apps/web/src/features/dictation/core/types.ts new file mode 100644 index 00000000000..bc4f6ac9f34 --- /dev/null +++ b/apps/web/src/features/dictation/core/types.ts @@ -0,0 +1,29 @@ +import type { VolumeLevel } from './volume'; + +export type DictationPhase = + | 'unavailable' + | 'idle' + | 'starting' + | 'listening' + | 'finishing' + | 'review'; + +export const ACTIVE_PHASES: readonly DictationPhase[] = [ + 'starting', + 'listening', + 'finishing', + 'review', +]; + +export interface DictationController { + phase(): DictationPhase; + active(): boolean; + volumeHistory(): readonly VolumeLevel[]; + message(): string; + label(): string; + disabled(): boolean; + start(): Promise; + /** Resolves once the text has been committed, or the attempt was abandoned. */ + confirm(): Promise; + cancel(): void; +} diff --git a/apps/web/src/features/dictation/core/volume.ts b/apps/web/src/features/dictation/core/volume.ts new file mode 100644 index 00000000000..4a79cf0d033 --- /dev/null +++ b/apps/web/src/features/dictation/core/volume.ts @@ -0,0 +1,13 @@ +/** Requested sampling cadence; keep a bounded history even during long recordings. */ +export const VOLUME_INTERVAL_MS = 200; +export const MAX_VOLUME_SAMPLES = 512; + +/** Normalized 0..1 microphone level for one interval. */ +export type VolumeLevel = number; + +export function appendLevel( + history: readonly VolumeLevel[], + level: VolumeLevel +): readonly VolumeLevel[] { + return [...history.slice(-(MAX_VOLUME_SAMPLES - 1)), level]; +} diff --git a/apps/web/src/features/dictation/primitives/create-recorded-dictation.test.ts b/apps/web/src/features/dictation/primitives/create-recorded-dictation.test.ts new file mode 100644 index 00000000000..d3a3798caf2 --- /dev/null +++ b/apps/web/src/features/dictation/primitives/create-recorded-dictation.test.ts @@ -0,0 +1,225 @@ +import { renderHook } from '@solidjs/testing-library'; +import { describe, expect, it, vi } from 'vitest'; +import type { AudioRecorderCallbacks, RecorderHandle } from '../core/recording'; +import { createRecordedDictation } from './create-recorded-dictation'; + +const audio = new Blob(['audio'], { type: 'audio/webm' }); + +/** A recorder whose events the test fires explicitly. */ +class FakeRecorder implements RecorderHandle { + static latest: FakeRecorder; + /** Overrides how the next constructed recorder acquires the microphone. */ + static nextStart: (() => Promise) | undefined; + start = vi.fn(FakeRecorder.nextStart ?? (async () => {})); + /** The primitive receives a completed recording through its recorder contract. */ + stop = vi.fn(() => this.callbacks.onRecording?.(audio)); + cancel = vi.fn(); + constructor(readonly callbacks: AudioRecorderCallbacks) { + FakeRecorder.latest = this; + } +} + +function setup() { + const transcribe = vi.fn( + async (_blob: Blob, _signal: AbortSignal) => 'recognized words' + ); + const onConfirm = vi.fn(); + const onCancel = vi.fn(); + const trace = { + event: vi.fn(), + run: (operation: () => T) => operation(), + setAttr: vi.fn(), + error: vi.fn(), + end: vi.fn(), + }; + const hook = renderHook(() => + createRecordedDictation({ + supported: true, + startTrace: () => trace, + createRecorder: (callbacks) => new FakeRecorder(callbacks), + transcribe, + onConfirm, + onCancel, + }) + ); + return { ...hook, transcribe, onConfirm, onCancel, trace }; +} + +describe('Whisper dictation', () => { + it('uploads only on confirm and appends the result once', async () => { + const test = setup(); + await test.result.start(); + expect(test.result.phase()).toBe('listening'); + expect(test.transcribe).not.toHaveBeenCalled(); + const first = test.result.confirm(); + const second = test.result.confirm(); + await Promise.all([first, second]); + expect(test.onConfirm).toHaveBeenCalledExactlyOnceWith('recognized words'); + expect(test.transcribe).toHaveBeenCalledExactlyOnceWith( + audio, + expect.any(AbortSignal), + test.trace + ); + expect(test.result.phase()).toBe('idle'); + expect(test.result.message()).toBe(''); + expect(test.trace.setAttr).toHaveBeenCalledWith( + 'dictation.outcome', + 'success' + ); + expect(test.trace.end).toHaveBeenCalledOnce(); + }); + + it('keeps a bounded, listening-only volume timeline', async () => { + const test = setup(); + await test.result.start(); + const { onLevel } = FakeRecorder.latest.callbacks; + onLevel?.(0.2); + onLevel?.(0.9); + expect(test.result.volumeHistory()).toEqual([0.2, 0.9]); + const confirmed = test.result.confirm(); + onLevel?.(0.5); + await confirmed; + expect(test.result.volumeHistory()).toEqual([]); + }); + + it('discards cancellation without uploading', async () => { + const test = setup(); + await test.result.start(); + test.result.cancel(); + expect(FakeRecorder.latest.cancel).toHaveBeenCalledOnce(); + expect(test.transcribe).not.toHaveBeenCalled(); + expect(test.onConfirm).not.toHaveBeenCalled(); + expect(test.onCancel).toHaveBeenCalledOnce(); + expect(test.result.phase()).toBe('idle'); + expect(test.trace.setAttr).toHaveBeenCalledWith( + 'dictation.outcome', + 'cancelled' + ); + expect(test.trace.end).toHaveBeenCalledOnce(); + }); + + it('aborts an upload and ignores a late transcription', async () => { + const test = setup(); + let resolve!: (text: string) => void; + test.transcribe.mockImplementation( + () => + new Promise((done) => { + resolve = done; + }) + ); + await test.result.start(); + const confirmed = test.result.confirm(); + expect(test.transcribe).toHaveBeenCalledOnce(); + const signal = test.transcribe.mock.calls[0][1]; + test.result.cancel(); + expect(signal.aborted).toBe(true); + resolve('must not insert'); + await confirmed; + expect(test.onConfirm).not.toHaveBeenCalled(); + expect(test.result.phase()).toBe('idle'); + expect(test.trace.setAttr).toHaveBeenCalledWith( + 'dictation.outcome', + 'cancelled' + ); + expect(test.trace.end).toHaveBeenCalledOnce(); + }); + + it('keeps audio for explicit retry after a provider failure', async () => { + const test = setup(); + test.transcribe.mockRejectedValueOnce(new Error('Try again')); + await test.result.start(); + await test.result.confirm(); + expect(test.result.phase()).toBe('review'); + expect(test.result.message()).toBe('Try again'); + expect(test.onConfirm).not.toHaveBeenCalled(); + expect(test.trace.event).toHaveBeenCalledWith('dictation.upload_failed', { + attempt: 1, + }); + expect(test.trace.end).not.toHaveBeenCalled(); + await test.result.confirm(); + expect(test.onConfirm).toHaveBeenCalledExactlyOnceWith('recognized words'); + expect(FakeRecorder.latest.stop).toHaveBeenCalledOnce(); + expect(test.trace.event).toHaveBeenCalledWith('dictation.upload_started', { + attempt: 2, + audioBytes: audio.size, + }); + expect(test.trace.setAttr).toHaveBeenCalledWith( + 'dictation.outcome', + 'success' + ); + expect(test.trace.end).toHaveBeenCalledOnce(); + }); + + it('waits for confirmation after the recorder stops at its limit', async () => { + const test = setup(); + await test.result.start(); + const recorder = FakeRecorder.latest; + recorder.callbacks.onLimit?.(); + recorder.callbacks.onRecording?.(audio); + expect(test.result.phase()).toBe('review'); + expect(test.result.message()).toMatch(/limit/i); + expect(test.transcribe).not.toHaveBeenCalled(); + await test.result.confirm(); + expect(test.onConfirm).toHaveBeenCalledOnce(); + }); + + it('returns to idle with a message when capture fails mid-recording', async () => { + const test = setup(); + await test.result.start(); + FakeRecorder.latest.callbacks.onError(new Error('Microphone unplugged')); + expect(test.result.phase()).toBe('idle'); + expect(test.result.message()).toBe('Microphone unplugged'); + expect(test.transcribe).not.toHaveBeenCalled(); + }); + + it('releases its composer when another session takes the microphone', async () => { + const test = setup(); + await test.result.start(); + const previous = FakeRecorder.latest; + previous.callbacks.onInterrupted(); + expect(test.result.phase()).toBe('idle'); + expect(test.result.active()).toBe(false); + expect(test.onCancel).not.toHaveBeenCalled(); + previous.callbacks.onRecording(audio); + expect(test.transcribe).not.toHaveBeenCalled(); + expect(test.onConfirm).not.toHaveBeenCalled(); + }); + + it('settles a pending confirmation when its recording is interrupted', async () => { + const test = setup(); + await test.result.start(); + FakeRecorder.latest.stop.mockImplementation(() => {}); + const confirmed = test.result.confirm(); + FakeRecorder.latest.callbacks.onInterrupted(); + await confirmed; + expect(test.result.phase()).toBe('idle'); + expect(test.transcribe).not.toHaveBeenCalled(); + }); + + it('reports permission failures without leaving the composer stuck', async () => { + const test = setup(); + FakeRecorder.nextStart = async () => { + throw new Error('denied'); + }; + await test.result.start(); + FakeRecorder.nextStart = undefined; + expect(test.result.phase()).toBe('idle'); + expect(test.result.message()).toMatch(/microphone/i); + }); + + it('cancels a recorder granted after disposal', async () => { + const test = setup(); + let grant!: () => void; + FakeRecorder.nextStart = () => + new Promise((resolve) => { + grant = resolve; + }); + const starting = test.result.start(); + FakeRecorder.nextStart = undefined; + test.cleanup(); + grant(); + await starting; + expect(FakeRecorder.latest.cancel).toHaveBeenCalled(); + expect(test.transcribe).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/src/features/dictation/primitives/create-recorded-dictation.ts b/apps/web/src/features/dictation/primitives/create-recorded-dictation.ts new file mode 100644 index 00000000000..52e71fb5cab --- /dev/null +++ b/apps/web/src/features/dictation/primitives/create-recorded-dictation.ts @@ -0,0 +1,236 @@ +import type { Span } from '@macro-inc/observability'; +import { createSignal, onCleanup } from 'solid-js'; +import type { + CreateRecorder, + RecorderHandle, + TranscribeAudio, +} from '../core/recording'; +import { + ACTIVE_PHASES, + type DictationController, + type DictationPhase, +} from '../core/types'; +import { appendLevel, type VolumeLevel } from '../core/volume'; + +/** + * Record in memory and upload to Whisper only on confirm. Cancel never + * uploads, and a response that arrives after cancel never edits the draft. + */ +export function createRecordedDictation(options: { + supported: boolean; + startTrace: () => Pick; + createRecorder: CreateRecorder; + transcribe: TranscribeAudio; + onConfirm: (text: string) => void; + onCancel?: () => void; +}): DictationController { + const [phase, setPhase] = createSignal( + options.supported ? 'idle' : 'unavailable' + ); + const [message, setMessage] = createSignal(''); + const [volumeHistory, setVolumeHistory] = createSignal< + readonly VolumeLevel[] + >([]); + let recorder: RecorderHandle | undefined; + let audio: Blob | undefined; + let upload: AbortController | undefined; + let commitOnStop = false; + let limitReached = false; + let pendingConfirm: (() => void) | undefined; + let disposed = false; + let trace: ReturnType | undefined; + let attempt = 0; + + const endTrace = (outcome: string) => { + trace?.setAttr('dictation.outcome', outcome); + trace?.end(); + trace = undefined; + }; + + const active = () => ACTIVE_PHASES.includes(phase()); + + const settleConfirm = () => { + pendingConfirm?.(); + pendingConfirm = undefined; + }; + + const reset = (next: DictationPhase) => { + upload?.abort(); + upload = undefined; + recorder?.cancel(); + recorder = undefined; + audio = undefined; + commitOnStop = false; + limitReached = false; + setVolumeHistory([]); + setPhase(next); + }; + + const transcribe = async (blob: Blob) => { + if (upload) return; + const current = new AbortController(); + upload = current; + setPhase('finishing'); + setMessage('Transcribing with OpenAI…'); + trace?.event('dictation.upload_started', { + attempt: ++attempt, + audioBytes: blob.size, + }); + try { + const request = () => options.transcribe(blob, current.signal, trace); + const text = (await (trace ? trace.run(request) : request())).trim(); + if (current.signal.aborted || disposed) return; + upload = undefined; + endTrace(text ? 'success' : 'empty_transcript'); + reset('idle'); + setMessage(text ? '' : 'No speech was detected. Try again.'); + if (text) options.onConfirm(text); + else options.onCancel?.(); + } catch (error) { + if (current.signal.aborted || disposed) return; + upload = undefined; + trace?.event('dictation.upload_failed', { attempt }); + setMessage( + error instanceof Error + ? error.message + : 'Transcription failed. Please try again.' + ); + // Keep the recording so the checkmark retries without re-recording. + setPhase('review'); + } finally { + if (!current.signal.aborted) settleConfirm(); + } + }; + + const onRecording = (blob: Blob) => { + recorder = undefined; + if (disposed) return; + trace?.event('dictation.recording_stopped', { audioBytes: blob.size }); + if (!blob.size) { + endTrace('empty_audio'); + reset('idle'); + setMessage('No audio was recorded. Please try again.'); + settleConfirm(); + return; + } + audio = blob; + if (commitOnStop) { + commitOnStop = false; + void transcribe(blob); + return; + } + setPhase('review'); + setMessage( + limitReached + ? 'Recording limit reached. Select the checkmark to transcribe with OpenAI.' + : 'Recording stopped. Select the checkmark to transcribe with OpenAI.' + ); + }; + + const start = async () => { + if (phase() !== 'idle' || disposed) return; + trace = options.startTrace(); + attempt = 0; + setVolumeHistory([]); + setPhase('starting'); + setMessage('Audio is sent to OpenAI Whisper only when you confirm.'); + const current = options.createRecorder({ + onLevel: (level) => { + if (recorder === current && phase() === 'listening') + setVolumeHistory((history) => appendLevel(history, level)); + }, + onRecording: (blob) => { + if (recorder === current) onRecording(blob); + }, + onLimit: () => { + if (recorder === current) { + limitReached = true; + trace?.event('dictation.recording_limit'); + } + }, + onError: (error) => { + if (recorder !== current) return; + trace?.error('recording_error'); + endTrace('recording_error'); + reset('idle'); + setMessage(error.message); + settleConfirm(); + }, + onInterrupted: () => { + if (recorder !== current) return; + endTrace('interrupted'); + reset('idle'); + setMessage('Dictation moved to another composer.'); + settleConfirm(); + }, + }); + recorder = current; + try { + await current.start(); + if (disposed || recorder !== current) { + current.cancel(); + return; + } + setPhase('listening'); + trace?.event('dictation.recording_started'); + } catch { + if (disposed || recorder !== current) return; + trace?.error('microphone_error'); + endTrace('microphone_error'); + reset('idle'); + setMessage( + 'Could not access your microphone. Check permissions and try again.' + ); + } + }; + + const confirm = () => + new Promise((resolve) => { + if (phase() === 'review' && audio) { + trace?.event('dictation.confirmed'); + pendingConfirm = resolve; + void transcribe(audio); + return; + } + if (phase() === 'listening' && recorder) { + trace?.event('dictation.confirmed'); + pendingConfirm = resolve; + commitOnStop = true; + setPhase('finishing'); + recorder.stop(); + return; + } + resolve(); + }); + + const cancel = () => { + if (!active()) return; + endTrace('cancelled'); + reset('idle'); + setMessage(''); + options.onCancel?.(); + settleConfirm(); + }; + + onCleanup(() => { + disposed = true; + endTrace('disposed'); + reset('idle'); + settleConfirm(); + }); + + return { + phase, + active, + volumeHistory, + message, + start, + cancel, + confirm, + disabled: () => phase() !== 'idle', + label: () => + options.supported + ? 'Start dictation with OpenAI Whisper' + : 'Dictation is unavailable in this browser', + }; +} diff --git a/apps/web/src/lib/core/component/AI/component/input/ChatInput.test.tsx b/apps/web/src/lib/core/component/AI/component/input/ChatInput.test.tsx index 3798a011bfd..85c41e51d50 100644 --- a/apps/web/src/lib/core/component/AI/component/input/ChatInput.test.tsx +++ b/apps/web/src/lib/core/component/AI/component/input/ChatInput.test.tsx @@ -1,6 +1,6 @@ import type { EditorConfigBuilder } from '@core/component/LexicalMarkdown/builder/MarkdownConfigBuilder'; import { cleanup, fireEvent, render, screen } from '@solidjs/testing-library'; -import { type JSX, onCleanup, onMount } from 'solid-js'; +import { createSignal, type JSX, onCleanup, onMount } from 'solid-js'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { ChatInput } from './ChatInput'; @@ -11,6 +11,27 @@ const mocks = vi.hoisted(() => ({ upload: vi.fn(), mount: vi.fn(), unmount: vi.fn(), + startDictation: () => {}, + stopDictation: () => {}, +})); + +vi.mock('@app/features/dictation/composer-dictation', () => ({ + createComposerDictation: () => { + const [active, setActive] = createSignal(false); + mocks.startDictation = () => setActive(true); + mocks.stopDictation = () => setActive(false); + return { + active, + phase: () => (active() ? 'listening' : 'idle'), + volumeHistory: () => [], + message: () => '', + label: () => 'Start dictation', + disabled: () => false, + start: async () => mocks.startDictation(), + confirm: async () => mocks.stopDictation(), + cancel: () => mocks.stopDictation(), + }; + }, })); vi.mock('@app/lib/analytics/analytics-context', () => ({ @@ -163,6 +184,27 @@ function setup(collapseOnBlur = true) { } describe('compact mobile chat drafts', () => { + it('stays expanded while focus moves to dictation controls and preserves the draft', () => { + const { wrapper, input, draft, onSend } = setup(); + expect(wrapper.classList.contains('max-h-5')).toBe(true); + mocks.startDictation(); + expect(wrapper.classList.contains('max-h-5')).toBe(false); + const cancel = document.querySelector( + '[label="Cancel dictation"]' + )!; + input.focus(); + cancel.focus(); + expect(wrapper.classList.contains('max-h-5')).toBe(false); + fireEvent.click(screen.getByRole('button', { name: 'Send' })); + expect(onSend).not.toHaveBeenCalled(); + fireEvent.click(cancel); + input.focus(); + screen.getByRole('textbox', { name: 'Outside' }).focus(); + expect(wrapper.classList.contains('max-h-5')).toBe(true); + expect(input.textContent).toBe(draft); + expect(mocks.unmount).not.toHaveBeenCalled(); + }); + it('expands on focus and collapses on blur without replacing the editor or draft', () => { const { wrapper, input, draft } = setup(); expect(wrapper.classList.contains('max-h-5')).toBe(true); diff --git a/apps/web/src/lib/core/component/AI/component/input/ChatInput.tsx b/apps/web/src/lib/core/component/AI/component/input/ChatInput.tsx index 38b3de741bd..3af454715c4 100644 --- a/apps/web/src/lib/core/component/AI/component/input/ChatInput.tsx +++ b/apps/web/src/lib/core/component/AI/component/input/ChatInput.tsx @@ -1,3 +1,9 @@ +import { + DictationButton, + DictationFeedback, + DictationPanel, +} from '@app/features/dictation/components/dictation-controls'; +import { createComposerDictation } from '@app/features/dictation/composer-dictation'; import { useAnalytics } from '@app/lib/analytics/analytics-context'; import { useHasPaidAccess } from '@core/auth/license'; import type { ChatSendInput } from '@core/component/AI/component/input/buildRequest'; @@ -134,6 +140,7 @@ export function ChatInput(props: ChatInputComponentProps) { const [markdownText, setMarkdownText] = createSignal(''); const [isFocused, setIsFocused] = createSignal(false); + const dictation = createComposerDictation(() => props.editor.lexical); createEffect(() => { const uploaded = uploadQueue.popComplete(); @@ -160,6 +167,7 @@ export function ChatInput(props: ChatInputComponentProps) { const hasAttachedFiles = () => attachments.attached().length > 0; const hasUploadingAttachments = () => uploadQueue.uploading().length > 0; const canSendMessage = () => + !dictation.active() && (!isEmptyInput() || hasAttachedFiles()) && !generating() && !hasUploadingAttachments(); @@ -172,6 +180,7 @@ export function ChatInput(props: ChatInputComponentProps) { (isTouchDevice() && props.variant !== 'default') ); const isCollapsed = () => + !dictation.active() && props.collapseOnBlur && isTouchDevice() && !isTallVariant() && @@ -218,7 +227,13 @@ export function ChatInput(props: ChatInputComponentProps) { } return true; }) - .onEscape((e) => props.onEscape?.(e) ?? false) + .onEscape((e) => { + if (dictation.active()) { + dictation.cancel(); + return true; + } + return props.onEscape?.(e) ?? false; + }) .onChange((md) => { setMarkdownText(md); props.onChange?.(md); @@ -299,6 +314,7 @@ export function ChatInput(props: ChatInputComponentProps) { onLocked={() => showPaywall(PaywallKey.O1_LIMIT)} compact={compactSelector()} /> + }> @@ -346,7 +362,7 @@ export function ChatInput(props: ChatInputComponentProps) { />
{ + if (dictation.active()) return; const next = e.relatedTarget as Node | null; if (next && containerRef.contains(next)) return; setIsFocused(false); @@ -409,6 +428,7 @@ export function ChatInput(props: ChatInputComponentProps) { onSelect={() => {}} />
+
+ +
); } diff --git a/apps/web/src/lib/queries/dictation/keys.ts b/apps/web/src/lib/queries/dictation/keys.ts new file mode 100644 index 00000000000..09cb5098c98 --- /dev/null +++ b/apps/web/src/lib/queries/dictation/keys.ts @@ -0,0 +1,5 @@ +import { createQueryKeys } from '@lukemorales/query-key-factory'; + +export const dictationKeys = createQueryKeys('dictation', { + transcribe: null, +}); diff --git a/apps/web/src/lib/queries/dictation/transcribe.test.ts b/apps/web/src/lib/queries/dictation/transcribe.test.ts new file mode 100644 index 00000000000..73d1f0c2e30 --- /dev/null +++ b/apps/web/src/lib/queries/dictation/transcribe.test.ts @@ -0,0 +1,214 @@ +import { + DictationCapacityError, + transcribeDictation, +} from '@service-storage/dictation'; +import { timeoutManager } from '@tanstack/solid-query'; +import { err, ok } from 'neverthrow'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { queryClient } from '../client'; +import { transcribeAudio } from './transcribe'; + +vi.mock('../client', async () => { + const { QueryClient } = await import('@tanstack/solid-query'); + return { queryClient: new QueryClient() }; +}); +vi.mock('@service-storage/dictation', async (original) => ({ + ...(await original()), + transcribeDictation: vi.fn(), +})); + +const upload = vi.mocked(transcribeDictation); +const audio = new Blob(['PRIVATE_AUDIO'], { type: 'audio/webm' }); + +/** Execute TanStack's scheduled callbacks explicitly; no clock advancement. */ +function controlRetries() { + type Retry = { delay: number; fire: () => void }; + const queued: Retry[] = []; + let listener: ((retry: Retry) => void) | undefined; + let id = 0; + vi.spyOn(timeoutManager, 'setTimeout').mockImplementation( + (callback, delay) => { + if (delay > 0) { + const retry = { delay, fire: () => callback() }; + if (listener) { + listener(retry); + listener = undefined; + } else queued.push(retry); + } + return ++id; + } + ); + vi.spyOn(timeoutManager, 'clearTimeout').mockImplementation(() => {}); + return { + async next(): Promise { + const retry = queued.shift(); + if (retry) return retry; + return new Promise((resolve) => { + listener = resolve; + }); + }, + }; +} + +function settled() { + return new Promise((resolve) => { + const unsubscribe = queryClient.getMutationCache().subscribe((event) => { + if ( + event.type === 'updated' && + ['success', 'error'].includes(event.mutation.state.status) + ) { + unsubscribe(); + resolve(); + } + }); + }); +} + +beforeEach(() => { + upload.mockReset(); + vi.spyOn(Math, 'random').mockReturnValue(0.5); +}); +afterEach(() => { + queryClient.clear(); + vi.restoreAllMocks(); +}); + +describe('dictation capacity retries', () => { + it('honors Retry-After, preserves tracing, and keeps content out of mutation state', async () => { + const retries = controlRetries(); + const run = vi.fn(); + const trace = { + run: (operation: () => T): T => { + run(); + return operation(); + }, + event: vi.fn(), + }; + upload.mockResolvedValueOnce(err([new DictationCapacityError(3_000)])); + upload.mockResolvedValueOnce(ok({ text: 'PRIVATE_TRANSCRIPT' })); + const controller = new AbortController(); + const result = transcribeAudio(audio, 'en-US', controller.signal, trace); + const retry = await retries.next(); + expect(retry.delay).toBe(3_125); + expect(upload).toHaveBeenCalledTimes(1); + const mutation = queryClient.getMutationCache().getAll()[0]; + expect(mutation.state.variables).toBeUndefined(); + expect(mutation.state.data).toBeUndefined(); + expect(JSON.stringify(mutation.state)).not.toContain('PRIVATE'); + retry.fire(); + await expect(result).resolves.toBe('PRIVATE_TRANSCRIPT'); + expect(upload).toHaveBeenNthCalledWith( + 2, + audio, + 'en-US', + controller.signal + ); + expect(run).toHaveBeenCalledTimes(2); + expect(trace.event).toHaveBeenCalledExactlyOnceWith( + 'dictation.upload_retry', + { attempt: 2 } + ); + expect(mutation.state.data).toBeUndefined(); + expect(mutation.state.variables).toBeUndefined(); + }); + + it('stops after two retries with exponential backoff and jitter', async () => { + const retries = controlRetries(); + upload.mockResolvedValue(err([new DictationCapacityError(1_000)])); + const result = transcribeAudio(audio, 'en', new AbortController().signal); + const rejected = expect(result).rejects.toThrow('Dictation is busy'); + const first = await retries.next(); + expect(first.delay).toBe(1_125); + first.fire(); + const second = await retries.next(); + expect(second.delay).toBe(2_125); + second.fire(); + await rejected; + expect(upload).toHaveBeenCalledTimes(3); + }); + + it.each([ + 'DICTATION_RATE_LIMITED', + 'UNAUTHORIZED', + 'FORBIDDEN', + 'HTTP_ERROR', + 'NETWORK_ERROR', + ] as const)('does not automatically replay %s failures', async (code) => { + upload.mockResolvedValue(err([{ code, message: 'Terminal failure' }])); + await expect( + transcribeAudio(audio, 'en', new AbortController().signal) + ).rejects.toThrow('Terminal failure'); + expect(upload).toHaveBeenCalledTimes(1); + }); + + it('does not wait for an excessive server delay', async () => { + upload.mockResolvedValue(err([new DictationCapacityError(60_000)])); + await expect( + transcribeAudio(audio, 'en', new AbortController().signal) + ).rejects.toThrow('Dictation is busy'); + expect(upload).toHaveBeenCalledTimes(1); + }); + + it('cancels during backoff immediately and never sends the scheduled retry', async () => { + const retries = controlRetries(); + upload.mockResolvedValue(err([new DictationCapacityError(1_000)])); + const controller = new AbortController(); + const done = settled(); + const result = transcribeAudio(audio, 'en', controller.signal); + const rejected = expect(result).rejects.toMatchObject({ + name: 'AbortError', + }); + const retry = await retries.next(); + controller.abort(); + await rejected; + retry.fire(); + await done; + expect(upload).toHaveBeenCalledTimes(1); + }); + + it('cancels an in-flight upload and ignores its late result', async () => { + let finish!: ( + result: Awaited> + ) => void; + let started!: () => void; + const requested = new Promise((resolve) => { + started = resolve; + }); + upload.mockImplementation( + () => + new Promise((resolve) => { + finish = resolve; + started(); + }) + ); + const controller = new AbortController(); + const done = settled(); + const result = transcribeAudio(audio, 'en', controller.signal); + const rejected = expect(result).rejects.toMatchObject({ + name: 'AbortError', + }); + await requested; + controller.abort(); + await rejected; + finish(ok({ text: 'PRIVATE_LATE_TRANSCRIPT' })); + await done; + expect(upload).toHaveBeenCalledTimes(1); + expect( + JSON.stringify( + queryClient + .getMutationCache() + .getAll() + .map((mutation) => mutation.state) + ) + ).not.toContain('PRIVATE'); + }); + + it('does not upload an already cancelled recording', async () => { + const controller = new AbortController(); + controller.abort(); + await expect( + transcribeAudio(audio, 'en', controller.signal) + ).rejects.toMatchObject({ name: 'AbortError' }); + expect(upload).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/src/lib/queries/dictation/transcribe.ts b/apps/web/src/lib/queries/dictation/transcribe.ts new file mode 100644 index 00000000000..9b6ed11d1d8 --- /dev/null +++ b/apps/web/src/lib/queries/dictation/transcribe.ts @@ -0,0 +1,85 @@ +import { ThrownResultError, throwOnErr } from '@core/util/result'; +import type { Span } from '@macro-inc/observability'; +import { + DictationCapacityError, + transcribeDictation, +} from '@service-storage/dictation'; +import { MutationObserver } from '@tanstack/solid-query'; +import { queryClient } from '../client'; +import { dictationKeys } from './keys'; + +const CAPACITY_RETRIES = 2; +const MAX_RETRY_AFTER_MS = 5_000; + +function capacityError(error: unknown): DictationCapacityError | undefined { + if (!(error instanceof ThrownResultError)) return; + return error.errors.find( + (error): error is DictationCapacityError => + error instanceof DictationCapacityError + ); +} + +/** TanStack owns retries; recording and transcript stay out of mutation state. */ +export async function transcribeAudio( + audio: Blob, + language: string, + signal: AbortSignal, + trace?: Pick +) { + signal.throwIfAborted(); + let recording: Blob | undefined = audio; + let transcript = ''; + let attempt = 0; + // A fresh observer gives each recording its own retry policy and cancellation + // scope. No mutation variables or data contain the audio or transcript. + const mutation = new MutationObserver(queryClient, { + mutationKey: dictationKeys.transcribe.queryKey, + gcTime: 0, + networkMode: 'always', + mutationFn: async () => { + signal.throwIfAborted(); + if (!recording) + throw new DOMException('Dictation cancelled', 'AbortError'); + const blob = recording; + if (++attempt > 1) trace?.event('dictation.upload_retry', { attempt }); + const upload = () => + throwOnErr(() => transcribeDictation(blob, language, signal)); + const result = await (trace ? trace.run(upload) : upload()); + signal.throwIfAborted(); + transcript = result.text; + }, + retry: (failureCount, error) => { + const capacity = capacityError(error); + return ( + !signal.aborted && + failureCount < CAPACITY_RETRIES && + capacity !== undefined && + capacity.retryAfterMs <= MAX_RETRY_AFTER_MS + ); + }, + retryDelay: (attempt, error) => + Math.max(capacityError(error)?.retryAfterMs ?? 0, 1_000 * 2 ** attempt) + + Math.floor(Math.random() * 250), + // Detach after settlement so GC cannot repeatedly reschedule a pending + // mutation while TanStack is waiting for its retry delay or browser focus. + onSettled: (): void => mutation.reset(), + }); + + let onAbort = () => {}; + const cancelled = new Promise((_, reject) => { + onAbort = () => { + recording = undefined; + transcript = ''; + reject(signal.reason); + }; + signal.addEventListener('abort', onAbort, { once: true }); + }); + try { + await Promise.race([mutation.mutate(), cancelled]); + return transcript; + } finally { + signal.removeEventListener('abort', onAbort); + recording = undefined; + transcript = ''; + } +} diff --git a/apps/web/src/lib/service-clients/service-cognition/generated/client.ts b/apps/web/src/lib/service-clients/service-cognition/generated/client.ts index 3346a770cd7..85fe086c055 100644 --- a/apps/web/src/lib/service-clients/service-cognition/generated/client.ts +++ b/apps/web/src/lib/service-clients/service-cognition/generated/client.ts @@ -68,6 +68,11 @@ export type setPricingHandlerResponse200 = { status: 200; }; +export type setPricingHandlerResponse400 = { + data: ErrorBody; + status: 400; +}; + export type setPricingHandlerResponse403 = { data: ErrorBody; status: 403; @@ -82,6 +87,7 @@ export type setPricingHandlerResponseSuccess = setPricingHandlerResponse200 & { headers: Headers; }; export type setPricingHandlerResponseError = ( + | setPricingHandlerResponse400 | setPricingHandlerResponse403 | setPricingHandlerResponse500 ) & { diff --git a/apps/web/src/lib/service-clients/service-cognition/generated/schemas/aiFeature.ts b/apps/web/src/lib/service-clients/service-cognition/generated/schemas/aiFeature.ts index 4fb7259edd0..03e5376dda8 100644 --- a/apps/web/src/lib/service-clients/service-cognition/generated/schemas/aiFeature.ts +++ b/apps/web/src/lib/service-clients/service-cognition/generated/schemas/aiFeature.ts @@ -25,4 +25,5 @@ export const AiFeature = { import: 'import', agent_session: 'agent_session', agent_repository_choice: 'agent_repository_choice', + dictation: 'dictation', } as const; diff --git a/apps/web/src/lib/service-clients/service-cognition/generated/schemas/completionUsage.ts b/apps/web/src/lib/service-clients/service-cognition/generated/schemas/completionUsage.ts index dd5ab30403b..2fb2a336ffa 100644 --- a/apps/web/src/lib/service-clients/service-cognition/generated/schemas/completionUsage.ts +++ b/apps/web/src/lib/service-clients/service-cognition/generated/schemas/completionUsage.ts @@ -11,16 +11,15 @@ import type { MacroUserIdStr } from './macroUserIdStr'; import type { Usage } from './usage'; /** - * A recorded completion: who, what feature, optional entity, and the cost. + * User and feature attribution for one invocation. */ export interface CompletionUsage { - /** Token usage and cost. */ + /** Measured usage and resolved cost. */ cost: Usage; - /** The entity the completion related to, if any. */ + /** Related entity, if any. */ entity?: CompletionUsageEntity; - /** The feature that performed the completion. */ + /** Feature that performed the invocation. */ feature: AiFeature; - /** The user the completion was performed for (the [system user](SYSTEM_USER_ID) -for background work). */ + /** User the invocation was performed for. */ user: MacroUserIdStr; } diff --git a/apps/web/src/lib/service-clients/service-cognition/generated/schemas/completionUsageEntity.ts b/apps/web/src/lib/service-clients/service-cognition/generated/schemas/completionUsageEntity.ts index d4d73acf11c..d24e6b5662a 100644 --- a/apps/web/src/lib/service-clients/service-cognition/generated/schemas/completionUsageEntity.ts +++ b/apps/web/src/lib/service-clients/service-cognition/generated/schemas/completionUsageEntity.ts @@ -6,6 +6,6 @@ */ /** - * The entity the completion related to, if any. + * Related entity, if any. */ export type CompletionUsageEntity = string | null; diff --git a/apps/web/src/lib/service-clients/service-cognition/generated/schemas/featureUsage.ts b/apps/web/src/lib/service-clients/service-cognition/generated/schemas/featureUsage.ts index d599cfc1a58..a407fcf5ca1 100644 --- a/apps/web/src/lib/service-clients/service-cognition/generated/schemas/featureUsage.ts +++ b/apps/web/src/lib/service-clients/service-cognition/generated/schemas/featureUsage.ts @@ -9,13 +9,13 @@ import type { AiFeature } from './aiFeature'; import type { CompletionUsage } from './completionUsage'; /** - * Usage for a single feature, with its rolled-up dollar total. + * Recorded invocations and total for one feature. */ export interface FeatureUsage { - /** The individual completions recorded for this feature. */ + /** Recorded invocations. */ entries: CompletionUsage[]; - /** The feature. */ + /** Feature attribution. */ feature: AiFeature; - /** Total cost across `entries` (USD). */ + /** Total cost (USD). */ total: number; } diff --git a/apps/web/src/lib/service-clients/service-cognition/generated/schemas/index.ts b/apps/web/src/lib/service-clients/service-cognition/generated/schemas/index.ts index 56cafc7a471..3169b50c849 100644 --- a/apps/web/src/lib/service-clients/service-cognition/generated/schemas/index.ts +++ b/apps/web/src/lib/service-clients/service-cognition/generated/schemas/index.ts @@ -177,6 +177,7 @@ export * from './pipedreamUpdateRequest'; export * from './pipedreamUpdateRequestEnabled'; export * from './pipedreamUpdateRequestServerName'; export * from './price'; +export * from './pricePricePerAudioMinute'; export * from './projectionStateResponse'; export * from './projectionStateResponseData'; export * from './projectionStateResponseError'; @@ -196,6 +197,9 @@ export * from './sendChatMessagePayloadAllOfAttachments'; export * from './sendChatMessageResponse'; export * from './serverResponse'; export * from './setPricingRequest'; +export * from './setPricingRequestPricePerAudioMinute'; +export * from './setPricingRequestPricePerMilIn'; +export * from './setPricingRequestPricePerMilOut'; export * from './sharePermissionV2'; export * from './sharePermissionV2ChannelSharePermissions'; export * from './sharePermissionV2LinkShare'; @@ -247,6 +251,7 @@ export * from './upsertProjectionRequestModel'; export * from './upsertProjectionRequestOutputSchema'; export * from './upsertProjectionRequestOutputSchemaAnyOf'; export * from './usage'; +export * from './usageAudioSeconds'; export * from './usagePrice'; export * from './usageRequest'; export * from './usageRequestFrom'; diff --git a/apps/web/src/lib/service-clients/service-cognition/generated/schemas/price.ts b/apps/web/src/lib/service-clients/service-cognition/generated/schemas/price.ts index 38ca8cefee0..f5023ea004b 100644 --- a/apps/web/src/lib/service-clients/service-cognition/generated/schemas/price.ts +++ b/apps/web/src/lib/service-clients/service-cognition/generated/schemas/price.ts @@ -4,15 +4,18 @@ * Document Cognition Service * OpenAPI spec version: 1.0.0 */ +import type { PricePricePerAudioMinute } from './pricePricePerAudioMinute'; /** - * Resolved price for one completion. + * Rates applied to an invocation and its resolved dollar cost. */ export interface Price { - /** Price per million input tokens (USD). */ + /** Price per audio minute (USD), absent for token billing. */ + price_per_audio_minute?: PricePricePerAudioMinute; + /** Price per million input tokens (USD); zero for audio billing. */ price_per_million_in: number; - /** Price per million output tokens (USD). */ + /** Price per million output tokens (USD); zero for audio billing. */ price_per_million_out: number; - /** Total cost of the completion (USD). */ + /** Total cost (USD). */ total: number; } diff --git a/apps/web/src/lib/service-clients/service-cognition/generated/schemas/pricePricePerAudioMinute.ts b/apps/web/src/lib/service-clients/service-cognition/generated/schemas/pricePricePerAudioMinute.ts new file mode 100644 index 00000000000..16a9115c96c --- /dev/null +++ b/apps/web/src/lib/service-clients/service-cognition/generated/schemas/pricePricePerAudioMinute.ts @@ -0,0 +1,11 @@ +/** + * Generated by orval v7.21.0 🍺 + * Do not edit manually. + * Document Cognition Service + * OpenAPI spec version: 1.0.0 + */ + +/** + * Price per audio minute (USD), absent for token billing. + */ +export type PricePricePerAudioMinute = number | null; diff --git a/apps/web/src/lib/service-clients/service-cognition/generated/schemas/setPricingRequest.ts b/apps/web/src/lib/service-clients/service-cognition/generated/schemas/setPricingRequest.ts index ac92a4d7d2e..5082dc795bd 100644 --- a/apps/web/src/lib/service-clients/service-cognition/generated/schemas/setPricingRequest.ts +++ b/apps/web/src/lib/service-clients/service-cognition/generated/schemas/setPricingRequest.ts @@ -4,6 +4,9 @@ * Document Cognition Service * OpenAPI spec version: 1.0.0 */ +import type { SetPricingRequestPricePerAudioMinute } from './setPricingRequestPricePerAudioMinute'; +import type { SetPricingRequestPricePerMilIn } from './setPricingRequestPricePerMilIn'; +import type { SetPricingRequestPricePerMilOut } from './setPricingRequestPricePerMilOut'; /** * Request body for [`set_pricing_handler`]. @@ -11,8 +14,10 @@ export interface SetPricingRequest { /** The model api id to (re)price. */ model: string; - /** New price per million input tokens (USD). */ - price_per_mil_in: number; - /** New price per million output tokens (USD). */ - price_per_mil_out: number; + /** Price per minute of audio (USD), or null for token-only pricing. */ + price_per_audio_minute?: SetPricingRequestPricePerAudioMinute; + /** New price per million input tokens (USD). Required for token pricing. */ + price_per_mil_in?: SetPricingRequestPricePerMilIn; + /** New price per million output tokens (USD). Required for token pricing. */ + price_per_mil_out?: SetPricingRequestPricePerMilOut; } diff --git a/apps/web/src/lib/service-clients/service-cognition/generated/schemas/setPricingRequestPricePerAudioMinute.ts b/apps/web/src/lib/service-clients/service-cognition/generated/schemas/setPricingRequestPricePerAudioMinute.ts new file mode 100644 index 00000000000..0ffb4ab639b --- /dev/null +++ b/apps/web/src/lib/service-clients/service-cognition/generated/schemas/setPricingRequestPricePerAudioMinute.ts @@ -0,0 +1,11 @@ +/** + * Generated by orval v7.21.0 🍺 + * Do not edit manually. + * Document Cognition Service + * OpenAPI spec version: 1.0.0 + */ + +/** + * Price per minute of audio (USD), or null for token-only pricing. + */ +export type SetPricingRequestPricePerAudioMinute = number | null; diff --git a/apps/web/src/lib/service-clients/service-cognition/generated/schemas/setPricingRequestPricePerMilIn.ts b/apps/web/src/lib/service-clients/service-cognition/generated/schemas/setPricingRequestPricePerMilIn.ts new file mode 100644 index 00000000000..a0366e61101 --- /dev/null +++ b/apps/web/src/lib/service-clients/service-cognition/generated/schemas/setPricingRequestPricePerMilIn.ts @@ -0,0 +1,11 @@ +/** + * Generated by orval v7.21.0 🍺 + * Do not edit manually. + * Document Cognition Service + * OpenAPI spec version: 1.0.0 + */ + +/** + * New price per million input tokens (USD). Required for token pricing. + */ +export type SetPricingRequestPricePerMilIn = number | null; diff --git a/apps/web/src/lib/service-clients/service-cognition/generated/schemas/setPricingRequestPricePerMilOut.ts b/apps/web/src/lib/service-clients/service-cognition/generated/schemas/setPricingRequestPricePerMilOut.ts new file mode 100644 index 00000000000..7febe89a438 --- /dev/null +++ b/apps/web/src/lib/service-clients/service-cognition/generated/schemas/setPricingRequestPricePerMilOut.ts @@ -0,0 +1,11 @@ +/** + * Generated by orval v7.21.0 🍺 + * Do not edit manually. + * Document Cognition Service + * OpenAPI spec version: 1.0.0 + */ + +/** + * New price per million output tokens (USD). Required for token pricing. + */ +export type SetPricingRequestPricePerMilOut = number | null; diff --git a/apps/web/src/lib/service-clients/service-cognition/generated/schemas/usage.ts b/apps/web/src/lib/service-clients/service-cognition/generated/schemas/usage.ts index d229e92d4ab..d80af98e777 100644 --- a/apps/web/src/lib/service-clients/service-cognition/generated/schemas/usage.ts +++ b/apps/web/src/lib/service-clients/service-cognition/generated/schemas/usage.ts @@ -4,23 +4,26 @@ * Document Cognition Service * OpenAPI spec version: 1.0.0 */ +import type { UsageAudioSeconds } from './usageAudioSeconds'; import type { UsagePrice } from './usagePrice'; /** - * The token usage and resolved cost of a single completion. + * Measured usage for one invocation. */ export interface Usage { - /** When the completion was recorded. */ + /** Audio duration in seconds, absent for token billing. */ + audio_seconds?: UsageAudioSeconds; + /** Recording timestamp. */ created_at: string; /** - * Tokens consumed by the input. + * Input tokens; zero for audio billing. * @minimum 0 */ input_tokens: number; - /** The model api id (e.g. `claude-opus-4-8`). */ + /** Provider model identifier. */ model: string; /** - * Tokens generated in the output. + * Output tokens; zero for audio billing. * @minimum 0 */ output_tokens: number; diff --git a/apps/web/src/lib/service-clients/service-cognition/generated/schemas/usageAudioSeconds.ts b/apps/web/src/lib/service-clients/service-cognition/generated/schemas/usageAudioSeconds.ts new file mode 100644 index 00000000000..ca432073c1a --- /dev/null +++ b/apps/web/src/lib/service-clients/service-cognition/generated/schemas/usageAudioSeconds.ts @@ -0,0 +1,11 @@ +/** + * Generated by orval v7.21.0 🍺 + * Do not edit manually. + * Document Cognition Service + * OpenAPI spec version: 1.0.0 + */ + +/** + * Audio duration in seconds, absent for token billing. + */ +export type UsageAudioSeconds = number | null; diff --git a/apps/web/src/lib/service-clients/service-cognition/generated/schemas/usageSummary.ts b/apps/web/src/lib/service-clients/service-cognition/generated/schemas/usageSummary.ts index 6985e371521..160480506b2 100644 --- a/apps/web/src/lib/service-clients/service-cognition/generated/schemas/usageSummary.ts +++ b/apps/web/src/lib/service-clients/service-cognition/generated/schemas/usageSummary.ts @@ -7,11 +7,11 @@ import type { FeatureUsage } from './featureUsage'; /** - * The result of a usage query: per-feature breakdown plus a grand total. + * Per-feature breakdown and grand total. */ export interface UsageSummary { /** Per-feature usage. */ entries: FeatureUsage[]; - /** Grand total cost across all features (USD). */ + /** Grand total (USD). */ total: number; } diff --git a/apps/web/src/lib/service-clients/service-cognition/openapi.json b/apps/web/src/lib/service-clients/service-cognition/openapi.json index 96393c74f1b..13f6531e342 100644 --- a/apps/web/src/lib/service-clients/service-cognition/openapi.json +++ b/apps/web/src/lib/service-clients/service-cognition/openapi.json @@ -30,6 +30,16 @@ "200": { "description": "Pricing updated" }, + "400": { + "description": "Invalid pricing", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorBody" + } + } + } + }, "403": { "description": "Admin access required", "content": { @@ -2132,7 +2142,8 @@ "ai_editing", "import", "agent_session", - "agent_repository_choice" + "agent_repository_choice", + "dictation" ] }, "AssistantMessagePart": { @@ -2806,25 +2817,25 @@ }, "CompletionUsage": { "type": "object", - "description": "A recorded completion: who, what feature, optional entity, and the cost.", + "description": "User and feature attribution for one invocation.", "required": ["feature", "user", "cost"], "properties": { "cost": { "$ref": "#/components/schemas/Usage", - "description": "Token usage and cost." + "description": "Measured usage and resolved cost." }, "entity": { "type": ["string", "null"], "format": "uuid", - "description": "The entity the completion related to, if any." + "description": "Related entity, if any." }, "feature": { "$ref": "#/components/schemas/AiFeature", - "description": "The feature that performed the completion." + "description": "Feature that performed the invocation." }, "user": { "$ref": "#/components/schemas/MacroUserIdStr", - "description": "The user the completion was performed for (the [system user](SYSTEM_USER_ID)\nfor background work)." + "description": "User the invocation was performed for." } } }, @@ -3006,7 +3017,7 @@ }, "FeatureUsage": { "type": "object", - "description": "Usage for a single feature, with its rolled-up dollar total.", + "description": "Recorded invocations and total for one feature.", "required": ["feature", "entries", "total"], "properties": { "entries": { @@ -3014,16 +3025,16 @@ "items": { "$ref": "#/components/schemas/CompletionUsage" }, - "description": "The individual completions recorded for this feature." + "description": "Recorded invocations." }, "feature": { "$ref": "#/components/schemas/AiFeature", - "description": "The feature." + "description": "Feature attribution." }, "total": { "type": "number", "format": "float", - "description": "Total cost across `entries` (USD)." + "description": "Total cost (USD)." } } }, @@ -4087,23 +4098,28 @@ }, "Price": { "type": "object", - "description": "Resolved price for one completion.", + "description": "Rates applied to an invocation and its resolved dollar cost.", "required": ["price_per_million_in", "price_per_million_out", "total"], "properties": { + "price_per_audio_minute": { + "type": ["number", "null"], + "format": "float", + "description": "Price per audio minute (USD), absent for token billing." + }, "price_per_million_in": { "type": "number", "format": "float", - "description": "Price per million input tokens (USD)." + "description": "Price per million input tokens (USD); zero for audio billing." }, "price_per_million_out": { "type": "number", "format": "float", - "description": "Price per million output tokens (USD)." + "description": "Price per million output tokens (USD); zero for audio billing." }, "total": { "type": "number", "format": "float", - "description": "Total cost of the completion (USD)." + "description": "Total cost (USD)." } } }, @@ -4312,21 +4328,26 @@ "SetPricingRequest": { "type": "object", "description": "Request body for [`set_pricing_handler`].", - "required": ["model", "price_per_mil_in", "price_per_mil_out"], + "required": ["model"], "properties": { "model": { "type": "string", "description": "The model api id to (re)price." }, + "price_per_audio_minute": { + "type": ["number", "null"], + "format": "float", + "description": "Price per minute of audio (USD), or null for token-only pricing." + }, "price_per_mil_in": { - "type": "number", + "type": ["number", "null"], "format": "float", - "description": "New price per million input tokens (USD)." + "description": "New price per million input tokens (USD). Required for token pricing." }, "price_per_mil_out": { - "type": "number", + "type": ["number", "null"], "format": "float", - "description": "New price per million output tokens (USD)." + "description": "New price per million output tokens (USD). Required for token pricing." } } }, @@ -4790,28 +4811,33 @@ }, "Usage": { "type": "object", - "description": "The token usage and resolved cost of a single completion.", + "description": "Measured usage for one invocation.", "required": ["input_tokens", "output_tokens", "model", "created_at"], "properties": { + "audio_seconds": { + "type": ["number", "null"], + "format": "double", + "description": "Audio duration in seconds, absent for token billing." + }, "created_at": { "type": "string", "format": "date-time", - "description": "When the completion was recorded." + "description": "Recording timestamp." }, "input_tokens": { "type": "integer", - "format": "int32", - "description": "Tokens consumed by the input.", + "format": "int64", + "description": "Input tokens; zero for audio billing.", "minimum": 0 }, "model": { "type": "string", - "description": "The model api id (e.g. `claude-opus-4-8`)." + "description": "Provider model identifier." }, "output_tokens": { "type": "integer", - "format": "int32", - "description": "Tokens generated in the output.", + "format": "int64", + "description": "Output tokens; zero for audio billing.", "minimum": 0 }, "price": { @@ -4821,7 +4847,7 @@ }, { "$ref": "#/components/schemas/Price", - "description": "Resolved price, or `None` when the model had no pricing at record time." + "description": "Applied pricing, if known." } ] } @@ -4859,7 +4885,7 @@ }, "UsageSummary": { "type": "object", - "description": "The result of a usage query: per-feature breakdown plus a grand total.", + "description": "Per-feature breakdown and grand total.", "required": ["entries", "total"], "properties": { "entries": { @@ -4872,7 +4898,7 @@ "total": { "type": "number", "format": "float", - "description": "Grand total cost across all features (USD)." + "description": "Grand total (USD)." } } }, diff --git a/apps/web/src/lib/service-clients/service-storage/dictation.test.ts b/apps/web/src/lib/service-clients/service-storage/dictation.test.ts new file mode 100644 index 00000000000..ac6c8fa37d8 --- /dev/null +++ b/apps/web/src/lib/service-clients/service-storage/dictation.test.ts @@ -0,0 +1,62 @@ +import { fetchWithToken } from '@core/util/fetchWithToken'; +import { ok } from 'neverthrow'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { DictationCapacityError, transcribeDictation } from './dictation'; + +vi.mock('@core/util/fetchWithToken', () => ({ fetchWithToken: vi.fn() })); +const fetch = vi.mocked(fetchWithToken); + +beforeEach(() => { + fetch.mockReset(); + fetch.mockResolvedValue(ok({ text: 'transcript' })); +}); + +describe('dictation HTTP adapter', () => { + it('sends encoded bytes unchanged with cancellation and no transport retries', async () => { + const audio = new Blob(['encoded'], { type: 'audio/webm' }); + const controller = new AbortController(); + await transcribeDictation(audio, 'en-US', controller.signal); + expect(fetch).toHaveBeenCalledExactlyOnceWith( + expect.stringContaining('/dictation/transcribe?language=en'), + expect.objectContaining({ + method: 'POST', + body: audio, + signal: controller.signal, + headers: { 'Content-Type': 'audio/webm' }, + retry: { maxTries: 1 }, + }) + ); + }); + + it.each([ + [503, '1', 'DICTATION_CAPACITY'], + [503, null, 'HTTP_ERROR'], + [503, '-1', 'HTTP_ERROR'], + [503, 'nonsense', 'HTTP_ERROR'], + [429, '1', 'DICTATION_RATE_LIMITED'], + [400, null, 'HTTP_ERROR'], + [502, '1', 'HTTP_ERROR'], + ])( + 'classifies status %s / Retry-After %s without retaining response content', + async (status, retryAfter, code) => { + await transcribeDictation(new Blob(), 'en', new AbortController().signal); + const handler = fetch.mock.calls[0][1]?.errorResponseHandler; + const headers = retryAfter + ? { 'Retry-After': String(retryAfter) } + : undefined; + const error = await handler!( + new Response('PRIVATE_PROVIDER_CONTENT', { + status: Number(status), + headers, + }) + ); + expect(error?.code).toBe(code); + expect(error?.message).not.toContain('PRIVATE'); + expect(JSON.stringify(error)).not.toContain('PRIVATE'); + if (code === 'DICTATION_CAPACITY') { + expect(error).toBeInstanceOf(DictationCapacityError); + expect((error as DictationCapacityError).retryAfterMs).toBe(1_000); + } + } + ); +}); diff --git a/apps/web/src/lib/service-clients/service-storage/dictation.ts b/apps/web/src/lib/service-clients/service-storage/dictation.ts new file mode 100644 index 00000000000..5c4b92c1519 --- /dev/null +++ b/apps/web/src/lib/service-clients/service-storage/dictation.ts @@ -0,0 +1,54 @@ +import { SERVER_HOSTS } from '@core/constant/servers'; +import { fetchWithToken } from '@core/util/fetchWithToken'; + +/** A rejected request that has not reached Whisper and can safely be retried. */ +export class DictationCapacityError extends Error { + readonly code = 'DICTATION_CAPACITY'; + + constructor(readonly retryAfterMs: number) { + super('Dictation is busy. Please try again shortly.'); + this.name = 'DictationCapacityError'; + } +} + +export function transcribeDictation( + audio: Blob, + language: string, + signal: AbortSignal +) { + const hint = language.split('-')[0].toLowerCase(); + const query = /^[a-z]{2}$/.test(hint) + ? `?language=${encodeURIComponent(hint)}` + : ''; + return fetchWithToken< + { text: string }, + 'DICTATION_CAPACITY' | 'DICTATION_RATE_LIMITED' + >( + `${SERVER_HOSTS['document-storage-service']}/dictation/transcribe${query}`, + { + method: 'POST', + body: audio, + signal, + headers: { 'Content-Type': audio.type }, + retry: { maxTries: 1 }, + errorResponseHandler: async (response) => { + const retryAfter = response.headers.get('Retry-After'); + // DSS sends delay-seconds only for capacity rejections, before Whisper. + if (response.status === 503 && retryAfter && /^\d+$/.test(retryAfter)) { + return new DictationCapacityError(Number(retryAfter) * 1_000); + } + if (response.status === 429) { + return { + code: 'DICTATION_RATE_LIMITED', + message: 'Dictation limit reached. Please try again later.', + }; + } + return { + code: 'HTTP_ERROR', + message: + 'Transcription failed. Select the checkmark to retry, or cancel.', + }; + }, + } + ); +} diff --git a/apps/web/src/lib/service-clients/service-storage/generated/schemas/index.ts b/apps/web/src/lib/service-clients/service-storage/generated/schemas/index.ts index aff5f960ff5..0675ef6579c 100644 --- a/apps/web/src/lib/service-clients/service-storage/generated/schemas/index.ts +++ b/apps/web/src/lib/service-clients/service-storage/generated/schemas/index.ts @@ -1410,6 +1410,8 @@ export * from './threadState'; export * from './threadStateAnchor'; export * from './threadStateDeletedAt'; export * from './threadUpdatedAt'; +export * from './transcribeDictationParams'; +export * from './transcribeResponse'; export * from './transcriptSegmentRequest'; export * from './transcriptSegmentRequestDiarizedSpeakerId'; export * from './transcriptSegmentRequestEmbedding'; diff --git a/apps/web/src/lib/service-clients/service-storage/generated/schemas/transcribeDictationParams.ts b/apps/web/src/lib/service-clients/service-storage/generated/schemas/transcribeDictationParams.ts new file mode 100644 index 00000000000..ecef1b16f21 --- /dev/null +++ b/apps/web/src/lib/service-clients/service-storage/generated/schemas/transcribeDictationParams.ts @@ -0,0 +1,13 @@ +/** + * Generated by orval v7.21.0 🍺 + * Do not edit manually. + * document_storage_service + * OpenAPI spec version: 0.1.0 + */ + +export type TranscribeDictationParams = { + /** + * ISO 639-1 language hint + */ + language?: string; +}; diff --git a/apps/web/src/lib/service-clients/service-storage/generated/schemas/transcribeResponse.ts b/apps/web/src/lib/service-clients/service-storage/generated/schemas/transcribeResponse.ts new file mode 100644 index 00000000000..3ddb5aee2bd --- /dev/null +++ b/apps/web/src/lib/service-clients/service-storage/generated/schemas/transcribeResponse.ts @@ -0,0 +1,14 @@ +/** + * Generated by orval v7.21.0 🍺 + * Do not edit manually. + * document_storage_service + * OpenAPI spec version: 0.1.0 + */ + +/** + * Transcription result. + */ +export interface TranscribeResponse { + /** Recognized text. */ + text: string; +} diff --git a/apps/web/src/lib/service-clients/service-storage/generated/zod.ts b/apps/web/src/lib/service-clients/service-storage/generated/zod.ts index 2ed21843520..0375a231f18 100644 --- a/apps/web/src/lib/service-clients/service-storage/generated/zod.ts +++ b/apps/web/src/lib/service-clients/service-storage/generated/zod.ts @@ -5473,6 +5473,21 @@ export const putCrmTeamStagesResponse = zod }) .describe("The team's custom stage set."); +/** + * Available to every signed-in user on every plan; does not consume chat +credits. Audio and transcripts are never persisted. + * @summary Transcribe a transient recording with OpenAI Whisper. + */ +export const transcribeDictationQueryParams = zod.object({ + language: zod.string().optional().describe('ISO 639-1 language hint'), +}); + +export const transcribeDictationResponse = zod + .object({ + text: zod.string().describe('Recognized text.'), + }) + .describe('Transcription result.'); + /** * @summary Gets the users documents to populate their recent document list */ diff --git a/apps/web/src/lib/service-clients/service-storage/openapi.json b/apps/web/src/lib/service-clients/service-storage/openapi.json index d0e667ea69d..1340f51280a 100644 --- a/apps/web/src/lib/service-clients/service-storage/openapi.json +++ b/apps/web/src/lib/service-clients/service-storage/openapi.json @@ -6284,6 +6284,116 @@ } } }, + "/dictation/transcribe": { + "post": { + "tags": ["dictation"], + "summary": "Transcribe a transient recording with OpenAI Whisper.", + "description": "Available to every signed-in user on every plan; does not consume chat\ncredits. Audio and transcripts are never persisted.", + "operationId": "transcribe_dictation", + "parameters": [ + { + "name": "language", + "in": "query", + "description": "ISO 639-1 language hint", + "required": false, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "description": "Encoded WebM, MP4, Ogg, or WAV audio up to 8 MiB and five minutes", + "content": { + "audio/webm": { + "schema": { + "type": "string", + "format": "binary", + "description": "OpenAPI representation of the raw encoded audio body extracted as `Bytes`." + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TranscribeResponse" + } + } + } + }, + "400": { + "description": "Empty, oversized, or malformed request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "401": { + "description": "Missing or invalid credentials", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "403": { + "description": "Only signed-in users may dictate", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "413": { + "description": "Body exceeds 8 MiB" + }, + "415": { + "description": "Unsupported audio container", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "429": { + "description": "Per-user hourly rate limit exceeded" + }, + "502": { + "description": "Provider failure", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "503": { + "description": "Transcription capacity exhausted; retry after the Retry-After delay", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + } + } + } + }, "/documents": { "get": { "tags": ["document"], @@ -29758,6 +29868,17 @@ } } }, + "TranscribeResponse": { + "type": "object", + "description": "Transcription result.", + "required": ["text"], + "properties": { + "text": { + "type": "string", + "description": "Recognized text." + } + } + }, "TranscriptSegmentRequest": { "type": "object", "description": "A transcript segment from LiveKit Inference STT.", diff --git a/bun.lock b/bun.lock index aa3c360af27..ab21d687c7d 100644 --- a/bun.lock +++ b/bun.lock @@ -69,6 +69,7 @@ "@solid-primitives/rootless": "^1.5.2", "@solid-primitives/scheduled": "^1.5.2", "@solid-primitives/storage": "^4.3.3", + "@solid-primitives/stream": "^0.7.4", "@solidjs/router": "^0.15.3", "@tailwindcss/vite": "^4.2.4", "@tanstack/pacer": "^0.17.3", @@ -1539,6 +1540,8 @@ "@solid-primitives/storage": ["@solid-primitives/storage@4.3.4", "", { "dependencies": { "@solid-primitives/utils": "^6.4.0" }, "peerDependencies": { "@tauri-apps/plugin-store": "*", "solid-js": "^1.6.12", "solid-start": "*" }, "optionalPeers": ["@tauri-apps/plugin-store", "solid-start"] }, "sha512-GSxPAIuyxhJWOcv7n10iv3aid5oHN3KUgyA9IV0GYWlPpgyGs43aS9E85b0VXDLoH+D4ThNK8+2WEJ8B/S6Ccg=="], + "@solid-primitives/stream": ["@solid-primitives/stream@0.7.4", "", { "dependencies": { "@solid-primitives/utils": "^6.4.1" }, "peerDependencies": { "solid-js": "^1.6.12" } }, "sha512-iOYay6stVIshOEplMUv0qqWl0gi120FK53YYthawVRbu+ryd5H/yCs2IHD6HubM5BAEKlx15XImin+BQEcQoEw=="], + "@solid-primitives/transition-group": ["@solid-primitives/transition-group@1.1.2", "", { "peerDependencies": { "solid-js": "^1.6.12" } }, "sha512-gnHS0OmcdjeoHN9n7Khu8KNrOlRc8a2weETDt2YT6o1zeW/XtUC6Db3Q9pkMU/9cCKdEmN4b0a/41MKAHRhzWA=="], "@solid-primitives/trigger": ["@solid-primitives/trigger@1.2.4", "", { "dependencies": { "@solid-primitives/utils": "^6.4.1" }, "peerDependencies": { "solid-js": "^1.6.12" } }, "sha512-Ju0e+ZOD7hpOp7nptJimvDSZHWFvIvF9iBWMvuwt30smX7c5wmB8Kmc0AdDuv0wOTi06PQ1J5IrP1WJbH2yUBQ=="], diff --git a/crates/ai_usage/src/domain/mod.rs b/crates/ai_usage/src/domain/mod.rs index 102bddaebbf..769dfae42ec 100644 --- a/crates/ai_usage/src/domain/mod.rs +++ b/crates/ai_usage/src/domain/mod.rs @@ -4,8 +4,8 @@ pub mod ports; pub mod service; pub use ports::{ - AiFeature, CompletionUsage, FeatureUsage, NoOpUsageRecorder, Price, Result, SYSTEM_USER_ID, - Usage, UsageApiParams, UsageContext, UsageError, UsageEvent, UsageRecorder, UsageRepo, - UsageService, UsageSummary, + AiFeature, CompletionUsage, FeatureUsage, ModelPricing, NoOpUsageRecorder, Price, Result, + SYSTEM_USER_ID, Usage, UsageAmount, UsageApiParams, UsageContext, UsageError, UsageEvent, + UsageRecorder, UsageRepo, UsageService, UsageSummary, }; pub use service::UsageServiceImpl; diff --git a/crates/ai_usage/src/domain/ports.rs b/crates/ai_usage/src/domain/ports.rs index e17d195f8f8..21e921074ba 100644 --- a/crates/ai_usage/src/domain/ports.rs +++ b/crates/ai_usage/src/domain/ports.rs @@ -64,49 +64,110 @@ pub enum AiFeature { AgentSession, /// Choosing the repository an agent session's first prompt belongs to. AgentRepositoryChoice, + /// User-confirmed audio transcription. + Dictation, } -/// Resolved price for one completion. -#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize, ToSchema)] +/// The billable quantity for one AI invocation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum UsageAmount { + /// Token-based inference. + Tokens { + /// Tokens consumed by the input. + input: u64, + /// Tokens generated in the output. + output: u64, + }, + /// Duration-based audio inference. + Audio { + /// Provider-reported duration of the audio. + duration: std::time::Duration, + }, +} + +/// Rates for a model's billing unit. +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum ModelPricing { + /// Prices per million input/output tokens (USD). + Tokens { + /// Input token rate. + input: f32, + /// Output token rate. + output: f32, + }, + /// Price per minute of audio (USD). + Audio { + /// Audio minute rate. + per_minute: f32, + }, +} + +impl ModelPricing { + /// Reject invalid prices before changing stored pricing or usage totals. + pub fn validate(self) -> Result { + let valid = |rate: f32| rate.is_finite() && rate >= 0.0; + let is_valid = match self { + Self::Tokens { input, output } => valid(input) && valid(output), + Self::Audio { per_minute } => valid(per_minute), + }; + if is_valid { + Ok(self) + } else { + Err(UsageError::InvalidPricing) + } + } +} + +/// Resolved price for one AI call, including the rates applied at record time. +#[derive(Debug, Clone, Copy, PartialEq)] pub struct Price { - /// Price per million input tokens (USD). - pub price_per_million_in: f32, - /// Price per million output tokens (USD). - pub price_per_million_out: f32, - /// Total cost of the completion (USD). + /// Rates matching the usage's billing unit. + pub pricing: ModelPricing, + /// Total cost (USD). pub total: f32, } impl Price { - /// Compute the price of a completion from per-million rates and token counts. - pub fn compute(price_per_million_in: f32, price_per_million_out: f32, usage: &Usage) -> Self { - let total = (usage.input_tokens as f32 / 1_000_000.0) * price_per_million_in - + (usage.output_tokens as f32 / 1_000_000.0) * price_per_million_out; - Self { - price_per_million_in, - price_per_million_out, - total, - } + /// Compute cost only when the rate and usage use the same billing unit. + pub fn compute(pricing: ModelPricing, amount: UsageAmount) -> Option { + let total = match (pricing, amount) { + ( + ModelPricing::Tokens { input, output }, + UsageAmount::Tokens { + input: input_tokens, + output: output_tokens, + }, + ) => { + input_tokens as f64 / 1_000_000.0 * f64::from(input) + + output_tokens as f64 / 1_000_000.0 * f64::from(output) + } + (ModelPricing::Audio { per_minute }, UsageAmount::Audio { duration }) => { + duration.as_secs_f64() / 60.0 * f64::from(per_minute) + } + _ => return None, + }; + Some(Self { + pricing, + total: total as f32, + }) } } -/// The token usage and resolved cost of a single completion. -#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +/// The measured usage and resolved cost of a single AI call. +#[derive(Debug, Clone)] pub struct Usage { - /// Tokens consumed by the input. - pub input_tokens: u32, - /// Tokens generated in the output. - pub output_tokens: u32, + /// The invocation's billable quantity. + pub amount: UsageAmount, /// The model api id (e.g. `claude-opus-4-8`). pub model: String, - /// Resolved price, or `None` when the model had no pricing at record time. + /// Resolved price, or `None` when compatible pricing was unavailable. pub price: Option, - /// When the completion was recorded. + /// When the call was recorded. pub created_at: DateTime, } /// A recorded completion: who, what feature, optional entity, and the cost. -#[derive(Debug, Clone, Serialize, ToSchema)] +#[derive(Debug, Clone)] pub struct CompletionUsage { /// The feature that performed the completion. pub feature: AiFeature, @@ -115,12 +176,12 @@ pub struct CompletionUsage { pub user: MacroUserIdStr<'static>, /// The entity the completion related to, if any. pub entity: Option, - /// Token usage and cost. + /// Token/audio usage and cost. pub cost: Usage, } /// Usage for a single feature, with its rolled-up dollar total. -#[derive(Debug, Clone, Serialize, ToSchema)] +#[derive(Debug, Clone)] pub struct FeatureUsage { /// The feature. pub feature: AiFeature, @@ -131,7 +192,7 @@ pub struct FeatureUsage { } /// The result of a usage query: per-feature breakdown plus a grand total. -#[derive(Debug, Clone, Serialize, ToSchema)] +#[derive(Debug, Clone)] pub struct UsageSummary { /// Per-feature usage. pub entries: Vec, @@ -152,7 +213,7 @@ pub struct UsageApiParams { pub features: Vec, } -/// A usage event handed to a [`UsageRecorder`] by the agent crate. The recorder +/// A usage event handed to a [`UsageRecorder`] by an AI caller. The recorder /// resolves pricing and persists it; callers never see the cost. #[derive(Debug, Clone)] pub struct UsageEvent { @@ -164,14 +225,12 @@ pub struct UsageEvent { pub entity: Option, /// The model api id. pub model: String, - /// Input tokens for this round-trip. - pub input_tokens: u64, - /// Output tokens for this round-trip. - pub output_tokens: u64, + /// The invocation's billable quantity. + pub amount: UsageAmount, } /// The constant attributes of a logical completion (everything except the model -/// and token counts, which are only known once the completion runs). +/// and measured usage, which are only known once the invocation runs). /// /// Threaded into agent functions so each call site declares which feature it is /// and who it is for. @@ -218,8 +277,21 @@ impl UsageContext { user: self.user, entity: self.entity, model, - input_tokens, - output_tokens, + amount: UsageAmount::Tokens { + input: input_tokens, + output: output_tokens, + }, + } + } + + /// Build a duration-based event without representing audio as tokens. + pub fn into_audio_event(self, model: String, duration: std::time::Duration) -> UsageEvent { + UsageEvent { + feature: self.feature, + user: self.user, + entity: self.entity, + model, + amount: UsageAmount::Audio { duration }, } } } @@ -227,6 +299,12 @@ impl UsageContext { /// Errors raised by the cost crate. #[derive(Debug, Error)] pub enum UsageError { + /// The actor cannot administer AI usage or pricing. + #[error("admin access required")] + Forbidden, + /// A rate was negative or non-finite. + #[error("prices must be finite and non-negative")] + InvalidPricing, /// A database error. #[error("database error: {0}")] Db(rootcause::Report), @@ -241,20 +319,18 @@ pub type Result = std::result::Result; /// Outbound storage port. pub trait UsageRepo: Send + Sync + 'static { /// Persist a fully-priced completion row. - // TODO: Consumers should have an easier call to make than this - // A consumer shouldn't have to know the pricing of the model the just need to supply (feature, input tokens, output tokens, model name) fn insert_usage(&self, usage: &CompletionUsage) -> impl Future> + Send; - /// Fetch the current per-million `(input, output)` rates for a model, if any. - fn get_pricing(&self, model: &str) -> impl Future>> + Send; + /// Fetch the current rate for a model's billing unit, if any. + fn get_pricing(&self, model: &str) + -> impl Future>> + Send; /// Upsert the pricing for a model and recompute the `total` of every /// existing `ai_usage` row for that model. fn set_pricing( &self, model: &str, - price_per_million_in: f32, - price_per_million_out: f32, + pricing: ModelPricing, ) -> impl Future> + Send; /// Query recorded completions matching `params`. @@ -287,14 +363,15 @@ pub trait UsageService: Send + Sync + 'static { /// Summarize recorded usage matching `params`. fn get_usage( &self, + actor: MacroUserIdStr<'static>, params: UsageApiParams, ) -> impl Future> + Send; /// Set the pricing for a model and recompute all of its recorded rows. fn set_pricing( &self, + actor: MacroUserIdStr<'static>, model: String, - price_per_million_in: f32, - price_per_million_out: f32, + pricing: ModelPricing, ) -> impl Future> + Send; } diff --git a/crates/ai_usage/src/domain/service.rs b/crates/ai_usage/src/domain/service.rs index 54670edc4a5..e621a3d7007 100644 --- a/crates/ai_usage/src/domain/service.rs +++ b/crates/ai_usage/src/domain/service.rs @@ -6,6 +6,15 @@ use chrono::Utc; use super::ports::*; +/// The existing Macro-admin policy for usage reporting and price changes. +fn require_admin(actor: ¯o_user_id::user_id::MacroUserIdStr<'_>) -> Result<()> { + if actor.email_str().ends_with("@macro.com") { + Ok(()) + } else { + Err(UsageError::Forbidden) + } +} + /// The cost service. Generic over the storage [`UsageRepo`]. /// /// Implements: @@ -31,17 +40,29 @@ where /// can run it on a background task. /// /// [`record`]: UsageRecorder::record + #[tracing::instrument(name = "ai_usage.record", skip_all, err, fields( + feature = %event.feature, + model = %event.model, + user_id = %event.user, + usage = ?event.amount, + usage.cost_usd = tracing::field::Empty, + usage.priced = tracing::field::Empty, + ))] async fn record_event(repo: &Repo, event: UsageEvent) -> Result<()> { let mut cost = Usage { - input_tokens: event.input_tokens.min(u32::MAX as u64) as u32, - output_tokens: event.output_tokens.min(u32::MAX as u64) as u32, + amount: event.amount, model: event.model.clone(), price: None, created_at: Utc::now(), }; - if let Some((per_in, per_out)) = repo.get_pricing(&event.model).await? { - cost.price = Some(Price::compute(per_in, per_out, &cost)); + if let Some(pricing) = repo.get_pricing(&event.model).await? { + cost.price = Price::compute(pricing, cost.amount); + } + let span = tracing::Span::current(); + span.record("usage.priced", cost.price.is_some()); + if let Some(price) = cost.price { + span.record("usage.cost_usd", price.total); } let row = CompletionUsage { @@ -62,11 +83,11 @@ where fn record(&self, event: UsageEvent) { let repo = self.repo.clone(); // Recording must never fail or delay the originating call. - tokio::spawn(async move { + tokio::spawn(tracing::Instrument::in_current_span(async move { if let Err(e) = Self::record_event(&repo, event).await { tracing::error!(error = ?e, "failed to record ai usage"); } - }); + })); } } @@ -75,7 +96,12 @@ where Repo: UsageRepo + Clone + 'static, { #[tracing::instrument(skip(self), err)] - async fn get_usage(&self, params: UsageApiParams) -> Result { + async fn get_usage( + &self, + actor: macro_user_id::user_id::MacroUserIdStr<'static>, + params: UsageApiParams, + ) -> Result { + require_admin(&actor)?; let rows = self.repo.query_usage(¶ms).await?; Ok(summarize(rows)) } @@ -83,13 +109,13 @@ where #[tracing::instrument(skip(self), err)] async fn set_pricing( &self, + actor: macro_user_id::user_id::MacroUserIdStr<'static>, model: String, - price_per_million_in: f32, - price_per_million_out: f32, + pricing: ModelPricing, ) -> Result<()> { - self.repo - .set_pricing(&model, price_per_million_in, price_per_million_out) - .await + require_admin(&actor)?; + let pricing = pricing.validate()?; + self.repo.set_pricing(&model, pricing).await } } diff --git a/crates/ai_usage/src/domain/service/test.rs b/crates/ai_usage/src/domain/service/test.rs index e0404a2b589..b118083fc59 100644 --- a/crates/ai_usage/src/domain/service/test.rs +++ b/crates/ai_usage/src/domain/service/test.rs @@ -11,12 +11,16 @@ fn completion(feature: AiFeature, total: Option) -> CompletionUsage { user: user(), entity: None, cost: Usage { - input_tokens: 1000, - output_tokens: 500, + amount: UsageAmount::Tokens { + input: 1000, + output: 500, + }, model: "claude-opus-4-8".to_string(), price: total.map(|t| Price { - price_per_million_in: 5.0, - price_per_million_out: 25.0, + pricing: ModelPricing::Tokens { + input: 5.0, + output: 25.0, + }, total: t, }), created_at: Utc::now(), @@ -27,13 +31,22 @@ fn completion(feature: AiFeature, total: Option) -> CompletionUsage { #[test] fn price_compute_uses_per_million_rates() { let usage = Usage { - input_tokens: 1_000_000, - output_tokens: 1_000_000, + amount: UsageAmount::Tokens { + input: 1_000_000, + output: 1_000_000, + }, model: "m".to_string(), price: None, created_at: Utc::now(), }; - let price = Price::compute(5.0, 25.0, &usage); + let price = Price::compute( + ModelPricing::Tokens { + input: 5.0, + output: 25.0, + }, + usage.amount, + ) + .unwrap(); assert!((price.total - 30.0).abs() < 1e-3); } @@ -84,3 +97,139 @@ fn ai_feature_roundtrips_through_snake_case() { fn system_user_is_valid() { assert_eq!(SYSTEM_USER_ID.as_ref(), "macro|ai-system@macro.com"); } + +#[derive(Clone, Default)] +struct FakeRepo { + pricing: std::sync::Arc>>, + rows: std::sync::Arc>>, +} + +impl UsageRepo for FakeRepo { + async fn insert_usage(&self, usage: &CompletionUsage) -> Result<()> { + self.rows.lock().unwrap().push(usage.clone()); + Ok(()) + } + + async fn get_pricing(&self, _model: &str) -> Result> { + Ok(*self.pricing.lock().unwrap()) + } + + async fn set_pricing(&self, _model: &str, pricing: ModelPricing) -> Result<()> { + *self.pricing.lock().unwrap() = Some(pricing); + Ok(()) + } + + async fn query_usage(&self, _params: &UsageApiParams) -> Result> { + Ok(self.rows.lock().unwrap().clone()) + } +} + +#[tokio::test] +async fn records_duration_with_the_model_rate_and_user_attribution() { + let repo = FakeRepo::default(); + *repo.pricing.lock().unwrap() = Some(ModelPricing::Audio { per_minute: 0.006 }); + let event = UsageContext::new(AiFeature::Dictation, user()) + .into_audio_event("whisper-1".into(), std::time::Duration::from_millis(90_500)); + UsageServiceImpl::record_event(&repo, event).await.unwrap(); + let rows = repo.rows.lock().unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].user, user()); + assert_eq!(rows[0].feature, AiFeature::Dictation); + assert_eq!(rows[0].cost.model, "whisper-1"); + assert_eq!( + rows[0].cost.amount, + UsageAmount::Audio { + duration: std::time::Duration::from_millis(90_500) + } + ); + let price = rows[0].cost.price.unwrap(); + assert_eq!(price.pricing, ModelPricing::Audio { per_minute: 0.006 }); + assert!((price.total - 0.00905).abs() < 1e-7); +} + +#[tokio::test] +async fn missing_audio_rate_keeps_usage_unpriced() { + for rates in [ + None, + Some(ModelPricing::Tokens { + input: 0.0, + output: 0.0, + }), + ] { + let repo = FakeRepo::default(); + *repo.pricing.lock().unwrap() = rates; + let event = UsageContext::new(AiFeature::Dictation, user()).into_audio_event( + "unpriced-audio-model".into(), + std::time::Duration::from_secs(60), + ); + UsageServiceImpl::record_event(&repo, event).await.unwrap(); + let rows = repo.rows.lock().unwrap(); + assert_eq!( + rows[0].cost.amount, + UsageAmount::Audio { + duration: std::time::Duration::from_secs(60) + } + ); + assert!(rows[0].cost.price.is_none()); + } +} + +#[tokio::test] +async fn admin_policy_protects_usage_and_price_changes() { + let repo = FakeRepo::default(); + let service = UsageServiceImpl::new(repo.clone()); + let other = MacroUserIdStr::try_from("macro|someone@example.com".to_owned()).unwrap(); + assert!(matches!( + service + .get_usage(other.clone(), UsageApiParams::default()) + .await, + Err(UsageError::Forbidden) + )); + assert!(matches!( + service + .set_pricing( + other, + "whisper-1".into(), + ModelPricing::Tokens { + input: 0.0, + output: 0.0 + } + ) + .await, + Err(UsageError::Forbidden) + )); + assert!(repo.pricing.lock().unwrap().is_none()); + service + .set_pricing( + user(), + "whisper-1".into(), + ModelPricing::Tokens { + input: 0.0, + output: 0.0, + }, + ) + .await + .unwrap(); + assert!( + service + .get_usage(user(), UsageApiParams::default()) + .await + .is_ok() + ); +} + +#[tokio::test] +async fn rejects_invalid_prices_before_writing() { + let repo = FakeRepo::default(); + let service = UsageServiceImpl::new(repo.clone()); + for price in [-0.001, f32::NAN, f32::INFINITY] { + let pricing = ModelPricing::Audio { per_minute: price }; + assert!(matches!( + service + .set_pricing(user(), "whisper-1".into(), pricing) + .await, + Err(UsageError::InvalidPricing) + )); + } + assert!(repo.pricing.lock().unwrap().is_none()); +} diff --git a/crates/ai_usage/src/inbound.rs b/crates/ai_usage/src/inbound.rs index 0b0cc8e0f42..68123427f23 100644 --- a/crates/ai_usage/src/inbound.rs +++ b/crates/ai_usage/src/inbound.rs @@ -1,5 +1,6 @@ //! Inbound adapters. pub mod axum_router; +pub mod models; pub use axum_router::{AiUsageRouterState, ai_usage_router}; diff --git a/crates/ai_usage/src/inbound/axum_router.rs b/crates/ai_usage/src/inbound/axum_router.rs index 9f87e3fb73c..e258ece8096 100644 --- a/crates/ai_usage/src/inbound/axum_router.rs +++ b/crates/ai_usage/src/inbound/axum_router.rs @@ -3,7 +3,8 @@ //! Every route is restricted to Macro admins — callers whose user id resolves //! to an `@macro.com` email. -use crate::domain::{AiFeature, UsageApiParams, UsageService, UsageSummary}; +use super::models::UsageSummary; +use crate::domain::{AiFeature, ModelPricing, UsageApiParams, UsageError, UsageService}; use axum::{ Json, Router, extract::{FromRef, State}, @@ -20,9 +21,6 @@ use serde::{Deserialize, Serialize}; use std::sync::Arc; use utoipa::ToSchema; -/// Domain suffix that identifies a Macro admin. -const ADMIN_EMAIL_SUFFIX: &str = "@macro.com"; - /// Request body for [`get_usage_handler`]. #[derive(Debug, Default, Deserialize, ToSchema)] pub struct UsageRequest { @@ -43,10 +41,32 @@ pub struct UsageRequest { pub struct SetPricingRequest { /// The model api id to (re)price. pub model: String, - /// New price per million input tokens (USD). - pub price_per_mil_in: f32, - /// New price per million output tokens (USD). - pub price_per_mil_out: f32, + /// New price per million input tokens (USD). Required for token pricing. + pub price_per_mil_in: Option, + /// New price per million output tokens (USD). Required for token pricing. + pub price_per_mil_out: Option, + /// Price per minute of audio (USD), or null for token-only pricing. + pub price_per_audio_minute: Option, +} + +impl SetPricingRequest { + fn pricing(&self) -> Result { + match ( + self.price_per_mil_in, + self.price_per_mil_out, + self.price_per_audio_minute, + ) { + (input, output, Some(per_minute)) + if input.is_none_or(|price| price == 0.0) + && output.is_none_or(|price| price == 0.0) => + { + Ok(ModelPricing::Audio { per_minute }) + } + (Some(input), Some(output), None) => Ok(ModelPricing::Tokens { input, output }), + (_, _, Some(_)) => Err("choose token pricing or audio pricing"), + _ => Err("provide both token prices or an audio price"), + } + } } /// Error response body. @@ -100,36 +120,22 @@ where .with_state(state) } -/// Returns `Some(403)` unless the caller is a Macro admin. -fn admin_rejection( - user: &MacroAuthorizationExtractor, -) -> Option { - if user - .authorization - .user - .macro_user_id - .email_str() - .ends_with(ADMIN_EMAIL_SUFFIX) - { - None - } else { - Some( - ( - StatusCode::FORBIDDEN, - Json(ErrorBody { - error: "admin access required".to_string(), - }), - ) - .into_response(), - ) - } -} - -fn internal_error(context: &str) -> Response { +fn error_response(error: UsageError, context: &str) -> Response { + let (status, message) = match error { + UsageError::Forbidden => (StatusCode::FORBIDDEN, "admin access required"), + UsageError::InvalidPricing => ( + StatusCode::BAD_REQUEST, + "prices must be finite and non-negative", + ), + error => { + tracing::error!(error = ?error, context, "AI usage request failed"); + (StatusCode::INTERNAL_SERVER_ERROR, context) + } + }; ( - StatusCode::INTERNAL_SERVER_ERROR, + status, Json(ErrorBody { - error: context.to_string(), + error: message.to_string(), }), ) .into_response() @@ -157,10 +163,6 @@ pub async fn get_usage_handler user: MacroAuthorizationExtractor, Json(req): Json, ) -> Response { - if let Some(resp) = admin_rejection(&user) { - return resp; - } - let include_users: std::result::Result>, _> = req .include_users .into_iter() @@ -186,12 +188,12 @@ pub async fn get_usage_handler features: req.features, }; - match service.get_usage(params).await { - Ok(summary) => Json(summary).into_response(), - Err(e) => { - tracing::error!(error = ?e, "failed to query ai usage"); - internal_error("failed to query usage") - } + match service + .get_usage(user.authorization.user.macro_user_id, params) + .await + { + Ok(summary) => Json(UsageSummary::from(summary)).into_response(), + Err(error) => error_response(error, "failed to query usage"), } } @@ -202,6 +204,7 @@ pub async fn get_usage_handler request_body = SetPricingRequest, responses( (status = 200, description = "Pricing updated"), + (status = 400, description = "Invalid pricing", body = ErrorBody), (status = 403, description = "Admin access required", body = ErrorBody), (status = 500, description = "Internal server error", body = ErrorBody), ), @@ -216,18 +219,26 @@ pub async fn set_pricing_handler, Json(req): Json, ) -> Response { - if let Some(resp) = admin_rejection(&user) { - return resp; - } - + let pricing = match req.pricing() { + Ok(pricing) => pricing, + Err(error) => { + return ( + StatusCode::BAD_REQUEST, + Json(ErrorBody { + error: error.into(), + }), + ) + .into_response(); + } + }; match service - .set_pricing(req.model, req.price_per_mil_in, req.price_per_mil_out) + .set_pricing(user.authorization.user.macro_user_id, req.model, pricing) .await { Ok(()) => StatusCode::OK.into_response(), - Err(e) => { - tracing::error!(error = ?e, "failed to set pricing"); - internal_error("failed to set pricing") - } + Err(error) => error_response(error, "failed to set pricing"), } } + +#[cfg(test)] +mod test; diff --git a/crates/ai_usage/src/inbound/axum_router/test.rs b/crates/ai_usage/src/inbound/axum_router/test.rs new file mode 100644 index 00000000000..4612874144d --- /dev/null +++ b/crates/ai_usage/src/inbound/axum_router/test.rs @@ -0,0 +1,67 @@ +use super::*; +use serde_json::{Value, json}; + +fn pricing(body: Value) -> Result { + serde_json::from_value::(body) + .unwrap() + .pricing() +} + +#[test] +fn omitted_token_prices_cannot_silently_reprice_usage_to_zero() { + for body in [ + json!({"model": "text-model"}), + json!({"model": "text-model", "price_per_audio_minute": null}), + json!({"model": "text-model", "price_per_mil_in": 1.0}), + json!({"model": "text-model", "price_per_mil_out": 2.0}), + ] { + assert_eq!( + pricing(body), + Err("provide both token prices or an audio price") + ); + } +} + +#[test] +fn token_prices_remain_explicit_including_free_models() { + for (input, output) in [(1.0, 2.0), (0.0, 0.0)] { + assert_eq!( + pricing(json!({ + "model": "text-model", + "price_per_mil_in": input, + "price_per_mil_out": output, + })), + Ok(ModelPricing::Tokens { input, output }) + ); + } +} + +#[test] +fn audio_prices_allow_omitted_or_zero_token_fields() { + for body in [ + json!({"model": "whisper-1", "price_per_audio_minute": 0.006}), + json!({ + "model": "whisper-1", + "price_per_mil_in": 0.0, + "price_per_mil_out": 0.0, + "price_per_audio_minute": 0.006, + }), + ] { + assert_eq!(pricing(body), Ok(ModelPricing::Audio { per_minute: 0.006 })); + } +} + +#[test] +fn mixed_billing_units_are_rejected() { + for (input, output) in [(1.0, 0.0), (0.0, 2.0)] { + assert_eq!( + pricing(json!({ + "model": "whisper-1", + "price_per_mil_in": input, + "price_per_mil_out": output, + "price_per_audio_minute": 0.006, + })), + Err("choose token pricing or audio pricing") + ); + } +} diff --git a/crates/ai_usage/src/inbound/models.rs b/crates/ai_usage/src/inbound/models.rs new file mode 100644 index 00000000000..d2ab36dbe95 --- /dev/null +++ b/crates/ai_usage/src/inbound/models.rs @@ -0,0 +1,133 @@ +//! HTTP representations of usage. Keep existing token fields compatible while +//! the domain represents billing units with enums. + +use crate::domain::{self, AiFeature, ModelPricing, UsageAmount}; +use chrono::{DateTime, Utc}; +use macro_user_id::user_id::MacroUserIdStr; +use macro_uuid::Uuid; +use serde::Serialize; +use utoipa::ToSchema; + +/// Rates applied to an invocation and its resolved dollar cost. +#[derive(Serialize, ToSchema)] +pub struct Price { + /// Price per million input tokens (USD); zero for audio billing. + pub price_per_million_in: f32, + /// Price per million output tokens (USD); zero for audio billing. + pub price_per_million_out: f32, + /// Price per audio minute (USD), absent for token billing. + pub price_per_audio_minute: Option, + /// Total cost (USD). + pub total: f32, +} + +impl From for Price { + fn from(price: domain::Price) -> Self { + let (input, output, audio) = match price.pricing { + ModelPricing::Tokens { input, output } => (input, output, None), + ModelPricing::Audio { per_minute } => (0.0, 0.0, Some(per_minute)), + }; + Self { + price_per_million_in: input, + price_per_million_out: output, + price_per_audio_minute: audio, + total: price.total, + } + } +} + +/// Measured usage for one invocation. +#[derive(Serialize, ToSchema)] +pub struct Usage { + /// Input tokens; zero for audio billing. + pub input_tokens: u64, + /// Output tokens; zero for audio billing. + pub output_tokens: u64, + /// Audio duration in seconds, absent for token billing. + pub audio_seconds: Option, + /// Provider model identifier. + pub model: String, + /// Applied pricing, if known. + pub price: Option, + /// Recording timestamp. + pub created_at: DateTime, +} + +impl From for Usage { + fn from(usage: domain::Usage) -> Self { + let (input_tokens, output_tokens, audio_seconds) = match usage.amount { + UsageAmount::Tokens { input, output } => (input, output, None), + UsageAmount::Audio { duration } => (0, 0, Some(duration.as_secs_f64())), + }; + Self { + input_tokens, + output_tokens, + audio_seconds, + model: usage.model, + price: usage.price.map(Into::into), + created_at: usage.created_at, + } + } +} + +/// User and feature attribution for one invocation. +#[derive(Serialize, ToSchema)] +pub struct CompletionUsage { + /// Feature that performed the invocation. + pub feature: AiFeature, + /// User the invocation was performed for. + pub user: MacroUserIdStr<'static>, + /// Related entity, if any. + pub entity: Option, + /// Measured usage and resolved cost. + pub cost: Usage, +} + +/// Recorded invocations and total for one feature. +#[derive(Serialize, ToSchema)] +pub struct FeatureUsage { + /// Feature attribution. + pub feature: AiFeature, + /// Recorded invocations. + pub entries: Vec, + /// Total cost (USD). + pub total: f32, +} + +/// Per-feature breakdown and grand total. +#[derive(Serialize, ToSchema)] +pub struct UsageSummary { + /// Per-feature usage. + pub entries: Vec, + /// Grand total (USD). + pub total: f32, +} + +impl From for UsageSummary { + fn from(summary: domain::UsageSummary) -> Self { + Self { + entries: summary + .entries + .into_iter() + .map(|feature| FeatureUsage { + feature: feature.feature, + entries: feature + .entries + .into_iter() + .map(|entry| CompletionUsage { + feature: entry.feature, + user: entry.user, + entity: entry.entity, + cost: entry.cost.into(), + }) + .collect(), + total: feature.total, + }) + .collect(), + total: summary.total, + } + } +} + +#[cfg(test)] +mod test; diff --git a/crates/ai_usage/src/inbound/models/test.rs b/crates/ai_usage/src/inbound/models/test.rs new file mode 100644 index 00000000000..7b96c2bc765 --- /dev/null +++ b/crates/ai_usage/src/inbound/models/test.rs @@ -0,0 +1,59 @@ +use super::*; +use serde_json::json; +use std::time::Duration; + +#[test] +fn token_usage_keeps_the_existing_http_shape() { + let created_at = "2026-09-18T00:00:00Z".parse().unwrap(); + let amount = UsageAmount::Tokens { + input: 1_000_000, + output: 2_000_000, + }; + let usage = Usage::from(domain::Usage { + amount, + model: "text-model".into(), + price: domain::Price::compute( + ModelPricing::Tokens { + input: 1.0, + output: 2.0, + }, + amount, + ), + created_at, + }); + assert_eq!( + serde_json::to_value(usage).unwrap(), + json!({ + "input_tokens": 1_000_000, + "output_tokens": 2_000_000, + "audio_seconds": null, + "model": "text-model", + "created_at": "2026-09-18T00:00:00Z", + "price": { + "price_per_million_in": 1.0, + "price_per_million_out": 2.0, + "price_per_audio_minute": null, + "total": 5.0 + } + }) + ); +} + +#[test] +fn audio_usage_exposes_duration_and_rate_without_inventing_tokens() { + let amount = UsageAmount::Audio { + duration: Duration::from_millis(90_500), + }; + let usage = Usage::from(domain::Usage { + amount, + model: "whisper-1".into(), + price: domain::Price::compute(ModelPricing::Audio { per_minute: 0.006 }, amount), + created_at: Utc::now(), + }); + let json = serde_json::to_value(usage).unwrap(); + assert_eq!(json["input_tokens"], 0); + assert_eq!(json["output_tokens"], 0); + assert_eq!(json["audio_seconds"], 90.5); + assert!((json["price"]["price_per_audio_minute"].as_f64().unwrap() - 0.006).abs() < 1e-7); + assert!((json["price"]["total"].as_f64().unwrap() - 0.00905).abs() < 1e-7); +} diff --git a/crates/ai_usage/src/lib.rs b/crates/ai_usage/src/lib.rs index 9f7c54dd58f..fe5fba4d0d0 100644 --- a/crates/ai_usage/src/lib.rs +++ b/crates/ai_usage/src/lib.rs @@ -4,7 +4,7 @@ //! //! The crate follows the hexagonal layout used elsewhere in this workspace: //! - [`domain`] holds the cost model, ports, and the service that computes -//! cost from a model id and token counts. +//! cost from a model id and token counts or audio duration. //! - [`outbound`] holds the Postgres storage adapter. //! - [`inbound`] holds the axum router mounted in the document cognition //! service (DCS). @@ -17,8 +17,9 @@ pub mod inbound; pub mod outbound; pub use domain::{ - AiFeature, CompletionUsage, FeatureUsage, NoOpUsageRecorder, Price, SYSTEM_USER_ID, Usage, - UsageApiParams, UsageContext, UsageEvent, UsageRecorder, UsageRepo, UsageService, UsageSummary, + AiFeature, CompletionUsage, FeatureUsage, ModelPricing, NoOpUsageRecorder, Price, + SYSTEM_USER_ID, Usage, UsageAmount, UsageApiParams, UsageContext, UsageEvent, UsageRecorder, + UsageRepo, UsageService, UsageSummary, }; use std::sync::Arc; diff --git a/crates/ai_usage/src/outbound/pg_usage_repo.rs b/crates/ai_usage/src/outbound/pg_usage_repo.rs index c7e971322f0..7ec42f1cfbe 100644 --- a/crates/ai_usage/src/outbound/pg_usage_repo.rs +++ b/crates/ai_usage/src/outbound/pg_usage_repo.rs @@ -4,7 +4,8 @@ mod test; use crate::domain::{ - AiFeature, CompletionUsage, Price, Result, Usage, UsageApiParams, UsageError, UsageRepo, + AiFeature, CompletionUsage, ModelPricing, Price, Result, Usage, UsageAmount, UsageApiParams, + UsageError, UsageRepo, }; use macro_user_id::user_id::MacroUserIdStr; use sqlx::PgPool; @@ -25,33 +26,46 @@ impl PgUsageRepo { impl UsageRepo for PgUsageRepo { async fn insert_usage(&self, usage: &CompletionUsage) -> Result<()> { let id = macro_uuid::generate_uuid_v7(); - let (per_in, per_out, total) = match &usage.cost.price { - Some(p) => ( - Some(p.price_per_million_in), - Some(p.price_per_million_out), - Some(p.total), + let (input_tokens, output_tokens, audio_seconds) = match usage.cost.amount { + UsageAmount::Tokens { input, output } => ( + i64::try_from(input).unwrap_or(i64::MAX), + i64::try_from(output).unwrap_or(i64::MAX), + None, ), - None => (None, None, None), + UsageAmount::Audio { duration } => (0, 0, Some(duration.as_secs_f64())), + }; + let (per_in, per_out, per_audio_minute, total) = match usage.cost.price { + Some(Price { + pricing: ModelPricing::Tokens { input, output }, + total, + }) => (Some(input), Some(output), None, Some(total)), + Some(Price { + pricing: ModelPricing::Audio { per_minute }, + total, + }) => (Some(0.0), Some(0.0), Some(per_minute), Some(total)), + None => (None, None, None, None), }; sqlx::query!( r#" INSERT INTO ai_usage ( id, feature, user_id, entity, model, - input_tokens, output_tokens, - price_per_million_in, price_per_million_out, total + input_tokens, output_tokens, audio_seconds, + price_per_million_in, price_per_million_out, price_per_audio_minute, total ) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) "#, id, usage.feature.to_string(), usage.user.as_ref(), usage.entity, usage.cost.model, - i64::from(usage.cost.input_tokens), - i64::from(usage.cost.output_tokens), + input_tokens, + output_tokens, + audio_seconds, per_in, per_out, + per_audio_minute, total, ) .execute(&self.inner) @@ -60,10 +74,10 @@ impl UsageRepo for PgUsageRepo { Ok(()) } - async fn get_pricing(&self, model: &str) -> Result> { + async fn get_pricing(&self, model: &str) -> Result> { let row = sqlx::query!( r#" - SELECT price_per_million_in, price_per_million_out + SELECT price_per_million_in, price_per_million_out, price_per_audio_minute FROM ai_pricing WHERE model = $1 "#, @@ -72,29 +86,36 @@ impl UsageRepo for PgUsageRepo { .fetch_optional(&self.inner) .await?; - Ok(row.map(|r| (r.price_per_million_in, r.price_per_million_out))) + Ok(row.map(|r| match r.price_per_audio_minute { + Some(per_minute) => ModelPricing::Audio { per_minute }, + None => ModelPricing::Tokens { + input: r.price_per_million_in, + output: r.price_per_million_out, + }, + })) } - async fn set_pricing( - &self, - model: &str, - price_per_million_in: f32, - price_per_million_out: f32, - ) -> Result<()> { + async fn set_pricing(&self, model: &str, pricing: ModelPricing) -> Result<()> { + let (per_in, per_out, per_audio_minute) = match pricing { + ModelPricing::Tokens { input, output } => (input, output, None), + ModelPricing::Audio { per_minute } => (0.0, 0.0, Some(per_minute)), + }; let mut tx = self.inner.begin().await?; sqlx::query!( r#" - INSERT INTO ai_pricing (model, price_per_million_in, price_per_million_out) - VALUES ($1, $2, $3) + INSERT INTO ai_pricing (model, price_per_million_in, price_per_million_out, price_per_audio_minute) + VALUES ($1, $2, $3, $4) ON CONFLICT (model) DO UPDATE SET price_per_million_in = EXCLUDED.price_per_million_in, price_per_million_out = EXCLUDED.price_per_million_out, + price_per_audio_minute = EXCLUDED.price_per_audio_minute, updated_at = NOW() "#, model, - price_per_million_in, - price_per_million_out, + per_in, + per_out, + per_audio_minute, ) .execute(&mut *tx) .await?; @@ -105,13 +126,17 @@ impl UsageRepo for PgUsageRepo { UPDATE ai_usage SET price_per_million_in = $2::real, price_per_million_out = $3::real, - total = (input_tokens::real / 1000000.0::real) * $2::real + price_per_audio_minute = $4::real, + total = CASE WHEN (audio_seconds IS NULL) <> ($4::real IS NULL) THEN NULL + ELSE (input_tokens::real / 1000000.0::real) * $2::real + (output_tokens::real / 1000000.0::real) * $3::real + + (COALESCE(audio_seconds, 0) / 60.0 * COALESCE($4::real, 0))::real END WHERE model = $1 "#, model, - price_per_million_in, - price_per_million_out, + per_in, + per_out, + per_audio_minute, ) .execute(&mut *tx) .await?; @@ -137,8 +162,10 @@ impl UsageRepo for PgUsageRepo { model, input_tokens, output_tokens, + audio_seconds, price_per_million_in, price_per_million_out, + price_per_audio_minute, total, created_at FROM ai_usage @@ -167,8 +194,13 @@ impl UsageRepo for PgUsageRepo { let price = match (r.price_per_million_in, r.price_per_million_out, r.total) { (Some(per_in), Some(per_out), Some(total)) => Some(Price { - price_per_million_in: per_in, - price_per_million_out: per_out, + pricing: match r.price_per_audio_minute { + Some(per_minute) => ModelPricing::Audio { per_minute }, + None => ModelPricing::Tokens { + input: per_in, + output: per_out, + }, + }, total, }), _ => None, @@ -179,8 +211,16 @@ impl UsageRepo for PgUsageRepo { user, entity: r.entity, cost: Usage { - input_tokens: r.input_tokens.clamp(0, i64::from(u32::MAX)) as u32, - output_tokens: r.output_tokens.clamp(0, i64::from(u32::MAX)) as u32, + amount: match r.audio_seconds { + Some(seconds) => UsageAmount::Audio { + duration: std::time::Duration::try_from_secs_f64(seconds) + .map_err(|error| UsageError::Other(error.into()))?, + }, + None => UsageAmount::Tokens { + input: r.input_tokens.max(0) as u64, + output: r.output_tokens.max(0) as u64, + }, + }, model: r.model, price, created_at: r.created_at, diff --git a/crates/ai_usage/src/outbound/pg_usage_repo/test.rs b/crates/ai_usage/src/outbound/pg_usage_repo/test.rs index 27abe68a3dc..f2ce7e762f0 100644 --- a/crates/ai_usage/src/outbound/pg_usage_repo/test.rs +++ b/crates/ai_usage/src/outbound/pg_usage_repo/test.rs @@ -4,14 +4,13 @@ use chrono::Utc; use macro_db_migrator::MACRO_DB_MIGRATIONS; use sqlx::PgPool; -fn completion(feature: AiFeature, model: &str, input: u32, output: u32) -> CompletionUsage { +fn completion(feature: AiFeature, model: &str, input: u64, output: u64) -> CompletionUsage { CompletionUsage { feature, user: SYSTEM_USER_ID.clone(), entity: None, cost: Usage { - input_tokens: input, - output_tokens: output, + amount: UsageAmount::Tokens { input, output }, model: model.to_string(), price: None, created_at: Utc::now(), @@ -23,9 +22,25 @@ fn completion(feature: AiFeature, model: &str, input: u32, output: u32) -> Compl async fn seeded_pricing_is_available(pool: PgPool) { let repo = PgUsageRepo::new(pool); let price = repo.get_pricing("claude-opus-4-8").await.unwrap(); - assert_eq!(price, Some((5.0, 25.0))); + assert_eq!( + price, + Some(ModelPricing::Tokens { + input: 5.0, + output: 25.0 + }) + ); let price = repo.get_pricing("claude-opus-5").await.unwrap(); - assert_eq!(price, Some((5.0, 25.0))); + assert_eq!( + price, + Some(ModelPricing::Tokens { + input: 5.0, + output: 25.0 + }) + ); + assert_eq!( + repo.get_pricing("whisper-1").await.unwrap(), + Some(ModelPricing::Audio { per_minute: 0.006 }) + ); assert_eq!(repo.get_pricing("nonexistent-model").await.unwrap(), None); } @@ -34,7 +49,10 @@ async fn insert_and_query_roundtrips(pool: PgPool) { let repo = PgUsageRepo::new(pool); let mut row = completion(AiFeature::Chat, "claude-opus-4-8", 1_000_000, 1_000_000); - row.cost.price = Some(Price::compute(5.0, 25.0, &row.cost)); + row.cost.price = Price::compute( + repo.get_pricing("claude-opus-4-8").await.unwrap().unwrap(), + row.cost.amount, + ); repo.insert_usage(&row).await.unwrap(); repo.insert_usage(&completion(AiFeature::Memory, "unknown-model", 10, 20)) .await @@ -73,14 +91,25 @@ async fn set_pricing_recomputes_existing_rows(pool: PgPool) { let before = repo.query_usage(&UsageApiParams::default()).await.unwrap(); assert!(before[0].cost.price.is_none()); - repo.set_pricing("brand-new-model", 2.0, 8.0).await.unwrap(); + repo.set_pricing( + "brand-new-model", + ModelPricing::Tokens { + input: 2.0, + output: 8.0, + }, + ) + .await + .unwrap(); let after = repo.query_usage(&UsageApiParams::default()).await.unwrap(); let price = after[0].cost.price.as_ref().unwrap(); assert!((price.total - 2.0).abs() < 1e-3); // 1M input * $2/1M = $2 assert_eq!( repo.get_pricing("brand-new-model").await.unwrap(), - Some((2.0, 8.0)) + Some(ModelPricing::Tokens { + input: 2.0, + output: 8.0 + }) ); } @@ -107,3 +136,102 @@ async fn query_filters_by_user(pool: PgPool) { assert_eq!(only_other.len(), 1); assert_eq!(only_other[0].user.as_ref(), "macro|someone@example.com"); } + +#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")] +async fn duration_usage_roundtrips_and_reprices_without_affecting_tokens(pool: PgPool) { + let repo = PgUsageRepo::new(pool); + let mut audio = completion(AiFeature::Dictation, "whisper-1", 0, 0); + let amount = UsageAmount::Audio { + duration: std::time::Duration::from_millis(90_500), + }; + audio.cost.amount = amount; + audio.cost.price = Price::compute( + repo.get_pricing("whisper-1").await.unwrap().unwrap(), + audio.cost.amount, + ); + repo.insert_usage(&audio).await.unwrap(); + repo.insert_usage(&completion( + AiFeature::Chat, + "claude-opus-4-8", + 1_000_000, + 0, + )) + .await + .unwrap(); + + let query = UsageApiParams { + features: vec![AiFeature::Dictation], + ..Default::default() + }; + let rows = repo.query_usage(&query).await.unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].cost.amount, amount); + assert!((rows[0].cost.price.unwrap().total - 0.00905).abs() < 1e-7); + + for rate in [None, Some(0.012)] { + repo.set_pricing( + "whisper-1", + match rate { + Some(per_minute) => ModelPricing::Audio { per_minute }, + None => ModelPricing::Tokens { + input: 0.0, + output: 0.0, + }, + }, + ) + .await + .unwrap(); + let rows = repo.query_usage(&query).await.unwrap(); + assert_eq!(rows[0].cost.amount, amount); + match rate { + None => assert!(rows[0].cost.price.is_none()), + Some(rate) => { + let price = rows[0].cost.price.unwrap(); + assert_eq!(price.pricing, ModelPricing::Audio { per_minute: rate }); + assert!((price.total - 0.0181).abs() < 1e-7); + } + } + } + let chat = repo + .query_usage(&UsageApiParams { + features: vec![AiFeature::Chat], + ..Default::default() + }) + .await + .unwrap(); + assert_eq!( + chat[0].cost.amount, + UsageAmount::Tokens { + input: 1_000_000, + output: 0 + } + ); + assert!( + chat[0].cost.price.is_none(), + "repricing another model must not touch chat rows" + ); +} + +#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")] +async fn repricing_cannot_apply_audio_rates_to_token_usage(pool: PgPool) { + let repo = PgUsageRepo::new(pool); + repo.insert_usage(&completion(AiFeature::Chat, "mixed-history", 1_000_000, 0)) + .await + .unwrap(); + repo.set_pricing("mixed-history", ModelPricing::Audio { per_minute: 0.006 }) + .await + .unwrap(); + let rows = repo.query_usage(&UsageApiParams::default()).await.unwrap(); + assert!(rows[0].cost.price.is_none()); + repo.set_pricing( + "mixed-history", + ModelPricing::Tokens { + input: 2.0, + output: 8.0, + }, + ) + .await + .unwrap(); + let rows = repo.query_usage(&UsageApiParams::default()).await.unwrap(); + assert_eq!(rows[0].cost.price.unwrap().total, 2.0); +} diff --git a/crates/dictation/Cargo.toml b/crates/dictation/Cargo.toml new file mode 100644 index 00000000000..345f485f62a --- /dev/null +++ b/crates/dictation/Cargo.toml @@ -0,0 +1,69 @@ +[package] +edition = "2024" +name = "dictation" +publish = false +version = "0.1.0" + +[features] +default = [] +inbound = [ + "dep:axum", + "dep:axum-extra", + "dep:macro_authorization", + "dep:model-error-response", + "dep:rate_limit", + "dep:rootcause", + "dep:serde", + "dep:utoipa", + "rate_limit/axum", +] +outbound = [ + "dep:async-openai", + "dep:backoff", + "dep:macro_env_var", + "dep:opus-pure", + "dep:reqwest", + "dep:serde", + "dep:symphonia", +] + +[dependencies] +ai_usage = { path = "../ai_usage" } +async-openai = { workspace = true, optional = true, features = ["audio"] } +axum = { workspace = true, optional = true } +axum-extra = { workspace = true, optional = true } +backoff = { workspace = true, optional = true } +bytes = { workspace = true } +infer = { workspace = true } +macro_authorization = { path = "../macro_authorization", optional = true, default-features = false, features = [ + "axum", +] } +macro_env_var = { path = "../macro_env_var", optional = true } +macro_user_id = { path = "../macro_user_id" } +model-error-response = { path = "../model-error-response", optional = true } +opus-pure = { workspace = true, optional = true } +rate_limit = { path = "../rate_limit", optional = true, default-features = false } +reqwest = { workspace = true, optional = true } +rootcause = { workspace = true, optional = true } +serde = { workspace = true, optional = true } +symphonia = { workspace = true, optional = true } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["sync"] } +tracing = { workspace = true } +utoipa = { workspace = true, optional = true } +workspace-hack = { version = "0.1", path = "../workspace-hack" } + +[dev-dependencies] +http-body-util = { workspace = true } +model_user = { path = "../model_user" } +rate_limit = { path = "../rate_limit", default-features = false, features = [ + "axum", + "testing", +] } +rootcause = { workspace = true } +serde_json = { workspace = true } +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } +tower = { workspace = true } +tracing-subscriber = { workspace = true } +uuid = { workspace = true } +wiremock = { workspace = true } diff --git a/crates/dictation/src/domain.rs b/crates/dictation/src/domain.rs new file mode 100644 index 00000000000..3ca5f0cc5a0 --- /dev/null +++ b/crates/dictation/src/domain.rs @@ -0,0 +1,13 @@ +//! Dictation domain: recording value objects, the provider port, and the +//! transcription use case. No transport or infrastructure lives here. + +pub mod models; +pub mod ports; +pub mod service; + +pub use models::{ + AudioFormat, DictationError, LanguageHint, MAX_AUDIO_BYTES, MAX_AUDIO_DURATION, Recording, + Transcript, +}; +pub use ports::{DictationService, RecordingInspector, TranscriptionProvider}; +pub use service::DictationServiceImpl; diff --git a/crates/dictation/src/domain/models.rs b/crates/dictation/src/domain/models.rs new file mode 100644 index 00000000000..f791bb713c0 --- /dev/null +++ b/crates/dictation/src/domain/models.rs @@ -0,0 +1,164 @@ +//! Value objects for a transient dictation recording. + +use bytes::Bytes; +use std::time::Duration; +use std::{fmt, str::FromStr}; + +/// Maximum encoded recording accepted by the service. +pub const MAX_AUDIO_BYTES: usize = 8 * 1024 * 1024; + +/// Maximum audio duration accepted before invoking the paid provider. +pub const MAX_AUDIO_DURATION: Duration = Duration::from_secs(5 * 60); + +/// Browser recording containers accepted for transcription. +/// +/// The container is detected from the encoded bytes with [`infer`] rather than +/// trusted from a caller-supplied content type, so the provider always receives +/// a filename whose extension matches the audio it is given. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AudioFormat { + /// WebM/Opus, produced by Chromium-based browsers. + Webm, + /// MP4/AAC (including M4A), produced by Safari. + Mp4, + /// Ogg/Opus, produced by Firefox. + Ogg, + /// PCM WAV. + Wav, +} + +impl AudioFormat { + /// Detect the container from the leading bytes of a recording. + pub fn detect(bytes: &[u8]) -> Result { + let kind = infer::get(bytes).ok_or(DictationError::UnsupportedAudio)?; + match kind.mime_type() { + "video/webm" => Ok(Self::Webm), + "video/mp4" | "audio/m4a" => Ok(Self::Mp4), + // Providers accept `.ogg` but not `.opus`, so Ogg Opus maps to Ogg. + "audio/ogg" | "audio/opus" => Ok(Self::Ogg), + "audio/x-wav" => Ok(Self::Wav), + _ => Err(DictationError::UnsupportedAudio), + } + } + + /// File extension the provider uses to pick a decoder. + pub fn extension(self) -> &'static str { + match self { + Self::Webm => "webm", + Self::Mp4 => "mp4", + Self::Ogg => "ogg", + Self::Wav => "wav", + } + } + + /// Fixed upload filename, independent of user input. + pub fn filename(self) -> String { + format!("dictation.{}", self.extension()) + } +} + +/// An ISO 639-1 language hint: exactly two lowercase ASCII letters. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LanguageHint(String); + +impl LanguageHint { + /// The validated two-letter code. + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl FromStr for LanguageHint { + type Err = DictationError; + + fn from_str(value: &str) -> Result { + let valid = value.len() == 2 && value.bytes().all(|byte| byte.is_ascii_lowercase()); + valid + .then(|| Self(value.to_owned())) + .ok_or(DictationError::InvalidLanguage) + } +} + +impl fmt::Display for LanguageHint { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +/// A validated, in-memory recording. Never persisted as a document or attachment. +#[derive(Clone, Debug)] +pub struct Recording { + bytes: Bytes, + format: AudioFormat, + language: Option, +} + +impl Recording { + /// Validate the size and detect the container of encoded audio. + pub fn new(bytes: Bytes, language: Option) -> Result { + if bytes.is_empty() || bytes.len() > MAX_AUDIO_BYTES { + return Err(DictationError::InvalidSize); + } + let format = AudioFormat::detect(&bytes)?; + Ok(Self { + bytes, + format, + language, + }) + } + + /// Encoded audio. + pub fn bytes(&self) -> &Bytes { + &self.bytes + } + + /// Detected container. + pub fn format(&self) -> AudioFormat { + self.format + } + + /// Optional recognition hint. + pub fn language(&self) -> Option<&LanguageHint> { + self.language.as_ref() + } + + /// Decompose into the provider request parts. + pub fn into_parts(self) -> (Bytes, AudioFormat, Option) { + (self.bytes, self.format, self.language) + } +} + +/// Provider result, including billable duration for operational metering. +#[derive(Clone, Debug, PartialEq)] +pub struct Transcript { + /// Recognized text in the original language. + pub text: String, + /// Audio duration reported by the provider, in seconds. + pub duration_seconds: f32, +} + +/// Errors exposed by the dictation use case. +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum DictationError { + /// Empty or oversized audio. + #[error("Recording must contain between 1 byte and 8 MB of audio")] + InvalidSize, + /// The bytes are not a supported audio container. + #[error("Unsupported audio format")] + UnsupportedAudio, + /// A recognized container contains no usable audio packets. + #[error("Recording is incomplete or contains no audio")] + InvalidAudio, + /// The encoded audio timeline exceeds the dictation duration limit. + #[error("Recording must be no longer than five minutes")] + TooLong, + /// Malformed language hint. + #[error("Language must be a two-letter ISO 639-1 code")] + InvalidLanguage, + /// Provider concurrency guard, independent of per-user rate limits. + #[error("Dictation is busy. Please try again")] + Busy, + /// The provider failed; the adapter logs status only, never audio or text. + #[error("Transcription failed. Please try again")] + Provider, +} diff --git a/crates/dictation/src/domain/ports.rs b/crates/dictation/src/domain/ports.rs new file mode 100644 index 00000000000..e9b3e831ec5 --- /dev/null +++ b/crates/dictation/src/domain/ports.rs @@ -0,0 +1,42 @@ +//! Ports: the use case exposed to inbound adapters and the capability it needs. + +use super::models::{DictationError, LanguageHint, Recording, Transcript}; +use bytes::Bytes; +use macro_user_id::user_id::MacroUserIdStr; +use std::{future::Future, time::Duration}; + +/// Media inspection without coupling the use case to a container parser. +pub trait RecordingInspector: Send + Sync + 'static { + /// Read the encoded audio timeline; malformed or empty audio is rejected. + fn duration( + &self, + recording: Recording, + ) -> impl Future> + Send; +} + +/// External speech-to-text capability, implemented by outbound adapters. +pub trait TranscriptionProvider: Send + Sync + 'static { + /// Model identifier used by the shared pricing and usage recorder. + fn model_id(&self) -> &'static str; + + /// Transcribe a validated in-memory recording. + fn transcribe( + &self, + recording: Recording, + ) -> impl Future> + Send; +} + +/// The dictation use case, called by inbound adapters on behalf of an +/// authenticated user. +pub trait DictationService: Send + Sync + 'static { + /// Validate encoded audio and transcribe it for `user`. + /// + /// Authentication happens at the edge; the user is passed in only for + /// usage metering. Dictation never consumes chat credits or checks plans. + fn transcribe( + &self, + user: MacroUserIdStr<'static>, + audio: Bytes, + language: Option, + ) -> impl Future> + Send; +} diff --git a/crates/dictation/src/domain/service.rs b/crates/dictation/src/domain/service.rs new file mode 100644 index 00000000000..94270c5ab61 --- /dev/null +++ b/crates/dictation/src/domain/service.rs @@ -0,0 +1,99 @@ +//! The transcription use case. + +use super::{ + models::{DictationError, LanguageHint, MAX_AUDIO_DURATION, Recording, Transcript}, + ports::{DictationService, RecordingInspector, TranscriptionProvider}, +}; +use ai_usage::{AiFeature, UsageContext, UsageRecorder}; +use bytes::Bytes; +use macro_user_id::user_id::MacroUserIdStr; +use std::{sync::Arc, time::Duration}; +use tokio::sync::Semaphore; + +/// Upper bound on in-flight inspections/provider requests per service process. +const MAX_CONCURRENT_TRANSCRIPTIONS: usize = 16; +/// Dictation for authenticated users on every plan. +pub struct DictationServiceImpl { + provider: P, + inspector: I, + recorder: Arc, + capacity: Semaphore, +} + +impl DictationServiceImpl { + /// Build a service with a bounded number of concurrent provider requests. + pub fn new(provider: P, inspector: I, recorder: Arc) -> Self { + Self { + provider, + inspector, + recorder, + capacity: Semaphore::new(MAX_CONCURRENT_TRANSCRIPTIONS), + } + } + + #[cfg(test)] + pub(super) fn capacity(&self) -> &Semaphore { + &self.capacity + } +} + +impl DictationService + for DictationServiceImpl +{ + #[tracing::instrument(name = "dictation.transcribe", skip_all, err, fields( + user_id = %user, + audio.bytes = audio.len(), + audio.format = tracing::field::Empty, + audio.duration_seconds = tracing::field::Empty, + dictation.stage = "validation", + dictation.in_flight = tracing::field::Empty, + ))] + async fn transcribe( + &self, + user: MacroUserIdStr<'static>, + audio: Bytes, + language: Option, + ) -> Result { + let span = tracing::Span::current(); + let recording = Recording::new(audio, language)?; + span.record("audio.format", recording.format().extension()); + span.record("dictation.stage", "admission"); + let _permit = self.capacity.try_acquire().map_err(|_| { + tracing::warn!( + limit = MAX_CONCURRENT_TRANSCRIPTIONS, + "dictation capacity exhausted" + ); + DictationError::Busy + })?; + span.record( + "dictation.in_flight", + MAX_CONCURRENT_TRANSCRIPTIONS - self.capacity.available_permits(), + ); + span.record("dictation.stage", "inspection"); + let duration = self.inspector.duration(recording.clone()).await?; + span.record("audio.duration_seconds", duration.as_secs_f64()); + if duration.is_zero() { + return Err(DictationError::InvalidAudio); + } + if duration > MAX_AUDIO_DURATION { + return Err(DictationError::TooLong); + } + span.record("dictation.stage", "provider"); + let transcript = self.provider.transcribe(recording).await?; + span.record("dictation.stage", "usage"); + let duration = Duration::try_from_secs_f32(transcript.duration_seconds) + .map_err(|_| DictationError::Provider)?; + self.recorder.record( + UsageContext::new(AiFeature::Dictation, user) + .into_audio_event(self.provider.model_id().to_owned(), duration), + ); + span.record("dictation.stage", "complete"); + Ok(Transcript { + text: transcript.text.trim().to_owned(), + duration_seconds: transcript.duration_seconds, + }) + } +} + +#[cfg(test)] +mod test; diff --git a/crates/dictation/src/domain/service/test.rs b/crates/dictation/src/domain/service/test.rs new file mode 100644 index 00000000000..9c27a3288b0 --- /dev/null +++ b/crates/dictation/src/domain/service/test.rs @@ -0,0 +1,270 @@ +use super::*; +use crate::domain::models::{AudioFormat, MAX_AUDIO_BYTES}; +use std::sync::{ + Arc, Mutex, + atomic::{AtomicUsize, Ordering}, +}; +use std::time::Duration; + +/// Minimal Ogg page header, as Firefox's MediaRecorder emits. +const OGG_HEADER: &[u8] = b"OggS\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00"; + +#[derive(Default)] +struct FakeProvider { + fails: bool, + calls: AtomicUsize, + seen: Mutex)>>, +} + +impl TranscriptionProvider for Arc { + fn model_id(&self) -> &'static str { + "whisper-1" + } + + async fn transcribe(&self, recording: Recording) -> Result { + self.calls.fetch_add(1, Ordering::SeqCst); + if self.fails { + return Err(DictationError::Provider); + } + let (_, format, language) = recording.into_parts(); + self.seen + .lock() + .unwrap() + .push((format, language.map(|hint| hint.to_string()))); + Ok(Transcript { + text: " hello world ".into(), + duration_seconds: 2.0, + }) + } +} + +struct FakeInspector(Result); + +impl RecordingInspector for FakeInspector { + async fn duration(&self, _recording: Recording) -> Result { + match &self.0 { + Ok(duration) => Ok(*duration), + Err(_) => Err(DictationError::InvalidAudio), + } + } +} + +fn service() -> ( + DictationServiceImpl, FakeInspector>, + Arc, +) { + let provider = Arc::new(FakeProvider::default()); + ( + DictationServiceImpl::new( + provider.clone(), + FakeInspector(Ok(Duration::from_secs(2))), + Arc::new(ai_usage::NoOpUsageRecorder), + ), + provider, + ) +} + +fn user() -> MacroUserIdStr<'static> { + MacroUserIdStr::try_from("macro|dictation-test@example.com".to_owned()).unwrap() +} + +fn language() -> Option { + Some("en".parse().unwrap()) +} + +#[tokio::test] +async fn transcribes_detected_audio_and_trims_text() { + let (service, provider) = service(); + let transcript = service + .transcribe(user(), Bytes::from_static(OGG_HEADER), language()) + .await + .unwrap(); + assert_eq!(transcript.text, "hello world"); + assert_eq!(transcript.duration_seconds, 2.0); + assert_eq!(provider.calls.load(Ordering::SeqCst), 1); + assert_eq!( + provider.seen.lock().unwrap().as_slice(), + [(AudioFormat::Ogg, Some("en".to_owned()))] + ); +} + +#[tokio::test] +async fn rejects_empty_or_oversized_audio_before_calling_provider() { + let (service, provider) = service(); + for bytes in [Bytes::new(), Bytes::from(vec![0; MAX_AUDIO_BYTES + 1])] { + assert_eq!( + service.transcribe(user(), bytes, language()).await, + Err(DictationError::InvalidSize) + ); + } + assert_eq!(provider.calls.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn rejects_bytes_that_are_not_a_supported_audio_container() { + let (service, provider) = service(); + for bytes in [ + Bytes::from_static(b""), + Bytes::from_static(b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR"), + ] { + assert_eq!( + service.transcribe(user(), bytes, None).await, + Err(DictationError::UnsupportedAudio) + ); + } + assert_eq!(provider.calls.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn capacity_guard_rejects_when_saturated_and_recovers() { + let (service, provider) = service(); + let permits = service + .capacity() + .acquire_many(MAX_CONCURRENT_TRANSCRIPTIONS as u32) + .await + .unwrap(); + assert_eq!( + service + .transcribe(user(), Bytes::from_static(OGG_HEADER), None) + .await, + Err(DictationError::Busy) + ); + assert_eq!(provider.calls.load(Ordering::SeqCst), 0); + drop(permits); + assert!( + service + .transcribe(user(), Bytes::from_static(OGG_HEADER), None) + .await + .is_ok() + ); +} + +#[tokio::test] +async fn rejects_invalid_and_long_audio_before_calling_the_provider() { + for (duration, expected) in [ + (Ok(Duration::ZERO), DictationError::InvalidAudio), + ( + Err(DictationError::InvalidAudio), + DictationError::InvalidAudio, + ), + ( + Ok(MAX_AUDIO_DURATION + Duration::from_millis(1)), + DictationError::TooLong, + ), + ] { + let provider = Arc::new(FakeProvider::default()); + let service = DictationServiceImpl::new( + provider.clone(), + FakeInspector(duration), + Arc::new(ai_usage::NoOpUsageRecorder), + ); + assert_eq!( + service + .transcribe(user(), Bytes::from_static(OGG_HEADER), None) + .await, + Err(expected) + ); + assert_eq!(provider.calls.load(Ordering::SeqCst), 0); + } +} + +#[tokio::test] +async fn accepts_the_exact_duration_limit() { + let provider = Arc::new(FakeProvider::default()); + let service = DictationServiceImpl::new( + provider.clone(), + FakeInspector(Ok(MAX_AUDIO_DURATION)), + Arc::new(ai_usage::NoOpUsageRecorder), + ); + assert!( + service + .transcribe(user(), Bytes::from_static(OGG_HEADER), None) + .await + .is_ok() + ); + assert_eq!(provider.calls.load(Ordering::SeqCst), 1); +} + +#[test] +fn language_hint_accepts_only_two_lowercase_letters() { + assert_eq!("en".parse::().unwrap().as_str(), "en"); + for invalid in ["EN", "en-US", "e", "", "1a"] { + assert_eq!( + invalid.parse::(), + Err(DictationError::InvalidLanguage) + ); + } +} + +#[test] +fn detects_browser_containers_and_names_upload_by_extension() { + let webm = [ + 0x1A, 0x45, 0xDF, 0xA3, 0x9F, 0x42, 0x86, 0x81, 0x01, 0x42, 0xF7, 0x81, + ]; + let mp4 = b"\x00\x00\x00\x1cftypiso5\x00\x00\x02\x00isomiso5"; + let wav = b"RIFF\x24\x00\x00\x00WAVEfmt "; + let mut ogg_opus = OGG_HEADER.to_vec(); + ogg_opus.resize(28, 0); + ogg_opus.extend_from_slice(b"OpusHead\x01\x02"); + for (bytes, format) in [ + (&webm[..], AudioFormat::Webm), + (&mp4[..], AudioFormat::Mp4), + (&wav[..], AudioFormat::Wav), + (OGG_HEADER, AudioFormat::Ogg), + (&ogg_opus[..], AudioFormat::Ogg), + ] { + assert_eq!(AudioFormat::detect(bytes), Ok(format), "{format:?}"); + } + assert_eq!(AudioFormat::Ogg.filename(), "dictation.ogg"); + assert_eq!( + AudioFormat::detect(b"ID3\x03\x00\x00\x00"), + Err(DictationError::UnsupportedAudio), + "mp3 uploads are not produced by browser recorders" + ); +} + +#[derive(Default)] +struct FakeRecorder(Mutex>); + +impl UsageRecorder for FakeRecorder { + fn record(&self, event: ai_usage::UsageEvent) { + self.0.lock().unwrap().push(event); + } +} + +#[tokio::test] +async fn records_provider_duration_only_for_successful_transcriptions() { + let recorder = Arc::new(FakeRecorder::default()); + for fails in [false, true] { + let provider = Arc::new(FakeProvider { + fails, + ..FakeProvider::default() + }); + let service = DictationServiceImpl::new( + provider, + FakeInspector(Ok(Duration::from_secs(3))), + recorder.clone(), + ); + let result = service + .transcribe(user(), Bytes::from_static(OGG_HEADER), None) + .await; + assert_eq!(result.is_err(), fails); + assert!( + service + .transcribe(user(), Bytes::new(), None) + .await + .is_err() + ); + } + let events = recorder.0.lock().unwrap(); + assert_eq!(events.len(), 1); + assert_eq!(events[0].user, user()); + assert_eq!(events[0].feature, AiFeature::Dictation); + assert_eq!(events[0].model, "whisper-1"); + assert_eq!( + events[0].amount, + ai_usage::UsageAmount::Audio { + duration: Duration::from_secs(2) + } + ); +} diff --git a/crates/dictation/src/inbound.rs b/crates/dictation/src/inbound.rs new file mode 100644 index 00000000000..caa6e347fe5 --- /dev/null +++ b/crates/dictation/src/inbound.rs @@ -0,0 +1,3 @@ +//! Inbound adapters that drive the dictation use case. + +pub mod axum_router; diff --git a/crates/dictation/src/inbound/axum_router.rs b/crates/dictation/src/inbound/axum_router.rs new file mode 100644 index 00000000000..2fd9e997990 --- /dev/null +++ b/crates/dictation/src/inbound/axum_router.rs @@ -0,0 +1,281 @@ +//! Authenticated HTTP endpoint for dictation. +//! +//! The adapter only extracts identity, syntax-validates the request, and maps +//! domain errors to status codes. Recording validation and provider policy live +//! in the domain service. + +use crate::domain::{DictationError, DictationService, LanguageHint, MAX_AUDIO_BYTES, Transcript}; +use axum::{ + Json, RequestPartsExt, Router, + body::Bytes, + extract::{DefaultBodyLimit, FromRef, FromRequestParts, Query, State}, + http::{StatusCode, header}, + response::{IntoResponse, Response}, + routing::post, +}; +use axum_extra::extract::Cached; +use macro_authorization::{ + MacroAuthorizationExtractor, MacroAuthorizationService, MacroAuthorizationState, UserOnly, +}; +use model_error_response::ErrorResponse; +use rate_limit::{ + RateLimitConfig, RateLimitKey, RateLimitResult, RateLimitService, + domain::models::RateLimitOk, + inbound::{RateLimitExtractable, RateLimitExtractor}, +}; +use rootcause::Report; +use serde::{Deserialize, Serialize}; +use std::{sync::Arc, time::Duration}; + +/// Authenticated attempts per hour, including provider failures and retries. +const PER_USER_TRANSCRIPTIONS_PER_HOUR: u64 = 60; + +/// State for the dictation router. +pub struct DictationRouterState { + service: Arc, + rate_limiter: R, + authorization_state: MacroAuthorizationState, +} + +impl Clone for DictationRouterState { + fn clone(&self) -> Self { + Self { + service: self.service.clone(), + rate_limiter: self.rate_limiter.clone(), + authorization_state: self.authorization_state.clone(), + } + } +} + +impl DictationRouterState { + /// Create dictation router state. + pub fn new( + service: S, + rate_limiter: R, + authorization_state: MacroAuthorizationState, + ) -> Self { + Self { + service: Arc::new(service), + rate_limiter, + authorization_state, + } + } +} + +impl FromRef> for Arc { + fn from_ref(state: &DictationRouterState) -> Self { + state.service.clone() + } +} + +impl FromRef> for MacroAuthorizationState { + fn from_ref(state: &DictationRouterState) -> Self { + state.authorization_state.clone() + } +} + +impl RateLimitService for DictationRouterState +where + S: Send + Sync + 'static, + R: RateLimitService, + Auth: MacroAuthorizationService, +{ + async fn check_rate_limit( + &self, + key: RateLimitKey, + config: RateLimitConfig, + ) -> Result { + let limit = config.max_count; + let window_seconds = config.window.as_secs(); + let result = self.rate_limiter.check_rate_limit(key, config).await?; + if result.is_err() { + tracing::warn!(limit, window_seconds, "dictation rate limit exceeded"); + } + Ok(result) + } + + async fn rollback_ticket(&self, ticket: RateLimitOk) -> Result<(), Report> { + self.rate_limiter.rollback_ticket(ticket).await + } +} + +/// Per-user transcription rate limit. Only signed-in users may dictate; bots, +/// harnesses, and internal callers are rejected by the `UserOnly` policy. +pub struct PerUserDictationRateLimit(MacroAuthorizationExtractor); + +impl RateLimitExtractable for PerUserDictationRateLimit +where + S: Send + Sync + 'static, + Auth: MacroAuthorizationService, + MacroAuthorizationState: FromRef, +{ + fn config() -> RateLimitConfig { + RateLimitConfig { + max_count: PER_USER_TRANSCRIPTIONS_PER_HOUR, + window: Duration::from_secs(3600), + } + } + + fn key(&self) -> RateLimitKey { + RateLimitKey::builder(&"per-user-dictation") + .append(&self.0.authorization.macro_user_id.as_ref()) + .finish() + } +} + +impl FromRequestParts for PerUserDictationRateLimit +where + S: Send + Sync + 'static, + Auth: MacroAuthorizationService, + MacroAuthorizationState: FromRef, +{ + type Rejection = Response; + + async fn from_request_parts( + parts: &mut axum::http::request::Parts, + state: &S, + ) -> Result { + let Cached(authorization): Cached> = parts + .extract_with_state(state) + .await + .map_err(IntoResponse::into_response)?; + // This runs before the body is buffered and before rate-limit rejection. + tracing::Span::current().record( + "user_id", + authorization.authorization.macro_user_id.as_ref(), + ); + Ok(Self(authorization)) + } +} + +/// Create the dictation router. Mount under `/dictation`. +pub fn dictation_router(state: DictationRouterState) -> Router +where + S: DictationService, + R: RateLimitService + Clone, + Auth: MacroAuthorizationService, + T: Send + Sync + 'static, +{ + Router::new() + .route("/transcribe", post(transcribe_handler::)) + .layer(DefaultBodyLimit::max(MAX_AUDIO_BYTES)) + .layer(axum::middleware::from_fn(trace_request)) + .with_state(state) +} + +/// Covers extractor failures as well as the handler, under the shared HTTP span. +#[tracing::instrument(name = "dictation.request", skip_all, fields( + user_id = tracing::field::Empty, + http.response.status_code = tracing::field::Empty, +))] +async fn trace_request(request: axum::extract::Request, next: axum::middleware::Next) -> Response { + let response = next.run(request).await; + tracing::Span::current().record("http.response.status_code", response.status().as_u16()); + response +} + +/// Query parameters for transcription. +#[derive(Debug, Deserialize)] +pub struct TranscribeQuery { + /// Optional ISO 639-1 recognition hint. + pub language: Option, +} + +/// Transcription result. +#[derive(Debug, Serialize, Deserialize, utoipa::ToSchema)] +pub struct TranscribeResponse { + /// Recognized text. + pub text: String, +} + +impl From for TranscribeResponse { + fn from(transcript: Transcript) -> Self { + Self { + text: transcript.text, + } + } +} + +/// OpenAPI representation of the raw encoded audio body extracted as `Bytes`. +#[derive(utoipa::ToSchema)] +#[schema(value_type = String, format = Binary)] +pub struct EncodedAudioBody( + /// Browser-encoded audio bytes. + pub Vec, +); + +/// Transcribe a transient recording with OpenAI Whisper. +/// +/// Available to every signed-in user on every plan; does not consume chat +/// credits. Audio and transcripts are never persisted. +#[utoipa::path( + post, + tag = "dictation", + operation_id = "transcribe_dictation", + path = "/dictation/transcribe", + params(("language" = Option, Query, description = "ISO 639-1 language hint")), + request_body(content = inline(EncodedAudioBody), content_type = "audio/webm", description = "Encoded WebM, MP4, Ogg, or WAV audio up to 8 MiB and five minutes"), + responses( + (status = 200, body = TranscribeResponse), + (status = 400, description = "Empty, oversized, or malformed request", body = ErrorResponse), + (status = 401, description = "Missing or invalid credentials", body = ErrorResponse), + (status = 403, description = "Only signed-in users may dictate", body = ErrorResponse), + (status = 413, description = "Body exceeds 8 MiB"), + (status = 415, description = "Unsupported audio container", body = ErrorResponse), + (status = 429, description = "Per-user hourly rate limit exceeded"), + (status = 502, description = "Provider failure", body = ErrorResponse), + (status = 503, description = "Transcription capacity exhausted; retry after the Retry-After delay", body = ErrorResponse), + ) +)] +pub async fn transcribe_handler( + Cached(user): Cached>, + _limit: RateLimitExtractor, DictationRouterState>, + State(service): State>, + Query(query): Query, + audio: Bytes, +) -> Result, DictationError> +where + S: DictationService, + R: RateLimitService + Clone, + Auth: MacroAuthorizationService, +{ + let language = query + .language + .as_deref() + .map(str::parse::) + .transpose() + .inspect_err(|error| tracing::warn!(error = ?error, "invalid dictation language hint"))?; + let transcript = service + .transcribe(user.authorization.macro_user_id, audio, language) + .await?; + Ok(Json(transcript.into())) +} + +impl IntoResponse for DictationError { + fn into_response(self) -> Response { + let status = match self { + Self::InvalidSize | Self::InvalidLanguage | Self::InvalidAudio | Self::TooLong => { + StatusCode::BAD_REQUEST + } + Self::UnsupportedAudio => StatusCode::UNSUPPORTED_MEDIA_TYPE, + Self::Busy => StatusCode::SERVICE_UNAVAILABLE, + Self::Provider => StatusCode::BAD_GATEWAY, + }; + let mut response = ( + status, + Json(ErrorResponse { + message: self.to_string().into(), + }), + ) + .into_response(); + if matches!(self, Self::Busy) { + response + .headers_mut() + .insert(header::RETRY_AFTER, header::HeaderValue::from_static("1")); + } + response + } +} + +#[cfg(test)] +mod test; diff --git a/crates/dictation/src/inbound/axum_router/test.rs b/crates/dictation/src/inbound/axum_router/test.rs new file mode 100644 index 00000000000..1829361d2df --- /dev/null +++ b/crates/dictation/src/inbound/axum_router/test.rs @@ -0,0 +1,330 @@ +use super::*; +use crate::domain::{DictationError, Transcript}; +use axum::{ + body::{Body, to_bytes}, + http::{Request, header}, +}; +use bytes::Bytes; +use macro_authorization::{InternalIdentityClaims, MacroAuthorizationError}; +use macro_user_id::user_id::MacroUserIdStr; +use model_user::UserContext; +use rate_limit::domain::models::RateLimitOk; +use std::sync::{ + Arc, Mutex, + atomic::{AtomicUsize, Ordering}, +}; +use tower::ServiceExt; + +const USER_ID: &str = "macro|dictation-test@example.com"; +const INTERNAL_KEY: &str = "valid-internal-key"; + +/// `(user id, audio bytes, language hint)` as seen by the service. +type ServiceCall = (String, usize, Option); + +#[derive(Clone, Default)] +struct FakeService { + calls: Arc>>, + response: Arc>>>, +} + +impl FakeService { + fn respond(&self, response: Result) { + *self.response.lock().unwrap() = Some(response); + } + + fn calls(&self) -> Vec { + self.calls.lock().unwrap().clone() + } +} + +impl DictationService for FakeService { + async fn transcribe( + &self, + user: MacroUserIdStr<'static>, + audio: Bytes, + language: Option, + ) -> Result { + self.calls.lock().unwrap().push(( + user.to_string(), + audio.len(), + language.map(|hint| hint.to_string()), + )); + self.response.lock().unwrap().take().unwrap_or_else(|| { + Ok(Transcript { + text: "hello".into(), + duration_seconds: 1.0, + }) + }) + } +} + +#[derive(Clone, Default)] +struct FakeRateLimiter { + exceeded: bool, + checks: Arc, + rollbacks: Arc, +} + +impl RateLimitService for FakeRateLimiter { + async fn check_rate_limit( + &self, + key: RateLimitKey, + config: RateLimitConfig, + ) -> Result { + self.checks.fetch_add(1, Ordering::SeqCst); + if self.exceeded { + return Ok(Err(rate_limit::RateLimitExceeded { + current_count: config.max_count, + max_count: config.max_count, + retry_after: config.window, + })); + } + Ok(Ok(RateLimitOk::new_testing_value(1, key, config))) + } + + async fn rollback_ticket(&self, _ticket: RateLimitOk) -> Result<(), Report> { + self.rollbacks.fetch_add(1, Ordering::SeqCst); + Ok(()) + } +} + +#[derive(Clone, Default)] +struct FakeAuthorizationService; + +impl MacroAuthorizationService for FakeAuthorizationService { + async fn authorize(&self, jwt: &str) -> Result> { + if jwt != "valid" { + return Err(Report::new(MacroAuthorizationError::InvalidCredentials)); + } + Ok(UserContext { + user_id: USER_ID.to_owned(), + ..UserContext::default() + }) + } + + async fn authorize_internal( + &self, + provided_key: &str, + claims: InternalIdentityClaims, + ) -> Result, Report> { + if provided_key != INTERNAL_KEY { + return Err(Report::new(MacroAuthorizationError::InvalidCredentials)); + } + Ok(claims.user_id.map(|user_id| UserContext { + user_id, + ..UserContext::default() + })) + } +} + +struct Harness { + service: FakeService, + limiter: FakeRateLimiter, +} + +impl Harness { + fn new() -> Self { + Self { + service: FakeService::default(), + limiter: FakeRateLimiter::default(), + } + } + + fn rate_limited() -> Self { + Self { + limiter: FakeRateLimiter { + exceeded: true, + ..FakeRateLimiter::default() + }, + ..Self::new() + } + } + + fn router(&self) -> Router { + dictation_router::<_, _, _, ()>(DictationRouterState::new( + self.service.clone(), + self.limiter.clone(), + MacroAuthorizationState::new(Arc::new(FakeAuthorizationService)), + )) + } + + async fn post( + &self, + uri: &str, + headers: &[(header::HeaderName, &str)], + body: impl Into, + ) -> Response { + let mut request = Request::builder().method("POST").uri(uri); + for (name, value) in headers { + request = request.header(name, *value); + } + self.router() + .oneshot(request.body(body.into()).unwrap()) + .await + .unwrap() + } +} + +async fn message(response: Response) -> String { + let bytes = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + serde_json::from_slice::(&bytes).unwrap()["message"] + .as_str() + .unwrap() + .to_owned() +} + +#[tokio::test] +async fn transcribes_for_an_authenticated_user_and_forwards_language() { + let harness = Harness::new(); + let response = harness + .post( + "/transcribe?language=en", + &[(header::AUTHORIZATION, "Bearer valid")], + "OggS-bytes", + ) + .await; + + assert_eq!(response.status(), StatusCode::OK); + let bytes = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + let body: TranscribeResponse = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(body.text, "hello"); + assert_eq!( + harness.service.calls(), + [(USER_ID.to_owned(), 10, Some("en".to_owned()))] + ); + assert_eq!(harness.limiter.checks.load(Ordering::SeqCst), 1); + assert_eq!(harness.limiter.rollbacks.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn rejects_missing_and_invalid_credentials_before_any_work() { + let harness = Harness::new(); + for headers in [&[][..], &[(header::AUTHORIZATION, "Bearer wrong")][..]] { + let response = harness.post("/transcribe", headers, "audio").await; + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + } + assert!(harness.service.calls().is_empty()); + assert_eq!(harness.limiter.checks.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn rejects_internal_callers_because_dictation_is_user_only() { + let harness = Harness::new(); + let response = harness + .post( + "/transcribe", + &[ + ( + header::HeaderName::from_static("x-internal-auth-key"), + INTERNAL_KEY, + ), + ( + header::HeaderName::from_static("x-internal-macro-user-id"), + USER_ID, + ), + ], + "audio", + ) + .await; + + assert_eq!(response.status(), StatusCode::FORBIDDEN); + assert!(harness.service.calls().is_empty()); +} + +#[tokio::test] +async fn enforces_the_per_user_rate_limit() { + let harness = Harness::rate_limited(); + let response = harness + .post( + "/transcribe", + &[(header::AUTHORIZATION, "Bearer valid")], + "audio", + ) + .await; + + assert_eq!(response.status(), StatusCode::TOO_MANY_REQUESTS); + assert!(harness.service.calls().is_empty()); +} + +#[tokio::test] +async fn failed_provider_attempts_still_consume_the_rate_limit() { + let harness = Harness::new(); + harness.service.respond(Err(DictationError::Provider)); + let response = harness + .post( + "/transcribe", + &[(header::AUTHORIZATION, "Bearer valid")], + "audio", + ) + .await; + + assert_eq!(response.status(), StatusCode::BAD_GATEWAY); + assert_eq!(harness.limiter.checks.load(Ordering::SeqCst), 1); + assert_eq!(harness.limiter.rollbacks.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn maps_domain_errors_to_status_codes_without_leaking_content() { + let cases = [ + (DictationError::InvalidSize, StatusCode::BAD_REQUEST), + (DictationError::InvalidAudio, StatusCode::BAD_REQUEST), + (DictationError::TooLong, StatusCode::BAD_REQUEST), + ( + DictationError::UnsupportedAudio, + StatusCode::UNSUPPORTED_MEDIA_TYPE, + ), + (DictationError::Busy, StatusCode::SERVICE_UNAVAILABLE), + (DictationError::Provider, StatusCode::BAD_GATEWAY), + ]; + for (error, status) in cases { + let harness = Harness::new(); + let expected = error.to_string(); + harness.service.respond(Err(error)); + let response = harness + .post( + "/transcribe", + &[(header::AUTHORIZATION, "Bearer valid")], + "audio", + ) + .await; + assert_eq!(response.status(), status); + assert_eq!( + response + .headers() + .get(header::RETRY_AFTER) + .map(|value| value.to_str().unwrap()), + (status == StatusCode::SERVICE_UNAVAILABLE).then_some("1") + ); + assert_eq!(message(response).await, expected); + } +} + +#[tokio::test] +async fn rejects_a_malformed_language_hint_in_the_adapter() { + let harness = Harness::new(); + let response = harness + .post( + "/transcribe?language=en-US", + &[(header::AUTHORIZATION, "Bearer valid")], + "audio", + ) + .await; + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + assert!(harness.service.calls().is_empty()); +} + +#[tokio::test] +async fn rejects_bodies_over_the_recording_cap() { + let harness = Harness::new(); + let response = harness + .post( + "/transcribe", + &[(header::AUTHORIZATION, "Bearer valid")], + vec![0u8; MAX_AUDIO_BYTES + 1], + ) + .await; + + assert_eq!(response.status(), StatusCode::PAYLOAD_TOO_LARGE); + assert!(harness.service.calls().is_empty()); +} diff --git a/crates/dictation/src/lib.rs b/crates/dictation/src/lib.rs new file mode 100644 index 00000000000..cd2299cf7ab --- /dev/null +++ b/crates/dictation/src/lib.rs @@ -0,0 +1,12 @@ +//! Private, transient audio transcription for composer dictation. +//! +//! Hexagonal layout: [`domain`] owns recording validation, the provider port, +//! and the use case; [`inbound`] exposes the authenticated, rate-limited HTTP +//! endpoint; [`outbound`] implements the port with OpenAI Whisper. +#![deny(missing_docs)] + +pub mod domain; +#[cfg(feature = "inbound")] +pub mod inbound; +#[cfg(feature = "outbound")] +pub mod outbound; diff --git a/crates/dictation/src/outbound.rs b/crates/dictation/src/outbound.rs new file mode 100644 index 00000000000..c27129d7513 --- /dev/null +++ b/crates/dictation/src/outbound.rs @@ -0,0 +1,7 @@ +//! Outbound adapters implementing the dictation ports. + +pub mod media; +pub mod whisper; + +pub use media::SymphoniaRecordingInspector; +pub use whisper::{OpenaiApiKey, WhisperTranscriber}; diff --git a/crates/dictation/src/outbound/media.rs b/crates/dictation/src/outbound/media.rs new file mode 100644 index 00000000000..5ef8e409341 --- /dev/null +++ b/crates/dictation/src/outbound/media.rs @@ -0,0 +1,128 @@ +//! Inspect browser recording containers without transcoding or temporary files. + +use crate::domain::{AudioFormat, DictationError, Recording, RecordingInspector}; +use std::{ + io::{Cursor, ErrorKind}, + time::Duration, +}; +use symphonia::core::{ + codecs::{CodecParameters, audio::well_known::CODEC_ID_OPUS}, + errors::Error, + formats::{FormatOptions, probe::Hint}, + io::MediaSourceStream, + meta::MetadataOptions, +}; + +/// Symphonia demuxes WebM/Opus, MP4/AAC, Ogg/Opus, and WAV in memory. +/// Reading packet timestamps does not require an Opus decoder or re-encoding. +pub struct SymphoniaRecordingInspector; + +impl RecordingInspector for SymphoniaRecordingInspector { + #[tracing::instrument(name = "dictation.inspect_audio", skip_all, err, fields( + audio.format = recording.format().extension(), + audio.bytes = recording.bytes().len(), + ))] + async fn duration(&self, recording: Recording) -> Result { + // Container parsing is CPU work; it must not block the async executor. + // Blocking tasks do not inherit the request's tracing context. + let span = tracing::Span::current(); + tokio::task::spawn_blocking(move || span.in_scope(|| inspect(recording))) + .await + .map_err(|error| { + tracing::error!(panicked = error.is_panic(), "audio inspection task failed"); + DictationError::InvalidAudio + })? + } +} + +#[tracing::instrument(name = "dictation.parse_audio", skip_all, err, fields( + audio.duration_seconds = tracing::field::Empty, + audio.packet_count = tracing::field::Empty, + dictation.inspection_stage = "probe", +))] +fn inspect(recording: Recording) -> Result { + let span = tracing::Span::current(); + let mut hint = Hint::new(); + hint.with_extension(recording.format().extension()); + let (bytes, format, _) = recording.into_parts(); + let source = MediaSourceStream::new(Box::new(Cursor::new(bytes)), Default::default()); + let mut reader = symphonia::default::get_probe() + .probe( + &hint, + source, + FormatOptions::default(), + MetadataOptions::default(), + ) + .map_err(|_| DictationError::InvalidAudio)?; + span.record("dictation.inspection_stage", "track"); + let track = reader + .tracks() + .iter() + .find(|track| { + track + .codec_params + .as_ref() + .is_some_and(CodecParameters::is_audio) + }) + .ok_or(DictationError::InvalidAudio)?; + let track_id = track.id; + let codec = track + .codec_params + .as_ref() + .and_then(CodecParameters::audio) + .ok_or(DictationError::InvalidAudio)? + .codec; + let time_base = track.time_base.ok_or(DictationError::InvalidAudio)?; + let mut end = 0.0_f64; + let mut first = None; + let mut packet_count = 0_u64; + span.record("dictation.inspection_stage", "packets"); + loop { + let packet = match reader.next_packet() { + Ok(Some(packet)) => packet, + Ok(None) => break, + // MediaRecorder leaves its streaming WebM segment open at EOF. + Err(Error::IoError(error)) + if format == AudioFormat::Webm && error.kind() == ErrorKind::UnexpectedEof => + { + break; + } + Err(_) => return Err(DictationError::InvalidAudio), + }; + if packet.track_id == track_id { + packet_count += 1; + let timestamp = time_base + .calc_time(packet.pts) + .ok_or(DictationError::InvalidAudio)? + .as_secs_f64(); + // Chrome omits BlockDuration/DefaultDuration. Read Opus framing + // through the codec library; no decoding or handwritten bit parser. + let duration = if packet.dur.is_zero() && codec == CODEC_ID_OPUS { + let samples = opus_pure::packet::samples_48k(&packet.data) + .map_err(|_| DictationError::InvalidAudio)?; + samples as f64 / 48_000.0 + } else { + time_base + .calc_duration(packet.dur) + .ok_or(DictationError::InvalidAudio)? + .as_secs_f64() + }; + first.get_or_insert(timestamp); + end = end.max(timestamp + duration); + } + } + span.record("audio.packet_count", packet_count); + span.record("dictation.inspection_stage", "duration"); + let seconds = end - first.ok_or(DictationError::InvalidAudio)?; + if seconds <= 0.0 { + return Err(DictationError::InvalidAudio); + } + let duration = + Duration::try_from_secs_f64(seconds).map_err(|_| DictationError::InvalidAudio)?; + span.record("audio.duration_seconds", duration.as_secs_f64()); + span.record("dictation.inspection_stage", "complete"); + Ok(duration) +} + +#[cfg(test)] +mod test; diff --git a/crates/dictation/src/outbound/media/test.rs b/crates/dictation/src/outbound/media/test.rs new file mode 100644 index 00000000000..ac24b65e4bd --- /dev/null +++ b/crates/dictation/src/outbound/media/test.rs @@ -0,0 +1,65 @@ +use super::*; +use bytes::Bytes; + +#[tokio::test] +async fn inspects_real_browser_containers_without_decoding_or_transcoding() { + for bytes in [ + include_bytes!("../../../tests/fixtures/tone.webm").as_slice(), + include_bytes!("../../../tests/fixtures/tone.ogg").as_slice(), + include_bytes!("../../../tests/fixtures/tone.mp4").as_slice(), + include_bytes!("../../../tests/fixtures/tone.wav").as_slice(), + ] { + let recording = Recording::new(Bytes::from_static(bytes), None).unwrap(); + let format = recording.format(); + let duration = SymphoniaRecordingInspector + .duration(recording) + .await + .unwrap(); + // Codec framing/padding differs; each fixture contains 100 ms of audio. + assert!( + (0.09..0.25).contains(&duration.as_secs_f64()), + "{format:?}: {duration:?}" + ); + } +} + +#[tokio::test] +async fn inspects_a_chrome_media_recorder_stream_without_container_duration() { + let recording = Recording::new( + Bytes::from_static(include_bytes!("../../../tests/fixtures/chrome.webm")), + None, + ) + .unwrap(); + let duration = SymphoniaRecordingInspector + .duration(recording) + .await + .unwrap(); + assert_eq!(duration, Duration::from_millis(240)); +} + +#[tokio::test] +async fn rejects_header_only_audio() { + let recording = Recording::new( + Bytes::from_static(b"OggS\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00"), + None, + ) + .unwrap(); + assert_eq!( + SymphoniaRecordingInspector.duration(recording).await, + Err(DictationError::InvalidAudio) + ); +} + +#[tokio::test] +async fn identifies_long_compressed_audio_even_below_the_byte_limit() { + let recording = Recording::new( + Bytes::from_static(include_bytes!("../../../tests/fixtures/long.ogg")), + None, + ) + .unwrap(); + let duration = SymphoniaRecordingInspector + .duration(recording) + .await + .unwrap(); + assert!(duration > crate::domain::MAX_AUDIO_DURATION); +} diff --git a/crates/dictation/src/outbound/whisper.rs b/crates/dictation/src/outbound/whisper.rs new file mode 100644 index 00000000000..147ad17ff79 --- /dev/null +++ b/crates/dictation/src/outbound/whisper.rs @@ -0,0 +1,158 @@ +//! OpenAI Whisper adapter for the [`TranscriptionProvider`] port. +//! +//! `/v1/audio/transcriptions` is eligible for OpenAI's Zero Data Retention and +//! has no abuse-monitoring retention by default, so an enterprise key with ZDR +//! can serve dictation without audio or transcripts being stored by the +//! provider. This adapter never logs either. + +use crate::domain::{DictationError, Recording, Transcript, TranscriptionProvider}; +use async_openai::{ + Client, + config::OpenAIConfig, + error::OpenAIError, + types::audio::{AudioInput, AudioResponseFormat, CreateTranscriptionRequest}, +}; +use macro_env_var::env_vars; +use serde::Deserialize; +use std::time::Duration; +use tracing::instrument::WithSubscriber; + +env_vars! { + /// Server-side OpenAI credential, injected as `OPENAI_API_KEY`. Never + /// exposed to the browser. + pub struct OpenaiApiKey; +} + +const MODEL: &str = "whisper-1"; +/// Overall request deadline, including the SDK's retries. +const REQUEST_TIMEOUT: Duration = Duration::from_secs(90); +/// Total budget for retrying provider 429/5xx responses before giving up. +const RETRY_BUDGET: Duration = Duration::from_secs(10); + +/// Speech-to-text through OpenAI Whisper. +pub struct WhisperTranscriber { + client: Client, +} + +/// The subset of the `verbose_json` response this adapter relies on. +#[derive(Deserialize)] +struct VerboseTranscription { + text: String, + duration: f32, +} + +impl WhisperTranscriber { + /// Construct once at startup against the public OpenAI API. + pub fn new(api_key: &OpenaiApiKey) -> Result { + Self::with_api_base(api_key, None) + } + + /// Construct against a custom API base, e.g. a test server. + pub fn with_api_base( + api_key: &OpenaiApiKey, + api_base: Option<&str>, + ) -> Result { + if api_key.trim().is_empty() { + return Err(WhisperConfigError::EmptyApiKey); + } + let config = OpenAIConfig::new() + .with_api_key(api_key.as_ref()) + .with_api_base(api_base.unwrap_or("https://api.openai.com/v1")); + let http_client = reqwest::Client::builder() + .timeout(REQUEST_TIMEOUT) + .build()?; + let backoff = backoff::ExponentialBackoffBuilder::new() + .with_max_elapsed_time(Some(RETRY_BUDGET)) + .build(); + Ok(Self { + client: Client::build(http_client, config, backoff), + }) + } +} + +/// Startup failures for [`WhisperTranscriber::new`]. +#[derive(Debug, thiserror::Error)] +pub enum WhisperConfigError { + /// The required credential contains no usable key. + #[error("OPENAI_API_KEY must not be empty")] + EmptyApiKey, + /// The HTTP client could not be constructed. + #[error(transparent)] + HttpClient(#[from] reqwest::Error), +} + +impl TranscriptionProvider for WhisperTranscriber { + fn model_id(&self) -> &'static str { + MODEL + } + + #[tracing::instrument(name = "transcribe whisper-1", skip_all, err, fields( + otel.kind = "client", + gen_ai.operation.name = "transcribe", + gen_ai.provider.name = "openai", + gen_ai.request.model = MODEL, + audio.format = recording.format().extension(), + audio.bytes = recording.bytes().len(), + audio.duration_seconds = tracing::field::Empty, + error.type = tracing::field::Empty, + ))] + async fn transcribe(&self, recording: Recording) -> Result { + let (bytes, format, language) = recording.into_parts(); + let request = CreateTranscriptionRequest { + file: AudioInput::from_bytes(format.filename(), bytes), + model: MODEL.to_owned(), + language: language.map(|hint| hint.to_string()), + response_format: Some(AudioResponseFormat::VerboseJson), + ..Default::default() + }; + let audio_api = self.client.audio(); + let transcriptions = audio_api.transcription(); + let transcription = + transcriptions.create_verbose_json_byot::<_, VerboseTranscription>(request); + // async-openai 0.36 logs raw response bodies on parse/HTTP failures. + // Scope a silent subscriber to this future's polls only; our surrounding + // span and sanitized diagnostics remain visible, including after await. + let response = tokio::time::timeout( + REQUEST_TIMEOUT, + transcription.with_subscriber(tracing::Dispatch::none()), + ) + .await + .map_err(|_| { + tracing::Span::current().record("error.type", "timeout"); + tracing::warn!(kind = "timeout", "Whisper request failed"); + DictationError::Provider + })? + .map_err(|error| { + let kind = match &error { + OpenAIError::Reqwest(error) if error.is_timeout() => "timeout", + OpenAIError::Reqwest(error) if error.is_connect() => "connection", + OpenAIError::Reqwest(_) => "http", + OpenAIError::ApiError(_) => "provider_rejection", + OpenAIError::JSONDeserialize(..) => "invalid_response", + _ => "client_error", + }; + tracing::Span::current().record("error.type", kind); + // ApiError discards HTTP headers/status. Only Reqwest retains status; + // never recover it by logging the SDK error or its raw body. + let status = match &error { + OpenAIError::Reqwest(error) => error.status().map(|status| status.as_u16()), + _ => None, + }; + tracing::warn!(kind, status, "Whisper request failed"); + DictationError::Provider + })?; + if !response.duration.is_finite() || response.duration <= 0.0 { + tracing::Span::current().record("error.type", "invalid_duration"); + tracing::warn!(kind = "invalid_duration", "Whisper response unusable"); + return Err(DictationError::Provider); + } + tracing::Span::current().record("audio.duration_seconds", response.duration); + Ok(Transcript { + text: response.text, + duration_seconds: response.duration, + }) + } +} + +#[cfg(test)] +mod test; diff --git a/crates/dictation/src/outbound/whisper/test.rs b/crates/dictation/src/outbound/whisper/test.rs new file mode 100644 index 00000000000..07ed331876b --- /dev/null +++ b/crates/dictation/src/outbound/whisper/test.rs @@ -0,0 +1,105 @@ +use super::*; +use crate::domain::AudioFormat; +use bytes::Bytes; +use serde_json::json; +use wiremock::matchers::{header, method, path}; +use wiremock::{Mock, MockServer, Request, ResponseTemplate}; + +/// Minimal Ogg page header so container detection yields `.ogg`. +const OGG_HEADER: &[u8] = b"OggS\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00"; + +fn transcriber(server: &MockServer) -> WhisperTranscriber { + let api_key = OpenaiApiKey::new_testing("test-server-key"); + WhisperTranscriber::with_api_base(&api_key, Some(&format!("{}/v1", server.uri()))).unwrap() +} + +fn recording() -> Recording { + Recording::new(Bytes::from_static(OGG_HEADER), Some("en".parse().unwrap())).unwrap() +} + +fn multipart_body(request: &Request) -> String { + String::from_utf8_lossy(&request.body).into_owned() +} + +#[tokio::test] +async fn posts_whisper_multipart_with_server_credential_and_reads_duration() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/audio/transcriptions")) + .and(header("authorization", "Bearer test-server-key")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "task": "transcribe", + "language": "english", + "duration": 2.5, + "text": "Hello.", + "segments": [] + }))) + .expect(1) + .mount(&server) + .await; + + let transcript = transcriber(&server).transcribe(recording()).await.unwrap(); + + assert_eq!(transcript.text, "Hello."); + assert_eq!(transcript.duration_seconds, 2.5); + let requests = server.received_requests().await.unwrap(); + let body = multipart_body(&requests[0]); + for expected in [ + "name=\"model\"\r\n\r\nwhisper-1", + "name=\"response_format\"\r\n\r\nverbose_json", + "name=\"language\"\r\n\r\nen", + "filename=\"dictation.ogg\"", + "OggS", + ] { + assert!(body.contains(expected), "missing {expected:?} in {body:?}"); + } + assert_eq!(recording().format(), AudioFormat::Ogg); +} + +#[tokio::test] +async fn rejects_responses_without_billable_duration() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/audio/transcriptions")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ "text": "Only text." }))) + .mount(&server) + .await; + + assert_eq!( + transcriber(&server).transcribe(recording()).await, + Err(DictationError::Provider) + ); +} + +#[test] +fn rejects_empty_credentials_at_construction() { + assert!(matches!( + WhisperTranscriber::new(&OpenaiApiKey::new_testing(" ")), + Err(WhisperConfigError::EmptyApiKey) + )); +} + +#[tokio::test] +async fn maps_provider_rejections_to_a_content_free_error() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/audio/transcriptions")) + .respond_with(ResponseTemplate::new(400).set_body_json(json!({ + "error": { + "message": "Invalid file format.", + "type": "invalid_request_error", + "param": "file", + "code": null + } + }))) + .expect(1) + .mount(&server) + .await; + + let error = transcriber(&server) + .transcribe(recording()) + .await + .unwrap_err(); + + assert_eq!(error, DictationError::Provider); +} diff --git a/crates/dictation/tests/fixtures/README.md b/crates/dictation/tests/fixtures/README.md new file mode 100644 index 00000000000..a761bc87a3f --- /dev/null +++ b/crates/dictation/tests/fixtures/README.md @@ -0,0 +1,35 @@ +# Dictation media fixtures + +These fixtures contain generated tones or silence, with no recorded speech. +They exercise container parsing, including Opus and fragmented MP4, without +requiring a codec decoder, FFmpeg, a microphone, or elapsed-time waits in tests. + +Generate each `tone` file with FFmpeg using this input: + +```sh +ffmpeg -f lavfi -i 'sine=frequency=440:sample_rate=16000:duration=0.1' ... +``` + +Output options: + +| File | Options | +| --- | --- | +| `tone.webm` | `-c:a libopus tone.webm` | +| `tone.ogg` | `-c:a libopus tone.ogg` | +| `tone.mp4` | `-c:a aac -movflags frag_keyframe+empty_moov+default_base_moof tone.mp4` | +| `tone.wav` | `-c:a pcm_s16le tone.wav` | + +The duration-limit fixture is generated with: + +```sh +ffmpeg -f lavfi -i 'anullsrc=r=16000:cl=mono' -t 301 -c:a libopus long.ogg +``` + +`chrome.webm` was captured in Chromium 152 using the production `AudioRecorder`, +with a Web Audio oscillator connected to a `MediaStreamAudioDestinationNode` +instead of a microphone. It uses WebM/Opus at 64 kbit/s and stops after the first +`dataavailable` event. Unlike the FFmpeg fixture, its container has no declared +duration or packet durations; the inspector must read packet timestamps and +Opus framing. FFprobe reports four 60 ms packets, with the last starting at +180 ms, for a total of 240 ms. + diff --git a/crates/dictation/tests/fixtures/chrome.webm b/crates/dictation/tests/fixtures/chrome.webm new file mode 100644 index 00000000000..7da5a9a3a1f Binary files /dev/null and b/crates/dictation/tests/fixtures/chrome.webm differ diff --git a/crates/dictation/tests/fixtures/long.ogg b/crates/dictation/tests/fixtures/long.ogg new file mode 100644 index 00000000000..a1b56426cff Binary files /dev/null and b/crates/dictation/tests/fixtures/long.ogg differ diff --git a/crates/dictation/tests/fixtures/tone.mp4 b/crates/dictation/tests/fixtures/tone.mp4 new file mode 100644 index 00000000000..12428000080 Binary files /dev/null and b/crates/dictation/tests/fixtures/tone.mp4 differ diff --git a/crates/dictation/tests/fixtures/tone.ogg b/crates/dictation/tests/fixtures/tone.ogg new file mode 100644 index 00000000000..9a25da72046 Binary files /dev/null and b/crates/dictation/tests/fixtures/tone.ogg differ diff --git a/crates/dictation/tests/fixtures/tone.wav b/crates/dictation/tests/fixtures/tone.wav new file mode 100644 index 00000000000..098d8365e2b Binary files /dev/null and b/crates/dictation/tests/fixtures/tone.wav differ diff --git a/crates/dictation/tests/fixtures/tone.webm b/crates/dictation/tests/fixtures/tone.webm new file mode 100644 index 00000000000..6c09c25da34 Binary files /dev/null and b/crates/dictation/tests/fixtures/tone.webm differ diff --git a/crates/dictation/tests/whisper_tracing.rs b/crates/dictation/tests/whisper_tracing.rs new file mode 100644 index 00000000000..9ec1056b288 --- /dev/null +++ b/crates/dictation/tests/whisper_tracing.rs @@ -0,0 +1,78 @@ +//! Exercise SDK logging with an isolated subscriber; unrelated parallel tests +//! must not alter callsite registration while the capture is active. +#![cfg(feature = "outbound")] + +use bytes::Bytes; +use dictation::{ + domain::{DictationError, Recording, TranscriptionProvider}, + outbound::{OpenaiApiKey, WhisperTranscriber}, +}; +use serde_json::json; +use tracing::instrument::WithSubscriber; +use wiremock::{Mock, MockServer, ResponseTemplate, matchers::method}; + +#[derive(Clone, Default)] +struct CapturedLogs(std::sync::Arc>>); + +impl std::io::Write for CapturedLogs { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.lock().unwrap().extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for CapturedLogs { + type Writer = Self; + fn make_writer(&'a self) -> Self { + self.clone() + } +} + +#[tokio::test] +async fn malformed_provider_responses_never_log_transcript_content() { + for status in [200, 400] { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(status).set_body_json( + json!({"text": "PRIVATE_TRANSCRIPT_SENTINEL", "duration": "invalid"}), + )) + .expect(1) + .mount(&server) + .await; + let logs = CapturedLogs::default(); + let subscriber = tracing_subscriber::fmt() + .without_time() + .with_ansi(false) + .with_span_events(tracing_subscriber::fmt::format::FmtSpan::CLOSE) + .with_max_level(tracing::Level::TRACE) + .with_writer(logs.clone()) + .finish(); + let transcriber = WhisperTranscriber::with_api_base( + &OpenaiApiKey::Comptime("test-server-key"), + Some(&format!("{}/v1", server.uri())), + ) + .unwrap(); + let recording = Recording::new( + Bytes::from_static(include_bytes!("fixtures/tone.ogg")), + None, + ) + .unwrap(); + let result = transcriber + .transcribe(recording) + .with_subscriber(subscriber) + .await; + assert_eq!(result, Err(DictationError::Provider)); + let output = String::from_utf8(logs.0.lock().unwrap().clone()).unwrap(); + assert!( + output.contains("Whisper request failed") && output.contains("invalid_response"), + "sanitized diagnostics must remain visible: {output}" + ); + assert!(!output.contains("PRIVATE_TRANSCRIPT_SENTINEL")); + assert!(!output.contains("test-server-key")); + assert!(output.contains("gen_ai.request.model=\"whisper-1\"")); + } +} diff --git a/crates/macro_db_client/migrations/20260918204004_add_audio_usage_pricing.sql b/crates/macro_db_client/migrations/20260918204004_add_audio_usage_pricing.sql new file mode 100644 index 00000000000..75388cb11f4 --- /dev/null +++ b/crates/macro_db_client/migrations/20260918204004_add_audio_usage_pricing.sql @@ -0,0 +1,17 @@ +-- Add duration billing alongside token billing; deployed token-only writers can +-- continue omitting every new column. +ALTER TABLE ai_pricing + ADD COLUMN price_per_audio_minute REAL + CHECK (price_per_audio_minute >= 0 AND price_per_audio_minute < 'Infinity'::real); + +ALTER TABLE ai_usage + ADD COLUMN audio_seconds DOUBLE PRECISION + CHECK (audio_seconds >= 0 AND audio_seconds < 'Infinity'::double precision), + ADD COLUMN price_per_audio_minute REAL + CHECK (price_per_audio_minute >= 0 AND price_per_audio_minute < 'Infinity'::real); + +-- https://platform.openai.com/docs/pricing — Whisper is priced per audio minute. +-- Preserve any rate already configured by an administrator. +INSERT INTO ai_pricing (model, price_per_million_in, price_per_million_out, price_per_audio_minute) +VALUES ('whisper-1', 0, 0, 0.006) +ON CONFLICT (model) DO NOTHING; diff --git a/crates/macro_db_migrator/src/bin/comment_import/test.rs b/crates/macro_db_migrator/src/bin/comment_import/test.rs index a20de15d6a0..b5651467851 100644 --- a/crates/macro_db_migrator/src/bin/comment_import/test.rs +++ b/crates/macro_db_migrator/src/bin/comment_import/test.rs @@ -567,7 +567,7 @@ async fn other_connection(pool: &PgPool) -> PgConnection { async fn release_import_lock(connection: &mut PgConnection) { // Closing the client does not wait for PostgreSQL to release session locks. - // Await an explicit unlock before another connection tries to run the import. + // Await an explicit unlock before another connection tries to acquire it. assert!( sqlx::query_scalar!(r#"SELECT pg_advisory_unlock($1) AS "released!""#, LOCK_ID) .fetch_one(connection) diff --git a/crates/workspace-hack/Cargo.toml b/crates/workspace-hack/Cargo.toml index 61dc98cc239..3489129c938 100644 --- a/crates/workspace-hack/Cargo.toml +++ b/crates/workspace-hack/Cargo.toml @@ -19,6 +19,7 @@ aes-gcm = { version = "0.10", features = ["zeroize"] } ahash = { version = "0.8" } aho-corasick = { version = "1" } anyhow = { version = "1" } +async-openai = { version = "0.36", features = ["audio", "byot", "chat-completion", "completions", "embedding"] } aws-config = { version = "1", features = ["behavior-version-latest"] } aws-credential-types = { version = "1", default-features = false, features = ["hardcoded-credentials", "test-util"] } aws-runtime = { version = "1", default-features = false, features = ["event-stream", "http-02x", "http-1x", "sigv4a"] } @@ -34,6 +35,7 @@ axum = { version = "0.8", features = ["macros", "multipart", "ws"] } axum-extra = { version = "0.10", features = ["cookie", "query", "typed-header"] } base64 = { version = "0.22" } bstr = { version = "1" } +bytemuck = { version = "1", default-features = false, features = ["extern_crate_alloc"] } byteorder = { version = "1" } bytes = { version = "1", features = ["serde"] } bzip2 = { version = "0.5", features = ["libbz2-rs-sys"] } @@ -152,6 +154,7 @@ aes-gcm = { version = "0.10", features = ["zeroize"] } ahash = { version = "0.8" } aho-corasick = { version = "1" } anyhow = { version = "1" } +async-openai = { version = "0.36", features = ["audio", "byot", "chat-completion", "completions", "embedding"] } aws-config = { version = "1", features = ["behavior-version-latest"] } aws-credential-types = { version = "1", default-features = false, features = ["hardcoded-credentials", "test-util"] } aws-runtime = { version = "1", default-features = false, features = ["event-stream", "http-02x", "http-1x", "sigv4a"] } @@ -167,6 +170,7 @@ axum = { version = "0.8", features = ["macros", "multipart", "ws"] } axum-extra = { version = "0.10", features = ["cookie", "query", "typed-header"] } base64 = { version = "0.22" } bstr = { version = "1" } +bytemuck = { version = "1", default-features = false, features = ["extern_crate_alloc"] } byteorder = { version = "1" } bytes = { version = "1", features = ["serde"] } bzip2 = { version = "0.5", features = ["libbz2-rs-sys"] } diff --git a/deny.toml b/deny.toml index 30a7c69b06c..ccf12256039 100644 --- a/deny.toml +++ b/deny.toml @@ -207,6 +207,7 @@ deny = [ "channels", # Dev logging only "coding_agent_worker", # Standalone daemon logging setup "cursor_cloud_agents", # Standalone ACP binary logging; stdout is the protocol, so macro_entrypoint cannot own it + "dictation", # Metadata tracing and content-redaction tests only "documents", # Dev logging only "email", # Dev logging only "genai_telemetry", # GenAI span tests diff --git a/docs/AGENT_GUIDE/ai-chat.md b/docs/AGENT_GUIDE/ai-chat.md index 14c5e999fe1..050820140af 100644 --- a/docs/AGENT_GUIDE/ai-chat.md +++ b/docs/AGENT_GUIDE/ai-chat.md @@ -263,6 +263,45 @@ notified when the AI responds). Legacy Home background sends preserve the submit ## Composer anatomy (a11y) +AI chat (including Home and doc-scoped chat) and agent session composers have +a **Start dictation with OpenAI Whisper** microphone beside Send. It records +in memory and uploads to the +authenticated `/dictation/transcribe` storage endpoint only on confirmation. +Whisper is available on all plans without consuming chat credits; its server +credential is never exposed to the browser. Unsupported recording environments +show a disabled microphone. Every supported browser uses Whisper; there are +no browser speech-recognition or language-pack installation flows. + +While dictating, a scrolling microphone-volume timeline and **Cancel dictation** / **Use dictation** +replace the composer controls. Cancel (or Escape) preserves the original draft. +Bars sample microphone volume as recording chunks arrive (normally every 200ms): silence stays dotted, louder speech +creates taller bars, and earlier levels move left without changing height. +Volume analysis stays on-device and stops on confirm, cancel, error, or close. +Use dictation stops recording, waits for Whisper, and appends plain text to +the draft without sending it. Existing rich text and attachments remain intact. +If the browser stops listening on its own, **Ready** waits for confirmation. +While **Finishing…**, the checkmark is disabled and Cancel remains available. +Mobile chat stays expanded when focus moves into dictation controls. +Starting dictation in another composer releases the previous session without +moving focus back to it. Closing the composer releases the microphone. Capture failures +appear below the composer. + +Whisper dictation supports WebM, MP4, and Ogg recording depending on browser. +Recordings stop just before five minutes or near 8 MB and wait for confirmation. +Cancel discards the recording; cancel during transcription aborts the request and +ignores any late result. If the service is temporarily at capacity, the composer +stays in **Finishing…** while TanStack retries up to twice with exponential backoff, +jitter, and the server's `Retry-After` delay. Cancel also cancels these retries. +Other failures keep the recording in memory for an explicit retry with the +checkmark. No audio or transcript is stored in the query cache or persisted by the dictation +endpoint. Provider diagnostics exclude response content. The server detects +the audio container and inspects its duration before contacting OpenAI, requires a +signed-in user (bots and internal callers are refused), and rate limits each +user to 60 attempts per hour (failed requests and retries count). Hourly limits +show “Dictation limit reached. Please try again later.” and are not automatically retried. +The backend records provider-reported audio seconds in the shared AI usage system +and uses Whisper's per-minute model pricing without charging user credits. + Desktop composer and conversation body text use 15px type. Mobile keeps its existing text sizing. diff --git a/docs/AGENT_GUIDE/channels.md b/docs/AGENT_GUIDE/channels.md index 4874574e8fe..78e923c2b8a 100644 --- a/docs/AGENT_GUIDE/channels.md +++ b/docs/AGENT_GUIDE/channels.md @@ -38,6 +38,16 @@ show whatever has arrived so far. ## Message composer +Channel messages and thread replies have a microphone next to Send, including +the collapsed composer. It uses the same OpenAI Whisper dictation and scrolling +volume timeline as AI chat. **Use dictation** appends text to the draft without +sending; **Cancel dictation** or Escape preserves the existing draft. Sending +is blocked during dictation, including keyboard and external send actions. +When local recognition is unavailable, confirming uploads the in-memory audio +to Whisper. This fallback is available on all plans without consuming chat credits. +Browsers without microphone recording show a disabled button. +The first click downloads a language pack when needed; click again to record. + Composer and conversation body text use `text-base` (15px at the default root size) on desktop and mobile. The shared scale uses 14px for `text-sm` and 12px for `text-xs`, with accessibility text scaling preserved. diff --git a/nix-support/node_modules-hashes.json b/nix-support/node_modules-hashes.json index c0b39f08763..973d8e68ed3 100644 --- a/nix-support/node_modules-hashes.json +++ b/nix-support/node_modules-hashes.json @@ -1,6 +1,6 @@ { "nodeModules": { - "aarch64-darwin": "sha256-UdoatrQ33Socze4bAzdZS/NvIC1Uc1Z+A7n9QXEFunA=", - "x86_64-linux": "sha256-aQvn+NvaP9aUvoZyz4fPhS9/3xcyP4/XPlV9etbUMOM=" + "x86_64-linux": "sha256-euQs05yQlRl3TE2TunEjX1dvQIAIAL+v16meI7VNa5A=", + "aarch64-darwin": "sha256-FO99wNeI8e3GYKkNShok27bFcEQxUddMB8VKaCmGck8=" } } diff --git a/packages/sdk/generated/cognition/types.gen.ts b/packages/sdk/generated/cognition/types.gen.ts index e038e212567..66d77e7b9c9 100644 --- a/packages/sdk/generated/cognition/types.gen.ts +++ b/packages/sdk/generated/cognition/types.gen.ts @@ -27,7 +27,7 @@ export type AddServerRequest = { * Everything we use AI for. The wire / DB form of each variant is its * `snake_case` name. */ -export type AiFeature = 'chat' | 'memory' | 'automation' | 'dynamic_completions_api' | 'chat_rename' | 'call_summary' | 'channel_bot' | 'ai_projection' | 'ai_editing' | 'import' | 'agent_session' | 'agent_repository_choice'; +export type AiFeature = 'chat' | 'memory' | 'automation' | 'dynamic_completions_api' | 'chat_rename' | 'call_summary' | 'channel_bot' | 'ai_projection' | 'ai_editing' | 'import' | 'agent_session' | 'agent_repository_choice' | 'dictation'; /** * A structured part within an assistant message. @@ -400,24 +400,23 @@ export type CompleteOnboardingRequest = { }; /** - * A recorded completion: who, what feature, optional entity, and the cost. + * User and feature attribution for one invocation. */ export type CompletionUsage = { /** - * Token usage and cost. + * Measured usage and resolved cost. */ cost: Usage; /** - * The entity the completion related to, if any. + * Related entity, if any. */ entity?: string | null; /** - * The feature that performed the completion. + * Feature that performed the invocation. */ feature: AiFeature; /** - * The user the completion was performed for (the [system user](SYSTEM_USER_ID) - * for background work). + * User the invocation was performed for. */ user: MacroUserIdStr; }; @@ -537,19 +536,19 @@ export type ErrorResponse = { export type Expiry = 'day' | 'week' | 'month'; /** - * Usage for a single feature, with its rolled-up dollar total. + * Recorded invocations and total for one feature. */ export type FeatureUsage = { /** - * The individual completions recorded for this feature. + * Recorded invocations. */ entries: Array; /** - * The feature. + * Feature attribution. */ feature: AiFeature; /** - * Total cost across `entries` (USD). + * Total cost (USD). */ total: number; }; @@ -1073,19 +1072,23 @@ export type PipedreamUpdateRequest = { }; /** - * Resolved price for one completion. + * Rates applied to an invocation and its resolved dollar cost. */ export type Price = { /** - * Price per million input tokens (USD). + * Price per audio minute (USD), absent for token billing. + */ + price_per_audio_minute?: number | null; + /** + * Price per million input tokens (USD); zero for audio billing. */ price_per_million_in: number; /** - * Price per million output tokens (USD). + * Price per million output tokens (USD); zero for audio billing. */ price_per_million_out: number; /** - * Total cost of the completion (USD). + * Total cost (USD). */ total: number; }; @@ -1263,13 +1266,17 @@ export type SetPricingRequest = { */ model: string; /** - * New price per million input tokens (USD). + * Price per minute of audio (USD), or null for token-only pricing. */ - price_per_mil_in: number; + price_per_audio_minute?: number | null; /** - * New price per million output tokens (USD). + * New price per million input tokens (USD). Required for token pricing. */ - price_per_mil_out: number; + price_per_mil_in?: number | null; + /** + * New price per million output tokens (USD). Required for token pricing. + */ + price_per_mil_out?: number | null; }; export type SharePermissionV2 = { @@ -1560,23 +1567,27 @@ export type UpsertProjectionRequest = { }; /** - * The token usage and resolved cost of a single completion. + * Measured usage for one invocation. */ export type Usage = { /** - * When the completion was recorded. + * Audio duration in seconds, absent for token billing. + */ + audio_seconds?: number | null; + /** + * Recording timestamp. */ created_at: string; /** - * Tokens consumed by the input. + * Input tokens; zero for audio billing. */ input_tokens: number; /** - * The model api id (e.g. `claude-opus-4-8`). + * Provider model identifier. */ model: string; /** - * Tokens generated in the output. + * Output tokens; zero for audio billing. */ output_tokens: number; price?: null | Price; @@ -1605,7 +1616,7 @@ export type UsageRequest = { }; /** - * The result of a usage query: per-feature breakdown plus a grand total. + * Per-feature breakdown and grand total. */ export type UsageSummary = { /** @@ -1613,7 +1624,7 @@ export type UsageSummary = { */ entries: Array; /** - * Grand total cost across all features (USD). + * Grand total (USD). */ total: number; }; @@ -1677,6 +1688,10 @@ export type SetPricingHandlerData = { }; export type SetPricingHandlerErrors = { + /** + * Invalid pricing + */ + 400: ErrorBody; /** * Admin access required */ diff --git a/packages/sdk/generated/storage/index.ts b/packages/sdk/generated/storage/index.ts index 043fd879794..65d9febdd33 100644 --- a/packages/sdk/generated/storage/index.ts +++ b/packages/sdk/generated/storage/index.ts @@ -1,4 +1,4 @@ // This file is auto-generated by @hey-api/openapi-ts export { type Options, Sdk } from './sdk.gen'; -export type { AccessLevel, ActiveCallsResponse, ActiveCallSummary, ActivityType, AddFavoriteData, AddFavoriteError, AddFavoriteErrors, AddFavoriteRequest, AddFavoriteResponse, AddFavoriteResponses, AddParticipantsData, AddParticipantsError, AddParticipantsErrors, AddParticipantsRequest, AddParticipantsResponses, AddPinHandlerData, AddPinHandlerError, AddPinHandlerErrors, AddPinHandlerResponse, AddPinHandlerResponses, AddPinRequest, Agent, AgentActionId, AgentChannelScope, AgentMcpServer, AgentMcpServers, AgentSessionFilters, AgentSessionLifecycleEvent, Anchor, AnchorId, AnchorRequest, AnchorResponse, AnnotationIncrementalUpdate, ApiActivity, ApiAttachmentChannelReference, ApiAttachmentEntityReference, ApiAttachmentGenericReference, ApiChannelAttachment, ApiChannelAttachmentsPage, ApiChannelContextMessage, ApiChannelDetail, ApiChannelListMessage, ApiChannelListPage, ApiChannelListParticipant, ApiChannelListType, ApiChannelMessage, ApiChannelMessageKind, ApiChannelMessagesPage, ApiChannelParticipant, ApiChannelWithLatest, ApiCountedReaction, ApiEntityFilterAst, ApiGroupByField, ApiGroupMeta, ApiMessageAttachment, ApiMessageSender, ApiMessageSenderType, ApiParticipantListRole, ApiParticipantRole, ApiPropertyEntityType, ApiResolvedChannelMessage, ApiThreadInfo, ApiThreadReply, ApproveHarnessPairingData, ApproveHarnessPairingError, ApproveHarnessPairingErrors, ApproveHarnessPairingResponse, ApproveHarnessPairingResponses, ApprovePairingRequest, AssignInitiativeTasksData, AssignInitiativeTasksError, AssignInitiativeTasksErrors, AssignInitiativeTasksResponse, AssignInitiativeTasksResponses, AssignTasksRequest, AssignTasksResponse, AssignTasksResult, AssignTaskStatus, AttachmentChange, AttendeeResponseStatus, BasicDocument, BasicDocumentSubType, BomPart, Bot, BotChannel, BotChannelType, BotId, BotKind, BotOwner, BotSenderProfile, BotToken, BranchNameResponse, BTreeMap, BulkWakeupRequest, BulkWakeupResponse, BulkWakeupSyncServiceDocumentsData, BulkWakeupSyncServiceDocumentsErrors, BulkWakeupSyncServiceDocumentsResponse, BulkWakeupSyncServiceDocumentsResponses, CalendarAttendee, CalendarEvent, CalendarEventFilters, CalendarEventSourceContent, CalendarMentionEvent, CalendarMentionPreviewItem, CalendarMentionPreviewKind, CalendarMentionPreviewRequest, CalendarMentionPreviewRequestItem, CalendarMentionPreviewResponse, CalendarOccurrence, CalendarOccurrenceItem, CalendarOccurrenceResponse, CalendarSyncStatus, CallActiveResponse, CallFilters, CallRecord, CallRecordParticipant, CallRecordPreview, CallRecordPreviewData, CallRecordTranscriptSegment, CallStatus, CallTokenResponse, CallWebhookData, CallWebhookError, CallWebhookErrors, CallWebhookResponses, Channel, ChannelCreatedMetadata, ChannelDeletedMetadata, ChannelEventAttachment, ChannelFilters, ChannelJoinCodeResponse, ChannelMentionedMetadata, ChannelMessage, ChannelMessageAttachmentCreatedMetadata, ChannelMessageAttachmentRemovedMetadata, ChannelMessageDeletedMetadata, ChannelMessageFilters, ChannelMessagePatchedMetadata, ChannelMessagePostedMetadata, ChannelParticipant, ChannelParticipantAddedMetadata, ChannelParticipantRemovedMetadata, ChannelPreview, ChannelPreviewData, ChannelSender, ChannelSharePermission, ChannelThreadFilters, ChannelTopicEvent, ChannelType, ChannelUpdatedMetadata, ChannelWebhookRequest, ChannelWebhookResponse, ChannelWithParticipants, Chat, ChatFilters, CheckActiveCallData, CheckActiveCallError, CheckActiveCallErrors, CheckActiveCallResponse, CheckActiveCallResponses, ClaimedPairing, ClaimHarnessPairingData, ClaimHarnessPairingError, ClaimHarnessPairingErrors, ClaimHarnessPairingResponse, ClaimHarnessPairingResponses, ClaimPairingRequest, ClientOptions, CloudStorageItemType, CollabSurfaceResponse, CollabSurfaceTokenResponse, Comment, CommentThread, ConferenceProvider, CopyDocumentData, CopyDocumentError, CopyDocumentErrors, CopyDocumentQueryParams, CopyDocumentRequest, CopyDocumentResponse, CopyDocumentResponse2, CopyDocumentResponses, CountedReaction, CreateAgentData, CreateAgentError, CreateAgentErrors, CreateAgentRequest, CreateAgentResponse, CreateAgentResponses, CreateAnchorData, CreateAnchorError, CreateAnchorErrors, CreateAnchorResponse, CreateAnchorResponses, CreateBotRequest, CreateBulkDocumentResponse, CreateBulkDocumentResponseData, CreateChannelData, CreateChannelError, CreateChannelErrors, CreateChannelRequest, CreateChannelResponse, CreateChannelResponse2, CreateChannelResponses, CreateChannelScopedBotData, CreateChannelScopedBotError, CreateChannelScopedBotErrors, CreateChannelScopedBotRequest, CreateChannelScopedBotResponse, CreateChannelScopedBotResponse2, CreateChannelScopedBotResponses, CreateCollabSurfaceTokenData, CreateCollabSurfaceTokenError, CreateCollabSurfaceTokenErrors, CreateCollabSurfaceTokenResponse, CreateCollabSurfaceTokenResponses, CreateCommentData, CreateCommentError, CreateCommentErrors, CreateCommentRequest, CreateCommentResponse, CreateCommentResponse2, CreateCommentResponses, CreateCrmCommentData, CreateCrmCommentError, CreateCrmCommentErrors, CreateCrmCommentRequest, CreateCrmCommentResponse, CreateCrmCommentResponses, CreateCrmCompanyData, CreateCrmCompanyError, CreateCrmCompanyErrors, CreateCrmCompanyRequest, CreateCrmCompanyResponse, CreateCrmCompanyResponses, CreateCrmContactData, CreateCrmContactError, CreateCrmContactErrors, CreateCrmContactRequest, CreateCrmContactResponse, CreateCrmContactResponses, CreateDocumentData, CreateDocumentError, CreateDocumentErrors, CreateDocumentRequest, CreateDocumentResponse, CreateDocumentResponseData, CreateDocumentResponses, CreatedPairing, CreatedUserApiKey, CreateEntityMentionData, CreateEntityMentionError, CreateEntityMentionErrors, CreateEntityMentionRequest, CreateEntityMentionResponse, CreateEntityMentionResponse2, CreateEntityMentionResponses, CreateHarnessPairingData, CreateHarnessPairingError, CreateHarnessPairingErrors, CreateHarnessPairingResponse, CreateHarnessPairingResponses, CreateInitiativeData, CreateInitiativeError, CreateInitiativeErrors, CreateInitiativeRequest, CreateInitiativeResponse, CreateInitiativeResponses, CreateInstructionsDocumentResponse, CreateInstructionsHandlerData, CreateInstructionsHandlerError, CreateInstructionsHandlerErrors, CreateInstructionsHandlerResponse, CreateInstructionsHandlerResponses, CreateMarkdownDocumentRequest, CreateMarkdownDocumentResponse, CreateMarkdownHandlerData, CreateMarkdownHandlerError, CreateMarkdownHandlerErrors, CreateMarkdownHandlerResponse, CreateMarkdownHandlerResponses, CreatePairingRequest, CreateProjectHandlerData, CreateProjectHandlerError, CreateProjectHandlerErrors, CreateProjectHandlerResponse, CreateProjectHandlerResponses, CreateProjectRequest, CreateProjectResponse, CreateReminderData, CreateReminderError, CreateReminderErrors, CreateReminderRequest, CreateReminderResponse, CreateReminderResponses, CreateSkillHandlerData, CreateSkillHandlerError, CreateSkillHandlerErrors, CreateSkillHandlerResponse, CreateSkillHandlerResponses, CreateSkillRequest, CreateSkillResponse, CreateSnippetHandlerData, CreateSnippetHandlerError, CreateSnippetHandlerErrors, CreateSnippetHandlerResponse, CreateSnippetHandlerResponses, CreateSnippetRequest, CreateSnippetResponse, CreateTaskHandlerData, CreateTaskHandlerError, CreateTaskHandlerErrors, CreateTaskHandlerResponse, CreateTaskHandlerResponses, CreateTaskRequest, CreateTaskResponse, CreateUnthreadedAnchorRequest, CreateUnthreadedAnchorResponse, CreateUnthreadedPdfAnchorRequest, CreateUserApiKeyData, CreateUserApiKeyError, CreateUserApiKeyErrors, CreateUserApiKeyRequest, CreateUserApiKeyResponse, CreateUserApiKeyResponses, CreateViewHandlerData, CreateViewHandlerError, CreateViewHandlerErrors, CreateViewHandlerResponse, CreateViewHandlerResponses, CreateViewRequest, CreateWebhookData, CreateWebhookError, CreateWebhookErrors, CreateWebhookRequest, CreateWebhookResponse, CreateWebhookResponse2, CreateWebhookResponses, CrmComment, CrmCommentEntityType, CrmCommentThread, CrmCompanyFilters, CrmCompanyResponse, CrmContactResponse, CrmDomainResponse, CrmPermissionRole, CrmStageInput, CrmStageResponse, CrmStagesResponse, CrmTeamSettingsResponse, CrmThread, CustomSpeakerAssignment, DataType, DeleteAnchorData, DeleteAnchorError, DeleteAnchorErrors, DeleteAnchorInfo, DeleteAnchorResponse, DeleteAnchorResponses, DeleteCallRecordData, DeleteCallRecordError, DeleteCallRecordErrors, DeleteCallRecordResponse, DeleteCallRecordResponses, DeleteChannelData, DeleteChannelError, DeleteChannelErrors, DeleteChannelResponse, DeleteChannelResponses, DeleteCollabSurfaceData, DeleteCollabSurfaceError, DeleteCollabSurfaceErrors, DeleteCollabSurfaceResponse, DeleteCollabSurfaceResponses, DeleteCommentData, DeleteCommentError, DeleteCommentErrors, DeleteCommentRequest, DeleteCommentResponse, DeleteCommentResponse2, DeleteCommentResponses, DeleteCrmCommentData, DeleteCrmCommentError, DeleteCrmCommentErrors, DeleteCrmCommentResponse, DeleteCrmCommentResponses, DeleteCrmCommentResult, DeleteDocumentData, DeleteDocumentError, DeleteDocumentErrors, DeleteDocumentResponse, DeleteDocumentResponses, DeleteEntityMentionData, DeleteEntityMentionError, DeleteEntityMentionErrors, DeleteEntityMentionResponse, DeleteEntityMentionResponse2, DeleteEntityMentionResponses, DeleteHarnessData, DeleteHarnessError, DeleteHarnessErrors, DeleteHarnessResponse, DeleteHarnessResponses, DeleteHistoryHandlerData, DeleteHistoryHandlerError, DeleteHistoryHandlerErrors, DeleteHistoryHandlerResponse, DeleteHistoryHandlerResponses, DeleteInitiativeData, DeleteInitiativeError, DeleteInitiativeErrors, DeleteInitiativeResponse, DeleteInitiativeResponses, DeleteMessageData, DeleteMessageError, DeleteMessageErrors, DeleteMessageQuery, DeleteMessageResponse, DeleteMessageResponses, DeleteProjectHandlerData, DeleteProjectHandlerError, DeleteProjectHandlerErrors, DeleteProjectHandlerResponse, DeleteProjectHandlerResponses, DeleteReminderData, DeleteReminderError, DeleteReminderErrors, DeleteReminderResponse, DeleteReminderResponses, DeleteSelfHarnessData, DeleteSelfHarnessError, DeleteSelfHarnessErrors, DeleteSelfHarnessResponse, DeleteSelfHarnessResponses, DeleteThreadInfo, DeleteUnthreadedAnchorRequest, DeleteUnthreadedAnchorResponse, DeleteUnthreadedPdfAnchorRequest, DeleteUserApiKeyData, DeleteUserApiKeyError, DeleteUserApiKeyErrors, DeleteUserApiKeyResponse, DeleteUserApiKeyResponses, DeleteUserDocumentViewLocationData, DeleteUserDocumentViewLocationError, DeleteUserDocumentViewLocationErrors, DeleteUserDocumentViewLocationResponse, DeleteUserDocumentViewLocationResponses, DeleteViewHandlerData, DeleteViewHandlerError, DeleteViewHandlerErrors, DeleteViewHandlerResponses, DeleteWebhookData, DeleteWebhookError, DeleteWebhookErrors, DeleteWebhookResponse, DeleteWebhookResponses, DescriptionDocumentId, DocumentBasic, DocumentContent, DocumentContentLocation, DocumentContentState, DocumentContentUploadedMetadata, DocumentCopiedMetadata, DocumentCreatedMetadata, DocumentDeletedMetadata, DocumentFilters, DocumentId, DocumentInteractionMetadata, DocumentMetadata, DocumentMetadataWithContent, DocumentPermissionsToken, DocumentPermissionsTokenRequest, DocumentPermissionsTokenResponse, DocumentPreview, DocumentPreviewData, DocumentPreviewDataSubType, DocumentPurgedMetadata, DocumentResponse, DocumentResponseMetadata, DocumentResponseMetadataWithContent, DocumentStorageServiceApiVersion, DocumentSubType, DocumentSyncContentUpdatedMetadata, DocumentTeamShareResponse, DocumentTopicEvent, DocumentUpdatedMetadata, EditAnchorData, EditAnchorError, EditAnchorErrors, EditAnchorRequest, EditAnchorResponse, EditAnchorResponse2, EditAnchorResponses, EditCallRecordData, EditCallRecordError, EditCallRecordErrors, EditCallRecordRequest, EditCallRecordResponse, EditCallRecordResponses, EditCallTranscriptData, EditCallTranscriptError, EditCallTranscriptErrors, EditCallTranscriptRequest, EditCallTranscriptResponse, EditCallTranscriptResponses, EditCommentData, EditCommentError, EditCommentErrors, EditCommentRequest, EditCommentResponse, EditCommentResponse2, EditCommentResponses, EditCrmCommentData, EditCrmCommentError, EditCrmCommentErrors, EditCrmCommentRequest, EditCrmCommentResponse, EditCrmCommentResponses, EditDocumentData, EditDocumentError, EditDocumentErrors, EditDocumentResponse, EditDocumentResponse2, EditDocumentResponses, EditDocumentServiceArgs, EditPdfAnchorRequest, EditPdfPlaceableCommentAnchorRequest, EditProjectV2Data, EditProjectV2Error, EditProjectV2Errors, EditProjectV2Response, EditProjectV2Responses, EditThreadV2Data, EditThreadV2Error, EditThreadV2Errors, EditThreadV2Response, EditThreadV2Responses, EmailFilters, EmptyResponse, EnsureCollabSurfaceData, EnsureCollabSurfaceError, EnsureCollabSurfaceErrors, EnsureCollabSurfaceRequest, EnsureCollabSurfaceResponse, EnsureCollabSurfaceResponses, EntityFilters, EntityMessageCreateData, EntityMessageCreateResponse, EntityMessageCreateResponses, EntityMessageDeleteMessageData, EntityMessageDeleteMessageResponse, EntityMessageDeleteMessageResponses, EntityMessageDeleteThreadData, EntityMessageDeleteThreadResponse, EntityMessageDeleteThreadResponses, EntityMessageEditData, EntityMessageEditResponse, EntityMessageEditResponses, EntityMessageGetMessageData, EntityMessageGetMessageResponse, EntityMessageGetMessageResponses, EntityMessageGetThreadData, EntityMessageGetThreadResponse, EntityMessageGetThreadResponses, EntityMessageLegacyData, EntityMessageLegacyResponse, EntityMessageLegacyResponses, EntityMessagePatchThreadData, EntityMessagePatchThreadResponse, EntityMessagePatchThreadResponses, EntityMessageReactData, EntityMessageReactResponse, EntityMessageReactResponses, EntityMessageReferencesData, EntityMessageReferencesResponse, EntityMessageReferencesResponses, EntityMessageTypingData, EntityMessageTypingResponse, EntityMessageTypingResponses, EntityPermission, EntityPermissionResponse, EntityReference, EntityType, ErrorResponse, EventReminderOverride, EventReminders, EventStatus, EventTime, EventTransparency, EventType, EventVisibility, ExcludedDefaultView, ExcludeDefaultViewHandlerData, ExcludeDefaultViewHandlerError, ExcludeDefaultViewHandlerErrors, ExcludeDefaultViewHandlerResponses, ExcludeDefaultViewRequest, ExportDocumentData, ExportDocumentError, ExportDocumentErrors, ExportDocumentResponse, ExportDocumentResponse2, ExportDocumentResponses, Favorite, FavoriteEntityRef, FavoritesList, FileSystemNodeWithIds, FileType, FileTypeUpdate, FolderItem, ForeignEntity, ForeignEntityFilters, GenericErrorResponse, GenericResponse, GenericSuccessResponse, GetActiveCallsData, GetActiveCallsError, GetActiveCallsErrors, GetActiveCallsResponse, GetActiveCallsResponses, GetActivityData, GetActivityError, GetActivityErrors, GetActivityResponse, GetActivityResponses, GetAttachmentReferencesData, GetAttachmentReferencesError, GetAttachmentReferencesErrors, GetAttachmentReferencesResponse, GetAttachmentReferencesResponse2, GetAttachmentReferencesResponses, GetBatchCallRecordPreviewData, GetBatchCallRecordPreviewError, GetBatchCallRecordPreviewErrors, GetBatchCallRecordPreviewRequest, GetBatchCallRecordPreviewResponse, GetBatchCallRecordPreviewResponse2, GetBatchCallRecordPreviewResponses, GetBatchChannelPreviewData, GetBatchChannelPreviewError, GetBatchChannelPreviewErrors, GetBatchChannelPreviewRequest, GetBatchChannelPreviewResponse, GetBatchChannelPreviewResponse2, GetBatchChannelPreviewResponses, GetBatchPreviewHandlerData, GetBatchPreviewHandlerError, GetBatchPreviewHandlerErrors, GetBatchPreviewHandlerResponse, GetBatchPreviewHandlerResponses, GetBatchPreviewRequest, GetBatchPreviewResponse, GetBatchProjectPreviewData, GetBatchProjectPreviewError, GetBatchProjectPreviewErrors, GetBatchProjectPreviewRequest, GetBatchProjectPreviewResponse, GetBatchProjectPreviewResponse2, GetBatchProjectPreviewResponses, GetCallRecordData, GetCallRecordError, GetCallRecordErrors, GetCallRecordResponse, GetCallRecordResponses, GetChannelAttachmentsData, GetChannelAttachmentsError, GetChannelAttachmentsErrors, GetChannelAttachmentsResponse, GetChannelAttachmentsResponses, GetChannelData, GetChannelError, GetChannelErrors, GetChannelJoinLinkData, GetChannelJoinLinkError, GetChannelJoinLinkErrors, GetChannelJoinLinkResponse, GetChannelJoinLinkResponses, GetChannelMessagesCatchUpData, GetChannelMessagesCatchUpError, GetChannelMessagesCatchUpErrors, GetChannelMessagesCatchUpResponse, GetChannelMessagesCatchUpResponses, GetChannelMessagesData, GetChannelMessagesError, GetChannelMessagesErrors, GetChannelMessagesResponse, GetChannelMessagesResponses, GetChannelParticipantsData, GetChannelParticipantsError, GetChannelParticipantsErrors, GetChannelParticipantsResponse, GetChannelParticipantsResponses, GetChannelResponse, GetChannelResponses, GetChannelsData, GetChannelsError, GetChannelsErrors, GetChannelsResponse, GetChannelsResponses, GetCollabSurfaceData, GetCollabSurfaceError, GetCollabSurfaceErrors, GetCollabSurfaceResponse, GetCollabSurfaceResponses, GetCompanyData, GetCompanyError, GetCompanyErrors, GetCompanyResponse, GetCompanyResponses, GetContactByEmailData, GetContactByEmailError, GetContactByEmailErrors, GetContactByEmailResponse, GetContactByEmailResponse2, GetContactByEmailResponses, GetContactData, GetContactError, GetContactErrors, GetContactResponse, GetContactResponses, GetCrmTeamSettingsData, GetCrmTeamSettingsError, GetCrmTeamSettingsErrors, GetCrmTeamSettingsResponse, GetCrmTeamSettingsResponses, GetDocumentAnchorsData, GetDocumentAnchorsError, GetDocumentAnchorsErrors, GetDocumentAnchorsResponse, GetDocumentAnchorsResponses, GetDocumentBranchNameData, GetDocumentBranchNameError, GetDocumentBranchNameErrors, GetDocumentBranchNameResponse, GetDocumentBranchNameResponses, GetDocumentByTeamSlugData, GetDocumentByTeamSlugError, GetDocumentByTeamSlugErrors, GetDocumentByTeamSlugResponse, GetDocumentByTeamSlugResponses, GetDocumentCommentsData, GetDocumentCommentsError, GetDocumentCommentsErrors, GetDocumentCommentsResponse, GetDocumentCommentsResponses, GetDocumentData, GetDocumentError, GetDocumentErrors, GetDocumentGithubPullRequestsData, GetDocumentGithubPullRequestsError, GetDocumentGithubPullRequestsErrors, GetDocumentGithubPullRequestsResponse, GetDocumentGithubPullRequestsResponses, GetDocumentKeyResponse, GetDocumentKeyResponseData, GetDocumentListHandlerData, GetDocumentListHandlerError, GetDocumentListHandlerErrors, GetDocumentListHandlerResponse, GetDocumentListHandlerResponses, GetDocumentListResult, GetDocumentLocationV3Data, GetDocumentLocationV3Error, GetDocumentLocationV3Errors, GetDocumentLocationV3Response, GetDocumentLocationV3Responses, GetDocumentPermissionsResponseDataV2, GetDocumentPermissionsTokenData, GetDocumentPermissionsTokenError, GetDocumentPermissionsTokenErrors, GetDocumentPermissionsTokenResponse, GetDocumentPermissionsTokenResponses, GetDocumentPermissionsV2Data, GetDocumentPermissionsV2Error, GetDocumentPermissionsV2Errors, GetDocumentPermissionsV2Response, GetDocumentPermissionsV2Responses, GetDocumentProcessingResult, GetDocumentProcessingResultData, GetDocumentProcessingResultError, GetDocumentProcessingResultErrors, GetDocumentProcessingResultResponse, GetDocumentProcessingResultResponse2, GetDocumentProcessingResultResponses, GetDocumentResponse, GetDocumentResponse2, GetDocumentResponseData, GetDocumentResponses, GetDocumentSearchResponse, GetDocumentShortIdData, GetDocumentShortIdError, GetDocumentShortIdErrors, GetDocumentShortIdResponse, GetDocumentShortIdResponses, GetDocumentsResponse, GetDocumentTeamShareData, GetDocumentTeamShareError, GetDocumentTeamShareErrors, GetDocumentTeamShareResponse, GetDocumentTeamShareResponses, GetDocumentUserAccessLevelResponse, GetDocumentVersionData, GetDocumentVersionError, GetDocumentVersionErrors, GetDocumentVersionResponse, GetDocumentVersionResponses, GetDocumentViewsHandlerData, GetDocumentViewsHandlerError, GetDocumentViewsHandlerErrors, GetDocumentViewsHandlerResponse, GetDocumentViewsHandlerResponses, GetEntityPermissionData, GetEntityPermissionErrors, GetEntityPermissionResponse, GetEntityPermissionResponses, GetForeignEntityBySourceData, GetForeignEntityBySourceError, GetForeignEntityBySourceErrors, GetForeignEntityBySourceResponse, GetForeignEntityBySourceResponses, GetForeignEntityData, GetForeignEntityError, GetForeignEntityErrors, GetForeignEntityResponse, GetForeignEntityResponses, GetHarnessPairingData, GetHarnessPairingError, GetHarnessPairingErrors, GetHarnessPairingResponse, GetHarnessPairingResponses, GetHistoryHandlerData, GetHistoryHandlerError, GetHistoryHandlerErrors, GetHistoryHandlerResponse, GetHistoryHandlerResponses, GetInitiativeData, GetInitiativeError, GetInitiativeErrors, GetInitiativeResponse, GetInitiativeResponses, GetInstructionsDocumentResponse, GetInstructionsHandlerData, GetInstructionsHandlerError, GetInstructionsHandlerErrors, GetInstructionsHandlerResponse, GetInstructionsHandlerResponses, GetItemsSoupData, GetItemsSoupError, GetItemsSoupErrors, GetItemsSoupResponse, GetItemsSoupResponses, GetLocationHandlerData, GetLocationHandlerError, GetLocationHandlerErrors, GetLocationHandlerResponse, GetLocationHandlerResponses, GetMessageWithContextData, GetMessageWithContextError, GetMessageWithContextErrors, GetMessageWithContextResponse, GetMessageWithContextResponse2, GetMessageWithContextResponses, GetOrCreateAction, GetOrCreateCallData, GetOrCreateCallError, GetOrCreateCallErrors, GetOrCreateCallResponse, GetOrCreateCallResponses, GetOrCreateChannelResponse, GetOrCreateDmData, GetOrCreateDmError, GetOrCreateDmErrors, GetOrCreateDmRequest, GetOrCreateDmResponse, GetOrCreateDmResponses, GetOrCreatePrivateData, GetOrCreatePrivateError, GetOrCreatePrivateErrors, GetOrCreatePrivateRequest, GetOrCreatePrivateResponse, GetOrCreatePrivateResponses, GetPendingProjectsHandlerData, GetPendingProjectsHandlerError, GetPendingProjectsHandlerErrors, GetPendingProjectsHandlerResponse, GetPendingProjectsHandlerResponses, GetPinsHandlerData, GetPinsHandlerError, GetPinsHandlerErrors, GetPinsHandlerResponse, GetPinsHandlerResponses, GetPinsResponse, GetProjectContentHandlerData, GetProjectContentHandlerError, GetProjectContentHandlerErrors, GetProjectContentHandlerResponse, GetProjectContentHandlerResponses, GetProjectContentResponse, GetProjectHandlerData, GetProjectHandlerError, GetProjectHandlerErrors, GetProjectHandlerResponse, GetProjectHandlerResponses, GetProjectPermissionsV2Data, GetProjectPermissionsV2Error, GetProjectPermissionsV2Errors, GetProjectPermissionsV2Response, GetProjectPermissionsV2Responses, GetProjectResponse, GetProjectResponseData, GetProjectsHandlerData, GetProjectsHandlerError, GetProjectsHandlerErrors, GetProjectsHandlerResponse, GetProjectsHandlerResponses, GetProjectsResponse, GetProjectUserAccessLevelData, GetProjectUserAccessLevelError, GetProjectUserAccessLevelErrors, GetProjectUserAccessLevelResponse, GetProjectUserAccessLevelResponses, GetReminderData, GetReminderError, GetReminderErrors, GetReminderResponse, GetReminderResponses, GetRingStatusData, GetRingStatusError, GetRingStatusErrors, GetRingStatusResponse, GetRingStatusResponses, GetSelfBotData, GetSelfBotError, GetSelfBotErrors, GetSelfBotResponse, GetSelfBotResponses, GetSelfHarnessData, GetSelfHarnessError, GetSelfHarnessErrors, GetSelfHarnessResponse, GetSelfHarnessResponses, GetSystemSkillsHandlerData, GetSystemSkillsHandlerError, GetSystemSkillsHandlerErrors, GetSystemSkillsHandlerResponse, GetSystemSkillsHandlerResponses, GetThreadRepliesData, GetThreadRepliesError, GetThreadRepliesErrors, GetThreadRepliesResponse, GetThreadRepliesResponses, GetUserDocumentsHandlerData, GetUserDocumentsHandlerError, GetUserDocumentsHandlerErrors, GetUserDocumentsHandlerResponse, GetUserDocumentsHandlerResponses, GetUserDocumentViewLocationData, GetUserDocumentViewLocationError, GetUserDocumentViewLocationErrors, GetUserDocumentViewLocationResponse, GetUserDocumentViewLocationResponses, GetUserHistoryResponse, GetViewsHandlerData, GetViewsHandlerError, GetViewsHandlerErrors, GetViewsHandlerResponse, GetViewsHandlerResponses, GetWebhookData, GetWebhookError, GetWebhookErrors, GetWebhookResponse, GetWebhookResponses, GithubPullRequest, GithubPullRequestCheckRun, GithubPullRequestComment, GithubPullRequestsResponse, GroupedInitialParams, GroupedPageParams, GroupedSoupGroupPage, GroupedSoupInitialPage, GroupedSoupPage, GroupedSoupSort, HandlerData, HandlerError, HandlerErrors, HandlerResponse, HandlerResponses, Harness, HarnessAgent, HarnessId, HarnessOwner, HarnessSession, HashMap, HealthHandlerData, HealthHandlerResponse, HealthHandlerResponses, HighlightType, ImportedAuthor, InFlightTurnSummary, IngestTranscriptData, IngestTranscriptError, IngestTranscriptErrors, IngestTranscriptResponses, InitializeUserDocumentsData, InitializeUserDocumentsError, InitializeUserDocumentsErrors, InitializeUserDocumentsResponse, InitializeUserDocumentsResponses, InitiativeDetail, InitiativeId, InitiativeList, InitiativeSummary, InputReceivedMetadata, InstallSyncData, InstallSyncErrors, InteractionReason, Item, ItemWithUserAccessLevel, JobProcessingResultHandlerData, JobProcessingResultHandlerError, JobProcessingResultHandlerErrors, JobProcessingResultHandlerResponse, JobProcessingResultHandlerResponses, JoinChannelByCodeData, JoinChannelByCodeError, JoinChannelByCodeErrors, JoinChannelByCodeResponses, JoinChannelData, JoinChannelError, JoinChannelErrors, JoinChannelResponses, LatestMessage, LeaveCallResponse, LeaveChannelData, LeaveChannelError, LeaveChannelErrors, LeaveChannelResponses, LeaveOrEndCallData, LeaveOrEndCallError, LeaveOrEndCallErrors, LeaveOrEndCallResponse, LeaveOrEndCallResponses, LinkShare, ListAgentsData, ListAgentsError, ListAgentsErrors, ListAgentsResponse, ListAgentsResponses, ListBotChannelsData, ListBotChannelsError, ListBotChannelsErrors, ListBotChannelsResponse, ListBotChannelsResponses, ListCompanyContactsData, ListCompanyContactsError, ListCompanyContactsErrors, ListCompanyContactsResponse, ListCompanyContactsResponses, ListCrmCommentsData, ListCrmCommentsError, ListCrmCommentsErrors, ListCrmCommentsResponse, ListCrmCommentsResponses, ListFavoritesData, ListFavoritesError, ListFavoritesErrors, ListFavoritesResponse, ListFavoritesResponses, ListHarnessAgentsData, ListHarnessAgentsError, ListHarnessAgentsErrors, ListHarnessAgentsResponse, ListHarnessAgentsResponses, ListHarnessesData, ListHarnessesError, ListHarnessesErrors, ListHarnessesResponse, ListHarnessesResponses, ListHarnessSessionsData, ListHarnessSessionsError, ListHarnessSessionsErrors, ListHarnessSessionsResponse, ListHarnessSessionsResponses, ListInitiativesData, ListInitiativesError, ListInitiativesErrors, ListInitiativesResponse, ListInitiativesResponses, ListOccurrencesData, ListOccurrencesErrors, ListOccurrencesResponse, ListOccurrencesResponses, ListRemindersData, ListRemindersError, ListRemindersErrors, ListRemindersResponse, ListRemindersResponses, ListTeamOutOfOfficeData, ListTeamOutOfOfficeErrors, ListTeamOutOfOfficeResponse, ListTeamOutOfOfficeResponses, ListUserApiKeysData, ListUserApiKeysError, ListUserApiKeysErrors, ListUserApiKeysResponse, ListUserApiKeysResponses, ListWebhooksData, ListWebhooksError, ListWebhooksErrors, ListWebhooksResponse, ListWebhooksResponse2, ListWebhooksResponses, LocationResponseData, LocationResponseV3, MacroUserIdStr, MentionPreviewsData, MentionPreviewsErrors, MentionPreviewsResponse, MentionPreviewsResponses, Mentions, Message, MessageAttachment, MessageChange, MessageCursor, MessageDirection, MessageEvent, MessageListItem, MessagePage, MessageParent, MessagePatch, MessageThread, MessageThreadPreview, MessageTimelineData, MessageTimelineQuery, MessageTimelineResponse, MessageTimelineResponses, NewAttachment, NewChannelAttachment, NewThreadAnchor, NotificationFilters, NotificationState, PairingDetails, Params, ParticipantRole, PatchBotRequest, PatchChannelData, PatchChannelError, PatchChannelErrors, PatchChannelRequest, PatchChannelResponse, PatchChannelResponses, PatchMessageData, PatchMessageError, PatchMessageErrors, PatchMessageNotificationPolicy, PatchMessageRequest, PatchMessageResponse, PatchMessageResponses, PatchProjectRequestV2, PatchThreadRequestV2, PatchViewHandlerData, PatchViewHandlerError, PatchViewHandlerErrors, PatchViewHandlerResponses, PatchWebhookData, PatchWebhookError, PatchWebhookErrors, PatchWebhookRequest, PatchWebhookResponse, PatchWebhookResponses, PdfAnchor, PdfAnchorId, PdfAnchorRequest, PdfHighlightAnchor, PdfHighlightAnchorRequest, PdfHighlightRect, PdfHighlightRectAnchorRequest, PdfPlaceableCommentAnchor, PdfPlaceableCommentAnchorRequest, PendingClaimResponse, PermanentlyDeleteDocumentData, PermanentlyDeleteDocumentError, PermanentlyDeleteDocumentErrors, PermanentlyDeleteDocumentResponse, PermanentlyDeleteDocumentResponses, PermanentlyDeleteProjectData, PermanentlyDeleteProjectError, PermanentlyDeleteProjectErrors, PermanentlyDeleteProjectResponse, PermanentlyDeleteProjectResponses, PinnedItem, PinRequest, PostActivityData, PostActivityError, PostActivityErrors, PostActivityRequest, PostActivityResponse, PostActivityResponses, PostChannelBotWebhookData, PostChannelBotWebhookError, PostChannelBotWebhookErrors, PostChannelBotWebhookResponse, PostChannelBotWebhookResponses, PostChannelMessagesData, PostChannelMessagesError, PostChannelMessagesErrors, PostChannelMessagesResponse, PostChannelMessagesResponses, PostGroupedSoupAstGroupPageRequest, PostGroupedSoupAstInitialRequest, PostGroupedSoupAstRequest, PostItemsSoupAstData, PostItemsSoupAstError, PostItemsSoupAstErrors, PostItemsSoupAstGroupedData, PostItemsSoupAstGroupedError, PostItemsSoupAstGroupedErrors, PostItemsSoupAstGroupedResponse, PostItemsSoupAstGroupedResponses, PostItemsSoupAstResponse, PostItemsSoupAstResponses, PostItemsSoupData, PostItemsSoupError, PostItemsSoupErrors, PostItemsSoupResponse, PostItemsSoupResponses, PostMessage, PostMessageData, PostMessageError, PostMessageErrors, PostMessageNotificationPolicy, PostMessageRequest, PostMessageResponse, PostMessageResponse2, PostMessageResponses, PostReactionData, PostReactionError, PostReactionErrors, PostReactionRequest, PostReactionResponse, PostReactionResponses, PostSoupAstRequest, PostSoupRequest, PostTypingData, PostTypingError, PostTypingErrors, PostTypingRequest, PostTypingResponse, PostTypingResponses, PresaveDocumentHandlerData, PresaveDocumentHandlerError, PresaveDocumentHandlerErrors, PresaveDocumentHandlerResponse, PresaveDocumentHandlerResponses, PreSaveDocumentRequest, PreSaveDocumentResponse, PreSaveDocumentResponseData, PresignedUrl, Project, ProjectDeleteResponseData, ProjectDocumentStatus, ProjectFilters, ProjectPreview, ProjectPreviewData, PropertyDefinition, PropertyFilter, PropertyInput, PropertyOwner, PropertyValue, PutCrmTeamSettingsData, PutCrmTeamSettingsError, PutCrmTeamSettingsErrors, PutCrmTeamSettingsResponse, PutCrmTeamSettingsResponses, PutCrmTeamStagesData, PutCrmTeamStagesError, PutCrmTeamStagesErrors, PutCrmTeamStagesResponse, PutCrmTeamStagesResponses, ReactionAction, ReactionInput, RecentlyDeletedData, RecentlyDeletedError, RecentlyDeletedErrors, RecentlyDeletedResponse, RecentlyDeletedResponseData, RecentlyDeletedResponses, ReferencedThread, ReferencedThreadPage, Reminder, ReminderFilters, ReminderSchedule, RemindersList, RemoveBotFromChannelByBotData, RemoveBotFromChannelByBotError, RemoveBotFromChannelByBotErrors, RemoveBotFromChannelByBotResponse, RemoveBotFromChannelByBotResponses, RemoveFavoriteByEntityData, RemoveFavoriteByEntityError, RemoveFavoriteByEntityErrors, RemoveFavoriteByEntityResponses, RemoveParticipantsData, RemoveParticipantsError, RemoveParticipantsErrors, RemoveParticipantsRequest, RemoveParticipantsResponses, RemovePinHandlerData, RemovePinHandlerError, RemovePinHandlerErrors, RemovePinHandlerResponse, RemovePinHandlerResponses, ReorderFavoritesData, ReorderFavoritesError, ReorderFavoritesErrors, ReorderFavoritesRequest, ReorderFavoritesResponses, ReorderPinRequest, ReorderPinsHandlerData, ReorderPinsHandlerError, ReorderPinsHandlerErrors, ReorderPinsHandlerResponse, ReorderPinsHandlerResponses, ReplaceCrmStagesRequest, RequestedHarnessScope, ResetCrmTeamStagesData, ResetCrmTeamStagesError, ResetCrmTeamStagesErrors, ResetCrmTeamStagesResponse, ResetCrmTeamStagesResponses, ResolveChannelMessageData, ResolveChannelMessageError, ResolveChannelMessageErrors, ResolveChannelMessageResponse, ResolveChannelMessageResponses, RevertDeleteDocumentData, RevertDeleteDocumentError, RevertDeleteDocumentErrors, RevertDeleteDocumentResponse, RevertDeleteDocumentResponses, RevertDeleteProjectData, RevertDeleteProjectError, RevertDeleteProjectErrors, RevertDeleteProjectResponse, RevertDeleteProjectResponses, RingStatus, RingStatusResponse, S3ObjectInfo, SaveBomPart, SaveDocumentHandlerData, SaveDocumentHandlerError, SaveDocumentHandlerErrors, SaveDocumentHandlerResponse, SaveDocumentHandlerResponses, SaveDocumentRequest, SaveDocumentResponse, SaveDocumentResponseData, SessionDeletedMetadata, SessionIdentity, SessionMentionedMetadata, SessionOpenedMetadata, SessionRenamedMetadata, SessionSettledMetadata, SessionStoppedMetadata, SetChannelPictureData, SetChannelPictureError, SetChannelPictureErrors, SetChannelPictureRequest, SetChannelPictureResponse, SetChannelPictureResponses, SetCompanyHiddenData, SetCompanyHiddenError, SetCompanyHiddenErrors, SetCompanyHiddenRequest, SetCompanyHiddenResponse, SetCompanyHiddenResponses, SetCompanyNameRequest, SetContactHiddenData, SetContactHiddenError, SetContactHiddenErrors, SetContactHiddenRequest, SetContactHiddenResponse, SetContactHiddenResponses, SetContactNameRequest, SetCrmCompanyNameData, SetCrmCompanyNameError, SetCrmCompanyNameErrors, SetCrmCompanyNameResponse, SetCrmCompanyNameResponses, SetCrmContactNameData, SetCrmContactNameError, SetCrmContactNameErrors, SetCrmContactNameResponse, SetCrmContactNameResponses, SetDocumentTeamShareData, SetDocumentTeamShareError, SetDocumentTeamShareErrors, SetDocumentTeamShareRequest, SetDocumentTeamShareResponse, SetDocumentTeamShareResponses, SetEmailSyncData, SetEmailSyncError, SetEmailSyncErrors, SetEmailSyncRequest, SetEmailSyncResponse, SetEmailSyncResponses, SetPropertyValue, SharedEmailFilter, SharePermissionV2, ShortIdResponse, SimpleMention, SimpleSaveData, SimpleSaveError, SimpleSaveErrors, SimpleSaveResponse, SimpleSaveResponses, SoupAgentSessionSoupPropertiesField, SoupApiItem, SoupApiSort, SoupApiSortDirection, SoupAttachment, SoupCalendarEventSoupPropertiesField, SoupCalendarEventTime, SoupCallRecordParticipant, SoupCallRecordSoupPropertiesField, SoupChannel, SoupChannelThread, SoupChatSoupPropertiesField, SoupContact, SoupCountedReaction, SoupCrmCompanySoupPropertiesField, SoupCrmDomain, SoupDocumentSoupPropertiesField, SoupDocumentSubType, SoupEmailThreadPreview, SoupEnrichedEmailThreadPreviewSoupPropertiesField, SoupForeignEntity, SoupItem, SoupLabel, SoupLabelListVisibility, SoupLabelType, SoupMessageAttachment, SoupMessageListVisibility, SoupMessageSender, SoupMessageSenderType, SoupPage, SoupProjectSoupPropertiesField, SoupPropertiesField, SoupProperty, SoupReminderReference, SoupReminderSchedule, SoupReminderSoupPropertiesField, SoupThreadInfo, SoupThreadReply, StarterDocumentsResponse, StreamEventsData, StreamEventsError, StreamEventsErrors, StreamEventsResponse, StreamEventsResponses, String, SuccessResponse, SurfaceState, SyncServiceVersionId, SystemSkillSummary, TagFilterMode, TaskFilters, TeamOutOfOfficeItem, TeamOutOfOfficeResponse, TeamRole, Thread, ThreadAnchor, ThreadOrigin, ThreadPatch, ThreadResponse, ThreadState, ToggleShareWithTeamData, ToggleShareWithTeamError, ToggleShareWithTeamErrors, ToggleShareWithTeamResponse, ToggleShareWithTeamResponses, TranscriptSegmentRequest, TurnEndedMetadata, TurnStartedMetadata, TurnSummary, TypedSuccessResponse, TypedSuccessResponseGetDocumentResponseData, TypingAction, TypingInput, UnassignInitiativeTaskData, UnassignInitiativeTaskError, UnassignInitiativeTaskErrors, UnassignInitiativeTaskResponse, UnassignInitiativeTaskResponses, UnthreadedPdfUuidRequest, UpdateAgentData, UpdateAgentError, UpdateAgentErrors, UpdateAgentRequest, UpdateAgentResponse, UpdateAgentResponses, UpdateChannelSharePermission, UpdateCrmTeamSettingsRequest, UpdateInitiativeData, UpdateInitiativeError, UpdateInitiativeErrors, UpdateInitiativeRequest, UpdateInitiativeResponse, UpdateInitiativeResponses, UpdateOperation, UpdateReminderData, UpdateReminderError, UpdateReminderErrors, UpdateReminderRequest, UpdateReminderResponse, UpdateReminderResponses, UpdateSharePermissionRequestV2, UploadDocumentStatus, UploadExtractFolderHandlerData, UploadExtractFolderHandlerError, UploadExtractFolderHandlerErrors, UploadExtractFolderHandlerResponse, UploadExtractFolderHandlerResponses, UploadExtractFolderRequest, UploadFolderHandlerData, UploadFolderHandlerError, UploadFolderHandlerErrors, UploadFolderHandlerResponse, UploadFolderHandlerResponses, UploadFolderRequest, UpsertHistoryHandlerData, UpsertHistoryHandlerError, UpsertHistoryHandlerErrors, UpsertHistoryHandlerResponse, UpsertHistoryHandlerResponses, UpsertUserDocumentViewLocationData, UpsertUserDocumentViewLocationError, UpsertUserDocumentViewLocationErrors, UpsertUserDocumentViewLocationRequest, UpsertUserDocumentViewLocationResponse, UpsertUserDocumentViewLocationResponses, UserApiKeyId, UserApiKeyInfo, UserApiKeysList, UserDocumentsResponse, UserDocumentViewLocation, UserDocumentViewLocationResponse, UserPinsResponse, UserViewsResponse, ValidateDocumentPermissionsTokenData, ValidateDocumentPermissionsTokenError, ValidateDocumentPermissionsTokenErrors, ValidateDocumentPermissionsTokenResponse, ValidateDocumentPermissionsTokenResponses, ValidateWebhookData, ValidateWebhookError, ValidateWebhookErrors, ValidateWebhookResponse, ValidateWebhookResponse2, ValidateWebhookResponses, Vec, View, ViewPatch, ViewsResponse, WaitingForInputMetadata, Webhook, WebhookEvent, WebhookFilter, WebhookScope, WebhookStatus, WebhookValidationTestEvent, WithCallId, WithChannelId, WithDocumentId, WithProjectId } from './types.gen'; +export type { AccessLevel, ActiveCallsResponse, ActiveCallSummary, ActivityType, AddFavoriteData, AddFavoriteError, AddFavoriteErrors, AddFavoriteRequest, AddFavoriteResponse, AddFavoriteResponses, AddParticipantsData, AddParticipantsError, AddParticipantsErrors, AddParticipantsRequest, AddParticipantsResponses, AddPinHandlerData, AddPinHandlerError, AddPinHandlerErrors, AddPinHandlerResponse, AddPinHandlerResponses, AddPinRequest, Agent, AgentActionId, AgentChannelScope, AgentMcpServer, AgentMcpServers, AgentSessionFilters, AgentSessionLifecycleEvent, Anchor, AnchorId, AnchorRequest, AnchorResponse, AnnotationIncrementalUpdate, ApiActivity, ApiAttachmentChannelReference, ApiAttachmentEntityReference, ApiAttachmentGenericReference, ApiChannelAttachment, ApiChannelAttachmentsPage, ApiChannelContextMessage, ApiChannelDetail, ApiChannelListMessage, ApiChannelListPage, ApiChannelListParticipant, ApiChannelListType, ApiChannelMessage, ApiChannelMessageKind, ApiChannelMessagesPage, ApiChannelParticipant, ApiChannelWithLatest, ApiCountedReaction, ApiEntityFilterAst, ApiGroupByField, ApiGroupMeta, ApiMessageAttachment, ApiMessageSender, ApiMessageSenderType, ApiParticipantListRole, ApiParticipantRole, ApiPropertyEntityType, ApiResolvedChannelMessage, ApiThreadInfo, ApiThreadReply, ApproveHarnessPairingData, ApproveHarnessPairingError, ApproveHarnessPairingErrors, ApproveHarnessPairingResponse, ApproveHarnessPairingResponses, ApprovePairingRequest, AssignInitiativeTasksData, AssignInitiativeTasksError, AssignInitiativeTasksErrors, AssignInitiativeTasksResponse, AssignInitiativeTasksResponses, AssignTasksRequest, AssignTasksResponse, AssignTasksResult, AssignTaskStatus, AttachmentChange, AttendeeResponseStatus, BasicDocument, BasicDocumentSubType, BomPart, Bot, BotChannel, BotChannelType, BotId, BotKind, BotOwner, BotSenderProfile, BotToken, BranchNameResponse, BTreeMap, BulkWakeupRequest, BulkWakeupResponse, BulkWakeupSyncServiceDocumentsData, BulkWakeupSyncServiceDocumentsErrors, BulkWakeupSyncServiceDocumentsResponse, BulkWakeupSyncServiceDocumentsResponses, CalendarAttendee, CalendarEvent, CalendarEventFilters, CalendarEventSourceContent, CalendarMentionEvent, CalendarMentionPreviewItem, CalendarMentionPreviewKind, CalendarMentionPreviewRequest, CalendarMentionPreviewRequestItem, CalendarMentionPreviewResponse, CalendarOccurrence, CalendarOccurrenceItem, CalendarOccurrenceResponse, CalendarSyncStatus, CallActiveResponse, CallFilters, CallRecord, CallRecordParticipant, CallRecordPreview, CallRecordPreviewData, CallRecordTranscriptSegment, CallStatus, CallTokenResponse, CallWebhookData, CallWebhookError, CallWebhookErrors, CallWebhookResponses, Channel, ChannelCreatedMetadata, ChannelDeletedMetadata, ChannelEventAttachment, ChannelFilters, ChannelJoinCodeResponse, ChannelMentionedMetadata, ChannelMessage, ChannelMessageAttachmentCreatedMetadata, ChannelMessageAttachmentRemovedMetadata, ChannelMessageDeletedMetadata, ChannelMessageFilters, ChannelMessagePatchedMetadata, ChannelMessagePostedMetadata, ChannelParticipant, ChannelParticipantAddedMetadata, ChannelParticipantRemovedMetadata, ChannelPreview, ChannelPreviewData, ChannelSender, ChannelSharePermission, ChannelThreadFilters, ChannelTopicEvent, ChannelType, ChannelUpdatedMetadata, ChannelWebhookRequest, ChannelWebhookResponse, ChannelWithParticipants, Chat, ChatFilters, CheckActiveCallData, CheckActiveCallError, CheckActiveCallErrors, CheckActiveCallResponse, CheckActiveCallResponses, ClaimedPairing, ClaimHarnessPairingData, ClaimHarnessPairingError, ClaimHarnessPairingErrors, ClaimHarnessPairingResponse, ClaimHarnessPairingResponses, ClaimPairingRequest, ClientOptions, CloudStorageItemType, CollabSurfaceResponse, CollabSurfaceTokenResponse, Comment, CommentThread, ConferenceProvider, CopyDocumentData, CopyDocumentError, CopyDocumentErrors, CopyDocumentQueryParams, CopyDocumentRequest, CopyDocumentResponse, CopyDocumentResponse2, CopyDocumentResponses, CountedReaction, CreateAgentData, CreateAgentError, CreateAgentErrors, CreateAgentRequest, CreateAgentResponse, CreateAgentResponses, CreateAnchorData, CreateAnchorError, CreateAnchorErrors, CreateAnchorResponse, CreateAnchorResponses, CreateBotRequest, CreateBulkDocumentResponse, CreateBulkDocumentResponseData, CreateChannelData, CreateChannelError, CreateChannelErrors, CreateChannelRequest, CreateChannelResponse, CreateChannelResponse2, CreateChannelResponses, CreateChannelScopedBotData, CreateChannelScopedBotError, CreateChannelScopedBotErrors, CreateChannelScopedBotRequest, CreateChannelScopedBotResponse, CreateChannelScopedBotResponse2, CreateChannelScopedBotResponses, CreateCollabSurfaceTokenData, CreateCollabSurfaceTokenError, CreateCollabSurfaceTokenErrors, CreateCollabSurfaceTokenResponse, CreateCollabSurfaceTokenResponses, CreateCommentData, CreateCommentError, CreateCommentErrors, CreateCommentRequest, CreateCommentResponse, CreateCommentResponse2, CreateCommentResponses, CreateCrmCommentData, CreateCrmCommentError, CreateCrmCommentErrors, CreateCrmCommentRequest, CreateCrmCommentResponse, CreateCrmCommentResponses, CreateCrmCompanyData, CreateCrmCompanyError, CreateCrmCompanyErrors, CreateCrmCompanyRequest, CreateCrmCompanyResponse, CreateCrmCompanyResponses, CreateCrmContactData, CreateCrmContactError, CreateCrmContactErrors, CreateCrmContactRequest, CreateCrmContactResponse, CreateCrmContactResponses, CreateDocumentData, CreateDocumentError, CreateDocumentErrors, CreateDocumentRequest, CreateDocumentResponse, CreateDocumentResponseData, CreateDocumentResponses, CreatedPairing, CreatedUserApiKey, CreateEntityMentionData, CreateEntityMentionError, CreateEntityMentionErrors, CreateEntityMentionRequest, CreateEntityMentionResponse, CreateEntityMentionResponse2, CreateEntityMentionResponses, CreateHarnessPairingData, CreateHarnessPairingError, CreateHarnessPairingErrors, CreateHarnessPairingResponse, CreateHarnessPairingResponses, CreateInitiativeData, CreateInitiativeError, CreateInitiativeErrors, CreateInitiativeRequest, CreateInitiativeResponse, CreateInitiativeResponses, CreateInstructionsDocumentResponse, CreateInstructionsHandlerData, CreateInstructionsHandlerError, CreateInstructionsHandlerErrors, CreateInstructionsHandlerResponse, CreateInstructionsHandlerResponses, CreateMarkdownDocumentRequest, CreateMarkdownDocumentResponse, CreateMarkdownHandlerData, CreateMarkdownHandlerError, CreateMarkdownHandlerErrors, CreateMarkdownHandlerResponse, CreateMarkdownHandlerResponses, CreatePairingRequest, CreateProjectHandlerData, CreateProjectHandlerError, CreateProjectHandlerErrors, CreateProjectHandlerResponse, CreateProjectHandlerResponses, CreateProjectRequest, CreateProjectResponse, CreateReminderData, CreateReminderError, CreateReminderErrors, CreateReminderRequest, CreateReminderResponse, CreateReminderResponses, CreateSkillHandlerData, CreateSkillHandlerError, CreateSkillHandlerErrors, CreateSkillHandlerResponse, CreateSkillHandlerResponses, CreateSkillRequest, CreateSkillResponse, CreateSnippetHandlerData, CreateSnippetHandlerError, CreateSnippetHandlerErrors, CreateSnippetHandlerResponse, CreateSnippetHandlerResponses, CreateSnippetRequest, CreateSnippetResponse, CreateTaskHandlerData, CreateTaskHandlerError, CreateTaskHandlerErrors, CreateTaskHandlerResponse, CreateTaskHandlerResponses, CreateTaskRequest, CreateTaskResponse, CreateUnthreadedAnchorRequest, CreateUnthreadedAnchorResponse, CreateUnthreadedPdfAnchorRequest, CreateUserApiKeyData, CreateUserApiKeyError, CreateUserApiKeyErrors, CreateUserApiKeyRequest, CreateUserApiKeyResponse, CreateUserApiKeyResponses, CreateViewHandlerData, CreateViewHandlerError, CreateViewHandlerErrors, CreateViewHandlerResponse, CreateViewHandlerResponses, CreateViewRequest, CreateWebhookData, CreateWebhookError, CreateWebhookErrors, CreateWebhookRequest, CreateWebhookResponse, CreateWebhookResponse2, CreateWebhookResponses, CrmComment, CrmCommentEntityType, CrmCommentThread, CrmCompanyFilters, CrmCompanyResponse, CrmContactResponse, CrmDomainResponse, CrmPermissionRole, CrmStageInput, CrmStageResponse, CrmStagesResponse, CrmTeamSettingsResponse, CrmThread, CustomSpeakerAssignment, DataType, DeleteAnchorData, DeleteAnchorError, DeleteAnchorErrors, DeleteAnchorInfo, DeleteAnchorResponse, DeleteAnchorResponses, DeleteCallRecordData, DeleteCallRecordError, DeleteCallRecordErrors, DeleteCallRecordResponse, DeleteCallRecordResponses, DeleteChannelData, DeleteChannelError, DeleteChannelErrors, DeleteChannelResponse, DeleteChannelResponses, DeleteCollabSurfaceData, DeleteCollabSurfaceError, DeleteCollabSurfaceErrors, DeleteCollabSurfaceResponse, DeleteCollabSurfaceResponses, DeleteCommentData, DeleteCommentError, DeleteCommentErrors, DeleteCommentRequest, DeleteCommentResponse, DeleteCommentResponse2, DeleteCommentResponses, DeleteCrmCommentData, DeleteCrmCommentError, DeleteCrmCommentErrors, DeleteCrmCommentResponse, DeleteCrmCommentResponses, DeleteCrmCommentResult, DeleteDocumentData, DeleteDocumentError, DeleteDocumentErrors, DeleteDocumentResponse, DeleteDocumentResponses, DeleteEntityMentionData, DeleteEntityMentionError, DeleteEntityMentionErrors, DeleteEntityMentionResponse, DeleteEntityMentionResponse2, DeleteEntityMentionResponses, DeleteHarnessData, DeleteHarnessError, DeleteHarnessErrors, DeleteHarnessResponse, DeleteHarnessResponses, DeleteHistoryHandlerData, DeleteHistoryHandlerError, DeleteHistoryHandlerErrors, DeleteHistoryHandlerResponse, DeleteHistoryHandlerResponses, DeleteInitiativeData, DeleteInitiativeError, DeleteInitiativeErrors, DeleteInitiativeResponse, DeleteInitiativeResponses, DeleteMessageData, DeleteMessageError, DeleteMessageErrors, DeleteMessageQuery, DeleteMessageResponse, DeleteMessageResponses, DeleteProjectHandlerData, DeleteProjectHandlerError, DeleteProjectHandlerErrors, DeleteProjectHandlerResponse, DeleteProjectHandlerResponses, DeleteReminderData, DeleteReminderError, DeleteReminderErrors, DeleteReminderResponse, DeleteReminderResponses, DeleteSelfHarnessData, DeleteSelfHarnessError, DeleteSelfHarnessErrors, DeleteSelfHarnessResponse, DeleteSelfHarnessResponses, DeleteThreadInfo, DeleteUnthreadedAnchorRequest, DeleteUnthreadedAnchorResponse, DeleteUnthreadedPdfAnchorRequest, DeleteUserApiKeyData, DeleteUserApiKeyError, DeleteUserApiKeyErrors, DeleteUserApiKeyResponse, DeleteUserApiKeyResponses, DeleteUserDocumentViewLocationData, DeleteUserDocumentViewLocationError, DeleteUserDocumentViewLocationErrors, DeleteUserDocumentViewLocationResponse, DeleteUserDocumentViewLocationResponses, DeleteViewHandlerData, DeleteViewHandlerError, DeleteViewHandlerErrors, DeleteViewHandlerResponses, DeleteWebhookData, DeleteWebhookError, DeleteWebhookErrors, DeleteWebhookResponse, DeleteWebhookResponses, DescriptionDocumentId, DocumentBasic, DocumentContent, DocumentContentLocation, DocumentContentState, DocumentContentUploadedMetadata, DocumentCopiedMetadata, DocumentCreatedMetadata, DocumentDeletedMetadata, DocumentFilters, DocumentId, DocumentInteractionMetadata, DocumentMetadata, DocumentMetadataWithContent, DocumentPermissionsToken, DocumentPermissionsTokenRequest, DocumentPermissionsTokenResponse, DocumentPreview, DocumentPreviewData, DocumentPreviewDataSubType, DocumentPurgedMetadata, DocumentResponse, DocumentResponseMetadata, DocumentResponseMetadataWithContent, DocumentStorageServiceApiVersion, DocumentSubType, DocumentSyncContentUpdatedMetadata, DocumentTeamShareResponse, DocumentTopicEvent, DocumentUpdatedMetadata, EditAnchorData, EditAnchorError, EditAnchorErrors, EditAnchorRequest, EditAnchorResponse, EditAnchorResponse2, EditAnchorResponses, EditCallRecordData, EditCallRecordError, EditCallRecordErrors, EditCallRecordRequest, EditCallRecordResponse, EditCallRecordResponses, EditCallTranscriptData, EditCallTranscriptError, EditCallTranscriptErrors, EditCallTranscriptRequest, EditCallTranscriptResponse, EditCallTranscriptResponses, EditCommentData, EditCommentError, EditCommentErrors, EditCommentRequest, EditCommentResponse, EditCommentResponse2, EditCommentResponses, EditCrmCommentData, EditCrmCommentError, EditCrmCommentErrors, EditCrmCommentRequest, EditCrmCommentResponse, EditCrmCommentResponses, EditDocumentData, EditDocumentError, EditDocumentErrors, EditDocumentResponse, EditDocumentResponse2, EditDocumentResponses, EditDocumentServiceArgs, EditPdfAnchorRequest, EditPdfPlaceableCommentAnchorRequest, EditProjectV2Data, EditProjectV2Error, EditProjectV2Errors, EditProjectV2Response, EditProjectV2Responses, EditThreadV2Data, EditThreadV2Error, EditThreadV2Errors, EditThreadV2Response, EditThreadV2Responses, EmailFilters, EmptyResponse, EnsureCollabSurfaceData, EnsureCollabSurfaceError, EnsureCollabSurfaceErrors, EnsureCollabSurfaceRequest, EnsureCollabSurfaceResponse, EnsureCollabSurfaceResponses, EntityFilters, EntityMessageCreateData, EntityMessageCreateResponse, EntityMessageCreateResponses, EntityMessageDeleteMessageData, EntityMessageDeleteMessageResponse, EntityMessageDeleteMessageResponses, EntityMessageDeleteThreadData, EntityMessageDeleteThreadResponse, EntityMessageDeleteThreadResponses, EntityMessageEditData, EntityMessageEditResponse, EntityMessageEditResponses, EntityMessageGetMessageData, EntityMessageGetMessageResponse, EntityMessageGetMessageResponses, EntityMessageGetThreadData, EntityMessageGetThreadResponse, EntityMessageGetThreadResponses, EntityMessageLegacyData, EntityMessageLegacyResponse, EntityMessageLegacyResponses, EntityMessagePatchThreadData, EntityMessagePatchThreadResponse, EntityMessagePatchThreadResponses, EntityMessageReactData, EntityMessageReactResponse, EntityMessageReactResponses, EntityMessageReferencesData, EntityMessageReferencesResponse, EntityMessageReferencesResponses, EntityMessageTypingData, EntityMessageTypingResponse, EntityMessageTypingResponses, EntityPermission, EntityPermissionResponse, EntityReference, EntityType, ErrorResponse, EventReminderOverride, EventReminders, EventStatus, EventTime, EventTransparency, EventType, EventVisibility, ExcludedDefaultView, ExcludeDefaultViewHandlerData, ExcludeDefaultViewHandlerError, ExcludeDefaultViewHandlerErrors, ExcludeDefaultViewHandlerResponses, ExcludeDefaultViewRequest, ExportDocumentData, ExportDocumentError, ExportDocumentErrors, ExportDocumentResponse, ExportDocumentResponse2, ExportDocumentResponses, Favorite, FavoriteEntityRef, FavoritesList, FileSystemNodeWithIds, FileType, FileTypeUpdate, FolderItem, ForeignEntity, ForeignEntityFilters, GenericErrorResponse, GenericResponse, GenericSuccessResponse, GetActiveCallsData, GetActiveCallsError, GetActiveCallsErrors, GetActiveCallsResponse, GetActiveCallsResponses, GetActivityData, GetActivityError, GetActivityErrors, GetActivityResponse, GetActivityResponses, GetAttachmentReferencesData, GetAttachmentReferencesError, GetAttachmentReferencesErrors, GetAttachmentReferencesResponse, GetAttachmentReferencesResponse2, GetAttachmentReferencesResponses, GetBatchCallRecordPreviewData, GetBatchCallRecordPreviewError, GetBatchCallRecordPreviewErrors, GetBatchCallRecordPreviewRequest, GetBatchCallRecordPreviewResponse, GetBatchCallRecordPreviewResponse2, GetBatchCallRecordPreviewResponses, GetBatchChannelPreviewData, GetBatchChannelPreviewError, GetBatchChannelPreviewErrors, GetBatchChannelPreviewRequest, GetBatchChannelPreviewResponse, GetBatchChannelPreviewResponse2, GetBatchChannelPreviewResponses, GetBatchPreviewHandlerData, GetBatchPreviewHandlerError, GetBatchPreviewHandlerErrors, GetBatchPreviewHandlerResponse, GetBatchPreviewHandlerResponses, GetBatchPreviewRequest, GetBatchPreviewResponse, GetBatchProjectPreviewData, GetBatchProjectPreviewError, GetBatchProjectPreviewErrors, GetBatchProjectPreviewRequest, GetBatchProjectPreviewResponse, GetBatchProjectPreviewResponse2, GetBatchProjectPreviewResponses, GetCallRecordData, GetCallRecordError, GetCallRecordErrors, GetCallRecordResponse, GetCallRecordResponses, GetChannelAttachmentsData, GetChannelAttachmentsError, GetChannelAttachmentsErrors, GetChannelAttachmentsResponse, GetChannelAttachmentsResponses, GetChannelData, GetChannelError, GetChannelErrors, GetChannelJoinLinkData, GetChannelJoinLinkError, GetChannelJoinLinkErrors, GetChannelJoinLinkResponse, GetChannelJoinLinkResponses, GetChannelMessagesCatchUpData, GetChannelMessagesCatchUpError, GetChannelMessagesCatchUpErrors, GetChannelMessagesCatchUpResponse, GetChannelMessagesCatchUpResponses, GetChannelMessagesData, GetChannelMessagesError, GetChannelMessagesErrors, GetChannelMessagesResponse, GetChannelMessagesResponses, GetChannelParticipantsData, GetChannelParticipantsError, GetChannelParticipantsErrors, GetChannelParticipantsResponse, GetChannelParticipantsResponses, GetChannelResponse, GetChannelResponses, GetChannelsData, GetChannelsError, GetChannelsErrors, GetChannelsResponse, GetChannelsResponses, GetCollabSurfaceData, GetCollabSurfaceError, GetCollabSurfaceErrors, GetCollabSurfaceResponse, GetCollabSurfaceResponses, GetCompanyData, GetCompanyError, GetCompanyErrors, GetCompanyResponse, GetCompanyResponses, GetContactByEmailData, GetContactByEmailError, GetContactByEmailErrors, GetContactByEmailResponse, GetContactByEmailResponse2, GetContactByEmailResponses, GetContactData, GetContactError, GetContactErrors, GetContactResponse, GetContactResponses, GetCrmTeamSettingsData, GetCrmTeamSettingsError, GetCrmTeamSettingsErrors, GetCrmTeamSettingsResponse, GetCrmTeamSettingsResponses, GetDocumentAnchorsData, GetDocumentAnchorsError, GetDocumentAnchorsErrors, GetDocumentAnchorsResponse, GetDocumentAnchorsResponses, GetDocumentBranchNameData, GetDocumentBranchNameError, GetDocumentBranchNameErrors, GetDocumentBranchNameResponse, GetDocumentBranchNameResponses, GetDocumentByTeamSlugData, GetDocumentByTeamSlugError, GetDocumentByTeamSlugErrors, GetDocumentByTeamSlugResponse, GetDocumentByTeamSlugResponses, GetDocumentCommentsData, GetDocumentCommentsError, GetDocumentCommentsErrors, GetDocumentCommentsResponse, GetDocumentCommentsResponses, GetDocumentData, GetDocumentError, GetDocumentErrors, GetDocumentGithubPullRequestsData, GetDocumentGithubPullRequestsError, GetDocumentGithubPullRequestsErrors, GetDocumentGithubPullRequestsResponse, GetDocumentGithubPullRequestsResponses, GetDocumentKeyResponse, GetDocumentKeyResponseData, GetDocumentListHandlerData, GetDocumentListHandlerError, GetDocumentListHandlerErrors, GetDocumentListHandlerResponse, GetDocumentListHandlerResponses, GetDocumentListResult, GetDocumentLocationV3Data, GetDocumentLocationV3Error, GetDocumentLocationV3Errors, GetDocumentLocationV3Response, GetDocumentLocationV3Responses, GetDocumentPermissionsResponseDataV2, GetDocumentPermissionsTokenData, GetDocumentPermissionsTokenError, GetDocumentPermissionsTokenErrors, GetDocumentPermissionsTokenResponse, GetDocumentPermissionsTokenResponses, GetDocumentPermissionsV2Data, GetDocumentPermissionsV2Error, GetDocumentPermissionsV2Errors, GetDocumentPermissionsV2Response, GetDocumentPermissionsV2Responses, GetDocumentProcessingResult, GetDocumentProcessingResultData, GetDocumentProcessingResultError, GetDocumentProcessingResultErrors, GetDocumentProcessingResultResponse, GetDocumentProcessingResultResponse2, GetDocumentProcessingResultResponses, GetDocumentResponse, GetDocumentResponse2, GetDocumentResponseData, GetDocumentResponses, GetDocumentSearchResponse, GetDocumentShortIdData, GetDocumentShortIdError, GetDocumentShortIdErrors, GetDocumentShortIdResponse, GetDocumentShortIdResponses, GetDocumentsResponse, GetDocumentTeamShareData, GetDocumentTeamShareError, GetDocumentTeamShareErrors, GetDocumentTeamShareResponse, GetDocumentTeamShareResponses, GetDocumentUserAccessLevelResponse, GetDocumentVersionData, GetDocumentVersionError, GetDocumentVersionErrors, GetDocumentVersionResponse, GetDocumentVersionResponses, GetDocumentViewsHandlerData, GetDocumentViewsHandlerError, GetDocumentViewsHandlerErrors, GetDocumentViewsHandlerResponse, GetDocumentViewsHandlerResponses, GetEntityPermissionData, GetEntityPermissionErrors, GetEntityPermissionResponse, GetEntityPermissionResponses, GetForeignEntityBySourceData, GetForeignEntityBySourceError, GetForeignEntityBySourceErrors, GetForeignEntityBySourceResponse, GetForeignEntityBySourceResponses, GetForeignEntityData, GetForeignEntityError, GetForeignEntityErrors, GetForeignEntityResponse, GetForeignEntityResponses, GetHarnessPairingData, GetHarnessPairingError, GetHarnessPairingErrors, GetHarnessPairingResponse, GetHarnessPairingResponses, GetHistoryHandlerData, GetHistoryHandlerError, GetHistoryHandlerErrors, GetHistoryHandlerResponse, GetHistoryHandlerResponses, GetInitiativeData, GetInitiativeError, GetInitiativeErrors, GetInitiativeResponse, GetInitiativeResponses, GetInstructionsDocumentResponse, GetInstructionsHandlerData, GetInstructionsHandlerError, GetInstructionsHandlerErrors, GetInstructionsHandlerResponse, GetInstructionsHandlerResponses, GetItemsSoupData, GetItemsSoupError, GetItemsSoupErrors, GetItemsSoupResponse, GetItemsSoupResponses, GetLocationHandlerData, GetLocationHandlerError, GetLocationHandlerErrors, GetLocationHandlerResponse, GetLocationHandlerResponses, GetMessageWithContextData, GetMessageWithContextError, GetMessageWithContextErrors, GetMessageWithContextResponse, GetMessageWithContextResponse2, GetMessageWithContextResponses, GetOrCreateAction, GetOrCreateCallData, GetOrCreateCallError, GetOrCreateCallErrors, GetOrCreateCallResponse, GetOrCreateCallResponses, GetOrCreateChannelResponse, GetOrCreateDmData, GetOrCreateDmError, GetOrCreateDmErrors, GetOrCreateDmRequest, GetOrCreateDmResponse, GetOrCreateDmResponses, GetOrCreatePrivateData, GetOrCreatePrivateError, GetOrCreatePrivateErrors, GetOrCreatePrivateRequest, GetOrCreatePrivateResponse, GetOrCreatePrivateResponses, GetPendingProjectsHandlerData, GetPendingProjectsHandlerError, GetPendingProjectsHandlerErrors, GetPendingProjectsHandlerResponse, GetPendingProjectsHandlerResponses, GetPinsHandlerData, GetPinsHandlerError, GetPinsHandlerErrors, GetPinsHandlerResponse, GetPinsHandlerResponses, GetPinsResponse, GetProjectContentHandlerData, GetProjectContentHandlerError, GetProjectContentHandlerErrors, GetProjectContentHandlerResponse, GetProjectContentHandlerResponses, GetProjectContentResponse, GetProjectHandlerData, GetProjectHandlerError, GetProjectHandlerErrors, GetProjectHandlerResponse, GetProjectHandlerResponses, GetProjectPermissionsV2Data, GetProjectPermissionsV2Error, GetProjectPermissionsV2Errors, GetProjectPermissionsV2Response, GetProjectPermissionsV2Responses, GetProjectResponse, GetProjectResponseData, GetProjectsHandlerData, GetProjectsHandlerError, GetProjectsHandlerErrors, GetProjectsHandlerResponse, GetProjectsHandlerResponses, GetProjectsResponse, GetProjectUserAccessLevelData, GetProjectUserAccessLevelError, GetProjectUserAccessLevelErrors, GetProjectUserAccessLevelResponse, GetProjectUserAccessLevelResponses, GetReminderData, GetReminderError, GetReminderErrors, GetReminderResponse, GetReminderResponses, GetRingStatusData, GetRingStatusError, GetRingStatusErrors, GetRingStatusResponse, GetRingStatusResponses, GetSelfBotData, GetSelfBotError, GetSelfBotErrors, GetSelfBotResponse, GetSelfBotResponses, GetSelfHarnessData, GetSelfHarnessError, GetSelfHarnessErrors, GetSelfHarnessResponse, GetSelfHarnessResponses, GetSystemSkillsHandlerData, GetSystemSkillsHandlerError, GetSystemSkillsHandlerErrors, GetSystemSkillsHandlerResponse, GetSystemSkillsHandlerResponses, GetThreadRepliesData, GetThreadRepliesError, GetThreadRepliesErrors, GetThreadRepliesResponse, GetThreadRepliesResponses, GetUserDocumentsHandlerData, GetUserDocumentsHandlerError, GetUserDocumentsHandlerErrors, GetUserDocumentsHandlerResponse, GetUserDocumentsHandlerResponses, GetUserDocumentViewLocationData, GetUserDocumentViewLocationError, GetUserDocumentViewLocationErrors, GetUserDocumentViewLocationResponse, GetUserDocumentViewLocationResponses, GetUserHistoryResponse, GetViewsHandlerData, GetViewsHandlerError, GetViewsHandlerErrors, GetViewsHandlerResponse, GetViewsHandlerResponses, GetWebhookData, GetWebhookError, GetWebhookErrors, GetWebhookResponse, GetWebhookResponses, GithubPullRequest, GithubPullRequestCheckRun, GithubPullRequestComment, GithubPullRequestsResponse, GroupedInitialParams, GroupedPageParams, GroupedSoupGroupPage, GroupedSoupInitialPage, GroupedSoupPage, GroupedSoupSort, HandlerData, HandlerError, HandlerErrors, HandlerResponse, HandlerResponses, Harness, HarnessAgent, HarnessId, HarnessOwner, HarnessSession, HashMap, HealthHandlerData, HealthHandlerResponse, HealthHandlerResponses, HighlightType, ImportedAuthor, InFlightTurnSummary, IngestTranscriptData, IngestTranscriptError, IngestTranscriptErrors, IngestTranscriptResponses, InitializeUserDocumentsData, InitializeUserDocumentsError, InitializeUserDocumentsErrors, InitializeUserDocumentsResponse, InitializeUserDocumentsResponses, InitiativeDetail, InitiativeId, InitiativeList, InitiativeSummary, InputReceivedMetadata, InstallSyncData, InstallSyncErrors, InteractionReason, Item, ItemWithUserAccessLevel, JobProcessingResultHandlerData, JobProcessingResultHandlerError, JobProcessingResultHandlerErrors, JobProcessingResultHandlerResponse, JobProcessingResultHandlerResponses, JoinChannelByCodeData, JoinChannelByCodeError, JoinChannelByCodeErrors, JoinChannelByCodeResponses, JoinChannelData, JoinChannelError, JoinChannelErrors, JoinChannelResponses, LatestMessage, LeaveCallResponse, LeaveChannelData, LeaveChannelError, LeaveChannelErrors, LeaveChannelResponses, LeaveOrEndCallData, LeaveOrEndCallError, LeaveOrEndCallErrors, LeaveOrEndCallResponse, LeaveOrEndCallResponses, LinkShare, ListAgentsData, ListAgentsError, ListAgentsErrors, ListAgentsResponse, ListAgentsResponses, ListBotChannelsData, ListBotChannelsError, ListBotChannelsErrors, ListBotChannelsResponse, ListBotChannelsResponses, ListCompanyContactsData, ListCompanyContactsError, ListCompanyContactsErrors, ListCompanyContactsResponse, ListCompanyContactsResponses, ListCrmCommentsData, ListCrmCommentsError, ListCrmCommentsErrors, ListCrmCommentsResponse, ListCrmCommentsResponses, ListFavoritesData, ListFavoritesError, ListFavoritesErrors, ListFavoritesResponse, ListFavoritesResponses, ListHarnessAgentsData, ListHarnessAgentsError, ListHarnessAgentsErrors, ListHarnessAgentsResponse, ListHarnessAgentsResponses, ListHarnessesData, ListHarnessesError, ListHarnessesErrors, ListHarnessesResponse, ListHarnessesResponses, ListHarnessSessionsData, ListHarnessSessionsError, ListHarnessSessionsErrors, ListHarnessSessionsResponse, ListHarnessSessionsResponses, ListInitiativesData, ListInitiativesError, ListInitiativesErrors, ListInitiativesResponse, ListInitiativesResponses, ListOccurrencesData, ListOccurrencesErrors, ListOccurrencesResponse, ListOccurrencesResponses, ListRemindersData, ListRemindersError, ListRemindersErrors, ListRemindersResponse, ListRemindersResponses, ListTeamOutOfOfficeData, ListTeamOutOfOfficeErrors, ListTeamOutOfOfficeResponse, ListTeamOutOfOfficeResponses, ListUserApiKeysData, ListUserApiKeysError, ListUserApiKeysErrors, ListUserApiKeysResponse, ListUserApiKeysResponses, ListWebhooksData, ListWebhooksError, ListWebhooksErrors, ListWebhooksResponse, ListWebhooksResponse2, ListWebhooksResponses, LocationResponseData, LocationResponseV3, MacroUserIdStr, MentionPreviewsData, MentionPreviewsErrors, MentionPreviewsResponse, MentionPreviewsResponses, Mentions, Message, MessageAttachment, MessageChange, MessageCursor, MessageDirection, MessageEvent, MessageListItem, MessagePage, MessageParent, MessagePatch, MessageThread, MessageThreadPreview, MessageTimelineData, MessageTimelineQuery, MessageTimelineResponse, MessageTimelineResponses, NewAttachment, NewChannelAttachment, NewThreadAnchor, NotificationFilters, NotificationState, PairingDetails, Params, ParticipantRole, PatchBotRequest, PatchChannelData, PatchChannelError, PatchChannelErrors, PatchChannelRequest, PatchChannelResponse, PatchChannelResponses, PatchMessageData, PatchMessageError, PatchMessageErrors, PatchMessageNotificationPolicy, PatchMessageRequest, PatchMessageResponse, PatchMessageResponses, PatchProjectRequestV2, PatchThreadRequestV2, PatchViewHandlerData, PatchViewHandlerError, PatchViewHandlerErrors, PatchViewHandlerResponses, PatchWebhookData, PatchWebhookError, PatchWebhookErrors, PatchWebhookRequest, PatchWebhookResponse, PatchWebhookResponses, PdfAnchor, PdfAnchorId, PdfAnchorRequest, PdfHighlightAnchor, PdfHighlightAnchorRequest, PdfHighlightRect, PdfHighlightRectAnchorRequest, PdfPlaceableCommentAnchor, PdfPlaceableCommentAnchorRequest, PendingClaimResponse, PermanentlyDeleteDocumentData, PermanentlyDeleteDocumentError, PermanentlyDeleteDocumentErrors, PermanentlyDeleteDocumentResponse, PermanentlyDeleteDocumentResponses, PermanentlyDeleteProjectData, PermanentlyDeleteProjectError, PermanentlyDeleteProjectErrors, PermanentlyDeleteProjectResponse, PermanentlyDeleteProjectResponses, PinnedItem, PinRequest, PostActivityData, PostActivityError, PostActivityErrors, PostActivityRequest, PostActivityResponse, PostActivityResponses, PostChannelBotWebhookData, PostChannelBotWebhookError, PostChannelBotWebhookErrors, PostChannelBotWebhookResponse, PostChannelBotWebhookResponses, PostChannelMessagesData, PostChannelMessagesError, PostChannelMessagesErrors, PostChannelMessagesResponse, PostChannelMessagesResponses, PostGroupedSoupAstGroupPageRequest, PostGroupedSoupAstInitialRequest, PostGroupedSoupAstRequest, PostItemsSoupAstData, PostItemsSoupAstError, PostItemsSoupAstErrors, PostItemsSoupAstGroupedData, PostItemsSoupAstGroupedError, PostItemsSoupAstGroupedErrors, PostItemsSoupAstGroupedResponse, PostItemsSoupAstGroupedResponses, PostItemsSoupAstResponse, PostItemsSoupAstResponses, PostItemsSoupData, PostItemsSoupError, PostItemsSoupErrors, PostItemsSoupResponse, PostItemsSoupResponses, PostMessage, PostMessageData, PostMessageError, PostMessageErrors, PostMessageNotificationPolicy, PostMessageRequest, PostMessageResponse, PostMessageResponse2, PostMessageResponses, PostReactionData, PostReactionError, PostReactionErrors, PostReactionRequest, PostReactionResponse, PostReactionResponses, PostSoupAstRequest, PostSoupRequest, PostTypingData, PostTypingError, PostTypingErrors, PostTypingRequest, PostTypingResponse, PostTypingResponses, PresaveDocumentHandlerData, PresaveDocumentHandlerError, PresaveDocumentHandlerErrors, PresaveDocumentHandlerResponse, PresaveDocumentHandlerResponses, PreSaveDocumentRequest, PreSaveDocumentResponse, PreSaveDocumentResponseData, PresignedUrl, Project, ProjectDeleteResponseData, ProjectDocumentStatus, ProjectFilters, ProjectPreview, ProjectPreviewData, PropertyDefinition, PropertyFilter, PropertyInput, PropertyOwner, PropertyValue, PutCrmTeamSettingsData, PutCrmTeamSettingsError, PutCrmTeamSettingsErrors, PutCrmTeamSettingsResponse, PutCrmTeamSettingsResponses, PutCrmTeamStagesData, PutCrmTeamStagesError, PutCrmTeamStagesErrors, PutCrmTeamStagesResponse, PutCrmTeamStagesResponses, ReactionAction, ReactionInput, RecentlyDeletedData, RecentlyDeletedError, RecentlyDeletedErrors, RecentlyDeletedResponse, RecentlyDeletedResponseData, RecentlyDeletedResponses, ReferencedThread, ReferencedThreadPage, Reminder, ReminderFilters, ReminderSchedule, RemindersList, RemoveBotFromChannelByBotData, RemoveBotFromChannelByBotError, RemoveBotFromChannelByBotErrors, RemoveBotFromChannelByBotResponse, RemoveBotFromChannelByBotResponses, RemoveFavoriteByEntityData, RemoveFavoriteByEntityError, RemoveFavoriteByEntityErrors, RemoveFavoriteByEntityResponses, RemoveParticipantsData, RemoveParticipantsError, RemoveParticipantsErrors, RemoveParticipantsRequest, RemoveParticipantsResponses, RemovePinHandlerData, RemovePinHandlerError, RemovePinHandlerErrors, RemovePinHandlerResponse, RemovePinHandlerResponses, ReorderFavoritesData, ReorderFavoritesError, ReorderFavoritesErrors, ReorderFavoritesRequest, ReorderFavoritesResponses, ReorderPinRequest, ReorderPinsHandlerData, ReorderPinsHandlerError, ReorderPinsHandlerErrors, ReorderPinsHandlerResponse, ReorderPinsHandlerResponses, ReplaceCrmStagesRequest, RequestedHarnessScope, ResetCrmTeamStagesData, ResetCrmTeamStagesError, ResetCrmTeamStagesErrors, ResetCrmTeamStagesResponse, ResetCrmTeamStagesResponses, ResolveChannelMessageData, ResolveChannelMessageError, ResolveChannelMessageErrors, ResolveChannelMessageResponse, ResolveChannelMessageResponses, RevertDeleteDocumentData, RevertDeleteDocumentError, RevertDeleteDocumentErrors, RevertDeleteDocumentResponse, RevertDeleteDocumentResponses, RevertDeleteProjectData, RevertDeleteProjectError, RevertDeleteProjectErrors, RevertDeleteProjectResponse, RevertDeleteProjectResponses, RingStatus, RingStatusResponse, S3ObjectInfo, SaveBomPart, SaveDocumentHandlerData, SaveDocumentHandlerError, SaveDocumentHandlerErrors, SaveDocumentHandlerResponse, SaveDocumentHandlerResponses, SaveDocumentRequest, SaveDocumentResponse, SaveDocumentResponseData, SessionDeletedMetadata, SessionIdentity, SessionMentionedMetadata, SessionOpenedMetadata, SessionRenamedMetadata, SessionSettledMetadata, SessionStoppedMetadata, SetChannelPictureData, SetChannelPictureError, SetChannelPictureErrors, SetChannelPictureRequest, SetChannelPictureResponse, SetChannelPictureResponses, SetCompanyHiddenData, SetCompanyHiddenError, SetCompanyHiddenErrors, SetCompanyHiddenRequest, SetCompanyHiddenResponse, SetCompanyHiddenResponses, SetCompanyNameRequest, SetContactHiddenData, SetContactHiddenError, SetContactHiddenErrors, SetContactHiddenRequest, SetContactHiddenResponse, SetContactHiddenResponses, SetContactNameRequest, SetCrmCompanyNameData, SetCrmCompanyNameError, SetCrmCompanyNameErrors, SetCrmCompanyNameResponse, SetCrmCompanyNameResponses, SetCrmContactNameData, SetCrmContactNameError, SetCrmContactNameErrors, SetCrmContactNameResponse, SetCrmContactNameResponses, SetDocumentTeamShareData, SetDocumentTeamShareError, SetDocumentTeamShareErrors, SetDocumentTeamShareRequest, SetDocumentTeamShareResponse, SetDocumentTeamShareResponses, SetEmailSyncData, SetEmailSyncError, SetEmailSyncErrors, SetEmailSyncRequest, SetEmailSyncResponse, SetEmailSyncResponses, SetPropertyValue, SharedEmailFilter, SharePermissionV2, ShortIdResponse, SimpleMention, SimpleSaveData, SimpleSaveError, SimpleSaveErrors, SimpleSaveResponse, SimpleSaveResponses, SoupAgentSessionSoupPropertiesField, SoupApiItem, SoupApiSort, SoupApiSortDirection, SoupAttachment, SoupCalendarEventSoupPropertiesField, SoupCalendarEventTime, SoupCallRecordParticipant, SoupCallRecordSoupPropertiesField, SoupChannel, SoupChannelThread, SoupChatSoupPropertiesField, SoupContact, SoupCountedReaction, SoupCrmCompanySoupPropertiesField, SoupCrmDomain, SoupDocumentSoupPropertiesField, SoupDocumentSubType, SoupEmailThreadPreview, SoupEnrichedEmailThreadPreviewSoupPropertiesField, SoupForeignEntity, SoupItem, SoupLabel, SoupLabelListVisibility, SoupLabelType, SoupMessageAttachment, SoupMessageListVisibility, SoupMessageSender, SoupMessageSenderType, SoupPage, SoupProjectSoupPropertiesField, SoupPropertiesField, SoupProperty, SoupReminderReference, SoupReminderSchedule, SoupReminderSoupPropertiesField, SoupThreadInfo, SoupThreadReply, StarterDocumentsResponse, StreamEventsData, StreamEventsError, StreamEventsErrors, StreamEventsResponse, StreamEventsResponses, String, SuccessResponse, SurfaceState, SyncServiceVersionId, SystemSkillSummary, TagFilterMode, TaskFilters, TeamOutOfOfficeItem, TeamOutOfOfficeResponse, TeamRole, Thread, ThreadAnchor, ThreadOrigin, ThreadPatch, ThreadResponse, ThreadState, ToggleShareWithTeamData, ToggleShareWithTeamError, ToggleShareWithTeamErrors, ToggleShareWithTeamResponse, ToggleShareWithTeamResponses, TranscribeDictationData, TranscribeDictationError, TranscribeDictationErrors, TranscribeDictationResponse, TranscribeDictationResponses, TranscribeResponse, TranscriptSegmentRequest, TurnEndedMetadata, TurnStartedMetadata, TurnSummary, TypedSuccessResponse, TypedSuccessResponseGetDocumentResponseData, TypingAction, TypingInput, UnassignInitiativeTaskData, UnassignInitiativeTaskError, UnassignInitiativeTaskErrors, UnassignInitiativeTaskResponse, UnassignInitiativeTaskResponses, UnthreadedPdfUuidRequest, UpdateAgentData, UpdateAgentError, UpdateAgentErrors, UpdateAgentRequest, UpdateAgentResponse, UpdateAgentResponses, UpdateChannelSharePermission, UpdateCrmTeamSettingsRequest, UpdateInitiativeData, UpdateInitiativeError, UpdateInitiativeErrors, UpdateInitiativeRequest, UpdateInitiativeResponse, UpdateInitiativeResponses, UpdateOperation, UpdateReminderData, UpdateReminderError, UpdateReminderErrors, UpdateReminderRequest, UpdateReminderResponse, UpdateReminderResponses, UpdateSharePermissionRequestV2, UploadDocumentStatus, UploadExtractFolderHandlerData, UploadExtractFolderHandlerError, UploadExtractFolderHandlerErrors, UploadExtractFolderHandlerResponse, UploadExtractFolderHandlerResponses, UploadExtractFolderRequest, UploadFolderHandlerData, UploadFolderHandlerError, UploadFolderHandlerErrors, UploadFolderHandlerResponse, UploadFolderHandlerResponses, UploadFolderRequest, UpsertHistoryHandlerData, UpsertHistoryHandlerError, UpsertHistoryHandlerErrors, UpsertHistoryHandlerResponse, UpsertHistoryHandlerResponses, UpsertUserDocumentViewLocationData, UpsertUserDocumentViewLocationError, UpsertUserDocumentViewLocationErrors, UpsertUserDocumentViewLocationRequest, UpsertUserDocumentViewLocationResponse, UpsertUserDocumentViewLocationResponses, UserApiKeyId, UserApiKeyInfo, UserApiKeysList, UserDocumentsResponse, UserDocumentViewLocation, UserDocumentViewLocationResponse, UserPinsResponse, UserViewsResponse, ValidateDocumentPermissionsTokenData, ValidateDocumentPermissionsTokenError, ValidateDocumentPermissionsTokenErrors, ValidateDocumentPermissionsTokenResponse, ValidateDocumentPermissionsTokenResponses, ValidateWebhookData, ValidateWebhookError, ValidateWebhookErrors, ValidateWebhookResponse, ValidateWebhookResponse2, ValidateWebhookResponses, Vec, View, ViewPatch, ViewsResponse, WaitingForInputMetadata, Webhook, WebhookEvent, WebhookFilter, WebhookScope, WebhookStatus, WebhookValidationTestEvent, WithCallId, WithChannelId, WithDocumentId, WithProjectId } from './types.gen'; diff --git a/packages/sdk/generated/storage/sdk.gen.ts b/packages/sdk/generated/storage/sdk.gen.ts index 183a9a840a2..630fd412c71 100644 --- a/packages/sdk/generated/storage/sdk.gen.ts +++ b/packages/sdk/generated/storage/sdk.gen.ts @@ -2,7 +2,7 @@ import type { Client, ClientMeta, Options as Options2, RequestResult, ServerSentEventsResult, TDataShape } from './client'; import { client } from './client.gen'; -import type { AddFavoriteData, AddFavoriteErrors, AddFavoriteResponses, AddParticipantsData, AddParticipantsErrors, AddParticipantsResponses, AddPinHandlerData, AddPinHandlerErrors, AddPinHandlerResponses, ApproveHarnessPairingData, ApproveHarnessPairingErrors, ApproveHarnessPairingResponses, AssignInitiativeTasksData, AssignInitiativeTasksErrors, AssignInitiativeTasksResponses, BulkWakeupSyncServiceDocumentsData, BulkWakeupSyncServiceDocumentsErrors, BulkWakeupSyncServiceDocumentsResponses, CallWebhookData, CallWebhookErrors, CallWebhookResponses, CheckActiveCallData, CheckActiveCallErrors, CheckActiveCallResponses, ClaimHarnessPairingData, ClaimHarnessPairingErrors, ClaimHarnessPairingResponses, CopyDocumentData, CopyDocumentErrors, CopyDocumentResponses, CreateAgentData, CreateAgentErrors, CreateAgentResponses, CreateAnchorData, CreateAnchorErrors, CreateAnchorResponses, CreateChannelData, CreateChannelErrors, CreateChannelResponses, CreateChannelScopedBotData, CreateChannelScopedBotErrors, CreateChannelScopedBotResponses, CreateCollabSurfaceTokenData, CreateCollabSurfaceTokenErrors, CreateCollabSurfaceTokenResponses, CreateCommentData, CreateCommentErrors, CreateCommentResponses, CreateCrmCommentData, CreateCrmCommentErrors, CreateCrmCommentResponses, CreateCrmCompanyData, CreateCrmCompanyErrors, CreateCrmCompanyResponses, CreateCrmContactData, CreateCrmContactErrors, CreateCrmContactResponses, CreateDocumentData, CreateDocumentErrors, CreateDocumentResponses, CreateEntityMentionData, CreateEntityMentionErrors, CreateEntityMentionResponses, CreateHarnessPairingData, CreateHarnessPairingErrors, CreateHarnessPairingResponses, CreateInitiativeData, CreateInitiativeErrors, CreateInitiativeResponses, CreateInstructionsHandlerData, CreateInstructionsHandlerErrors, CreateInstructionsHandlerResponses, CreateMarkdownHandlerData, CreateMarkdownHandlerErrors, CreateMarkdownHandlerResponses, CreateProjectHandlerData, CreateProjectHandlerErrors, CreateProjectHandlerResponses, CreateReminderData, CreateReminderErrors, CreateReminderResponses, CreateSkillHandlerData, CreateSkillHandlerErrors, CreateSkillHandlerResponses, CreateSnippetHandlerData, CreateSnippetHandlerErrors, CreateSnippetHandlerResponses, CreateTaskHandlerData, CreateTaskHandlerErrors, CreateTaskHandlerResponses, CreateUserApiKeyData, CreateUserApiKeyErrors, CreateUserApiKeyResponses, CreateViewHandlerData, CreateViewHandlerErrors, CreateViewHandlerResponses, CreateWebhookData, CreateWebhookErrors, CreateWebhookResponses, DeleteAnchorData, DeleteAnchorErrors, DeleteAnchorResponses, DeleteCallRecordData, DeleteCallRecordErrors, DeleteCallRecordResponses, DeleteChannelData, DeleteChannelErrors, DeleteChannelResponses, DeleteCollabSurfaceData, DeleteCollabSurfaceErrors, DeleteCollabSurfaceResponses, DeleteCommentData, DeleteCommentErrors, DeleteCommentResponses, DeleteCrmCommentData, DeleteCrmCommentErrors, DeleteCrmCommentResponses, DeleteDocumentData, DeleteDocumentErrors, DeleteDocumentResponses, DeleteEntityMentionData, DeleteEntityMentionErrors, DeleteEntityMentionResponses, DeleteHarnessData, DeleteHarnessErrors, DeleteHarnessResponses, DeleteHistoryHandlerData, DeleteHistoryHandlerErrors, DeleteHistoryHandlerResponses, DeleteInitiativeData, DeleteInitiativeErrors, DeleteInitiativeResponses, DeleteMessageData, DeleteMessageErrors, DeleteMessageResponses, DeleteProjectHandlerData, DeleteProjectHandlerErrors, DeleteProjectHandlerResponses, DeleteReminderData, DeleteReminderErrors, DeleteReminderResponses, DeleteSelfHarnessData, DeleteSelfHarnessErrors, DeleteSelfHarnessResponses, DeleteUserApiKeyData, DeleteUserApiKeyErrors, DeleteUserApiKeyResponses, DeleteUserDocumentViewLocationData, DeleteUserDocumentViewLocationErrors, DeleteUserDocumentViewLocationResponses, DeleteViewHandlerData, DeleteViewHandlerErrors, DeleteViewHandlerResponses, DeleteWebhookData, DeleteWebhookErrors, DeleteWebhookResponses, EditAnchorData, EditAnchorErrors, EditAnchorResponses, EditCallRecordData, EditCallRecordErrors, EditCallRecordResponses, EditCallTranscriptData, EditCallTranscriptErrors, EditCallTranscriptResponses, EditCommentData, EditCommentErrors, EditCommentResponses, EditCrmCommentData, EditCrmCommentErrors, EditCrmCommentResponses, EditDocumentData, EditDocumentErrors, EditDocumentResponses, EditProjectV2Data, EditProjectV2Errors, EditProjectV2Responses, EditThreadV2Data, EditThreadV2Errors, EditThreadV2Responses, EnsureCollabSurfaceData, EnsureCollabSurfaceErrors, EnsureCollabSurfaceResponses, EntityMessageCreateData, EntityMessageCreateResponses, EntityMessageDeleteMessageData, EntityMessageDeleteMessageResponses, EntityMessageDeleteThreadData, EntityMessageDeleteThreadResponses, EntityMessageEditData, EntityMessageEditResponses, EntityMessageGetMessageData, EntityMessageGetMessageResponses, EntityMessageGetThreadData, EntityMessageGetThreadResponses, EntityMessageLegacyData, EntityMessageLegacyResponses, EntityMessagePatchThreadData, EntityMessagePatchThreadResponses, EntityMessageReactData, EntityMessageReactResponses, EntityMessageReferencesData, EntityMessageReferencesResponses, EntityMessageTypingData, EntityMessageTypingResponses, ExcludeDefaultViewHandlerData, ExcludeDefaultViewHandlerErrors, ExcludeDefaultViewHandlerResponses, ExportDocumentData, ExportDocumentErrors, ExportDocumentResponses, GetActiveCallsData, GetActiveCallsErrors, GetActiveCallsResponses, GetActivityData, GetActivityErrors, GetActivityResponses, GetAttachmentReferencesData, GetAttachmentReferencesErrors, GetAttachmentReferencesResponses, GetBatchCallRecordPreviewData, GetBatchCallRecordPreviewErrors, GetBatchCallRecordPreviewResponses, GetBatchChannelPreviewData, GetBatchChannelPreviewErrors, GetBatchChannelPreviewResponses, GetBatchPreviewHandlerData, GetBatchPreviewHandlerErrors, GetBatchPreviewHandlerResponses, GetBatchProjectPreviewData, GetBatchProjectPreviewErrors, GetBatchProjectPreviewResponses, GetCallRecordData, GetCallRecordErrors, GetCallRecordResponses, GetChannelAttachmentsData, GetChannelAttachmentsErrors, GetChannelAttachmentsResponses, GetChannelData, GetChannelErrors, GetChannelJoinLinkData, GetChannelJoinLinkErrors, GetChannelJoinLinkResponses, GetChannelMessagesCatchUpData, GetChannelMessagesCatchUpErrors, GetChannelMessagesCatchUpResponses, GetChannelMessagesData, GetChannelMessagesErrors, GetChannelMessagesResponses, GetChannelParticipantsData, GetChannelParticipantsErrors, GetChannelParticipantsResponses, GetChannelResponses, GetChannelsData, GetChannelsErrors, GetChannelsResponses, GetCollabSurfaceData, GetCollabSurfaceErrors, GetCollabSurfaceResponses, GetCompanyData, GetCompanyErrors, GetCompanyResponses, GetContactByEmailData, GetContactByEmailErrors, GetContactByEmailResponses, GetContactData, GetContactErrors, GetContactResponses, GetCrmTeamSettingsData, GetCrmTeamSettingsErrors, GetCrmTeamSettingsResponses, GetDocumentAnchorsData, GetDocumentAnchorsErrors, GetDocumentAnchorsResponses, GetDocumentBranchNameData, GetDocumentBranchNameErrors, GetDocumentBranchNameResponses, GetDocumentByTeamSlugData, GetDocumentByTeamSlugErrors, GetDocumentByTeamSlugResponses, GetDocumentCommentsData, GetDocumentCommentsErrors, GetDocumentCommentsResponses, GetDocumentData, GetDocumentErrors, GetDocumentGithubPullRequestsData, GetDocumentGithubPullRequestsErrors, GetDocumentGithubPullRequestsResponses, GetDocumentListHandlerData, GetDocumentListHandlerErrors, GetDocumentListHandlerResponses, GetDocumentLocationV3Data, GetDocumentLocationV3Errors, GetDocumentLocationV3Responses, GetDocumentPermissionsTokenData, GetDocumentPermissionsTokenErrors, GetDocumentPermissionsTokenResponses, GetDocumentPermissionsV2Data, GetDocumentPermissionsV2Errors, GetDocumentPermissionsV2Responses, GetDocumentProcessingResultData, GetDocumentProcessingResultErrors, GetDocumentProcessingResultResponses, GetDocumentResponses, GetDocumentShortIdData, GetDocumentShortIdErrors, GetDocumentShortIdResponses, GetDocumentTeamShareData, GetDocumentTeamShareErrors, GetDocumentTeamShareResponses, GetDocumentVersionData, GetDocumentVersionErrors, GetDocumentVersionResponses, GetDocumentViewsHandlerData, GetDocumentViewsHandlerErrors, GetDocumentViewsHandlerResponses, GetEntityPermissionData, GetEntityPermissionErrors, GetEntityPermissionResponses, GetForeignEntityBySourceData, GetForeignEntityBySourceErrors, GetForeignEntityBySourceResponses, GetForeignEntityData, GetForeignEntityErrors, GetForeignEntityResponses, GetHarnessPairingData, GetHarnessPairingErrors, GetHarnessPairingResponses, GetHistoryHandlerData, GetHistoryHandlerErrors, GetHistoryHandlerResponses, GetInitiativeData, GetInitiativeErrors, GetInitiativeResponses, GetInstructionsHandlerData, GetInstructionsHandlerErrors, GetInstructionsHandlerResponses, GetItemsSoupData, GetItemsSoupErrors, GetItemsSoupResponses, GetLocationHandlerData, GetLocationHandlerErrors, GetLocationHandlerResponses, GetMessageWithContextData, GetMessageWithContextErrors, GetMessageWithContextResponses, GetOrCreateCallData, GetOrCreateCallErrors, GetOrCreateCallResponses, GetOrCreateDmData, GetOrCreateDmErrors, GetOrCreateDmResponses, GetOrCreatePrivateData, GetOrCreatePrivateErrors, GetOrCreatePrivateResponses, GetPendingProjectsHandlerData, GetPendingProjectsHandlerErrors, GetPendingProjectsHandlerResponses, GetPinsHandlerData, GetPinsHandlerErrors, GetPinsHandlerResponses, GetProjectContentHandlerData, GetProjectContentHandlerErrors, GetProjectContentHandlerResponses, GetProjectHandlerData, GetProjectHandlerErrors, GetProjectHandlerResponses, GetProjectPermissionsV2Data, GetProjectPermissionsV2Errors, GetProjectPermissionsV2Responses, GetProjectsHandlerData, GetProjectsHandlerErrors, GetProjectsHandlerResponses, GetProjectUserAccessLevelData, GetProjectUserAccessLevelErrors, GetProjectUserAccessLevelResponses, GetReminderData, GetReminderErrors, GetReminderResponses, GetRingStatusData, GetRingStatusErrors, GetRingStatusResponses, GetSelfBotData, GetSelfBotErrors, GetSelfBotResponses, GetSelfHarnessData, GetSelfHarnessErrors, GetSelfHarnessResponses, GetSystemSkillsHandlerData, GetSystemSkillsHandlerErrors, GetSystemSkillsHandlerResponses, GetThreadRepliesData, GetThreadRepliesErrors, GetThreadRepliesResponses, GetUserDocumentsHandlerData, GetUserDocumentsHandlerErrors, GetUserDocumentsHandlerResponses, GetUserDocumentViewLocationData, GetUserDocumentViewLocationErrors, GetUserDocumentViewLocationResponses, GetViewsHandlerData, GetViewsHandlerErrors, GetViewsHandlerResponses, GetWebhookData, GetWebhookErrors, GetWebhookResponses, HandlerData, HandlerErrors, HandlerResponses, HealthHandlerData, HealthHandlerResponses, IngestTranscriptData, IngestTranscriptErrors, IngestTranscriptResponses, InitializeUserDocumentsData, InitializeUserDocumentsErrors, InitializeUserDocumentsResponses, InstallSyncData, InstallSyncErrors, JobProcessingResultHandlerData, JobProcessingResultHandlerErrors, JobProcessingResultHandlerResponses, JoinChannelByCodeData, JoinChannelByCodeErrors, JoinChannelByCodeResponses, JoinChannelData, JoinChannelErrors, JoinChannelResponses, LeaveChannelData, LeaveChannelErrors, LeaveChannelResponses, LeaveOrEndCallData, LeaveOrEndCallErrors, LeaveOrEndCallResponses, ListAgentsData, ListAgentsErrors, ListAgentsResponses, ListBotChannelsData, ListBotChannelsErrors, ListBotChannelsResponses, ListCompanyContactsData, ListCompanyContactsErrors, ListCompanyContactsResponses, ListCrmCommentsData, ListCrmCommentsErrors, ListCrmCommentsResponses, ListFavoritesData, ListFavoritesErrors, ListFavoritesResponses, ListHarnessAgentsData, ListHarnessAgentsErrors, ListHarnessAgentsResponses, ListHarnessesData, ListHarnessesErrors, ListHarnessesResponses, ListHarnessSessionsData, ListHarnessSessionsErrors, ListHarnessSessionsResponses, ListInitiativesData, ListInitiativesErrors, ListInitiativesResponses, ListOccurrencesData, ListOccurrencesErrors, ListOccurrencesResponses, ListRemindersData, ListRemindersErrors, ListRemindersResponses, ListTeamOutOfOfficeData, ListTeamOutOfOfficeErrors, ListTeamOutOfOfficeResponses, ListUserApiKeysData, ListUserApiKeysErrors, ListUserApiKeysResponses, ListWebhooksData, ListWebhooksErrors, ListWebhooksResponses, MentionPreviewsData, MentionPreviewsErrors, MentionPreviewsResponses, MessageTimelineData, MessageTimelineResponses, PatchChannelData, PatchChannelErrors, PatchChannelResponses, PatchMessageData, PatchMessageErrors, PatchMessageResponses, PatchViewHandlerData, PatchViewHandlerErrors, PatchViewHandlerResponses, PatchWebhookData, PatchWebhookErrors, PatchWebhookResponses, PermanentlyDeleteDocumentData, PermanentlyDeleteDocumentErrors, PermanentlyDeleteDocumentResponses, PermanentlyDeleteProjectData, PermanentlyDeleteProjectErrors, PermanentlyDeleteProjectResponses, PostActivityData, PostActivityErrors, PostActivityResponses, PostChannelBotWebhookData, PostChannelBotWebhookErrors, PostChannelBotWebhookResponses, PostChannelMessagesData, PostChannelMessagesErrors, PostChannelMessagesResponses, PostItemsSoupAstData, PostItemsSoupAstErrors, PostItemsSoupAstGroupedData, PostItemsSoupAstGroupedErrors, PostItemsSoupAstGroupedResponses, PostItemsSoupAstResponses, PostItemsSoupData, PostItemsSoupErrors, PostItemsSoupResponses, PostMessageData, PostMessageErrors, PostMessageResponses, PostReactionData, PostReactionErrors, PostReactionResponses, PostTypingData, PostTypingErrors, PostTypingResponses, PresaveDocumentHandlerData, PresaveDocumentHandlerErrors, PresaveDocumentHandlerResponses, PutCrmTeamSettingsData, PutCrmTeamSettingsErrors, PutCrmTeamSettingsResponses, PutCrmTeamStagesData, PutCrmTeamStagesErrors, PutCrmTeamStagesResponses, RecentlyDeletedData, RecentlyDeletedErrors, RecentlyDeletedResponses, RemoveBotFromChannelByBotData, RemoveBotFromChannelByBotErrors, RemoveBotFromChannelByBotResponses, RemoveFavoriteByEntityData, RemoveFavoriteByEntityErrors, RemoveFavoriteByEntityResponses, RemoveParticipantsData, RemoveParticipantsErrors, RemoveParticipantsResponses, RemovePinHandlerData, RemovePinHandlerErrors, RemovePinHandlerResponses, ReorderFavoritesData, ReorderFavoritesErrors, ReorderFavoritesResponses, ReorderPinsHandlerData, ReorderPinsHandlerErrors, ReorderPinsHandlerResponses, ResetCrmTeamStagesData, ResetCrmTeamStagesErrors, ResetCrmTeamStagesResponses, ResolveChannelMessageData, ResolveChannelMessageErrors, ResolveChannelMessageResponses, RevertDeleteDocumentData, RevertDeleteDocumentErrors, RevertDeleteDocumentResponses, RevertDeleteProjectData, RevertDeleteProjectErrors, RevertDeleteProjectResponses, SaveDocumentHandlerData, SaveDocumentHandlerErrors, SaveDocumentHandlerResponses, SetChannelPictureData, SetChannelPictureErrors, SetChannelPictureResponses, SetCompanyHiddenData, SetCompanyHiddenErrors, SetCompanyHiddenResponses, SetContactHiddenData, SetContactHiddenErrors, SetContactHiddenResponses, SetCrmCompanyNameData, SetCrmCompanyNameErrors, SetCrmCompanyNameResponses, SetCrmContactNameData, SetCrmContactNameErrors, SetCrmContactNameResponses, SetDocumentTeamShareData, SetDocumentTeamShareErrors, SetDocumentTeamShareResponses, SetEmailSyncData, SetEmailSyncErrors, SetEmailSyncResponses, SimpleSaveData, SimpleSaveErrors, SimpleSaveResponses, StreamEventsData, StreamEventsErrors, StreamEventsResponse, StreamEventsResponses, ToggleShareWithTeamData, ToggleShareWithTeamErrors, ToggleShareWithTeamResponses, UnassignInitiativeTaskData, UnassignInitiativeTaskErrors, UnassignInitiativeTaskResponses, UpdateAgentData, UpdateAgentErrors, UpdateAgentResponses, UpdateInitiativeData, UpdateInitiativeErrors, UpdateInitiativeResponses, UpdateReminderData, UpdateReminderErrors, UpdateReminderResponses, UploadExtractFolderHandlerData, UploadExtractFolderHandlerErrors, UploadExtractFolderHandlerResponses, UploadFolderHandlerData, UploadFolderHandlerErrors, UploadFolderHandlerResponses, UpsertHistoryHandlerData, UpsertHistoryHandlerErrors, UpsertHistoryHandlerResponses, UpsertUserDocumentViewLocationData, UpsertUserDocumentViewLocationErrors, UpsertUserDocumentViewLocationResponses, ValidateDocumentPermissionsTokenData, ValidateDocumentPermissionsTokenErrors, ValidateDocumentPermissionsTokenResponses, ValidateWebhookData, ValidateWebhookErrors, ValidateWebhookResponses } from './types.gen'; +import type { AddFavoriteData, AddFavoriteErrors, AddFavoriteResponses, AddParticipantsData, AddParticipantsErrors, AddParticipantsResponses, AddPinHandlerData, AddPinHandlerErrors, AddPinHandlerResponses, ApproveHarnessPairingData, ApproveHarnessPairingErrors, ApproveHarnessPairingResponses, AssignInitiativeTasksData, AssignInitiativeTasksErrors, AssignInitiativeTasksResponses, BulkWakeupSyncServiceDocumentsData, BulkWakeupSyncServiceDocumentsErrors, BulkWakeupSyncServiceDocumentsResponses, CallWebhookData, CallWebhookErrors, CallWebhookResponses, CheckActiveCallData, CheckActiveCallErrors, CheckActiveCallResponses, ClaimHarnessPairingData, ClaimHarnessPairingErrors, ClaimHarnessPairingResponses, CopyDocumentData, CopyDocumentErrors, CopyDocumentResponses, CreateAgentData, CreateAgentErrors, CreateAgentResponses, CreateAnchorData, CreateAnchorErrors, CreateAnchorResponses, CreateChannelData, CreateChannelErrors, CreateChannelResponses, CreateChannelScopedBotData, CreateChannelScopedBotErrors, CreateChannelScopedBotResponses, CreateCollabSurfaceTokenData, CreateCollabSurfaceTokenErrors, CreateCollabSurfaceTokenResponses, CreateCommentData, CreateCommentErrors, CreateCommentResponses, CreateCrmCommentData, CreateCrmCommentErrors, CreateCrmCommentResponses, CreateCrmCompanyData, CreateCrmCompanyErrors, CreateCrmCompanyResponses, CreateCrmContactData, CreateCrmContactErrors, CreateCrmContactResponses, CreateDocumentData, CreateDocumentErrors, CreateDocumentResponses, CreateEntityMentionData, CreateEntityMentionErrors, CreateEntityMentionResponses, CreateHarnessPairingData, CreateHarnessPairingErrors, CreateHarnessPairingResponses, CreateInitiativeData, CreateInitiativeErrors, CreateInitiativeResponses, CreateInstructionsHandlerData, CreateInstructionsHandlerErrors, CreateInstructionsHandlerResponses, CreateMarkdownHandlerData, CreateMarkdownHandlerErrors, CreateMarkdownHandlerResponses, CreateProjectHandlerData, CreateProjectHandlerErrors, CreateProjectHandlerResponses, CreateReminderData, CreateReminderErrors, CreateReminderResponses, CreateSkillHandlerData, CreateSkillHandlerErrors, CreateSkillHandlerResponses, CreateSnippetHandlerData, CreateSnippetHandlerErrors, CreateSnippetHandlerResponses, CreateTaskHandlerData, CreateTaskHandlerErrors, CreateTaskHandlerResponses, CreateUserApiKeyData, CreateUserApiKeyErrors, CreateUserApiKeyResponses, CreateViewHandlerData, CreateViewHandlerErrors, CreateViewHandlerResponses, CreateWebhookData, CreateWebhookErrors, CreateWebhookResponses, DeleteAnchorData, DeleteAnchorErrors, DeleteAnchorResponses, DeleteCallRecordData, DeleteCallRecordErrors, DeleteCallRecordResponses, DeleteChannelData, DeleteChannelErrors, DeleteChannelResponses, DeleteCollabSurfaceData, DeleteCollabSurfaceErrors, DeleteCollabSurfaceResponses, DeleteCommentData, DeleteCommentErrors, DeleteCommentResponses, DeleteCrmCommentData, DeleteCrmCommentErrors, DeleteCrmCommentResponses, DeleteDocumentData, DeleteDocumentErrors, DeleteDocumentResponses, DeleteEntityMentionData, DeleteEntityMentionErrors, DeleteEntityMentionResponses, DeleteHarnessData, DeleteHarnessErrors, DeleteHarnessResponses, DeleteHistoryHandlerData, DeleteHistoryHandlerErrors, DeleteHistoryHandlerResponses, DeleteInitiativeData, DeleteInitiativeErrors, DeleteInitiativeResponses, DeleteMessageData, DeleteMessageErrors, DeleteMessageResponses, DeleteProjectHandlerData, DeleteProjectHandlerErrors, DeleteProjectHandlerResponses, DeleteReminderData, DeleteReminderErrors, DeleteReminderResponses, DeleteSelfHarnessData, DeleteSelfHarnessErrors, DeleteSelfHarnessResponses, DeleteUserApiKeyData, DeleteUserApiKeyErrors, DeleteUserApiKeyResponses, DeleteUserDocumentViewLocationData, DeleteUserDocumentViewLocationErrors, DeleteUserDocumentViewLocationResponses, DeleteViewHandlerData, DeleteViewHandlerErrors, DeleteViewHandlerResponses, DeleteWebhookData, DeleteWebhookErrors, DeleteWebhookResponses, EditAnchorData, EditAnchorErrors, EditAnchorResponses, EditCallRecordData, EditCallRecordErrors, EditCallRecordResponses, EditCallTranscriptData, EditCallTranscriptErrors, EditCallTranscriptResponses, EditCommentData, EditCommentErrors, EditCommentResponses, EditCrmCommentData, EditCrmCommentErrors, EditCrmCommentResponses, EditDocumentData, EditDocumentErrors, EditDocumentResponses, EditProjectV2Data, EditProjectV2Errors, EditProjectV2Responses, EditThreadV2Data, EditThreadV2Errors, EditThreadV2Responses, EnsureCollabSurfaceData, EnsureCollabSurfaceErrors, EnsureCollabSurfaceResponses, EntityMessageCreateData, EntityMessageCreateResponses, EntityMessageDeleteMessageData, EntityMessageDeleteMessageResponses, EntityMessageDeleteThreadData, EntityMessageDeleteThreadResponses, EntityMessageEditData, EntityMessageEditResponses, EntityMessageGetMessageData, EntityMessageGetMessageResponses, EntityMessageGetThreadData, EntityMessageGetThreadResponses, EntityMessageLegacyData, EntityMessageLegacyResponses, EntityMessagePatchThreadData, EntityMessagePatchThreadResponses, EntityMessageReactData, EntityMessageReactResponses, EntityMessageReferencesData, EntityMessageReferencesResponses, EntityMessageTypingData, EntityMessageTypingResponses, ExcludeDefaultViewHandlerData, ExcludeDefaultViewHandlerErrors, ExcludeDefaultViewHandlerResponses, ExportDocumentData, ExportDocumentErrors, ExportDocumentResponses, GetActiveCallsData, GetActiveCallsErrors, GetActiveCallsResponses, GetActivityData, GetActivityErrors, GetActivityResponses, GetAttachmentReferencesData, GetAttachmentReferencesErrors, GetAttachmentReferencesResponses, GetBatchCallRecordPreviewData, GetBatchCallRecordPreviewErrors, GetBatchCallRecordPreviewResponses, GetBatchChannelPreviewData, GetBatchChannelPreviewErrors, GetBatchChannelPreviewResponses, GetBatchPreviewHandlerData, GetBatchPreviewHandlerErrors, GetBatchPreviewHandlerResponses, GetBatchProjectPreviewData, GetBatchProjectPreviewErrors, GetBatchProjectPreviewResponses, GetCallRecordData, GetCallRecordErrors, GetCallRecordResponses, GetChannelAttachmentsData, GetChannelAttachmentsErrors, GetChannelAttachmentsResponses, GetChannelData, GetChannelErrors, GetChannelJoinLinkData, GetChannelJoinLinkErrors, GetChannelJoinLinkResponses, GetChannelMessagesCatchUpData, GetChannelMessagesCatchUpErrors, GetChannelMessagesCatchUpResponses, GetChannelMessagesData, GetChannelMessagesErrors, GetChannelMessagesResponses, GetChannelParticipantsData, GetChannelParticipantsErrors, GetChannelParticipantsResponses, GetChannelResponses, GetChannelsData, GetChannelsErrors, GetChannelsResponses, GetCollabSurfaceData, GetCollabSurfaceErrors, GetCollabSurfaceResponses, GetCompanyData, GetCompanyErrors, GetCompanyResponses, GetContactByEmailData, GetContactByEmailErrors, GetContactByEmailResponses, GetContactData, GetContactErrors, GetContactResponses, GetCrmTeamSettingsData, GetCrmTeamSettingsErrors, GetCrmTeamSettingsResponses, GetDocumentAnchorsData, GetDocumentAnchorsErrors, GetDocumentAnchorsResponses, GetDocumentBranchNameData, GetDocumentBranchNameErrors, GetDocumentBranchNameResponses, GetDocumentByTeamSlugData, GetDocumentByTeamSlugErrors, GetDocumentByTeamSlugResponses, GetDocumentCommentsData, GetDocumentCommentsErrors, GetDocumentCommentsResponses, GetDocumentData, GetDocumentErrors, GetDocumentGithubPullRequestsData, GetDocumentGithubPullRequestsErrors, GetDocumentGithubPullRequestsResponses, GetDocumentListHandlerData, GetDocumentListHandlerErrors, GetDocumentListHandlerResponses, GetDocumentLocationV3Data, GetDocumentLocationV3Errors, GetDocumentLocationV3Responses, GetDocumentPermissionsTokenData, GetDocumentPermissionsTokenErrors, GetDocumentPermissionsTokenResponses, GetDocumentPermissionsV2Data, GetDocumentPermissionsV2Errors, GetDocumentPermissionsV2Responses, GetDocumentProcessingResultData, GetDocumentProcessingResultErrors, GetDocumentProcessingResultResponses, GetDocumentResponses, GetDocumentShortIdData, GetDocumentShortIdErrors, GetDocumentShortIdResponses, GetDocumentTeamShareData, GetDocumentTeamShareErrors, GetDocumentTeamShareResponses, GetDocumentVersionData, GetDocumentVersionErrors, GetDocumentVersionResponses, GetDocumentViewsHandlerData, GetDocumentViewsHandlerErrors, GetDocumentViewsHandlerResponses, GetEntityPermissionData, GetEntityPermissionErrors, GetEntityPermissionResponses, GetForeignEntityBySourceData, GetForeignEntityBySourceErrors, GetForeignEntityBySourceResponses, GetForeignEntityData, GetForeignEntityErrors, GetForeignEntityResponses, GetHarnessPairingData, GetHarnessPairingErrors, GetHarnessPairingResponses, GetHistoryHandlerData, GetHistoryHandlerErrors, GetHistoryHandlerResponses, GetInitiativeData, GetInitiativeErrors, GetInitiativeResponses, GetInstructionsHandlerData, GetInstructionsHandlerErrors, GetInstructionsHandlerResponses, GetItemsSoupData, GetItemsSoupErrors, GetItemsSoupResponses, GetLocationHandlerData, GetLocationHandlerErrors, GetLocationHandlerResponses, GetMessageWithContextData, GetMessageWithContextErrors, GetMessageWithContextResponses, GetOrCreateCallData, GetOrCreateCallErrors, GetOrCreateCallResponses, GetOrCreateDmData, GetOrCreateDmErrors, GetOrCreateDmResponses, GetOrCreatePrivateData, GetOrCreatePrivateErrors, GetOrCreatePrivateResponses, GetPendingProjectsHandlerData, GetPendingProjectsHandlerErrors, GetPendingProjectsHandlerResponses, GetPinsHandlerData, GetPinsHandlerErrors, GetPinsHandlerResponses, GetProjectContentHandlerData, GetProjectContentHandlerErrors, GetProjectContentHandlerResponses, GetProjectHandlerData, GetProjectHandlerErrors, GetProjectHandlerResponses, GetProjectPermissionsV2Data, GetProjectPermissionsV2Errors, GetProjectPermissionsV2Responses, GetProjectsHandlerData, GetProjectsHandlerErrors, GetProjectsHandlerResponses, GetProjectUserAccessLevelData, GetProjectUserAccessLevelErrors, GetProjectUserAccessLevelResponses, GetReminderData, GetReminderErrors, GetReminderResponses, GetRingStatusData, GetRingStatusErrors, GetRingStatusResponses, GetSelfBotData, GetSelfBotErrors, GetSelfBotResponses, GetSelfHarnessData, GetSelfHarnessErrors, GetSelfHarnessResponses, GetSystemSkillsHandlerData, GetSystemSkillsHandlerErrors, GetSystemSkillsHandlerResponses, GetThreadRepliesData, GetThreadRepliesErrors, GetThreadRepliesResponses, GetUserDocumentsHandlerData, GetUserDocumentsHandlerErrors, GetUserDocumentsHandlerResponses, GetUserDocumentViewLocationData, GetUserDocumentViewLocationErrors, GetUserDocumentViewLocationResponses, GetViewsHandlerData, GetViewsHandlerErrors, GetViewsHandlerResponses, GetWebhookData, GetWebhookErrors, GetWebhookResponses, HandlerData, HandlerErrors, HandlerResponses, HealthHandlerData, HealthHandlerResponses, IngestTranscriptData, IngestTranscriptErrors, IngestTranscriptResponses, InitializeUserDocumentsData, InitializeUserDocumentsErrors, InitializeUserDocumentsResponses, InstallSyncData, InstallSyncErrors, JobProcessingResultHandlerData, JobProcessingResultHandlerErrors, JobProcessingResultHandlerResponses, JoinChannelByCodeData, JoinChannelByCodeErrors, JoinChannelByCodeResponses, JoinChannelData, JoinChannelErrors, JoinChannelResponses, LeaveChannelData, LeaveChannelErrors, LeaveChannelResponses, LeaveOrEndCallData, LeaveOrEndCallErrors, LeaveOrEndCallResponses, ListAgentsData, ListAgentsErrors, ListAgentsResponses, ListBotChannelsData, ListBotChannelsErrors, ListBotChannelsResponses, ListCompanyContactsData, ListCompanyContactsErrors, ListCompanyContactsResponses, ListCrmCommentsData, ListCrmCommentsErrors, ListCrmCommentsResponses, ListFavoritesData, ListFavoritesErrors, ListFavoritesResponses, ListHarnessAgentsData, ListHarnessAgentsErrors, ListHarnessAgentsResponses, ListHarnessesData, ListHarnessesErrors, ListHarnessesResponses, ListHarnessSessionsData, ListHarnessSessionsErrors, ListHarnessSessionsResponses, ListInitiativesData, ListInitiativesErrors, ListInitiativesResponses, ListOccurrencesData, ListOccurrencesErrors, ListOccurrencesResponses, ListRemindersData, ListRemindersErrors, ListRemindersResponses, ListTeamOutOfOfficeData, ListTeamOutOfOfficeErrors, ListTeamOutOfOfficeResponses, ListUserApiKeysData, ListUserApiKeysErrors, ListUserApiKeysResponses, ListWebhooksData, ListWebhooksErrors, ListWebhooksResponses, MentionPreviewsData, MentionPreviewsErrors, MentionPreviewsResponses, MessageTimelineData, MessageTimelineResponses, PatchChannelData, PatchChannelErrors, PatchChannelResponses, PatchMessageData, PatchMessageErrors, PatchMessageResponses, PatchViewHandlerData, PatchViewHandlerErrors, PatchViewHandlerResponses, PatchWebhookData, PatchWebhookErrors, PatchWebhookResponses, PermanentlyDeleteDocumentData, PermanentlyDeleteDocumentErrors, PermanentlyDeleteDocumentResponses, PermanentlyDeleteProjectData, PermanentlyDeleteProjectErrors, PermanentlyDeleteProjectResponses, PostActivityData, PostActivityErrors, PostActivityResponses, PostChannelBotWebhookData, PostChannelBotWebhookErrors, PostChannelBotWebhookResponses, PostChannelMessagesData, PostChannelMessagesErrors, PostChannelMessagesResponses, PostItemsSoupAstData, PostItemsSoupAstErrors, PostItemsSoupAstGroupedData, PostItemsSoupAstGroupedErrors, PostItemsSoupAstGroupedResponses, PostItemsSoupAstResponses, PostItemsSoupData, PostItemsSoupErrors, PostItemsSoupResponses, PostMessageData, PostMessageErrors, PostMessageResponses, PostReactionData, PostReactionErrors, PostReactionResponses, PostTypingData, PostTypingErrors, PostTypingResponses, PresaveDocumentHandlerData, PresaveDocumentHandlerErrors, PresaveDocumentHandlerResponses, PutCrmTeamSettingsData, PutCrmTeamSettingsErrors, PutCrmTeamSettingsResponses, PutCrmTeamStagesData, PutCrmTeamStagesErrors, PutCrmTeamStagesResponses, RecentlyDeletedData, RecentlyDeletedErrors, RecentlyDeletedResponses, RemoveBotFromChannelByBotData, RemoveBotFromChannelByBotErrors, RemoveBotFromChannelByBotResponses, RemoveFavoriteByEntityData, RemoveFavoriteByEntityErrors, RemoveFavoriteByEntityResponses, RemoveParticipantsData, RemoveParticipantsErrors, RemoveParticipantsResponses, RemovePinHandlerData, RemovePinHandlerErrors, RemovePinHandlerResponses, ReorderFavoritesData, ReorderFavoritesErrors, ReorderFavoritesResponses, ReorderPinsHandlerData, ReorderPinsHandlerErrors, ReorderPinsHandlerResponses, ResetCrmTeamStagesData, ResetCrmTeamStagesErrors, ResetCrmTeamStagesResponses, ResolveChannelMessageData, ResolveChannelMessageErrors, ResolveChannelMessageResponses, RevertDeleteDocumentData, RevertDeleteDocumentErrors, RevertDeleteDocumentResponses, RevertDeleteProjectData, RevertDeleteProjectErrors, RevertDeleteProjectResponses, SaveDocumentHandlerData, SaveDocumentHandlerErrors, SaveDocumentHandlerResponses, SetChannelPictureData, SetChannelPictureErrors, SetChannelPictureResponses, SetCompanyHiddenData, SetCompanyHiddenErrors, SetCompanyHiddenResponses, SetContactHiddenData, SetContactHiddenErrors, SetContactHiddenResponses, SetCrmCompanyNameData, SetCrmCompanyNameErrors, SetCrmCompanyNameResponses, SetCrmContactNameData, SetCrmContactNameErrors, SetCrmContactNameResponses, SetDocumentTeamShareData, SetDocumentTeamShareErrors, SetDocumentTeamShareResponses, SetEmailSyncData, SetEmailSyncErrors, SetEmailSyncResponses, SimpleSaveData, SimpleSaveErrors, SimpleSaveResponses, StreamEventsData, StreamEventsErrors, StreamEventsResponse, StreamEventsResponses, ToggleShareWithTeamData, ToggleShareWithTeamErrors, ToggleShareWithTeamResponses, TranscribeDictationData, TranscribeDictationErrors, TranscribeDictationResponses, UnassignInitiativeTaskData, UnassignInitiativeTaskErrors, UnassignInitiativeTaskResponses, UpdateAgentData, UpdateAgentErrors, UpdateAgentResponses, UpdateInitiativeData, UpdateInitiativeErrors, UpdateInitiativeResponses, UpdateReminderData, UpdateReminderErrors, UpdateReminderResponses, UploadExtractFolderHandlerData, UploadExtractFolderHandlerErrors, UploadExtractFolderHandlerResponses, UploadFolderHandlerData, UploadFolderHandlerErrors, UploadFolderHandlerResponses, UpsertHistoryHandlerData, UpsertHistoryHandlerErrors, UpsertHistoryHandlerResponses, UpsertUserDocumentViewLocationData, UpsertUserDocumentViewLocationErrors, UpsertUserDocumentViewLocationResponses, ValidateDocumentPermissionsTokenData, ValidateDocumentPermissionsTokenErrors, ValidateDocumentPermissionsTokenResponses, ValidateWebhookData, ValidateWebhookErrors, ValidateWebhookResponses } from './types.gen'; export type Options = Options2 & { /** @@ -1085,6 +1085,24 @@ export class Sdk extends HeyApiClient { }); } + /** + * Transcribe a transient recording with OpenAI Whisper. + * + * Available to every signed-in user on every plan; does not consume chat + * credits. Audio and transcripts are never persisted. + */ + public transcribeDictation(options: Options): RequestResult { + return (options.client ?? this.client).post({ + bodySerializer: null, + url: '/dictation/transcribe', + ...options, + headers: { + 'Content-Type': 'audio/webm', + ...options.headers + } + }); + } + /** * Gets the users documents to populate their recent document list */ diff --git a/packages/sdk/generated/storage/types.gen.ts b/packages/sdk/generated/storage/types.gen.ts index c487d888fe6..93671d32a63 100644 --- a/packages/sdk/generated/storage/types.gen.ts +++ b/packages/sdk/generated/storage/types.gen.ts @@ -9921,6 +9921,16 @@ export type ThreadState = { user_id: string; }; +/** + * Transcription result. + */ +export type TranscribeResponse = { + /** + * Recognized text. + */ + text: string; +}; + /** * A transcript segment from LiveKit Inference STT. */ @@ -13128,6 +13138,64 @@ export type PutCrmTeamStagesResponses = { export type PutCrmTeamStagesResponse = PutCrmTeamStagesResponses[keyof PutCrmTeamStagesResponses]; +export type TranscribeDictationData = { + /** + * OpenAPI representation of the raw encoded audio body extracted as `Bytes`. + */ + body: Blob | File; + path?: never; + query?: { + /** + * ISO 639-1 language hint + */ + language?: string; + }; + url: '/dictation/transcribe'; +}; + +export type TranscribeDictationErrors = { + /** + * Empty, oversized, or malformed request + */ + 400: ErrorResponse; + /** + * Missing or invalid credentials + */ + 401: ErrorResponse; + /** + * Only signed-in users may dictate + */ + 403: ErrorResponse; + /** + * Body exceeds 8 MiB + */ + 413: unknown; + /** + * Unsupported audio container + */ + 415: ErrorResponse; + /** + * Per-user hourly rate limit exceeded + */ + 429: unknown; + /** + * Provider failure + */ + 502: ErrorResponse; + /** + * Transcription capacity exhausted; retry after the Retry-After delay + */ + 503: ErrorResponse; +}; + +export type TranscribeDictationError = TranscribeDictationErrors[keyof TranscribeDictationErrors]; + +export type TranscribeDictationResponses = { + 200: TranscribeResponse; +}; + +export type TranscribeDictationResponse = TranscribeDictationResponses[keyof TranscribeDictationResponses]; + export type GetUserDocumentsHandlerData = { body?: never; path?: never; diff --git a/packages/sdk/specs/cognition.json b/packages/sdk/specs/cognition.json index 96393c74f1b..13f6531e342 100644 --- a/packages/sdk/specs/cognition.json +++ b/packages/sdk/specs/cognition.json @@ -30,6 +30,16 @@ "200": { "description": "Pricing updated" }, + "400": { + "description": "Invalid pricing", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorBody" + } + } + } + }, "403": { "description": "Admin access required", "content": { @@ -2132,7 +2142,8 @@ "ai_editing", "import", "agent_session", - "agent_repository_choice" + "agent_repository_choice", + "dictation" ] }, "AssistantMessagePart": { @@ -2806,25 +2817,25 @@ }, "CompletionUsage": { "type": "object", - "description": "A recorded completion: who, what feature, optional entity, and the cost.", + "description": "User and feature attribution for one invocation.", "required": ["feature", "user", "cost"], "properties": { "cost": { "$ref": "#/components/schemas/Usage", - "description": "Token usage and cost." + "description": "Measured usage and resolved cost." }, "entity": { "type": ["string", "null"], "format": "uuid", - "description": "The entity the completion related to, if any." + "description": "Related entity, if any." }, "feature": { "$ref": "#/components/schemas/AiFeature", - "description": "The feature that performed the completion." + "description": "Feature that performed the invocation." }, "user": { "$ref": "#/components/schemas/MacroUserIdStr", - "description": "The user the completion was performed for (the [system user](SYSTEM_USER_ID)\nfor background work)." + "description": "User the invocation was performed for." } } }, @@ -3006,7 +3017,7 @@ }, "FeatureUsage": { "type": "object", - "description": "Usage for a single feature, with its rolled-up dollar total.", + "description": "Recorded invocations and total for one feature.", "required": ["feature", "entries", "total"], "properties": { "entries": { @@ -3014,16 +3025,16 @@ "items": { "$ref": "#/components/schemas/CompletionUsage" }, - "description": "The individual completions recorded for this feature." + "description": "Recorded invocations." }, "feature": { "$ref": "#/components/schemas/AiFeature", - "description": "The feature." + "description": "Feature attribution." }, "total": { "type": "number", "format": "float", - "description": "Total cost across `entries` (USD)." + "description": "Total cost (USD)." } } }, @@ -4087,23 +4098,28 @@ }, "Price": { "type": "object", - "description": "Resolved price for one completion.", + "description": "Rates applied to an invocation and its resolved dollar cost.", "required": ["price_per_million_in", "price_per_million_out", "total"], "properties": { + "price_per_audio_minute": { + "type": ["number", "null"], + "format": "float", + "description": "Price per audio minute (USD), absent for token billing." + }, "price_per_million_in": { "type": "number", "format": "float", - "description": "Price per million input tokens (USD)." + "description": "Price per million input tokens (USD); zero for audio billing." }, "price_per_million_out": { "type": "number", "format": "float", - "description": "Price per million output tokens (USD)." + "description": "Price per million output tokens (USD); zero for audio billing." }, "total": { "type": "number", "format": "float", - "description": "Total cost of the completion (USD)." + "description": "Total cost (USD)." } } }, @@ -4312,21 +4328,26 @@ "SetPricingRequest": { "type": "object", "description": "Request body for [`set_pricing_handler`].", - "required": ["model", "price_per_mil_in", "price_per_mil_out"], + "required": ["model"], "properties": { "model": { "type": "string", "description": "The model api id to (re)price." }, + "price_per_audio_minute": { + "type": ["number", "null"], + "format": "float", + "description": "Price per minute of audio (USD), or null for token-only pricing." + }, "price_per_mil_in": { - "type": "number", + "type": ["number", "null"], "format": "float", - "description": "New price per million input tokens (USD)." + "description": "New price per million input tokens (USD). Required for token pricing." }, "price_per_mil_out": { - "type": "number", + "type": ["number", "null"], "format": "float", - "description": "New price per million output tokens (USD)." + "description": "New price per million output tokens (USD). Required for token pricing." } } }, @@ -4790,28 +4811,33 @@ }, "Usage": { "type": "object", - "description": "The token usage and resolved cost of a single completion.", + "description": "Measured usage for one invocation.", "required": ["input_tokens", "output_tokens", "model", "created_at"], "properties": { + "audio_seconds": { + "type": ["number", "null"], + "format": "double", + "description": "Audio duration in seconds, absent for token billing." + }, "created_at": { "type": "string", "format": "date-time", - "description": "When the completion was recorded." + "description": "Recording timestamp." }, "input_tokens": { "type": "integer", - "format": "int32", - "description": "Tokens consumed by the input.", + "format": "int64", + "description": "Input tokens; zero for audio billing.", "minimum": 0 }, "model": { "type": "string", - "description": "The model api id (e.g. `claude-opus-4-8`)." + "description": "Provider model identifier." }, "output_tokens": { "type": "integer", - "format": "int32", - "description": "Tokens generated in the output.", + "format": "int64", + "description": "Output tokens; zero for audio billing.", "minimum": 0 }, "price": { @@ -4821,7 +4847,7 @@ }, { "$ref": "#/components/schemas/Price", - "description": "Resolved price, or `None` when the model had no pricing at record time." + "description": "Applied pricing, if known." } ] } @@ -4859,7 +4885,7 @@ }, "UsageSummary": { "type": "object", - "description": "The result of a usage query: per-feature breakdown plus a grand total.", + "description": "Per-feature breakdown and grand total.", "required": ["entries", "total"], "properties": { "entries": { @@ -4872,7 +4898,7 @@ "total": { "type": "number", "format": "float", - "description": "Grand total cost across all features (USD)." + "description": "Grand total (USD)." } } }, diff --git a/packages/sdk/specs/storage.json b/packages/sdk/specs/storage.json index d0e667ea69d..1340f51280a 100644 --- a/packages/sdk/specs/storage.json +++ b/packages/sdk/specs/storage.json @@ -6284,6 +6284,116 @@ } } }, + "/dictation/transcribe": { + "post": { + "tags": ["dictation"], + "summary": "Transcribe a transient recording with OpenAI Whisper.", + "description": "Available to every signed-in user on every plan; does not consume chat\ncredits. Audio and transcripts are never persisted.", + "operationId": "transcribe_dictation", + "parameters": [ + { + "name": "language", + "in": "query", + "description": "ISO 639-1 language hint", + "required": false, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "description": "Encoded WebM, MP4, Ogg, or WAV audio up to 8 MiB and five minutes", + "content": { + "audio/webm": { + "schema": { + "type": "string", + "format": "binary", + "description": "OpenAPI representation of the raw encoded audio body extracted as `Bytes`." + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TranscribeResponse" + } + } + } + }, + "400": { + "description": "Empty, oversized, or malformed request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "401": { + "description": "Missing or invalid credentials", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "403": { + "description": "Only signed-in users may dictate", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "413": { + "description": "Body exceeds 8 MiB" + }, + "415": { + "description": "Unsupported audio container", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "429": { + "description": "Per-user hourly rate limit exceeded" + }, + "502": { + "description": "Provider failure", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "503": { + "description": "Transcription capacity exhausted; retry after the Retry-After delay", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + } + } + } + }, "/documents": { "get": { "tags": ["document"], @@ -29758,6 +29868,17 @@ } } }, + "TranscribeResponse": { + "type": "object", + "description": "Transcription result.", + "required": ["text"], + "properties": { + "text": { + "type": "string", + "description": "Recognized text." + } + } + }, "TranscriptSegmentRequest": { "type": "object", "description": "A transcript segment from LiveKit Inference STT.", diff --git a/packages/sdk/src/coverage/skipped.ts b/packages/sdk/src/coverage/skipped.ts index 3d61791dece..097d3a1285d 100644 --- a/packages/sdk/src/coverage/skipped.ts +++ b/packages/sdk/src/coverage/skipped.ts @@ -322,6 +322,8 @@ export const storageExcluded = [ 'postItemsSoupAstGrouped', 'removeBotFromChannelByBot', 'resolveChannelMessage', + // Composer dictation is an app-internal, user-only upload flow. + 'transcribeDictation', 'uploadExtractFolderHandler', 'uploadFolderHandler', 'upsertHistoryHandler', diff --git a/services/document_cognition_service/src/api/swagger.rs b/services/document_cognition_service/src/api/swagger.rs index 7f51459e3ae..223f49e20d6 100644 --- a/services/document_cognition_service/src/api/swagger.rs +++ b/services/document_cognition_service/src/api/swagger.rs @@ -219,11 +219,11 @@ use utoipa::OpenApi; ai_usage_api::UsageRequest, ai_usage_api::SetPricingRequest, ai_usage_api::ErrorBody, - ai_usage::UsageSummary, - ai_usage::FeatureUsage, - ai_usage::CompletionUsage, - ai_usage::Usage, - ai_usage::Price, + ai_usage::inbound::models::UsageSummary, + ai_usage::inbound::models::FeatureUsage, + ai_usage::inbound::models::CompletionUsage, + ai_usage::inbound::models::Usage, + ai_usage::inbound::models::Price, ai_usage::AiFeature, // AI projections diff --git a/services/document_storage_service/Cargo.toml b/services/document_storage_service/Cargo.toml index 5ce675ea013..ad35ef06017 100644 --- a/services/document_storage_service/Cargo.toml +++ b/services/document_storage_service/Cargo.toml @@ -27,6 +27,7 @@ local_auth = ["macro_authorization/local_auth"] location_check = [] [dependencies] +dictation = { path = "../../crates/dictation", features = ["inbound", "outbound"] } agent_session = { path = "../../crates/agent_session", features = ["outbound"] } activity = { path = "../../crates/activity" } ai_tools = { path = "../../crates/ai_tools" } diff --git a/services/document_storage_service/src/api.rs b/services/document_storage_service/src/api.rs index c1cc5b10cfd..b2d6986b821 100644 --- a/services/document_storage_service/src/api.rs +++ b/services/document_storage_service/src/api.rs @@ -58,7 +58,7 @@ pub async fn setup_and_serve(state: ApiContext) -> anyhow::Result<()> { }, validate_api_version, )) - .layer(macro_cors::cors_layer()) + .layer(macro_cors::cors_layer().expose_headers([axum::http::header::RETRY_AFTER])) .layer(CompressionLayer::new().gzip(true)), ) // The health router is attached here so we don't attach the logging middleware to it @@ -113,6 +113,10 @@ fn api_router(state: ApiContext) -> Router { ); let internal_router = Router::new() + .nest( + "/dictation", + dictation::inbound::axum_router::dictation_router(state.dictation_state.clone()), + ) .nest( "/github", github::inbound::github_sync_router::github_sync_router( diff --git a/services/document_storage_service/src/api/context.rs b/services/document_storage_service/src/api/context.rs index c32d518c1b7..add28ce3c91 100644 --- a/services/document_storage_service/src/api/context.rs +++ b/services/document_storage_service/src/api/context.rs @@ -554,8 +554,19 @@ pub(crate) type DssSseStreamService = pub(crate) type DssSseStreamState = WebhookStreamRouterState; +/// Type alias for the dictation router state; shares the webhook Redis limiter. +pub(crate) type DssDictationState = dictation::inbound::axum_router::DictationRouterState< + dictation::domain::DictationServiceImpl< + dictation::outbound::WhisperTranscriber, + dictation::outbound::SymphoniaRecordingInspector, + >, + DssWebhookRateLimiter, + AuthorizationService, +>; + #[derive(Clone, FromRef)] pub(crate) struct ApiContext { + pub dictation_state: DssDictationState, pub db: PgPool, pub readonly_db: ReadOnlyPool, pub redis_client: Arc, diff --git a/services/document_storage_service/src/api/swagger.rs b/services/document_storage_service/src/api/swagger.rs index 9522795a12b..f83f3889d0f 100644 --- a/services/document_storage_service/src/api/swagger.rs +++ b/services/document_storage_service/src/api/swagger.rs @@ -150,6 +150,7 @@ use utoipa::OpenApi; terms_of_service = "https://macro.com/terms", ), paths( + dictation::inbound::axum_router::transcribe_handler, health::health_handler, calendar_events::inbound::axum_router::list_occurrences, calendar_events::inbound::axum_router::mention_previews, @@ -720,6 +721,7 @@ use utoipa::OpenApi; webhook::domain::models::WebhookFilter, webhook::domain::models::WebhookStatus, webhook::domain::models::WebhookValidationTestEvent, + dictation::inbound::axum_router::TranscribeResponse, DocumentSubType, diff --git a/services/document_storage_service/src/config.rs b/services/document_storage_service/src/config.rs index 4a75de0c4c1..78f2e5fdc1d 100644 --- a/services/document_storage_service/src/config.rs +++ b/services/document_storage_service/src/config.rs @@ -1,4 +1,5 @@ use anyhow::Context; +pub use dictation::outbound::OpenaiApiKey; use macro_auth::InternalApiKey; pub use macro_env::Environment; use macro_env_var::{env_vars, maybe_env_vars}; @@ -34,10 +35,6 @@ env_vars! { pub struct LivekitServerUrl; pub struct LivekitApiKey; pub struct LivekitApiSecret; - /// OpenAI API key used to generate task-dedup embeddings. Required — - /// injected as `OPENAI_API_KEY` from the `openai-key` secret by the - /// infra stack, the same way `document_cognition_service` consumes it. - pub struct OpenaiApiKey; /// Cohere API key used by the task-dedup reranker. Required — injected /// as `COHERE_API_KEY`, following the same pattern as `OPENAI_API_KEY`. pub struct CohereApiKey; @@ -108,6 +105,7 @@ pub struct Config { pub livekit_server_url: LivekitServerUrl, pub livekit_api_key: LivekitApiKey, pub livekit_api_secret: LivekitApiSecret, + /// Shared server credential for task embeddings and Whisper, supplied by Doppler. pub openai_api_key: OpenaiApiKey, pub cohere_api_key: CohereApiKey, pub github_webhook_secret_key: LocalOrRemoteSecret, diff --git a/services/document_storage_service/src/main.rs b/services/document_storage_service/src/main.rs index 7bdd7a8c52f..a4349aab957 100644 --- a/services/document_storage_service/src/main.rs +++ b/services/document_storage_service/src/main.rs @@ -942,20 +942,31 @@ async fn run() -> anyhow::Result<()> { let sqs_client = Arc::new(sqs_client); let conn_gateway_client = Arc::new(conn_gateway_client); - // The OpenAI key is injected as the required `OPENAI_API_KEY` env var - // (resolved from the `openai-key` secret at deploy time by the infra stack), - // the same way `document_cognition_service` consumes it. Fail fast if it's - // empty so the service never starts with a broken task-dedup embedder. + // MacroConfig reads the shared OPENAI_API_KEY from Doppler's APP_SECRETS_JSON + // or the local environment. Validate once before constructing consumers. let openai_api_key = config.openai_api_key.as_ref().to_owned(); anyhow::ensure!( !openai_api_key.trim().is_empty(), - "OpenAI API key is required for task dedup embeddings", + "OpenAI API key is required for task dedup embeddings and dictation", ); let cohere_api_key = config.cohere_api_key.as_ref().to_owned(); anyhow::ensure!( !cohere_api_key.trim().is_empty(), "Cohere API key is required for task dedup reranking", ); + let dictation_state = dictation::inbound::axum_router::DictationRouterState::new( + dictation::domain::DictationServiceImpl::new( + dictation::outbound::WhisperTranscriber::new(&config.openai_api_key)?, + dictation::outbound::SymphoniaRecordingInspector, + ai_usage::pg_recorder(db.clone()), + ), + RateLimitServiceImpl { + repo: RedisRateLimitAdapter { + redis: redis_client.clone(), + }, + }, + authorization_state.clone(), + ); let task_dedup_service = Arc::new(TaskDedupService::new( TextEmbedding3Small::new(openai_api_key), PgTaskVectorDb::new(db.clone()), @@ -1473,6 +1484,7 @@ async fn run() -> anyhow::Result<()> { )); let api_context = ApiContext { + dictation_state, contacts_ingress: contacts_ingress.clone(), soup_router_state: SoupRouterState::from_arc( soup_service.clone(), diff --git a/services/email_service/Cargo.toml b/services/email_service/Cargo.toml index 0c762cd515e..ccab98ca519 100644 --- a/services/email_service/Cargo.toml +++ b/services/email_service/Cargo.toml @@ -110,7 +110,7 @@ gmail_client = { path = "../../crates/gmail_client" } google_token = { path = "../../crates/google_token" } html-escape = { workspace = true } http-body-util = { workspace = true } -infer = "0.19.0" +infer = { workspace = true } item_filters = { path = "../../crates/item_filters" } lol_html = { workspace = true } macro_auth = { path = "../../crates/macro_auth" }