diff --git a/.changeset/clean-toes-melt.md b/.changeset/clean-toes-melt.md new file mode 100644 index 000000000..b2d442362 --- /dev/null +++ b/.changeset/clean-toes-melt.md @@ -0,0 +1,6 @@ +--- +"@livekit/components-core": patch +"@livekit/components-react": patch +--- + +Add useTranscription hook diff --git a/packages/core/etc/components-core.api.md b/packages/core/etc/components-core.api.md index 6ba2034d8..9e040a3e8 100644 --- a/packages/core/etc/components-core.api.md +++ b/packages/core/etc/components-core.api.md @@ -159,6 +159,7 @@ export const cssPrefix = "lk"; // @public (undocumented) export const DataTopic: { readonly CHAT: "lk.chat"; + readonly TRANSCRIPTION: "lk.transcription"; }; // @public (undocumented) diff --git a/packages/core/src/components/chat.ts b/packages/core/src/components/chat.ts index 6689f0476..6214b8a6d 100644 --- a/packages/core/src/components/chat.ts +++ b/packages/core/src/components/chat.ts @@ -8,6 +8,7 @@ import { sendMessage, setupDataMessageHandler, } from '../observables/dataChannel'; +import { log } from '../logger'; /** @public */ export type { ChatMessage }; @@ -165,10 +166,14 @@ export function setupChat(room: Room, options?: ChatOptions) { ...chatMsg, ignoreLegacy: serverSupportsDataStreams(), }); - await sendMessage(room.localParticipant, encodedLegacyMsg, { - reliable: true, - topic: legacyTopic, - }); + try { + await sendMessage(room.localParticipant, encodedLegacyMsg, { + reliable: true, + topic: legacyTopic, + }); + } catch (error) { + log.info('could not send message in legacy chat format', error); + } return chatMsg; } finally { isSending$.next(false); diff --git a/packages/core/src/components/textStream.ts b/packages/core/src/components/textStream.ts index 83841ef28..9fa124d01 100644 --- a/packages/core/src/components/textStream.ts +++ b/packages/core/src/components/textStream.ts @@ -97,6 +97,7 @@ export function setupTextStream(room: Room, topic: string): Observable { + room.unregisterTextStreamHandler(topic); textStreamsSubject.complete(); getObservableCache().delete(cacheKey); }); diff --git a/packages/core/src/observables/dataChannel.ts b/packages/core/src/observables/dataChannel.ts index e86d5ff25..04f6bf9f4 100644 --- a/packages/core/src/observables/dataChannel.ts +++ b/packages/core/src/observables/dataChannel.ts @@ -13,6 +13,7 @@ import { ReceivedChatMessage } from '../components/chat'; export const DataTopic = { CHAT: 'lk.chat', + TRANSCRIPTION: 'lk.transcription', } as const; /** @deprecated */ diff --git a/packages/react/etc/components-react.api.md b/packages/react/etc/components-react.api.md index a070ddf38..d60622cc6 100644 --- a/packages/react/etc/components-react.api.md +++ b/packages/react/etc/components-react.api.md @@ -1213,6 +1213,17 @@ export function useTrackTranscription(trackRef: TrackReferenceOrPlaceholder_4 | // @alpha export function useTrackVolume(trackOrTrackReference?: LocalAudioTrack | RemoteAudioTrack | TrackReference_3, options?: AudioAnalyserOptions): number; +// @beta +export function useTranscriptions(opts?: UseTranscriptionsOptions): TextStreamData_2[]; + +// @beta (undocumented) +export interface UseTranscriptionsOptions { + // (undocumented) + participantIdentities?: string[]; + // (undocumented) + trackSids?: string[]; +} + // @public export function useVisualStableUpdate( trackReferences: TrackReferenceOrPlaceholder_4[], maxItemsOnPage: number, options?: UseVisualStableUpdateOptions): TrackReferenceOrPlaceholder_4[]; diff --git a/packages/react/src/hooks/index.ts b/packages/react/src/hooks/index.ts index 071235ffa..cdd9ed342 100644 --- a/packages/react/src/hooks/index.ts +++ b/packages/react/src/hooks/index.ts @@ -55,3 +55,4 @@ export * from './useVoiceAssistant'; export * from './useParticipantAttributes'; export * from './useIsRecording'; export * from './useTextStream'; +export * from './useTranscriptions'; diff --git a/packages/react/src/hooks/useTrackTranscription.ts b/packages/react/src/hooks/useTrackTranscription.ts index 809dd37b4..8c917330b 100644 --- a/packages/react/src/hooks/useTrackTranscription.ts +++ b/packages/react/src/hooks/useTrackTranscription.ts @@ -35,7 +35,7 @@ const TRACK_TRANSCRIPTION_DEFAULTS = { } as const satisfies TrackTranscriptionOptions; /** - * @returns An object consisting of `segments` with maximum length of opts.windowLength and `activeSegments` that are valid for the current track timestamp + * @returns An object consisting of `segments` with maximum length of opts.bufferSize * @alpha */ export function useTrackTranscription( @@ -44,10 +44,7 @@ export function useTrackTranscription( ) { const opts = { ...TRACK_TRANSCRIPTION_DEFAULTS, ...options }; const [segments, setSegments] = React.useState>([]); - // const [activeSegments, setActiveSegments] = React.useState>( - // [], - // ); - // const prevActiveSegments = React.useRef([]); + const syncTimestamps = useTrackSyncTime(trackRef); const handleSegmentMessage = (newSegments: TranscriptionSegment[]) => { opts.onTranscription?.(newSegments); @@ -72,20 +69,5 @@ export function useTrackTranscription( }; }, [trackRef && getTrackReferenceId(trackRef), handleSegmentMessage]); - // React.useEffect(() => { - // if (syncTimestamps) { - // const newActiveSegments = getActiveTranscriptionSegments( - // segments, - // syncTimestamps, - // opts.maxAge, - // ); - // // only update active segment array if content actually changed - // if (didActiveSegmentsChange(prevActiveSegments.current, newActiveSegments)) { - // setActiveSegments(newActiveSegments); - // prevActiveSegments.current = newActiveSegments; - // } - // } - // }, [syncTimestamps, segments, opts.maxAge]); - return { segments }; } diff --git a/packages/react/src/hooks/useTranscriptions.ts b/packages/react/src/hooks/useTranscriptions.ts new file mode 100644 index 000000000..a256cbb31 --- /dev/null +++ b/packages/react/src/hooks/useTranscriptions.ts @@ -0,0 +1,44 @@ +import * as React from 'react'; +import { useTextStream } from './useTextStream'; +import { DataTopic } from '@livekit/components-core'; + +/** + * @beta + */ +export interface UseTranscriptionsOptions { + participantIdentities?: string[]; + trackSids?: string[]; +} + +/** + * @beta + * useTranscriptions is a hook that returns the transcriptions for the given participant identities and track sids, + * if no options are provided, it will return all transcriptions + * @example + * ```tsx + * const transcriptions = useTranscriptions(); + * return
{transcriptions.map((transcription) => transcription.text)}
; + * ``` + */ +export function useTranscriptions(opts?: UseTranscriptionsOptions) { + const { participantIdentities, trackSids } = opts ?? {}; + const { textStreams } = useTextStream(DataTopic.TRANSCRIPTION); + + const filteredMessages = React.useMemo( + () => + textStreams + .filter((stream) => + participantIdentities + ? participantIdentities.includes(stream.participantInfo.identity) + : true, + ) + .filter((stream) => + trackSids + ? trackSids.includes(stream.streamInfo.attributes?.['lk.transcribed_track_id'] ?? '') + : true, + ), + [textStreams, participantIdentities, trackSids], + ); + + return filteredMessages; +}