Agent SDK - ported on top of components-js primitives (#1207)

Author: 1egoman

.changeset/spotty-squids-search.md

@@ -0,0 +1,6 @@
+---
+'@livekit/components-react': patch
+'@livekit/components-core': patch
+---
+
+Add initial version of agents sdk (useSession, etc)

packages/core/package.json

@@ -45,7 +45,7 @@
   },
   "devDependencies": {
     "@livekit/components-styles": "workspace:*",
-    "@livekit/protocol": "^1.38.0",
+    "@livekit/protocol": "catalog:",
     "@microsoft/api-extractor": "^7.36.0",
     "@size-limit/file": "^11.0.2",
     "@size-limit/webpack": "^11.0.2",

packages/core/src/components/chat.ts

@@ -1,5 +1,5 @@
 /* eslint-disable camelcase */
-import type { Participant, Room, ChatMessage, SendTextOptions } from 'livekit-client';
+import type { Room, SendTextOptions } from 'livekit-client';
 import { compareVersions, RoomEvent } from 'livekit-client';
 import { BehaviorSubject, Subject, scan, map, takeUntil, from, filter } from 'rxjs';
 import {
@@ -9,15 +9,10 @@ import {
   setupDataMessageHandler,
 } from '../observables/dataChannel';
 import { log } from '../logger';
+import { ChatMessage, ReceivedChatMessage } from '../messages/types';
 
 /** @public */
-export type { ChatMessage };
-
-/** @public */
-export interface ReceivedChatMessage extends ChatMessage {
-  from?: Participant;
-  attributes?: Record<string, string>;
-}
+export type { ChatMessage, ReceivedChatMessage };
 
 export interface LegacyChatMessage extends ChatMessage {
   ignoreLegacy?: boolean;
@@ -55,7 +50,9 @@ function isIgnorableChatMessage(msg: ReceivedChatMessage | LegacyReceivedChatMes
 }
 
 const decodeLegacyMsg = (message: Uint8Array) =>
-  JSON.parse(new TextDecoder().decode(message)) as LegacyReceivedChatMessage | ReceivedChatMessage;
+  JSON.parse(new TextDecoder().decode(message)) as
+    | LegacyReceivedChatMessage
+    | Exclude<ReceivedChatMessage, 'type'>;
 
 const encodeLegacyMsg = (message: LegacyChatMessage) =>
   new TextEncoder().encode(JSON.stringify(message));
@@ -93,8 +90,9 @@ export function setupChat(room: Room, options?: ChatOptions) {
             timestamp,
             message: chunk,
             from: room.getParticipantByIdentity(participantInfo.identity),
+            type: 'chatMessage',
             // editTimestamp: type === 'update' ? timestamp : undefined,
-          } as ReceivedChatMessage;
+          } satisfies ReceivedChatMessage;
         }),
       );
       streamObservable.subscribe({
@@ -111,7 +109,11 @@ export function setupChat(room: Room, options?: ChatOptions) {
           if (isIgnorableChatMessage(parsedMessage)) {
             return undefined;
           }
-          const newMessage: ReceivedChatMessage = { ...parsedMessage, from: msg.from };
+          const newMessage: ReceivedChatMessage = {
+            ...parsedMessage,
+            type: 'chatMessage',
+            from: msg.from,
+          };
           return newMessage;
         }),
         filter((msg) => !!msg),
@@ -169,6 +171,7 @@ export function setupChat(room: Room, options?: ChatOptions) {
 
       const receivedChatMsg: ReceivedChatMessage = {
         ...chatMsg,
+        type: 'chatMessage',
         from: room.localParticipant,
         attributes: options.attributes,
       };

packages/core/src/components/textStream.ts

@@ -2,6 +2,7 @@ import { RoomEvent, type Room } from 'livekit-client';
 import type { TextStreamInfo } from 'livekit-client/dist/src/room/types';
 import { from, scan, Subject, type Observable } from 'rxjs';
 import { share, tap } from 'rxjs/operators';
+import { ParticipantAgentAttributes } from '../helper';
 
 export interface TextStreamData {
   text: string;
@@ -57,7 +58,7 @@ export function setupTextStream(room: Room, topic: string): Observable<TextStrea
   const textStreamsSubject = new Subject<TextStreamData[]>();
   let textStreams: TextStreamData[] = [];
 
-  const segmentAttribute = 'lk.segment_id';
+  const segmentAttribute = ParticipantAgentAttributes.TranscriptionSegmentId;
 
   // Create shared observable and store in cache
   const sharedObservable = textStreamsSubject.pipe(

packages/core/src/helper/index.ts

@@ -13,3 +13,4 @@ export {
 export { setDifference } from './set-helper';
 export { supportsScreenSharing } from './featureDetection';
 export * from './transcriptions';
+export * from './participant-attributes';

packages/core/src/helper/participant-attributes.ts

@@ -0,0 +1,9 @@
+/** An enum of first party livekit attributes generated by the serverside agents sdk */
+export enum ParticipantAgentAttributes {
+  AgentState = 'lk.agent.state',
+  PublishOnBehalf = 'lk.publish_on_behalf',
+
+  TranscriptionFinal = 'lk.transcription_final',
+  TranscriptionSegmentId = 'lk.segment_id',
+  TranscribedTrackId = 'lk.transcribed_track_id',
+}

packages/core/src/index.ts

@@ -4,6 +4,7 @@ export * from './helper';
 export * from './types';
 export * from './sorting';
 export * from './track-reference';
+export * from './messages';
 
 export * from './components/mediaToggle';
 export * from './components/mediaDeviceSelect';

packages/core/src/messages/index.ts

@@ -0,0 +1 @@
+export * from './types';

packages/core/src/messages/types.ts

@@ -0,0 +1,46 @@
+import type { Participant, ChatMessage } from 'livekit-client';
+
+/** @public */
+export type { ChatMessage };
+
+export type SentMessage = ChatMessage;
+
+type ReceivedMessageWithType<Type extends string, Metadata extends object = object> = {
+  id: string;
+  timestamp: number;
+
+  type: Type;
+
+  from?: Participant;
+  attributes?: Record<string, string>;
+} & Metadata;
+
+/** @public */
+export type ReceivedChatMessage = ReceivedMessageWithType<
+  'chatMessage',
+  ChatMessage & {
+    from?: Participant;
+    attributes?: Record<string, string>;
+  }
+>;
+
+export type ReceivedUserTranscriptionMessage = ReceivedMessageWithType<
+  'userTranscript',
+  {
+    message: string;
+  }
+>;
+
+export type ReceivedAgentTranscriptionMessage = ReceivedMessageWithType<
+  'agentTranscript',
+  {
+    message: string;
+  }
+>;
+
+/** @public */
+export type ReceivedMessage =
+  | ReceivedUserTranscriptionMessage
+  | ReceivedAgentTranscriptionMessage
+  | ReceivedChatMessage;
+// TODO: images? attachments? rpc?

packages/core/src/observables/dataChannel.ts

@@ -92,7 +92,12 @@ export function setupChatMessageHandler(room: Room) {
   const send = async (text: string, options: SendTextOptions): Promise<ReceivedChatMessage> => {
     const msg = await room.localParticipant.sendChatMessage(text, options);
     await room.localParticipant.sendText(text, options);
-    return { ...msg, from: room.localParticipant, attachedFiles: options.attachments };
+    return {
+      ...msg,
+      type: 'chatMessage',
+      from: room.localParticipant,
+      attachedFiles: options.attachments,
+    };
   };
 
   const edit = async (text: string, originalMsg: ChatMessage) => {